diff --git a/.changeset/alert-renotify-interval.md b/.changeset/alert-renotify-interval.md new file mode 100644 index 0000000000..9218a3d25f --- /dev/null +++ b/.changeset/alert-renotify-interval.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/app': minor +'@hyperdx/api': minor +--- + +feat: Alerts now notify once on the OK -> ALERT transition instead of on every evaluation while firing. A sustained breach used to send a duplicate notification on every evaluation interval; a new `renotifyIntervalMinutes` setting ("Re-notify every" under Advanced Settings) controls repeats instead — leave it unset for transition-only (the new default), set `0` to restore the old notify-on-every-evaluation behavior, or set N to re-notify every N minutes. Resolve notifications are unchanged. This is a behavior change for existing alerts: set `renotifyIntervalMinutes: 0` to keep the old behavior. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index f6ea5912f6..2f9cfdaa3d 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -419,6 +419,13 @@ "nullable": true, "description": "Fire the alert only after its condition has been met for this many consecutive evaluation windows. While the condition is met but fewer than this many consecutive windows have violated, the alert is in the PENDING state.", "example": 3 + }, + "renotifyIntervalMinutes": { + "type": "integer", + "minimum": 0, + "nullable": true, + "description": "How often to repeat the notification while the alert keeps firing. Omit or set to null to notify only once when the alert transitions from OK to ALERT, 0 to notify on every evaluation, or a positive number to re-notify at most once every that many minutes. A resolve notification is always sent when the alert returns to OK.", + "example": 60 } } }, diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index 0e8762e48c..65d00668a8 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -319,6 +319,7 @@ export const makeAlert = ( // Multi-window alerting numConsecutiveWindows: alert.numConsecutiveWindows ?? null, + renotifyIntervalMinutes: alert.renotifyIntervalMinutes ?? null, }; }; diff --git a/packages/api/src/mcp/tools/alerts/saveAlert.ts b/packages/api/src/mcp/tools/alerts/saveAlert.ts index 9c5962026d..4b358ee460 100644 --- a/packages/api/src/mcp/tools/alerts/saveAlert.ts +++ b/packages/api/src/mcp/tools/alerts/saveAlert.ts @@ -55,7 +55,8 @@ export function registerSaveAlert({ '(webhook IDs come from clickstack_get_webhook). Updates replace the ' + 'alert configuration rather than merging it: channels you do not resend are ' + 'removed, omitted name and message are cleared, omitted displayName and tags ' + - 'are re-derived from the source, and settings this tool cannot set ' + + 'are re-derived from the source, omitted renotifyIntervalMinutes resets ' + + 'to notify-on-transition-only, and settings this tool cannot set ' + '(note, consecutive-window count) are cleared. Omitting both ' + 'scheduleStartAt and scheduleOffsetMinutes keeps the stored schedule; ' + 'clearing scheduleStartAt without an offset resets the offset to 0. Read the alert with ' + @@ -112,6 +113,7 @@ export function registerSaveAlert({ thresholdMax: input.thresholdMax, scheduleOffsetMinutes: input.scheduleOffsetMinutes, scheduleStartAt: input.scheduleStartAt, + renotifyIntervalMinutes: input.renotifyIntervalMinutes, name: input.name, message: input.message, displayName: input.displayName, diff --git a/packages/api/src/mcp/tools/alerts/schemas.ts b/packages/api/src/mcp/tools/alerts/schemas.ts index 65503aa072..fe98e9bd59 100644 --- a/packages/api/src/mcp/tools/alerts/schemas.ts +++ b/packages/api/src/mcp/tools/alerts/schemas.ts @@ -233,6 +233,17 @@ export const mcpSaveAlertSchema = z.object({ 'boundary. Must be less than the interval. Ignored when ' + 'scheduleStartAt is set.', ), + renotifyIntervalMinutes: z + .number() + .int() + .min(0) + .nullish() + .describe( + 'How often to repeat the notification while the alert keeps firing. ' + + 'Omit or send null to notify only once on the OK to ALERT ' + + 'transition, 0 to notify on every evaluation, or N to re-notify at ' + + 'most once every N minutes. A resolve notification is always sent.', + ), scheduleStartAt: z .string() .datetime() diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index 3e62d19eab..e87392a450 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -127,6 +127,9 @@ export interface IAlert { // Multi-window alerting: fire only after N violations in M consecutive windows numConsecutiveWindows?: number | null; + // Re-notify while firing: null = transition only, 0 = every evaluation, N = every N minutes. + renotifyIntervalMinutes?: number | null; + // Errors recorded during the most recent execution executionErrors?: IAlertError[]; createdAt: Date; @@ -256,6 +259,11 @@ const AlertSchema = new Schema( required: false, min: 1, }, + renotifyIntervalMinutes: { + type: Number, + required: false, + min: 0, + }, silenced: { required: false, type: { diff --git a/packages/api/src/models/alertHistory.ts b/packages/api/src/models/alertHistory.ts index a0b9fbb3b3..91db410823 100644 --- a/packages/api/src/models/alertHistory.ts +++ b/packages/api/src/models/alertHistory.ts @@ -48,6 +48,7 @@ export interface IAlertHistory { errors?: IAlertError[]; /** Diagnostics for the evaluation that wrote this record. */ analytics?: IAlertHistoryAnalytics; + lastNotifiedAt?: Date; } const AlertHistorySchema = new Schema({ @@ -125,6 +126,10 @@ const AlertHistorySchema = new Schema({ required: false, default: undefined, }, + lastNotifiedAt: { + type: Date, + required: false, + }, }); AlertHistorySchema.index( diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts index 3085e3d764..09dea521f0 100644 --- a/packages/api/src/routers/api/alerts.ts +++ b/packages/api/src/routers/api/alerts.ts @@ -117,6 +117,7 @@ const formatAlertResponse = ( 'updatedAt', 'executionErrors', 'numConsecutiveWindows', + 'renotifyIntervalMinutes', ]), tileId: alert.tileId ?? undefined, groupBy: alert.groupBy ?? undefined, diff --git a/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts b/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts index 0d79f5ef43..d8e9f6440a 100644 --- a/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts @@ -481,6 +481,7 @@ describe('External API Alerts', () => { message: 'This is a test alert for format verification', note: null, numConsecutiveWindows: null, + renotifyIntervalMinutes: null, threshold: 123, interval: '15m', source: AlertSource.TILE, @@ -2725,6 +2726,37 @@ describe('External API Alerts', () => { expect(getResponse.body.data.numConsecutiveWindows).toBeNull(); }); }); + describe('Re-notification interval (renotifyIntervalMinutes)', () => { + it('should create an alert with renotifyIntervalMinutes and return it on GET', async () => { + const { alert } = await createTestAlert({ renotifyIntervalMinutes: 60 }); + expect(alert.renotifyIntervalMinutes).toBe(60); + + const getResponse = await authRequest( + 'get', + `${ALERTS_BASE_URL}/${alert.id}`, + ).expect(200); + expect(getResponse.body.data.renotifyIntervalMinutes).toBe(60); + }); + + it('should update an alert to set and then clear renotifyIntervalMinutes', async () => { + const { alert, alertInput } = await createTestAlert(); + expect(alert.renotifyIntervalMinutes).toBeNull(); + + const getRenotify = async () => + (await authRequest('get', `${ALERTS_BASE_URL}/${alert.id}`).expect(200)) + .body.data.renotifyIntervalMinutes; + + await authRequest('put', `${ALERTS_BASE_URL}/${alert.id}`) + .send({ ...alertInput, renotifyIntervalMinutes: 0 }) + .expect(200); + expect(await getRenotify()).toBe(0); + + await authRequest('put', `${ALERTS_BASE_URL}/${alert.id}`) + .send({ ...alertInput, renotifyIntervalMinutes: null }) + .expect(200); + expect(await getRenotify()).toBeNull(); + }); + }); describe('Multiple notification channels', () => { const makeWebhookNamed = async (name: string) => await Webhook.findOneAndUpdate( diff --git a/packages/api/src/routers/external-api/v2/alerts.ts b/packages/api/src/routers/external-api/v2/alerts.ts index 7a23df9d1a..e0f956f23f 100644 --- a/packages/api/src/routers/external-api/v2/alerts.ts +++ b/packages/api/src/routers/external-api/v2/alerts.ts @@ -328,6 +328,12 @@ function toInternalAlertInput(body: ExternalAlertInput): InternalAlertInput { * nullable: true * description: Fire the alert only after its condition has been met for this many consecutive evaluation windows. While the condition is met but fewer than this many consecutive windows have violated, the alert is in the PENDING state. * example: 3 + * renotifyIntervalMinutes: + * type: integer + * minimum: 0 + * nullable: true + * description: How often to repeat the notification while the alert keeps firing. Omit or set to null to notify only once when the alert transitions from OK to ALERT, 0 to notify on every evaluation, or a positive number to re-notify at most once every that many minutes. A resolve notification is always sent when the alert returns to OK. + * example: 60 * * AlertResponse: * allOf: diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 8ef7412f5c..bc89e5721e 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -2449,6 +2449,8 @@ describe('checkAlerts', () => { // check if webhook was triggered // We're only checking the general structure here since the exact text includes timestamps + // Transition + resolve only; the second ALERT window doesn't re-notify. + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); expect(slack.postMessageToWebhook).toHaveBeenNthCalledWith( 1, 'https://hooks.slack.com/services/123', @@ -2466,7 +2468,7 @@ describe('checkAlerts', () => { 2, 'https://hooks.slack.com/services/123', { - text: '🚨 Alert for "My Search" - 1 lines found', + text: '✅ Alert for "My Search" - 0 lines found', blocks: [ { text: expect.any(Object), @@ -6652,9 +6654,8 @@ describe('checkAlerts', () => { // Check webhook calls: // 1-2: First run alerts for service-a and service-b - // 3: Second run alert for service-a - // 4: Second run resolution notification for service-b - expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); + // 3: Second run resolution notification for service-b (service-a doesn't re-notify) + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); // Verify the resolution notification was sent for service-b const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; @@ -7211,9 +7212,8 @@ describe('checkAlerts', () => { // Check webhook calls: // 1-2: First run alerts for service-a and service-b - // 3: Second run alert for service-a (continues alerting) - // 4: Second run resolution notification for service-b - expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); + // 3: Second run resolution notification for service-b (service-a doesn't re-notify) + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); // Verify the resolution notification was sent for service-b const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; @@ -11450,6 +11450,609 @@ describe('checkAlerts', () => { expect(serviceBHistories[0].fired).toBeFalsy(); }); }); + + describe('re-notification (renotifyIntervalMinutes)', () => { + // Data in every 5m window, so the alert stays ALERT and never resolves. + const insertSustainedBreach = () => + bulkInsertLogs( + ['00:05:00', '00:10:00', '00:15:00', '00:20:00'].map(time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + const EVALUATION_TIMES = [ + '2024-01-01T00:12:00Z', + '2024-01-01T00:17:00Z', + '2024-01-01T00:22:00Z', + '2024-01-01T00:27:00Z', + ].map(t => new Date(t)); + + const runSustainedBreach = async ( + renotifyIntervalMinutes?: number | null, + ) => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + // threshold=1: ABOVE is inclusive, so 0 would breach empty windows too. + threshold: 1, + savedSearchId: savedSearch.id, + renotifyIntervalMinutes, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + await insertSustainedBreach(); + + const notificationCounts: number[] = []; + for (const now of EVALUATION_TIMES) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + notificationCounts.push( + jest.mocked(slack.postMessageToWebhook).mock.calls.length, + ); + } + + const histories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + + return { notificationCounts, histories }; + }; + + it('notifies once on the OK->ALERT transition by default', async () => { + const { notificationCounts, histories } = await runSustainedBreach(); + + expect(notificationCounts).toEqual([1, 1, 1, 1]); + expect(histories.map(h => h.state)).toEqual([ + 'ALERT', + 'ALERT', + 'ALERT', + 'ALERT', + ]); + // lastNotifiedAt stays pinned to the notifying window while silent. + expect(histories.map(h => h.lastNotifiedAt?.toISOString())).toEqual( + Array(4).fill('2024-01-01T00:10:00.000Z'), + ); + }); + + it('notifies on every evaluation when renotifyIntervalMinutes is 0', async () => { + const { notificationCounts, histories } = await runSustainedBreach(0); + + expect(notificationCounts).toEqual([1, 2, 3, 4]); + expect(histories.map(h => h.lastNotifiedAt?.toISOString())).toEqual([ + '2024-01-01T00:10:00.000Z', + '2024-01-01T00:15:00.000Z', + '2024-01-01T00:20:00.000Z', + '2024-01-01T00:25:00.000Z', + ]); + }); + + it('re-notifies only after renotifyIntervalMinutes has elapsed', async () => { + // 10m interval, 5m ticks: notify 00:10, skip 00:15, notify 00:20, skip 00:25. + const { notificationCounts, histories } = await runSustainedBreach(10); + + expect(notificationCounts).toEqual([1, 1, 2, 2]); + expect(histories.map(h => h.lastNotifiedAt?.toISOString())).toEqual([ + '2024-01-01T00:10:00.000Z', + '2024-01-01T00:10:00.000Z', + '2024-01-01T00:20:00.000Z', + '2024-01-01T00:20:00.000Z', + ]); + }); + + it('notifies again on a new OK->ALERT transition after resolving', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // Breach, then a quiet window, then breach again. + await bulkInsertLogs( + ['00:05:00', '00:15:00'].map(time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + for (const now of EVALUATION_TIMES.slice(0, 3)) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + const histories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + expect(histories.map(h => h.state)).toEqual(['ALERT', 'OK', 'ALERT']); + + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); + }); + + it('retries a failed first notification of a later incident', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // ALERT ok, resolve ok, second incident's ALERT fails, then ok. + // mockReset drops the Once value queued by the shared beforeEach. + jest + .spyOn(slack, 'postMessageToWebhook') + .mockReset() + .mockResolvedValueOnce({ text: 'ok' }) + .mockResolvedValueOnce({ text: 'ok' }) + .mockRejectedValueOnce(new Error('webhook down')) + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + await bulkInsertLogs( + ['00:05:00', '00:15:00', '00:20:00'].map(time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + for (const now of EVALUATION_TIMES) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); + const histories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + expect(histories.at(-1)?.lastNotifiedAt?.toISOString()).toBe( + '2024-01-01T00:25:00.000Z', + ); + }); + it('notifies raw SQL number alerts once on the OK->ALERT transition', async () => { + const { + team, + webhook, + connection, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const dashboard = await new Dashboard({ + name: 'Renotify Number Dashboard', + team: team._id, + tiles: [ + { + id: 'number1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'number', + sqlTemplate: RAW_SQL_NUMBER_ALERT_TEMPLATE, + connection: connection.id, + }, + }, + ], + }).save(); + const tile = dashboard.tiles?.find((t: any) => t.id === 'number1'); + if (!tile) throw new Error('tile not found'); + + const details = await createAlertDetails( + team, + undefined, + { + source: AlertSource.TILE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + dashboardId: dashboard.id, + tileId: 'number1', + }, + { taskType: AlertTaskType.TILE, tile, dashboard }, + ); + + await insertSustainedBreach(); + + const notificationCounts: number[] = []; + for (const now of EVALUATION_TIMES) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + notificationCounts.push( + jest.mocked(slack.postMessageToWebhook).mock.calls.length, + ); + } + + expect(notificationCounts).toEqual([1, 1, 1, 1]); + }); + + it('does not retry every evaluation when one channel is a deleted webhook', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channels: [ + { type: 'webhook', webhookId: webhook._id.toString() }, + { + type: 'webhook', + webhookId: new mongoose.Types.ObjectId().toString(), + }, + ], + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + await insertSustainedBreach(); + + for (const now of EVALUATION_TIMES) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1); + }); + + it('notifies a fresh incident that starts within a catch-up run', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // Breach at 00:05, below threshold at 00:10, breach again at 00:15. + await bulkInsertLogs( + ['00:05:00', '00:05:00', '00:10:00', '00:15:00', '00:15:00'].map( + time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + }), + ), + ); + + // The 00:17 evaluation is missed, so 00:22 covers both 00:10 and 00:15. + for (const now of [EVALUATION_TIMES[0], EVALUATION_TIMES[2]]) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); + }); + + it('notifies a fresh incident that also resolves within a catch-up run', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // Breach at 00:05, then OK / breach / OK at 00:10, 00:15, 00:20. + await bulkInsertLogs( + [ + '00:05:00', + '00:05:00', + '00:10:00', + '00:15:00', + '00:15:00', + '00:20:00', + ].map(time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + for (const now of [EVALUATION_TIMES[0], EVALUATION_TIMES[3]]) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + // First ALERT, then the catch-up run's ALERT and its resolve. + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); + }); + + it('notifies a group-by incident when the group was absent mid catch-up run', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 2, + savedSearchId: savedSearch.id, + groupBy: 'ServiceName', + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // api breaches at 00:05 and 00:15; at 00:10 only web (below threshold) has rows. + await bulkInsertLogs( + [ + ['api', '00:05:00'], + ['api', '00:05:00'], + ['web', '00:10:00'], + ['api', '00:15:00'], + ['api', '00:15:00'], + ].map(([service, time]) => ({ + ServiceName: service, + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + for (const now of [EVALUATION_TIMES[0], EVALUATION_TIMES[2]]) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); + }); + + it('tracks notifications per group for group-by alerts', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + groupBy: 'ServiceName', + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // api breaches in every window; web starts two windows later. + await insertSustainedBreach(); + await bulkInsertLogs( + ['00:15:00', '00:20:00'].map(time => ({ + ServiceName: 'web', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + const notificationCounts: number[] = []; + for (const now of EVALUATION_TIMES) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + notificationCounts.push( + jest.mocked(slack.postMessageToWebhook).mock.calls.length, + ); + } + + expect(notificationCounts).toEqual([1, 1, 2, 2]); + const webHistories = await AlertHistory.find({ + alert: details.alert.id, + group: 'ServiceName:web', + }).sort({ createdAt: 1 }); + expect(webHistories.map(h => h.lastNotifiedAt?.toISOString())).toEqual( + Array(2).fill('2024-01-01T00:20:00.000Z'), + ); + }); + }); }); describe('processAlert with materialized views', () => { diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 17a6b66c35..8bcda4135e 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -975,6 +975,15 @@ export const alertQueryAttribution = ( : {}), }); +// `fired` is missing on histories written before it existed, so only an +// explicit `false` means "never notified". +const hasUnresolvedNotification = ( + previousHistory: AggregatedAlertHistory | undefined, +): boolean => + (previousHistory?.state === AlertState.ALERT || + previousHistory?.state === AlertState.PENDING) && + previousHistory.fired !== false; + export const processAlert = async ( now: Date, details: AlertDetails, @@ -1283,6 +1292,13 @@ export const processAlert = async ( // Track state per group (or one history if no groupBy) const histories = new Map(); + // First OK and latest ALERT bucket per group; OK before ALERT means a new incident. + const firstOkBucket = new Map(); + const markOk = (groupKey: string, bucketIndex: number) => { + if (!firstOkBucket.has(groupKey)) + firstOkBucket.set(groupKey, bucketIndex); + }; + const lastAlertBucket = new Map(); const latestAlertContext = new Map< string, { value: number; attributes: Record; startTime: Date } @@ -1344,7 +1360,7 @@ export const processAlert = async ( group: string; startTime?: Date; attributes?: Record; - }) => { + }): Promise => { // KNOWN LIMITATION: Alert data (including silenced state) is fetched when // the task is queued via AlertProvider, not when it processes. If a user // silences an alert after it's queued but before it processes, this @@ -1360,7 +1376,7 @@ export const processAlert = async ( }, 'Skipped firing alert due to silence', ); - return; + return false; } alertEvaluationsCounter.add(1, { @@ -1422,6 +1438,8 @@ export const processAlert = async ( ); executionErrors.push(makeNotificationAlertError(failure)); } + // Only failed dispatches retry; unresolvable targets would retry forever. + return timings.every(t => t.ok); } catch (e) { // Render-level failures (title/link building, template compile) — // nothing was dispatched. @@ -1430,7 +1448,37 @@ export const processAlert = async ( 'Failed to fire channel event', ); executionErrors.push(makeWebhookAlertError(e)); + return false; + } + }; + + const shouldNotifyWhileFiring = ( + previousHistory: AggregatedAlertHistory | undefined, + ): boolean => { + if (!hasUnresolvedNotification(previousHistory)) { + return true; } + const { renotifyIntervalMinutes } = alert; + const lastNotifiedAt = previousHistory?.lastNotifiedAt; + const shouldNotify = + lastNotifiedAt == null || + (renotifyIntervalMinutes != null && + (renotifyIntervalMinutes <= 0 || + nowInMinsRoundDown.getTime() - lastNotifiedAt.getTime() >= + renotifyIntervalMinutes * 60_000)); + + if (!shouldNotify) { + alertEvaluationsCounter.add(1, { outcome: 'skipped_renotify' }); + logger.debug( + { + alertId: alert.id, + lastNotifiedAt, + renotifyIntervalMinutes, + }, + 'Skipped re-notifying an already-notified alert', + ); + } + return shouldNotify; }; const numWindowsToLookBack = alert.numConsecutiveWindows ?? 1; @@ -1462,9 +1510,7 @@ export const processAlert = async ( groupKey: string, ) => { if ( - (previousHistory?.state === AlertState.ALERT || - previousHistory?.state === AlertState.PENDING) && - previousHistory?.fired !== false && + hasUnresolvedNotification(previousHistory) && currentHistory.state === AlertState.OK ) { const lastValue = @@ -1502,17 +1548,27 @@ export const processAlert = async ( history.lastValues.push({ count: value, startTime: alertTimestamp }); const previous = previousMap.get(computeHistoryMapKey(alert.id, '')); + // Carry only within an unresolved incident, so a later incident's failed + // first send is retried. + history.lastNotifiedAt = hasUnresolvedNotification(previous) + ? previous?.lastNotifiedAt + : undefined; if (doesExceedThreshold(alert, value)) { history.counts += 1; if (shouldFireBasedOnConsecutiveWindows()) { history.state = AlertState.ALERT; history.fired = true; - await trySendNotification({ - state: AlertState.ALERT, - group: '', - totalCount: value, - startTime: alertTimestamp, - }); + if ( + shouldNotifyWhileFiring(previous) && + (await trySendNotification({ + state: AlertState.ALERT, + group: '', + totalCount: value, + startTime: alertTimestamp, + })) + ) { + history.lastNotifiedAt = nowInMinsRoundDown; + } } else { history.state = AlertState.PENDING; // Carry forward fired=true if a notification was previously sent and not yet resolved. @@ -1567,7 +1623,18 @@ export const processAlert = async ( checkDataByBucket.get(bucketStart.getTime())!.push(checkData); } - for (const bucketStart of expectedBuckets) { + // An absent group is OK unless a zero value breaches. + const previousGroups = Array.from(previousMap.keys(), key => + extractGroupKeyFromMapKey(key, alert.id), + ); + const markAbsentGroupsOk = (bucketIndex: number, present: Set) => { + if (!hasGroupBy || doesExceedThreshold(alert, 0)) return; + for (const groupKey of [...previousGroups, ...histories.keys()]) { + if (!present.has(groupKey)) markOk(groupKey, bucketIndex); + } + }; + + for (const [bucketIndex, bucketStart] of expectedBuckets.entries()) { const dataForBucket = checkDataByBucket.get(bucketStart.getTime()); // Handle case where no data is available for this bucket @@ -1596,6 +1663,7 @@ export const processAlert = async ( if (shouldFireBasedOnConsecutiveWindows()) { history.state = AlertState.ALERT; history.fired = true; + lastAlertBucket.set('', bucketIndex); latestAlertContext.set('', { value: 0, attributes: {}, @@ -1614,6 +1682,13 @@ export const processAlert = async ( const history = getOrCreateHistory(''); history.lastValues.push({ count: 0, startTime: bucketStart }); } + if (!zeroValueIsAlert) { + if (hasGroupBy) { + markAbsentGroupsOk(bucketIndex, new Set()); + } else { + markOk('', bucketIndex); + } + } continue; } @@ -1625,9 +1700,15 @@ export const processAlert = async ( string, { value: number; attributes: Record; exceeds: boolean } >(); + const bucketGroups = new Set(); for (const checkData of dataForBucket) { const { value, extraFields } = parseAlertData(checkData, meta); + const groupKey = hasGroupBy + ? extraFields.map(([k, v]) => `${k}:${v}`).join(', ') + : ''; + bucketGroups.add(groupKey); + // NULL means no data: a metric series with no row at this bucket, or // a ratio with a missing/zero denominator. Skip the row instead of // fabricating a state from a gap. @@ -1635,9 +1716,6 @@ export const processAlert = async ( continue; } - const groupKey = hasGroupBy - ? extraFields.map(([k, v]) => `${k}:${v}`).join(', ') - : ''; const attributes = hasGroupBy ? Object.fromEntries(extraFields) : {}; groupAttributes.set(groupKey, attributes); @@ -1649,6 +1727,8 @@ export const processAlert = async ( } } + markAbsentGroupsOk(bucketIndex, bucketGroups); + for (const [groupKey, evaluation] of bucketEvaluations.entries()) { const history = getOrCreateHistory(groupKey); @@ -1657,6 +1737,7 @@ export const processAlert = async ( if (shouldFireBasedOnConsecutiveWindows(groupKey)) { history.state = AlertState.ALERT; history.fired = true; + lastAlertBucket.set(groupKey, bucketIndex); latestAlertContext.set(groupKey, { value: evaluation.value, attributes: evaluation.attributes, @@ -1675,6 +1756,7 @@ export const processAlert = async ( // a subsequent OK window correctly resolves it before the notification phase. history.state = AlertState.OK; history.counts = 0; + markOk(groupKey, bucketIndex); } history.lastValues.push({ count: evaluation.value, @@ -1718,21 +1800,33 @@ export const processAlert = async ( // Check for state transitions and send notifications for (const [groupKey, history] of histories.entries()) { const previousKey = computeHistoryMapKey(alert.id, groupKey); - let groupPrevious = previousMap.get(previousKey); + const isNewIncidentInRun = + (firstOkBucket.get(groupKey) ?? Infinity) < + (lastAlertBucket.get(groupKey) ?? -Infinity); + let groupPrevious = isNewIncidentInRun + ? undefined + : previousMap.get(previousKey); const hitAlertThisRun = latestAlertContext.has(groupKey); + history.lastNotifiedAt = hasUnresolvedNotification(groupPrevious) + ? groupPrevious?.lastNotifiedAt + : undefined; - // If it hit ALERT during this run, send the notification (re-notifying every tick if it continuously breaches) + // If it hit ALERT during this run, send the notification if (hitAlertThisRun) { const context = latestAlertContext.get(groupKey); - if (context) { - await trySendNotification({ - state: AlertState.ALERT, - group: groupKey, - totalCount: context.value, - startTime: context.startTime, - attributes: context.attributes, - }); + if (context && shouldNotifyWhileFiring(groupPrevious)) { + if ( + await trySendNotification({ + state: AlertState.ALERT, + group: groupKey, + totalCount: context.value, + startTime: context.startTime, + attributes: context.attributes, + }) + ) { + history.lastNotifiedAt = nowInMinsRoundDown; + } // Inject a mock previous history so the resolve check below catches it // if the final state for this group is OK (i.e. it breached then resolved). @@ -1822,6 +1916,7 @@ export interface AggregatedAlertHistory { state: AlertState; group?: string; fired?: boolean; + lastNotifiedAt?: Date; } /** @@ -1883,6 +1978,7 @@ export const getPreviousAlertHistories = async ( createdAt: { $first: '$createdAt' }, state: { $first: '$state' }, fired: { $first: '$fired' }, + lastNotifiedAt: { $first: '$lastNotifiedAt' }, }, }, { @@ -1892,6 +1988,7 @@ export const getPreviousAlertHistories = async ( state: 1, group: '$_id.group', fired: 1, + lastNotifiedAt: 1, }, }, ]); diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 187abd7a24..63a573c165 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -337,6 +337,7 @@ export type ExternalAlert = { scheduleOffsetMinutes?: number; scheduleStartAt?: string | null; numConsecutiveWindows?: number | null; + renotifyIntervalMinutes?: number | null; thresholdType: AlertThresholdType; source?: string; state: AlertState; @@ -510,6 +511,7 @@ export function translateAlertDocumentToExternalAlert( }), scheduleStartAt: transformScheduleStartAt(alertObj.scheduleStartAt), numConsecutiveWindows: alertObj.numConsecutiveWindows ?? null, + renotifyIntervalMinutes: alertObj.renotifyIntervalMinutes ?? null, thresholdType: alertObj.thresholdType, source: alertObj.source, state: alertObj.state, diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 8ec394afdf..c970e2540d 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -1223,6 +1223,7 @@ const alertBaseSchema = z.object({ displayName: alertDisplayNameSchema, tags: alertTagsSchema, numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }); // External v2 alert schema. Inline alerts carry their chart config in the diff --git a/packages/app/src/DBSearchPageAlertModal.tsx b/packages/app/src/DBSearchPageAlertModal.tsx index ab748a0312..c4c9b0b2a1 100644 --- a/packages/app/src/DBSearchPageAlertModal.tsx +++ b/packages/app/src/DBSearchPageAlertModal.tsx @@ -90,6 +90,7 @@ const SavedSearchAlertFormSchema = z // nullish() (not optional()): persisted alerts store this as null, which // optional() would reject. numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }) .passthrough() .superRefine(validateAlertScheduleOffsetMinutes) @@ -144,6 +145,8 @@ const AlertForm = ({ // Persisted null -> undefined for the NumberInput. numConsecutiveWindows: defaultValues.numConsecutiveWindows ?? undefined, + renotifyIntervalMinutes: + defaultValues.renotifyIntervalMinutes ?? undefined, } : { interval: '5m', @@ -174,6 +177,10 @@ const AlertForm = ({ control, name: 'numConsecutiveWindows', }); + const renotifyIntervalMinutes = useWatch({ + control, + name: 'renotifyIntervalMinutes', + }); const maxScheduleOffsetMinutes = Math.max( intervalToMinutes(interval ?? '5m') - 1, 0, @@ -308,6 +315,8 @@ const AlertForm = ({ offsetWindowLabel={`from each ${intervalLabel} window`} numConsecutiveWindowsName="numConsecutiveWindows" numConsecutiveWindows={numConsecutiveWindows ?? undefined} + renotifyIntervalName="renotifyIntervalMinutes" + renotifyIntervalMinutes={renotifyIntervalMinutes ?? undefined} /> grouped by diff --git a/packages/app/src/components/AlertScheduleFields.tsx b/packages/app/src/components/AlertScheduleFields.tsx index 774baca58a..3d4b2fd542 100644 --- a/packages/app/src/components/AlertScheduleFields.tsx +++ b/packages/app/src/components/AlertScheduleFields.tsx @@ -38,6 +38,8 @@ type AlertScheduleFieldsProps = { offsetWindowLabel: string; numConsecutiveWindowsName?: FieldPath; numConsecutiveWindows?: number; + renotifyIntervalName?: FieldPath; + renotifyIntervalMinutes?: number; }; export function AlertScheduleFields({ @@ -50,6 +52,8 @@ export function AlertScheduleFields({ offsetWindowLabel, numConsecutiveWindowsName, numConsecutiveWindows, + renotifyIntervalName, + renotifyIntervalMinutes, }: AlertScheduleFieldsProps) { const showScheduleOffsetInput = maxScheduleOffsetMinutes > 0; const scheduleStartAtValue = useWatch({ @@ -60,7 +64,8 @@ export function AlertScheduleFields({ const hasAdvancedScheduleValues = (scheduleOffsetMinutes ?? 0) > 0 || hasScheduleStartAtAnchor || - (numConsecutiveWindows ?? 1) > 1; + (numConsecutiveWindows ?? 1) > 1 || + renotifyIntervalMinutes != null; const [opened, setOpened] = useState(hasAdvancedScheduleValues); useEffect(() => { @@ -149,6 +154,48 @@ export function AlertScheduleFields({ )} + {renotifyIntervalName && ( + + + + Re-notify every + + + + + + + + ( + + field.onChange(typeof v === 'number' ? v : undefined) + } + min={0} + allowDecimal={false} + allowNegative={false} + placeholder="Never" + size="xs" + w={90} + /> + )} + /> + + minutes + + + )} {showScheduleOffsetInput && ( <> diff --git a/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx b/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx index 5b3c225464..28dafe87a8 100644 --- a/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx @@ -97,6 +97,10 @@ export function TileAlertEditor({ control, name: 'alert.numConsecutiveWindows', }); + const alertRenotifyIntervalMinutes = useWatch({ + control, + name: 'alert.renotifyIntervalMinutes', + }); const maxAlertScheduleOffsetMinutes = alert?.interval ? Math.max(intervalToMinutes(alert.interval) - 1, 0) : 0; @@ -292,6 +296,8 @@ export function TileAlertEditor({ } numConsecutiveWindowsName="alert.numConsecutiveWindows" numConsecutiveWindows={alertnumConsecutiveWindows ?? undefined} + renotifyIntervalName="alert.renotifyIntervalMinutes" + renotifyIntervalMinutes={alertRenotifyIntervalMinutes ?? undefined} /> Send to diff --git a/packages/app/src/components/alerts/AlertPropertiesSummary.tsx b/packages/app/src/components/alerts/AlertPropertiesSummary.tsx index 233b082ebc..947c8686cf 100644 --- a/packages/app/src/components/alerts/AlertPropertiesSummary.tsx +++ b/packages/app/src/components/alerts/AlertPropertiesSummary.tsx @@ -204,6 +204,16 @@ export function AlertPropertiesSummary({ )} + {alert.renotifyIntervalMinutes != null && ( + <> + · + + {alert.renotifyIntervalMinutes === 0 + ? 'Notifies on every evaluation' + : `Re-notifies every ${alert.renotifyIntervalMinutes} min while firing`} + + + )} )} · diff --git a/packages/app/src/components/alerts/EditAlertModal.tsx b/packages/app/src/components/alerts/EditAlertModal.tsx index 2d670aa2cd..9fe5a16ba8 100644 --- a/packages/app/src/components/alerts/EditAlertModal.tsx +++ b/packages/app/src/components/alerts/EditAlertModal.tsx @@ -77,6 +77,7 @@ const EditAlertFormSchema = z // nullish() (not optional()): persisted alerts store this as null, which // optional() would reject. numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }) .passthrough() .superRefine(validateAlertChannelSelection) @@ -115,6 +116,7 @@ function alertToFormValues(alert: AlertsPageItem): Alert { tags: alert.tags, // Persisted null -> undefined for the NumberInput. numConsecutiveWindows: alert.numConsecutiveWindows ?? undefined, + renotifyIntervalMinutes: alert.renotifyIntervalMinutes ?? undefined, }; if (alert.source === AlertSource.TILE) { @@ -216,6 +218,10 @@ export function EditAlertModal({ control, name: 'numConsecutiveWindows', }); + const renotifyIntervalMinutes = useWatch({ + control, + name: 'renotifyIntervalMinutes', + }); const maxScheduleOffsetMinutes = Math.max( intervalToMinutes(interval ?? '5m') - 1, @@ -419,6 +425,8 @@ export function EditAlertModal({ offsetWindowLabel={`from each ${intervalLabel} window`} numConsecutiveWindowsName="numConsecutiveWindows" numConsecutiveWindows={numConsecutiveWindows ?? undefined} + renotifyIntervalName="renotifyIntervalMinutes" + renotifyIntervalMinutes={renotifyIntervalMinutes ?? undefined} /> {!isChartAlert && ( <> diff --git a/packages/app/src/components/alerts/EditInlineAlertModal.tsx b/packages/app/src/components/alerts/EditInlineAlertModal.tsx index 12aecdd0cb..4cc88a1c2a 100644 --- a/packages/app/src/components/alerts/EditInlineAlertModal.tsx +++ b/packages/app/src/components/alerts/EditInlineAlertModal.tsx @@ -56,6 +56,7 @@ export function inlineAlertToChartConfig( note: alert.note ?? null, // Persisted null -> undefined for the NumberInput. numConsecutiveWindows: alert.numConsecutiveWindows ?? undefined, + renotifyIntervalMinutes: alert.renotifyIntervalMinutes ?? undefined, }, }; } diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index f1c006a299..985d9400a5 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1126,6 +1126,7 @@ export const AlertBaseObjectSchema = z.object({ }) .optional(), numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }); // Keep AlertBaseSchema as a ZodObject for backwards compatibility with @@ -3099,6 +3100,7 @@ export const AlertsPageItemSchema = z.object({ .optional(), executionErrors: z.array(AlertErrorSchema).optional(), numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }); export type AlertsPageItem = z.infer;