From bf34c4bd8fd4160e14b55a76e41e43d5b6343c7f Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Thu, 30 Jul 2026 11:54:35 +0530 Subject: [PATCH 01/11] feat(alerts): add re-notify interval, stop notifying on every evaluation Signed-off-by: Niladri Adhikary --- .changeset/alert-renotify-interval.md | 7 + packages/api/openapi.json | 7 + packages/api/src/controllers/alerts.ts | 3 + packages/api/src/models/alert.ts | 8 + packages/api/src/models/alertHistory.ts | 5 + packages/api/src/routers/api/alerts.ts | 1 + .../api/src/routers/external-api/v2/alerts.ts | 6 + .../__tests__/checkAlerts.int.test.ts | 199 +++++++++++++++++- packages/api/src/tasks/checkAlerts/index.ts | 93 ++++++-- packages/api/src/utils/externalApi.ts | 2 + packages/api/src/utils/zod.ts | 1 + packages/app/src/DBSearchPageAlertModal.tsx | 9 + .../src/components/AlertScheduleFields.tsx | 47 ++++- .../DBEditTimeChartForm/TileAlertEditor.tsx | 6 + packages/common-utils/src/types.ts | 2 + 15 files changed, 369 insertions(+), 27 deletions(-) create mode 100644 .changeset/alert-renotify-interval.md diff --git a/.changeset/alert-renotify-interval.md b/.changeset/alert-renotify-interval.md new file mode 100644 index 0000000000..b8ddc6e1bd --- /dev/null +++ b/.changeset/alert-renotify-interval.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/app': minor +'@hyperdx/api': minor +--- + +feat: alerts now notify once on the OK -> ALERT transition instead of on every evaluation while firing. A sustained breach used to send a duplicate notification on every evaluation interval; a new `renotifyIntervalMinutes` setting ("Re-notify every" under Advanced Settings) controls repeats instead — leave it unset for transition-only (the new default), set `0` to restore the old notify-on-every-evaluation behavior, or set N to re-notify every N minutes. Resolve notifications are unchanged. This is a behavior change for existing alerts: set `renotifyIntervalMinutes: 0` to keep the old behavior. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 4c076f92fa..11a0e2b7c5 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -300,6 +300,13 @@ "nullable": true, "description": "Fire the alert only after its condition has been met for this many consecutive evaluation windows. While the condition is met but fewer than this many consecutive windows have violated, the alert is in the PENDING state.", "example": 3 + }, + "renotifyIntervalMinutes": { + "type": "integer", + "minimum": 0, + "nullable": true, + "description": "How often to repeat the notification while the alert keeps firing. Omit or set to null to notify only once when the alert transitions from OK to ALERT, 0 to notify on every evaluation, or a positive number to re-notify at most once every that many minutes. A resolve notification is always sent when the alert returns to OK.", + "example": 60 } } }, diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index ff430657c9..87cb7cad99 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -158,6 +158,9 @@ const makeAlert = (alert: AlertInput, userId?: ObjectId): Partial => { // Multi-window alerting numConsecutiveWindows: alert.numConsecutiveWindows ?? null, + + // Re-notification while firing + renotifyIntervalMinutes: alert.renotifyIntervalMinutes ?? null, }; }; diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index f84bc30020..ed1bd32574 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -88,6 +88,9 @@ export interface IAlert { // Multi-window alerting: fire only after N violations in M consecutive windows numConsecutiveWindows?: number | null; + // Re-notify while firing: null = transition only, 0 = every evaluation, N = every N minutes. + renotifyIntervalMinutes?: number | null; + // Errors recorded during the most recent execution executionErrors?: IAlertError[]; createdAt: Date; @@ -199,6 +202,11 @@ const AlertSchema = new Schema( required: false, min: 1, }, + renotifyIntervalMinutes: { + type: Number, + required: false, + min: 0, + }, silenced: { required: false, type: { diff --git a/packages/api/src/models/alertHistory.ts b/packages/api/src/models/alertHistory.ts index 5cf8630f81..97d29279e6 100644 --- a/packages/api/src/models/alertHistory.ts +++ b/packages/api/src/models/alertHistory.ts @@ -13,6 +13,7 @@ export interface IAlertHistory { lastValues: { startTime: Date; count: number }[]; group?: string; // For group-by alerts, stores the group identifier fired?: boolean; + lastNotifiedAt?: Date; } const AlertHistorySchema = new Schema({ @@ -50,6 +51,10 @@ const AlertHistorySchema = new Schema({ type: Boolean, required: false, }, + lastNotifiedAt: { + type: Date, + required: false, + }, }); AlertHistorySchema.index( diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts index 1ffe35bbb4..d5da470c2d 100644 --- a/packages/api/src/routers/api/alerts.ts +++ b/packages/api/src/routers/api/alerts.ts @@ -87,6 +87,7 @@ const formatAlertResponse = ( 'updatedAt', 'executionErrors', 'numConsecutiveWindows', + 'renotifyIntervalMinutes', ]), }; }; diff --git a/packages/api/src/routers/external-api/v2/alerts.ts b/packages/api/src/routers/external-api/v2/alerts.ts index e53aedf4bc..38a5a8c996 100644 --- a/packages/api/src/routers/external-api/v2/alerts.ts +++ b/packages/api/src/routers/external-api/v2/alerts.ts @@ -198,6 +198,12 @@ import { alertSchema, objectIdSchema } from '@/utils/zod'; * nullable: true * description: Fire the alert only after its condition has been met for this many consecutive evaluation windows. While the condition is met but fewer than this many consecutive windows have violated, the alert is in the PENDING state. * example: 3 + * renotifyIntervalMinutes: + * type: integer + * minimum: 0 + * nullable: true + * description: How often to repeat the notification while the alert keeps firing. Omit or set to null to notify only once when the alert transitions from OK to ALERT, 0 to notify on every evaluation, or a positive number to re-notify at most once every that many minutes. A resolve notification is always sent when the alert returns to OK. + * example: 60 * * AlertResponse: * allOf: diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 0e03d17cdb..b7c17873d3 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -2380,6 +2380,8 @@ describe('checkAlerts', () => { // check if webhook was triggered // We're only checking the general structure here since the exact text includes timestamps + // Transition + resolve only; the second ALERT window doesn't re-notify. + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); expect(slack.postMessageToWebhook).toHaveBeenNthCalledWith( 1, 'https://hooks.slack.com/services/123', @@ -2397,7 +2399,7 @@ describe('checkAlerts', () => { 2, 'https://hooks.slack.com/services/123', { - text: '🚨 Alert for "My Search" - 1 lines found', + text: '✅ Alert for "My Search" - 0 lines found', blocks: [ { text: expect.any(Object), @@ -5130,9 +5132,8 @@ describe('checkAlerts', () => { // Check webhook calls: // 1-2: First run alerts for service-a and service-b - // 3: Second run alert for service-a - // 4: Second run resolution notification for service-b - expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); + // 3: Second run resolution notification for service-b (service-a doesn't re-notify) + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); // Verify the resolution notification was sent for service-b const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; @@ -5606,9 +5607,8 @@ describe('checkAlerts', () => { // Check webhook calls: // 1-2: First run alerts for service-a and service-b - // 3: Second run alert for service-a (continues alerting) - // 4: Second run resolution notification for service-b - expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); + // 3: Second run resolution notification for service-b (service-a doesn't re-notify) + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); // Verify the resolution notification was sent for service-b const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; @@ -9096,6 +9096,191 @@ describe('checkAlerts', () => { expect(serviceBHistories[0].fired).toBeFalsy(); }); }); + + describe('re-notification (renotifyIntervalMinutes)', () => { + // A sustained breach across four consecutive 5m windows: every evaluation + // below sees data in its own window, so the alert stays in ALERT + // throughout without ever returning to OK. + const insertSustainedBreach = () => + bulkInsertLogs( + ['00:05:00', '00:10:00', '00:15:00', '00:20:00'].map(time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + const EVALUATION_TIMES = [ + '2024-01-01T00:12:00Z', + '2024-01-01T00:17:00Z', + '2024-01-01T00:22:00Z', + '2024-01-01T00:27:00Z', + ].map(t => new Date(t)); + + const runSustainedBreach = async ( + renotifyIntervalMinutes?: number | null, + ) => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + // threshold=1 (not 0): ABOVE is inclusive, so threshold=0 would make + // an empty window breach too and the alert could never resolve. + threshold: 1, + savedSearchId: savedSearch.id, + renotifyIntervalMinutes, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + await insertSustainedBreach(); + + // Notification count observed after each successive evaluation. + const notificationCounts: number[] = []; + for (const now of EVALUATION_TIMES) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + notificationCounts.push( + jest.mocked(slack.postMessageToWebhook).mock.calls.length, + ); + } + + const histories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + + return { notificationCounts, histories }; + }; + + it('notifies once on the OK->ALERT transition by default', async () => { + const { notificationCounts, histories } = await runSustainedBreach(); + + // One notification on the first breach, silence for the rest. + expect(notificationCounts).toEqual([1, 1, 1, 1]); + expect(histories.map(h => h.state)).toEqual([ + 'ALERT', + 'ALERT', + 'ALERT', + 'ALERT', + ]); + // lastNotifiedAt is pinned to the window that notified and carried + // forward unchanged across the silent evaluations. + expect(histories.map(h => h.lastNotifiedAt?.toISOString())).toEqual( + Array(4).fill('2024-01-01T00:10:00.000Z'), + ); + }); + + it('notifies on every evaluation when renotifyIntervalMinutes is 0', async () => { + const { notificationCounts, histories } = await runSustainedBreach(0); + + expect(notificationCounts).toEqual([1, 2, 3, 4]); + expect(histories.map(h => h.lastNotifiedAt?.toISOString())).toEqual([ + '2024-01-01T00:10:00.000Z', + '2024-01-01T00:15:00.000Z', + '2024-01-01T00:20:00.000Z', + '2024-01-01T00:25:00.000Z', + ]); + }); + + it('re-notifies only after renotifyIntervalMinutes has elapsed', async () => { + // 10m re-notification on a 5m interval: notify at 00:10, skip 00:15, + // re-notify at 00:20, skip 00:25. + const { notificationCounts, histories } = await runSustainedBreach(10); + + expect(notificationCounts).toEqual([1, 1, 2, 2]); + expect(histories.map(h => h.lastNotifiedAt?.toISOString())).toEqual([ + '2024-01-01T00:10:00.000Z', + '2024-01-01T00:10:00.000Z', + '2024-01-01T00:20:00.000Z', + '2024-01-01T00:20:00.000Z', + ]); + }); + + it('notifies again on a new OK->ALERT transition after resolving', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // Breach, then a quiet window, then breach again. + await bulkInsertLogs( + ['00:05:00', '00:15:00'].map(time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + for (const now of EVALUATION_TIMES.slice(0, 3)) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + const histories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + expect(histories.map(h => h.state)).toEqual(['ALERT', 'OK', 'ALERT']); + + // ALERT, resolved OK, then ALERT again: the second firing episode is a + // fresh transition, so it notifies even with re-notification off. + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); + }); + }); }); describe('processAlert with materialized views', () => { diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index a5259b39de..654bf30477 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -773,6 +773,18 @@ export const parseAlertData = ( return { value, extraFields }; }; +/** + * Whether the group already has a notification that was sent and has not been + * resolved yet. `fired` is absent on histories written before it was + * introduced, so only an explicit `false` counts as "never notified". + */ +const hasUnresolvedNotification = ( + previousHistory: AggregatedAlertHistory | undefined, +): boolean => + (previousHistory?.state === AlertState.ALERT || + previousHistory?.state === AlertState.PENDING) && + previousHistory.fired !== false; + export const processAlert = async ( now: Date, details: AlertDetails, @@ -1057,7 +1069,7 @@ export const processAlert = async ( }, 'Skipped firing alert due to silence', ); - return; + return false; } alertEvaluationsCounter.add(1, { @@ -1100,6 +1112,37 @@ export const processAlert = async ( ); executionErrors.push(makeWebhookAlertError(e)); } + return true; + }; + + // Gates re-notification of an already-firing group by renotifyIntervalMinutes. + const shouldNotifyWhileFiring = ( + previousHistory: AggregatedAlertHistory | undefined, + ): boolean => { + if (!hasUnresolvedNotification(previousHistory)) { + return true; + } + const { renotifyIntervalMinutes } = alert; + const lastNotifiedAt = previousHistory?.lastNotifiedAt; + const shouldNotify = + renotifyIntervalMinutes != null && + (renotifyIntervalMinutes <= 0 || + lastNotifiedAt == null || + nowInMinsRoundDown.getTime() - lastNotifiedAt.getTime() >= + renotifyIntervalMinutes * 60_000); + + if (!shouldNotify) { + alertEvaluationsCounter.add(1, { outcome: 'skipped_renotify' }); + logger.debug( + { + alertId: alert.id, + lastNotifiedAt, + renotifyIntervalMinutes, + }, + 'Skipped re-notifying an already-notified alert', + ); + } + return shouldNotify; }; const numWindowsToLookBack = alert.numConsecutiveWindows ?? 1; @@ -1131,9 +1174,7 @@ export const processAlert = async ( groupKey: string, ) => { if ( - (previousHistory?.state === AlertState.ALERT || - previousHistory?.state === AlertState.PENDING) && - previousHistory?.fired !== false && + hasUnresolvedNotification(previousHistory) && currentHistory.state === AlertState.OK ) { const lastValue = @@ -1170,17 +1211,23 @@ export const processAlert = async ( history.lastValues.push({ count: value, startTime: alertTimestamp }); const previous = previousMap.get(computeHistoryMapKey(alert.id, '')); + history.lastNotifiedAt = previous?.lastNotifiedAt; if (doesExceedThreshold(alert, value)) { history.counts += 1; if (shouldFireBasedOnConsecutiveWindows()) { history.state = AlertState.ALERT; history.fired = true; - await trySendNotification({ - state: AlertState.ALERT, - group: '', - totalCount: value, - startTime: alertTimestamp, - }); + if ( + shouldNotifyWhileFiring(previous) && + (await trySendNotification({ + state: AlertState.ALERT, + group: '', + totalCount: value, + startTime: alertTimestamp, + })) + ) { + history.lastNotifiedAt = nowInMinsRoundDown; + } } else { history.state = AlertState.PENDING; // Carry forward fired=true if a notification was previously sent and not yet resolved. @@ -1376,18 +1423,23 @@ export const processAlert = async ( const hitAlertThisRun = latestAlertContext.has(groupKey); const wasAlertingBefore = groupPrevious?.state === AlertState.ALERT; + history.lastNotifiedAt = groupPrevious?.lastNotifiedAt; - // If it hit ALERT during this run, send the notification (re-notifying every tick if it continuously breaches) + // If it hit ALERT during this run, send the notification if (hitAlertThisRun) { const context = latestAlertContext.get(groupKey); - if (context) { - await trySendNotification({ - state: AlertState.ALERT, - group: groupKey, - totalCount: context.value, - startTime: context.startTime, - attributes: context.attributes, - }); + if (context && shouldNotifyWhileFiring(groupPrevious)) { + if ( + await trySendNotification({ + state: AlertState.ALERT, + group: groupKey, + totalCount: context.value, + startTime: context.startTime, + attributes: context.attributes, + }) + ) { + history.lastNotifiedAt = nowInMinsRoundDown; + } // Inject a mock previous history so the resolve check below catches it // if the final state for this group is OK (i.e. it breached then resolved). @@ -1464,6 +1516,7 @@ export interface AggregatedAlertHistory { state: AlertState; group?: string; fired?: boolean; + lastNotifiedAt?: Date; } /** @@ -1521,6 +1574,7 @@ export const getPreviousAlertHistories = async ( createdAt: { $first: '$createdAt' }, state: { $first: '$state' }, fired: { $first: '$fired' }, + lastNotifiedAt: { $first: '$lastNotifiedAt' }, }, }, { @@ -1530,6 +1584,7 @@ export const getPreviousAlertHistories = async ( state: 1, group: '$_id.group', fired: 1, + lastNotifiedAt: 1, }, }, ]); diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index c25e76cbcc..5ef9f269ec 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -238,6 +238,7 @@ export type ExternalAlert = { scheduleOffsetMinutes?: number; scheduleStartAt?: string | null; numConsecutiveWindows?: number | null; + renotifyIntervalMinutes?: number | null; thresholdType: AlertThresholdType; source?: string; state: AlertState; @@ -340,6 +341,7 @@ export function translateAlertDocumentToExternalAlert( }), scheduleStartAt: transformScheduleStartAt(alertObj.scheduleStartAt), numConsecutiveWindows: alertObj.numConsecutiveWindows ?? null, + renotifyIntervalMinutes: alertObj.renotifyIntervalMinutes ?? null, thresholdType: alertObj.thresholdType, source: alertObj.source, state: alertObj.state, diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 42f11afeab..f8a77be31e 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -672,6 +672,7 @@ export const alertSchema = z message: z.string().min(1).max(4096).nullish(), note: alertNoteSchema, numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }) .and(zSavedSearchAlert.or(zTileAlert)) .superRefine(validateAlertScheduleOffsetMinutes) diff --git a/packages/app/src/DBSearchPageAlertModal.tsx b/packages/app/src/DBSearchPageAlertModal.tsx index 34c994fce5..4562dd417d 100644 --- a/packages/app/src/DBSearchPageAlertModal.tsx +++ b/packages/app/src/DBSearchPageAlertModal.tsx @@ -80,6 +80,7 @@ const SavedSearchAlertFormSchema = z // nullish() (not optional()): persisted alerts store this as null, which // optional() would reject. numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }) .passthrough() .superRefine(validateAlertScheduleOffsetMinutes) @@ -128,6 +129,8 @@ const AlertForm = ({ // Persisted null -> undefined for the NumberInput. numConsecutiveWindows: defaultValues.numConsecutiveWindows ?? undefined, + renotifyIntervalMinutes: + defaultValues.renotifyIntervalMinutes ?? undefined, } : { interval: '5m', @@ -160,6 +163,10 @@ const AlertForm = ({ control, name: 'numConsecutiveWindows', }); + const renotifyIntervalMinutes = useWatch({ + control, + name: 'renotifyIntervalMinutes', + }); const maxScheduleOffsetMinutes = Math.max( intervalToMinutes(interval ?? '5m') - 1, 0, @@ -284,6 +291,8 @@ const AlertForm = ({ offsetWindowLabel={`from each ${intervalLabel} window`} numConsecutiveWindowsName="numConsecutiveWindows" numConsecutiveWindows={numConsecutiveWindows ?? undefined} + renotifyIntervalName="renotifyIntervalMinutes" + renotifyIntervalMinutes={renotifyIntervalMinutes ?? undefined} /> grouped by diff --git a/packages/app/src/components/AlertScheduleFields.tsx b/packages/app/src/components/AlertScheduleFields.tsx index 774baca58a..b089a6fe7d 100644 --- a/packages/app/src/components/AlertScheduleFields.tsx +++ b/packages/app/src/components/AlertScheduleFields.tsx @@ -38,6 +38,8 @@ type AlertScheduleFieldsProps = { offsetWindowLabel: string; numConsecutiveWindowsName?: FieldPath; numConsecutiveWindows?: number; + renotifyIntervalName?: FieldPath; + renotifyIntervalMinutes?: number; }; export function AlertScheduleFields({ @@ -50,6 +52,8 @@ export function AlertScheduleFields({ offsetWindowLabel, numConsecutiveWindowsName, numConsecutiveWindows, + renotifyIntervalName, + renotifyIntervalMinutes, }: AlertScheduleFieldsProps) { const showScheduleOffsetInput = maxScheduleOffsetMinutes > 0; const scheduleStartAtValue = useWatch({ @@ -60,7 +64,8 @@ export function AlertScheduleFields({ const hasAdvancedScheduleValues = (scheduleOffsetMinutes ?? 0) > 0 || hasScheduleStartAtAnchor || - (numConsecutiveWindows ?? 1) > 1; + (numConsecutiveWindows ?? 1) > 1 || + renotifyIntervalMinutes != null; const [opened, setOpened] = useState(hasAdvancedScheduleValues); useEffect(() => { @@ -149,6 +154,46 @@ export function AlertScheduleFields({ )} + {renotifyIntervalName && ( + + + + Re-notify every + + + + + + + + ( + + field.onChange(typeof v === 'number' ? v : undefined) + } + min={0} + placeholder="Never" + size="xs" + w={90} + /> + )} + /> + + minutes + + + )} {showScheduleOffsetInput && ( <> diff --git a/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx b/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx index 8bfb0180fa..9104bf99e1 100644 --- a/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx @@ -76,6 +76,10 @@ export function TileAlertEditor({ control, name: 'alert.numConsecutiveWindows', }); + const alertRenotifyIntervalMinutes = useWatch({ + control, + name: 'alert.renotifyIntervalMinutes', + }); const maxAlertScheduleOffsetMinutes = alert?.interval ? Math.max(intervalToMinutes(alert.interval) - 1, 0) : 0; @@ -243,6 +247,8 @@ export function TileAlertEditor({ } numConsecutiveWindowsName="alert.numConsecutiveWindows" numConsecutiveWindows={alertnumConsecutiveWindows ?? undefined} + renotifyIntervalName="alert.renotifyIntervalMinutes" + renotifyIntervalMinutes={alertRenotifyIntervalMinutes ?? undefined} /> Send to diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 9ef47c1da4..814544d83e 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -774,6 +774,7 @@ export const AlertBaseObjectSchema = z.object({ }) .optional(), numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }); // Keep AlertBaseSchema as a ZodObject for backwards compatibility with @@ -2144,6 +2145,7 @@ export const AlertsPageItemSchema = z.object({ .optional(), executionErrors: z.array(AlertErrorSchema).optional(), numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }); export type AlertsPageItem = z.infer; From 07d077a0bd8a7e75cee1f544d0f03fa9c3482feb Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Thu, 30 Jul 2026 12:22:22 +0530 Subject: [PATCH 02/11] chore: refactoring comments Signed-off-by: Niladri Adhikary --- .../__tests__/checkAlerts.int.test.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index b7c17873d3..18b7c3bebc 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -9098,9 +9098,7 @@ describe('checkAlerts', () => { }); describe('re-notification (renotifyIntervalMinutes)', () => { - // A sustained breach across four consecutive 5m windows: every evaluation - // below sees data in its own window, so the alert stays in ALERT - // throughout without ever returning to OK. + // Data in every 5m window, so the alert stays ALERT and never resolves. const insertSustainedBreach = () => bulkInsertLogs( ['00:05:00', '00:10:00', '00:15:00', '00:20:00'].map(time => ({ @@ -9143,8 +9141,7 @@ describe('checkAlerts', () => { channel: { type: 'webhook', webhookId: webhook._id.toString() }, interval: '5m', thresholdType: AlertThresholdType.ABOVE, - // threshold=1 (not 0): ABOVE is inclusive, so threshold=0 would make - // an empty window breach too and the alert could never resolve. + // threshold=1: ABOVE is inclusive, so 0 would breach empty windows too. threshold: 1, savedSearchId: savedSearch.id, renotifyIntervalMinutes, @@ -9188,8 +9185,7 @@ describe('checkAlerts', () => { 'ALERT', 'ALERT', ]); - // lastNotifiedAt is pinned to the window that notified and carried - // forward unchanged across the silent evaluations. + // lastNotifiedAt stays pinned to the notifying window while silent. expect(histories.map(h => h.lastNotifiedAt?.toISOString())).toEqual( Array(4).fill('2024-01-01T00:10:00.000Z'), ); @@ -9208,8 +9204,7 @@ describe('checkAlerts', () => { }); it('re-notifies only after renotifyIntervalMinutes has elapsed', async () => { - // 10m re-notification on a 5m interval: notify at 00:10, skip 00:15, - // re-notify at 00:20, skip 00:25. + // 10m interval, 5m ticks: notify 00:10, skip 00:15, notify 00:20, skip 00:25. const { notificationCounts, histories } = await runSustainedBreach(10); expect(notificationCounts).toEqual([1, 1, 2, 2]); @@ -9276,8 +9271,7 @@ describe('checkAlerts', () => { }).sort({ createdAt: 1 }); expect(histories.map(h => h.state)).toEqual(['ALERT', 'OK', 'ALERT']); - // ALERT, resolved OK, then ALERT again: the second firing episode is a - // fresh transition, so it notifies even with re-notification off. + // Second ALERT is a fresh transition after resolving, so it notifies. expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); }); }); From 94e4f747bd7bde1ff827a5acc9f88b1bfbd560d9 Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Thu, 30 Jul 2026 12:33:34 +0530 Subject: [PATCH 03/11] fix: corrected return Signed-off-by: Niladri Adhikary --- packages/api/src/tasks/checkAlerts/index.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 654bf30477..288a0aebff 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1111,6 +1111,7 @@ export const processAlert = async ( 'Failed to fire channel event', ); executionErrors.push(makeWebhookAlertError(e)); + return false; } return true; }; @@ -1125,11 +1126,11 @@ export const processAlert = async ( const { renotifyIntervalMinutes } = alert; const lastNotifiedAt = previousHistory?.lastNotifiedAt; const shouldNotify = - renotifyIntervalMinutes != null && - (renotifyIntervalMinutes <= 0 || - lastNotifiedAt == null || - nowInMinsRoundDown.getTime() - lastNotifiedAt.getTime() >= - renotifyIntervalMinutes * 60_000); + lastNotifiedAt == null || + (renotifyIntervalMinutes != null && + (renotifyIntervalMinutes <= 0 || + nowInMinsRoundDown.getTime() - lastNotifiedAt.getTime() >= + renotifyIntervalMinutes * 60_000)); if (!shouldNotify) { alertEvaluationsCounter.add(1, { outcome: 'skipped_renotify' }); From c316ffad739dfa9dcad2e8632951043b9f774084 Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Sat, 1 Aug 2026 07:42:36 +0530 Subject: [PATCH 04/11] fix: missing field in test Signed-off-by: Niladri Adhikary --- .../api/src/routers/external-api/__tests__/alerts.int.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts b/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts index f241e405ef..8817e524ef 100644 --- a/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts @@ -224,6 +224,7 @@ describe('External API Alerts', () => { message: 'This is a test alert for format verification', note: null, numConsecutiveWindows: null, + renotifyIntervalMinutes: null, threshold: 123, interval: '15m', source: AlertSource.TILE, From a1d9129cc4e3f105c6ec159b79657d48a345a474 Mon Sep 17 00:00:00 2001 From: Baris Can Vural Date: Thu, 29 Oct 2026 18:35:02 +0300 Subject: [PATCH 05/11] feat(alerts): keep renotifyIntervalMinutes on the alerts page edit modals EditAlertModal and EditInlineAlertModal (added on main after this branch was cut) rebuild the PUT payload from a fixed field list, so saving an alert there dropped renotifyIntervalMinutes back to transition-only. Carry it like numConsecutiveWindows, and show it in AlertPropertiesSummary. --- .../src/components/alerts/AlertPropertiesSummary.tsx | 10 ++++++++++ packages/app/src/components/alerts/EditAlertModal.tsx | 8 ++++++++ .../app/src/components/alerts/EditInlineAlertModal.tsx | 1 + 3 files changed, 19 insertions(+) diff --git a/packages/app/src/components/alerts/AlertPropertiesSummary.tsx b/packages/app/src/components/alerts/AlertPropertiesSummary.tsx index 233b082ebc..947c8686cf 100644 --- a/packages/app/src/components/alerts/AlertPropertiesSummary.tsx +++ b/packages/app/src/components/alerts/AlertPropertiesSummary.tsx @@ -204,6 +204,16 @@ export function AlertPropertiesSummary({ )} + {alert.renotifyIntervalMinutes != null && ( + <> + · + + {alert.renotifyIntervalMinutes === 0 + ? 'Notifies on every evaluation' + : `Re-notifies every ${alert.renotifyIntervalMinutes} min while firing`} + + + )} )} · diff --git a/packages/app/src/components/alerts/EditAlertModal.tsx b/packages/app/src/components/alerts/EditAlertModal.tsx index 2d670aa2cd..9fe5a16ba8 100644 --- a/packages/app/src/components/alerts/EditAlertModal.tsx +++ b/packages/app/src/components/alerts/EditAlertModal.tsx @@ -77,6 +77,7 @@ const EditAlertFormSchema = z // nullish() (not optional()): persisted alerts store this as null, which // optional() would reject. numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }) .passthrough() .superRefine(validateAlertChannelSelection) @@ -115,6 +116,7 @@ function alertToFormValues(alert: AlertsPageItem): Alert { tags: alert.tags, // Persisted null -> undefined for the NumberInput. numConsecutiveWindows: alert.numConsecutiveWindows ?? undefined, + renotifyIntervalMinutes: alert.renotifyIntervalMinutes ?? undefined, }; if (alert.source === AlertSource.TILE) { @@ -216,6 +218,10 @@ export function EditAlertModal({ control, name: 'numConsecutiveWindows', }); + const renotifyIntervalMinutes = useWatch({ + control, + name: 'renotifyIntervalMinutes', + }); const maxScheduleOffsetMinutes = Math.max( intervalToMinutes(interval ?? '5m') - 1, @@ -419,6 +425,8 @@ export function EditAlertModal({ offsetWindowLabel={`from each ${intervalLabel} window`} numConsecutiveWindowsName="numConsecutiveWindows" numConsecutiveWindows={numConsecutiveWindows ?? undefined} + renotifyIntervalName="renotifyIntervalMinutes" + renotifyIntervalMinutes={renotifyIntervalMinutes ?? undefined} /> {!isChartAlert && ( <> diff --git a/packages/app/src/components/alerts/EditInlineAlertModal.tsx b/packages/app/src/components/alerts/EditInlineAlertModal.tsx index 12aecdd0cb..4cc88a1c2a 100644 --- a/packages/app/src/components/alerts/EditInlineAlertModal.tsx +++ b/packages/app/src/components/alerts/EditInlineAlertModal.tsx @@ -56,6 +56,7 @@ export function inlineAlertToChartConfig( note: alert.note ?? null, // Persisted null -> undefined for the NumberInput. numConsecutiveWindows: alert.numConsecutiveWindows ?? undefined, + renotifyIntervalMinutes: alert.renotifyIntervalMinutes ?? undefined, }, }; } From 2bba5f12d10ba5b0dce4c007988f4f5167e6153b Mon Sep 17 00:00:00 2001 From: Baris Can Vural Date: Thu, 29 Oct 2026 18:35:02 +0300 Subject: [PATCH 06/11] test(alerts): per-group re-notification for group-by alerts --- .../__tests__/checkAlerts.int.test.ts | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 8ad2eeb4f8..78279cd5f2 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -11628,6 +11628,72 @@ describe('checkAlerts', () => { // Second ALERT is a fresh transition after resolving, so it notifies. expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); }); + it('tracks notifications per group for group-by alerts', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + groupBy: 'ServiceName', + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // api breaches in every window; web starts two windows later. + await insertSustainedBreach(); + await bulkInsertLogs( + ['00:15:00', '00:20:00'].map(time => ({ + ServiceName: 'web', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + const notificationCounts: number[] = []; + for (const now of EVALUATION_TIMES) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + notificationCounts.push( + jest.mocked(slack.postMessageToWebhook).mock.calls.length, + ); + } + + // Each group notifies once, on its own OK->ALERT transition. + expect(notificationCounts).toEqual([1, 1, 2, 2]); + const webHistories = await AlertHistory.find({ + alert: details.alert.id, + group: 'ServiceName:web', + }).sort({ createdAt: 1 }); + expect(webHistories.map(h => h.lastNotifiedAt?.toISOString())).toEqual( + Array(2).fill('2024-01-01T00:20:00.000Z'), + ); + }); }); }); From ae619d6923f86b2bce25fe33821646edace8d590 Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Fri, 30 Oct 2026 02:16:10 +0530 Subject: [PATCH 07/11] feat(mcp): accept renotifyIntervalMinutes in save_alert --- packages/api/src/mcp/tools/alerts/saveAlert.ts | 1 + packages/api/src/mcp/tools/alerts/schemas.ts | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/packages/api/src/mcp/tools/alerts/saveAlert.ts b/packages/api/src/mcp/tools/alerts/saveAlert.ts index 9c5962026d..9d444386e9 100644 --- a/packages/api/src/mcp/tools/alerts/saveAlert.ts +++ b/packages/api/src/mcp/tools/alerts/saveAlert.ts @@ -112,6 +112,7 @@ export function registerSaveAlert({ thresholdMax: input.thresholdMax, scheduleOffsetMinutes: input.scheduleOffsetMinutes, scheduleStartAt: input.scheduleStartAt, + renotifyIntervalMinutes: input.renotifyIntervalMinutes, name: input.name, message: input.message, displayName: input.displayName, diff --git a/packages/api/src/mcp/tools/alerts/schemas.ts b/packages/api/src/mcp/tools/alerts/schemas.ts index 65503aa072..5d4b8bff1e 100644 --- a/packages/api/src/mcp/tools/alerts/schemas.ts +++ b/packages/api/src/mcp/tools/alerts/schemas.ts @@ -233,6 +233,16 @@ export const mcpSaveAlertSchema = z.object({ 'boundary. Must be less than the interval. Ignored when ' + 'scheduleStartAt is set.', ), + renotifyIntervalMinutes: z + .number() + .int() + .min(0) + .optional() + .describe( + 'Minutes between repeat notifications while the alert stays firing. ' + + 'Omit to notify only on the OK to ALERT transition; 0 notifies on ' + + 'every evaluation.', + ), scheduleStartAt: z .string() .datetime() From 0e0cea219b52200774bd82e68599347117eb6b44 Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Fri, 30 Oct 2026 02:16:11 +0530 Subject: [PATCH 08/11] fix(alerts): retry renotify when the first send of an incident fails --- .../__tests__/checkAlerts.int.test.ts | 64 +++++++++++++++++++ packages/api/src/tasks/checkAlerts/index.ts | 14 +++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 78279cd5f2..c720335f60 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -11628,6 +11628,70 @@ describe('checkAlerts', () => { // Second ALERT is a fresh transition after resolving, so it notifies. expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); }); + + it('retries a failed first notification of a later incident', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // ALERT ok, resolve ok, second incident's ALERT fails, then ok. + // mockReset drops the Once value queued by the shared beforeEach. + jest + .spyOn(slack, 'postMessageToWebhook') + .mockReset() + .mockResolvedValueOnce({ text: 'ok' }) + .mockResolvedValueOnce({ text: 'ok' }) + .mockRejectedValueOnce(new Error('webhook down')) + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + await bulkInsertLogs( + ['00:05:00', '00:15:00', '00:20:00'].map(time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + for (const now of EVALUATION_TIMES) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); + const histories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + expect(histories.at(-1)?.lastNotifiedAt?.toISOString()).toBe( + '2024-01-01T00:25:00.000Z', + ); + }); it('tracks notifications per group for group-by alerts', async () => { const { team, diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index aebb6b74ff..0563c5b6c5 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1434,6 +1434,9 @@ export const processAlert = async ( ); executionErrors.push(makeNotificationAlertError(failure)); } + // Delivered if any target got it; when every target failed, report + // not-notified so renotify gating retries on the next evaluation. + return timings.some(t => t.ok); } catch (e) { // Render-level failures (title/link building, template compile) — // nothing was dispatched. @@ -1444,7 +1447,6 @@ export const processAlert = async ( executionErrors.push(makeWebhookAlertError(e)); return false; } - return true; }; // Gates re-notification of an already-firing group by renotifyIntervalMinutes. @@ -1544,7 +1546,11 @@ export const processAlert = async ( history.lastValues.push({ count: value, startTime: alertTimestamp }); const previous = previousMap.get(computeHistoryMapKey(alert.id, '')); - history.lastNotifiedAt = previous?.lastNotifiedAt; + // Carry only within an unresolved incident, so a failed or silenced first + // notification of a later incident is retried instead of suppressed. + history.lastNotifiedAt = hasUnresolvedNotification(previous) + ? previous?.lastNotifiedAt + : undefined; if (doesExceedThreshold(alert, value)) { history.counts += 1; if (shouldFireBasedOnConsecutiveWindows()) { @@ -1769,7 +1775,9 @@ export const processAlert = async ( let groupPrevious = previousMap.get(previousKey); const hitAlertThisRun = latestAlertContext.has(groupKey); - history.lastNotifiedAt = groupPrevious?.lastNotifiedAt; + history.lastNotifiedAt = hasUnresolvedNotification(groupPrevious) + ? groupPrevious?.lastNotifiedAt + : undefined; // If it hit ALERT during this run, send the notification if (hitAlertThisRun) { From b127940921c3321e037059381781cfaf0eb7589b Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Fri, 30 Oct 2026 02:37:49 +0530 Subject: [PATCH 09/11] fix(alerts): address review feedback on renotify gating --- .changeset/alert-renotify-interval.md | 2 +- packages/api/src/controllers/alerts.ts | 2 - packages/api/src/mcp/tools/alerts/schemas.ts | 9 +- .../external-api/__tests__/alerts.int.test.ts | 31 +++++ .../__tests__/checkAlerts.int.test.ts | 130 +++++++++++++++++- packages/api/src/tasks/checkAlerts/index.ts | 33 +++-- .../src/components/AlertScheduleFields.tsx | 2 + 7 files changed, 186 insertions(+), 23 deletions(-) diff --git a/.changeset/alert-renotify-interval.md b/.changeset/alert-renotify-interval.md index b8ddc6e1bd..9218a3d25f 100644 --- a/.changeset/alert-renotify-interval.md +++ b/.changeset/alert-renotify-interval.md @@ -4,4 +4,4 @@ '@hyperdx/api': minor --- -feat: alerts now notify once on the OK -> ALERT transition instead of on every evaluation while firing. A sustained breach used to send a duplicate notification on every evaluation interval; a new `renotifyIntervalMinutes` setting ("Re-notify every" under Advanced Settings) controls repeats instead — leave it unset for transition-only (the new default), set `0` to restore the old notify-on-every-evaluation behavior, or set N to re-notify every N minutes. Resolve notifications are unchanged. This is a behavior change for existing alerts: set `renotifyIntervalMinutes: 0` to keep the old behavior. +feat: Alerts now notify once on the OK -> ALERT transition instead of on every evaluation while firing. A sustained breach used to send a duplicate notification on every evaluation interval; a new `renotifyIntervalMinutes` setting ("Re-notify every" under Advanced Settings) controls repeats instead — leave it unset for transition-only (the new default), set `0` to restore the old notify-on-every-evaluation behavior, or set N to re-notify every N minutes. Resolve notifications are unchanged. This is a behavior change for existing alerts: set `renotifyIntervalMinutes: 0` to keep the old behavior. diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index 693cf70981..65d00668a8 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -319,8 +319,6 @@ export const makeAlert = ( // Multi-window alerting numConsecutiveWindows: alert.numConsecutiveWindows ?? null, - - // Re-notification while firing renotifyIntervalMinutes: alert.renotifyIntervalMinutes ?? null, }; }; diff --git a/packages/api/src/mcp/tools/alerts/schemas.ts b/packages/api/src/mcp/tools/alerts/schemas.ts index 5d4b8bff1e..fe98e9bd59 100644 --- a/packages/api/src/mcp/tools/alerts/schemas.ts +++ b/packages/api/src/mcp/tools/alerts/schemas.ts @@ -237,11 +237,12 @@ export const mcpSaveAlertSchema = z.object({ .number() .int() .min(0) - .optional() + .nullish() .describe( - 'Minutes between repeat notifications while the alert stays firing. ' + - 'Omit to notify only on the OK to ALERT transition; 0 notifies on ' + - 'every evaluation.', + 'How often to repeat the notification while the alert keeps firing. ' + + 'Omit or send null to notify only once on the OK to ALERT ' + + 'transition, 0 to notify on every evaluation, or N to re-notify at ' + + 'most once every N minutes. A resolve notification is always sent.', ), scheduleStartAt: z .string() diff --git a/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts b/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts index 810126e441..d8e9f6440a 100644 --- a/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts @@ -2726,6 +2726,37 @@ describe('External API Alerts', () => { expect(getResponse.body.data.numConsecutiveWindows).toBeNull(); }); }); + describe('Re-notification interval (renotifyIntervalMinutes)', () => { + it('should create an alert with renotifyIntervalMinutes and return it on GET', async () => { + const { alert } = await createTestAlert({ renotifyIntervalMinutes: 60 }); + expect(alert.renotifyIntervalMinutes).toBe(60); + + const getResponse = await authRequest( + 'get', + `${ALERTS_BASE_URL}/${alert.id}`, + ).expect(200); + expect(getResponse.body.data.renotifyIntervalMinutes).toBe(60); + }); + + it('should update an alert to set and then clear renotifyIntervalMinutes', async () => { + const { alert, alertInput } = await createTestAlert(); + expect(alert.renotifyIntervalMinutes).toBeNull(); + + const getRenotify = async () => + (await authRequest('get', `${ALERTS_BASE_URL}/${alert.id}`).expect(200)) + .body.data.renotifyIntervalMinutes; + + await authRequest('put', `${ALERTS_BASE_URL}/${alert.id}`) + .send({ ...alertInput, renotifyIntervalMinutes: 0 }) + .expect(200); + expect(await getRenotify()).toBe(0); + + await authRequest('put', `${ALERTS_BASE_URL}/${alert.id}`) + .send({ ...alertInput, renotifyIntervalMinutes: null }) + .expect(200); + expect(await getRenotify()).toBeNull(); + }); + }); describe('Multiple notification channels', () => { const makeWebhookNamed = async (name: string) => await Webhook.findOneAndUpdate( diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index c720335f60..f4ae224584 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -11505,7 +11505,6 @@ describe('checkAlerts', () => { await insertSustainedBreach(); - // Notification count observed after each successive evaluation. const notificationCounts: number[] = []; for (const now of EVALUATION_TIMES) { await processAlertAtTime( @@ -11531,7 +11530,6 @@ describe('checkAlerts', () => { it('notifies once on the OK->ALERT transition by default', async () => { const { notificationCounts, histories } = await runSustainedBreach(); - // One notification on the first breach, silence for the rest. expect(notificationCounts).toEqual([1, 1, 1, 1]); expect(histories.map(h => h.state)).toEqual([ 'ALERT', @@ -11625,7 +11623,6 @@ describe('checkAlerts', () => { }).sort({ createdAt: 1 }); expect(histories.map(h => h.state)).toEqual(['ALERT', 'OK', 'ALERT']); - // Second ALERT is a fresh transition after resolving, so it notifies. expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); }); @@ -11692,6 +11689,132 @@ describe('checkAlerts', () => { '2024-01-01T00:25:00.000Z', ); }); + it('notifies raw SQL number alerts once on the OK->ALERT transition', async () => { + const { + team, + webhook, + connection, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const dashboard = await new Dashboard({ + name: 'Renotify Number Dashboard', + team: team._id, + tiles: [ + { + id: 'number1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'sql', + displayType: 'number', + sqlTemplate: RAW_SQL_NUMBER_ALERT_TEMPLATE, + connection: connection.id, + }, + }, + ], + }).save(); + const tile = dashboard.tiles?.find((t: any) => t.id === 'number1'); + if (!tile) throw new Error('tile not found'); + + const details = await createAlertDetails( + team, + undefined, + { + source: AlertSource.TILE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + dashboardId: dashboard.id, + tileId: 'number1', + }, + { taskType: AlertTaskType.TILE, tile, dashboard }, + ); + + await insertSustainedBreach(); + + const notificationCounts: number[] = []; + for (const now of EVALUATION_TIMES) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + notificationCounts.push( + jest.mocked(slack.postMessageToWebhook).mock.calls.length, + ); + } + + expect(notificationCounts).toEqual([1, 1, 1, 1]); + }); + + it('notifies a fresh incident that starts within a catch-up run', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // Breach at 00:05, below threshold at 00:10, breach again at 00:15. + await bulkInsertLogs( + ['00:05:00', '00:05:00', '00:10:00', '00:15:00', '00:15:00'].map( + time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + }), + ), + ); + + // The 00:17 evaluation is missed, so 00:22 covers both 00:10 and 00:15. + for (const now of [EVALUATION_TIMES[0], EVALUATION_TIMES[2]]) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); + }); + it('tracks notifications per group for group-by alerts', async () => { const { team, @@ -11748,7 +11871,6 @@ describe('checkAlerts', () => { ); } - // Each group notifies once, on its own OK->ALERT transition. expect(notificationCounts).toEqual([1, 1, 2, 2]); const webHistories = await AlertHistory.find({ alert: details.alert.id, diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 0563c5b6c5..79a986370a 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -975,11 +975,8 @@ export const alertQueryAttribution = ( : {}), }); -/** - * Whether the group already has a notification that was sent and has not been - * resolved yet. `fired` is absent on histories written before it was - * introduced, so only an explicit `false` counts as "never notified". - */ +// `fired` is missing on histories written before it existed, so only an +// explicit `false` means "never notified". const hasUnresolvedNotification = ( previousHistory: AggregatedAlertHistory | undefined, ): boolean => @@ -1295,6 +1292,10 @@ export const processAlert = async ( // Track state per group (or one history if no groupBy) const histories = new Map(); + // A catch-up run can go OK then ALERT again: that's a new incident, not a + // continuation of the previous run's notification. + const resolvedInRun = new Set(); + const reAlertedInRun = new Set(); const latestAlertContext = new Map< string, { value: number; attributes: Record; startTime: Date } @@ -1434,9 +1435,9 @@ export const processAlert = async ( ); executionErrors.push(makeNotificationAlertError(failure)); } - // Delivered if any target got it; when every target failed, report - // not-notified so renotify gating retries on the next evaluation. - return timings.some(t => t.ok); + // Retry on any failed target; a duplicate beats a channel missing the + // whole incident. + return failures.length === 0; } catch (e) { // Render-level failures (title/link building, template compile) — // nothing was dispatched. @@ -1449,7 +1450,6 @@ export const processAlert = async ( } }; - // Gates re-notification of an already-firing group by renotifyIntervalMinutes. const shouldNotifyWhileFiring = ( previousHistory: AggregatedAlertHistory | undefined, ): boolean => { @@ -1546,8 +1546,8 @@ export const processAlert = async ( history.lastValues.push({ count: value, startTime: alertTimestamp }); const previous = previousMap.get(computeHistoryMapKey(alert.id, '')); - // Carry only within an unresolved incident, so a failed or silenced first - // notification of a later incident is retried instead of suppressed. + // Carry only within an unresolved incident, so a later incident's failed + // first send is retried. history.lastNotifiedAt = hasUnresolvedNotification(previous) ? previous?.lastNotifiedAt : undefined; @@ -1667,6 +1667,9 @@ export const processAlert = async ( // we will handle creating a history as part of auto-resolve later const history = getOrCreateHistory(''); history.lastValues.push({ count: 0, startTime: bucketStart }); + if (!hasGroupBy) { + resolvedInRun.add(''); + } } continue; @@ -1711,6 +1714,9 @@ export const processAlert = async ( if (shouldFireBasedOnConsecutiveWindows(groupKey)) { history.state = AlertState.ALERT; history.fired = true; + if (resolvedInRun.has(groupKey)) { + reAlertedInRun.add(groupKey); + } latestAlertContext.set(groupKey, { value: evaluation.value, attributes: evaluation.attributes, @@ -1729,6 +1735,7 @@ export const processAlert = async ( // a subsequent OK window correctly resolves it before the notification phase. history.state = AlertState.OK; history.counts = 0; + resolvedInRun.add(groupKey); } history.lastValues.push({ count: evaluation.value, @@ -1772,7 +1779,9 @@ export const processAlert = async ( // Check for state transitions and send notifications for (const [groupKey, history] of histories.entries()) { const previousKey = computeHistoryMapKey(alert.id, groupKey); - let groupPrevious = previousMap.get(previousKey); + let groupPrevious = reAlertedInRun.has(groupKey) + ? undefined + : previousMap.get(previousKey); const hitAlertThisRun = latestAlertContext.has(groupKey); history.lastNotifiedAt = hasUnresolvedNotification(groupPrevious) diff --git a/packages/app/src/components/AlertScheduleFields.tsx b/packages/app/src/components/AlertScheduleFields.tsx index b089a6fe7d..3d4b2fd542 100644 --- a/packages/app/src/components/AlertScheduleFields.tsx +++ b/packages/app/src/components/AlertScheduleFields.tsx @@ -183,6 +183,8 @@ export function AlertScheduleFields({ field.onChange(typeof v === 'number' ? v : undefined) } min={0} + allowDecimal={false} + allowNegative={false} placeholder="Never" size="xs" w={90} From ff80322b7c822b96656458af2a95c2fe6f2c5419 Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Fri, 30 Oct 2026 03:01:53 +0530 Subject: [PATCH 10/11] fix(alerts): skip renotify retries for undeliverable targets, cover catch-up gaps --- .../api/src/mcp/tools/alerts/saveAlert.ts | 3 +- .../__tests__/checkAlerts.int.test.ts | 111 ++++++++++++++++++ packages/api/src/tasks/checkAlerts/index.ts | 58 ++++++--- 3 files changed, 152 insertions(+), 20 deletions(-) diff --git a/packages/api/src/mcp/tools/alerts/saveAlert.ts b/packages/api/src/mcp/tools/alerts/saveAlert.ts index 9d444386e9..4b358ee460 100644 --- a/packages/api/src/mcp/tools/alerts/saveAlert.ts +++ b/packages/api/src/mcp/tools/alerts/saveAlert.ts @@ -55,7 +55,8 @@ export function registerSaveAlert({ '(webhook IDs come from clickstack_get_webhook). Updates replace the ' + 'alert configuration rather than merging it: channels you do not resend are ' + 'removed, omitted name and message are cleared, omitted displayName and tags ' + - 'are re-derived from the source, and settings this tool cannot set ' + + 'are re-derived from the source, omitted renotifyIntervalMinutes resets ' + + 'to notify-on-transition-only, and settings this tool cannot set ' + '(note, consecutive-window count) are cleared. Omitting both ' + 'scheduleStartAt and scheduleOffsetMinutes keeps the stored schedule; ' + 'clearing scheduleStartAt without an offset resets the offset to 0. Read the alert with ' + diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index f4ae224584..b901976391 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -11759,6 +11759,57 @@ describe('checkAlerts', () => { expect(notificationCounts).toEqual([1, 1, 1, 1]); }); + it('does not retry every evaluation when one channel is a deleted webhook', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channels: [ + { type: 'webhook', webhookId: webhook._id.toString() }, + { + type: 'webhook', + webhookId: new mongoose.Types.ObjectId().toString(), + }, + ], + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + await insertSustainedBreach(); + + for (const now of EVALUATION_TIMES) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1); + }); + it('notifies a fresh incident that starts within a catch-up run', async () => { const { team, @@ -11815,6 +11866,66 @@ describe('checkAlerts', () => { expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); }); + it('notifies a group-by incident when the group was absent mid catch-up run', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 2, + savedSearchId: savedSearch.id, + groupBy: 'ServiceName', + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // api breaches at 00:05 and 00:15; at 00:10 only web (below threshold) has rows. + await bulkInsertLogs( + [ + ['api', '00:05:00'], + ['api', '00:05:00'], + ['web', '00:10:00'], + ['api', '00:15:00'], + ['api', '00:15:00'], + ].map(([service, time]) => ({ + ServiceName: service, + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + for (const now of [EVALUATION_TIMES[0], EVALUATION_TIMES[2]]) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); + }); + it('tracks notifications per group for group-by alerts', async () => { const { team, diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 79a986370a..6dc62ce535 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1292,10 +1292,9 @@ export const processAlert = async ( // Track state per group (or one history if no groupBy) const histories = new Map(); - // A catch-up run can go OK then ALERT again: that's a new incident, not a - // continuation of the previous run's notification. - const resolvedInRun = new Set(); - const reAlertedInRun = new Set(); + // Latest OK/ALERT bucket per group; OK before ALERT means a new incident. + const lastOkBucket = new Map(); + const lastAlertBucket = new Map(); const latestAlertContext = new Map< string, { value: number; attributes: Record; startTime: Date } @@ -1357,7 +1356,7 @@ export const processAlert = async ( group: string; startTime?: Date; attributes?: Record; - }) => { + }): Promise => { // KNOWN LIMITATION: Alert data (including silenced state) is fetched when // the task is queued via AlertProvider, not when it processes. If a user // silences an alert after it's queued but before it processes, this @@ -1435,9 +1434,8 @@ export const processAlert = async ( ); executionErrors.push(makeNotificationAlertError(failure)); } - // Retry on any failed target; a duplicate beats a channel missing the - // whole incident. - return failures.length === 0; + // Only failed dispatches retry; unresolvable targets would retry forever. + return timings.every(t => t.ok); } catch (e) { // Render-level failures (title/link building, template compile) — // nothing was dispatched. @@ -1621,7 +1619,18 @@ export const processAlert = async ( checkDataByBucket.get(bucketStart.getTime())!.push(checkData); } - for (const bucketStart of expectedBuckets) { + // An absent group is OK unless a zero value breaches. + const previousGroups = Array.from(previousMap.keys(), key => + extractGroupKeyFromMapKey(key, alert.id), + ); + const markAbsentGroupsOk = (bucketIndex: number, present: Set) => { + if (!hasGroupBy || doesExceedThreshold(alert, 0)) return; + for (const groupKey of [...previousGroups, ...histories.keys()]) { + if (!present.has(groupKey)) lastOkBucket.set(groupKey, bucketIndex); + } + }; + + for (const [bucketIndex, bucketStart] of expectedBuckets.entries()) { const dataForBucket = checkDataByBucket.get(bucketStart.getTime()); // Handle case where no data is available for this bucket @@ -1650,6 +1659,7 @@ export const processAlert = async ( if (shouldFireBasedOnConsecutiveWindows()) { history.state = AlertState.ALERT; history.fired = true; + lastAlertBucket.set('', bucketIndex); latestAlertContext.set('', { value: 0, attributes: {}, @@ -1667,8 +1677,12 @@ export const processAlert = async ( // we will handle creating a history as part of auto-resolve later const history = getOrCreateHistory(''); history.lastValues.push({ count: 0, startTime: bucketStart }); - if (!hasGroupBy) { - resolvedInRun.add(''); + } + if (!zeroValueIsAlert) { + if (hasGroupBy) { + markAbsentGroupsOk(bucketIndex, new Set()); + } else { + lastOkBucket.set('', bucketIndex); } } @@ -1682,9 +1696,15 @@ export const processAlert = async ( string, { value: number; attributes: Record; exceeds: boolean } >(); + const bucketGroups = new Set(); for (const checkData of dataForBucket) { const { value, extraFields } = parseAlertData(checkData, meta); + const groupKey = hasGroupBy + ? extraFields.map(([k, v]) => `${k}:${v}`).join(', ') + : ''; + bucketGroups.add(groupKey); + // NULL means no data: a metric series with no row at this bucket, or // a ratio with a missing/zero denominator. Skip the row instead of // fabricating a state from a gap. @@ -1692,9 +1712,6 @@ export const processAlert = async ( continue; } - const groupKey = hasGroupBy - ? extraFields.map(([k, v]) => `${k}:${v}`).join(', ') - : ''; const attributes = hasGroupBy ? Object.fromEntries(extraFields) : {}; groupAttributes.set(groupKey, attributes); @@ -1706,6 +1723,8 @@ export const processAlert = async ( } } + markAbsentGroupsOk(bucketIndex, bucketGroups); + for (const [groupKey, evaluation] of bucketEvaluations.entries()) { const history = getOrCreateHistory(groupKey); @@ -1714,9 +1733,7 @@ export const processAlert = async ( if (shouldFireBasedOnConsecutiveWindows(groupKey)) { history.state = AlertState.ALERT; history.fired = true; - if (resolvedInRun.has(groupKey)) { - reAlertedInRun.add(groupKey); - } + lastAlertBucket.set(groupKey, bucketIndex); latestAlertContext.set(groupKey, { value: evaluation.value, attributes: evaluation.attributes, @@ -1735,7 +1752,7 @@ export const processAlert = async ( // a subsequent OK window correctly resolves it before the notification phase. history.state = AlertState.OK; history.counts = 0; - resolvedInRun.add(groupKey); + lastOkBucket.set(groupKey, bucketIndex); } history.lastValues.push({ count: evaluation.value, @@ -1779,7 +1796,10 @@ export const processAlert = async ( // Check for state transitions and send notifications for (const [groupKey, history] of histories.entries()) { const previousKey = computeHistoryMapKey(alert.id, groupKey); - let groupPrevious = reAlertedInRun.has(groupKey) + const isNewIncidentInRun = + (lastOkBucket.get(groupKey) ?? Infinity) < + (lastAlertBucket.get(groupKey) ?? -Infinity); + let groupPrevious = isNewIncidentInRun ? undefined : previousMap.get(previousKey); From 467cb39ef1114f8404a1af1bb28756bba1ce9ae1 Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Fri, 30 Oct 2026 16:23:26 +0530 Subject: [PATCH 11/11] fix(alerts): catch-up run that resolves after a fresh breach still notifies --- .../__tests__/checkAlerts.int.test.ts | 61 +++++++++++++++++++ packages/api/src/tasks/checkAlerts/index.ts | 16 +++-- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index b901976391..bc89e5721e 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -11866,6 +11866,67 @@ describe('checkAlerts', () => { expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); }); + it('notifies a fresh incident that also resolves within a catch-up run', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 2, + savedSearchId: savedSearch.id, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // Breach at 00:05, then OK / breach / OK at 00:10, 00:15, 00:20. + await bulkInsertLogs( + [ + '00:05:00', + '00:05:00', + '00:10:00', + '00:15:00', + '00:15:00', + '00:20:00', + ].map(time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + for (const now of [EVALUATION_TIMES[0], EVALUATION_TIMES[3]]) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + // First ALERT, then the catch-up run's ALERT and its resolve. + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); + }); + it('notifies a group-by incident when the group was absent mid catch-up run', async () => { const { team, diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 6dc62ce535..8bcda4135e 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1292,8 +1292,12 @@ export const processAlert = async ( // Track state per group (or one history if no groupBy) const histories = new Map(); - // Latest OK/ALERT bucket per group; OK before ALERT means a new incident. - const lastOkBucket = new Map(); + // First OK and latest ALERT bucket per group; OK before ALERT means a new incident. + const firstOkBucket = new Map(); + const markOk = (groupKey: string, bucketIndex: number) => { + if (!firstOkBucket.has(groupKey)) + firstOkBucket.set(groupKey, bucketIndex); + }; const lastAlertBucket = new Map(); const latestAlertContext = new Map< string, @@ -1626,7 +1630,7 @@ export const processAlert = async ( const markAbsentGroupsOk = (bucketIndex: number, present: Set) => { if (!hasGroupBy || doesExceedThreshold(alert, 0)) return; for (const groupKey of [...previousGroups, ...histories.keys()]) { - if (!present.has(groupKey)) lastOkBucket.set(groupKey, bucketIndex); + if (!present.has(groupKey)) markOk(groupKey, bucketIndex); } }; @@ -1682,7 +1686,7 @@ export const processAlert = async ( if (hasGroupBy) { markAbsentGroupsOk(bucketIndex, new Set()); } else { - lastOkBucket.set('', bucketIndex); + markOk('', bucketIndex); } } @@ -1752,7 +1756,7 @@ export const processAlert = async ( // a subsequent OK window correctly resolves it before the notification phase. history.state = AlertState.OK; history.counts = 0; - lastOkBucket.set(groupKey, bucketIndex); + markOk(groupKey, bucketIndex); } history.lastValues.push({ count: evaluation.value, @@ -1797,7 +1801,7 @@ export const processAlert = async ( for (const [groupKey, history] of histories.entries()) { const previousKey = computeHistoryMapKey(alert.id, groupKey); const isNewIncidentInRun = - (lastOkBucket.get(groupKey) ?? Infinity) < + (firstOkBucket.get(groupKey) ?? Infinity) < (lastAlertBucket.get(groupKey) ?? -Infinity); let groupPrevious = isNewIncidentInRun ? undefined