Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/alert-renotify-interval.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@hyperdx/common-utils': minor
'@hyperdx/app': minor
'@hyperdx/api': minor
---

feat: Alerts now notify once on the OK -> ALERT transition instead of on every evaluation while firing. A sustained breach used to send a duplicate notification on every evaluation interval; a new `renotifyIntervalMinutes` setting ("Re-notify every" under Advanced Settings) controls repeats instead — leave it unset for transition-only (the new default), set `0` to restore the old notify-on-every-evaluation behavior, or set N to re-notify every N minutes. Resolve notifications are unchanged. This is a behavior change for existing alerts: set `renotifyIntervalMinutes: 0` to keep the old behavior.
7 changes: 7 additions & 0 deletions packages/api/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,13 @@
"nullable": true,
"description": "Fire the alert only after its condition has been met for this many consecutive evaluation windows. While the condition is met but fewer than this many consecutive windows have violated, the alert is in the PENDING state.",
"example": 3
},
"renotifyIntervalMinutes": {
"type": "integer",
"minimum": 0,
"nullable": true,
"description": "How often to repeat the notification while the alert keeps firing. Omit or set to null to notify only once when the alert transitions from OK to ALERT, 0 to notify on every evaluation, or a positive number to re-notify at most once every that many minutes. A resolve notification is always sent when the alert returns to OK.",
"example": 60
}
}
},
Expand Down
1 change: 1 addition & 0 deletions packages/api/src/controllers/alerts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,7 @@ export const makeAlert = (

// Multi-window alerting
numConsecutiveWindows: alert.numConsecutiveWindows ?? null,
renotifyIntervalMinutes: alert.renotifyIntervalMinutes ?? null,
};
};

Expand Down
4 changes: 3 additions & 1 deletion packages/api/src/mcp/tools/alerts/saveAlert.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ export function registerSaveAlert({
'(webhook IDs come from clickstack_get_webhook). Updates replace the ' +
'alert configuration rather than merging it: channels you do not resend are ' +
'removed, omitted name and message are cleared, omitted displayName and tags ' +
'are re-derived from the source, and settings this tool cannot set ' +
'are re-derived from the source, omitted renotifyIntervalMinutes resets ' +
'to notify-on-transition-only, and settings this tool cannot set ' +
'(note, consecutive-window count) are cleared. Omitting both ' +
'scheduleStartAt and scheduleOffsetMinutes keeps the stored schedule; ' +
'clearing scheduleStartAt without an offset resets the offset to 0. Read the alert with ' +
Expand Down Expand Up @@ -112,6 +113,7 @@ export function registerSaveAlert({
thresholdMax: input.thresholdMax,
scheduleOffsetMinutes: input.scheduleOffsetMinutes,
scheduleStartAt: input.scheduleStartAt,
renotifyIntervalMinutes: input.renotifyIntervalMinutes,
name: input.name,
message: input.message,
displayName: input.displayName,
Expand Down
11 changes: 11 additions & 0 deletions packages/api/src/mcp/tools/alerts/schemas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,17 @@ export const mcpSaveAlertSchema = z.object({
'boundary. Must be less than the interval. Ignored when ' +
'scheduleStartAt is set.',
),
renotifyIntervalMinutes: z
.number()
.int()
.min(0)
.nullish()
.describe(
'How often to repeat the notification while the alert keeps firing. ' +
'Omit or send null to notify only once on the OK to ALERT ' +
'transition, 0 to notify on every evaluation, or N to re-notify at ' +
'most once every N minutes. A resolve notification is always sent.',
),
scheduleStartAt: z
.string()
.datetime()
Expand Down
8 changes: 8 additions & 0 deletions packages/api/src/models/alert.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,9 @@ export interface IAlert {
// Multi-window alerting: fire only after N violations in M consecutive windows
numConsecutiveWindows?: number | null;

// Re-notify while firing: null = transition only, 0 = every evaluation, N = every N minutes.
renotifyIntervalMinutes?: number | null;

// Errors recorded during the most recent execution
executionErrors?: IAlertError[];
createdAt: Date;
Expand Down Expand Up @@ -256,6 +259,11 @@ const AlertSchema = new Schema<IAlert>(
required: false,
min: 1,
},
renotifyIntervalMinutes: {
type: Number,
required: false,
min: 0,
},
silenced: {
required: false,
type: {
Expand Down
5 changes: 5 additions & 0 deletions packages/api/src/models/alertHistory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ export interface IAlertHistory {
errors?: IAlertError[];
/** Diagnostics for the evaluation that wrote this record. */
analytics?: IAlertHistoryAnalytics;
lastNotifiedAt?: Date;
}

const AlertHistorySchema = new Schema<IAlertHistory>({
Expand Down Expand Up @@ -125,6 +126,10 @@ const AlertHistorySchema = new Schema<IAlertHistory>({
required: false,
default: undefined,
},
lastNotifiedAt: {
type: Date,
required: false,
},
});

AlertHistorySchema.index(
Expand Down
1 change: 1 addition & 0 deletions packages/api/src/routers/api/alerts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ const formatAlertResponse = (
'updatedAt',
'executionErrors',
'numConsecutiveWindows',
'renotifyIntervalMinutes',
]),
tileId: alert.tileId ?? undefined,
groupBy: alert.groupBy ?? undefined,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,7 @@ describe('External API Alerts', () => {
message: 'This is a test alert for format verification',
note: null,
numConsecutiveWindows: null,
renotifyIntervalMinutes: null,
threshold: 123,
interval: '15m',
source: AlertSource.TILE,
Expand Down Expand Up @@ -2725,6 +2726,37 @@ describe('External API Alerts', () => {
expect(getResponse.body.data.numConsecutiveWindows).toBeNull();
});
});
describe('Re-notification interval (renotifyIntervalMinutes)', () => {
it('should create an alert with renotifyIntervalMinutes and return it on GET', async () => {
const { alert } = await createTestAlert({ renotifyIntervalMinutes: 60 });
expect(alert.renotifyIntervalMinutes).toBe(60);

const getResponse = await authRequest(
'get',
`${ALERTS_BASE_URL}/${alert.id}`,
).expect(200);
expect(getResponse.body.data.renotifyIntervalMinutes).toBe(60);
});

it('should update an alert to set and then clear renotifyIntervalMinutes', async () => {
const { alert, alertInput } = await createTestAlert();
expect(alert.renotifyIntervalMinutes).toBeNull();

const getRenotify = async () =>
(await authRequest('get', `${ALERTS_BASE_URL}/${alert.id}`).expect(200))
.body.data.renotifyIntervalMinutes;

await authRequest('put', `${ALERTS_BASE_URL}/${alert.id}`)
.send({ ...alertInput, renotifyIntervalMinutes: 0 })
.expect(200);
expect(await getRenotify()).toBe(0);

await authRequest('put', `${ALERTS_BASE_URL}/${alert.id}`)
.send({ ...alertInput, renotifyIntervalMinutes: null })
.expect(200);
expect(await getRenotify()).toBeNull();
});
});
describe('Multiple notification channels', () => {
const makeWebhookNamed = async (name: string) =>
await Webhook.findOneAndUpdate(
Expand Down
6 changes: 6 additions & 0 deletions packages/api/src/routers/external-api/v2/alerts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,12 @@ function toInternalAlertInput(body: ExternalAlertInput): InternalAlertInput {
* nullable: true
* description: Fire the alert only after its condition has been met for this many consecutive evaluation windows. While the condition is met but fewer than this many consecutive windows have violated, the alert is in the PENDING state.
* example: 3
* renotifyIntervalMinutes:
* type: integer
* minimum: 0
* nullable: true
* description: How often to repeat the notification while the alert keeps firing. Omit or set to null to notify only once when the alert transitions from OK to ALERT, 0 to notify on every evaluation, or a positive number to re-notify at most once every that many minutes. A resolve notification is always sent when the alert returns to OK.
* example: 60
*
* AlertResponse:
* allOf:
Expand Down
Loading
Loading