From 88d54889b9a8754902b95fac9213565245e723b8 Mon Sep 17 00:00:00 2001 From: Jorik Date: Sat, 26 Sep 2026 16:53:51 +0200 Subject: [PATCH 1/3] feat(api): provision tile alerts from dashboard files The dashboard file provisioner stored a tile's `config.alert` inside the tile but never created the alert, so a provisioned alert passed validation and never fired. Create or update each declared tile alert on every sync, validated like the dashboards API validates them, and flag it `provisioned: true` so the provisioner can remove it once its file no longer declares it. An alert that fails validation keeps its last valid version; alerts created in the app are left alone. Co-Authored-By: Claude Opus 5.5 --- .changeset/provision-tile-alerts.md | 16 ++ .../api/docs/auto_provision/AUTO_PROVISION.md | 37 +++++ packages/api/src/controllers/alerts.ts | 4 + packages/api/src/models/alert.ts | 7 + .../__tests__/provisionDashboards.int.test.ts | 147 ++++++++++++++++++ .../src/tasks/provisionDashboards/index.ts | 74 ++++++++- 6 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 .changeset/provision-tile-alerts.md diff --git a/.changeset/provision-tile-alerts.md b/.changeset/provision-tile-alerts.md new file mode 100644 index 0000000000..c29637ba57 --- /dev/null +++ b/.changeset/provision-tile-alerts.md @@ -0,0 +1,16 @@ +--- +'@hyperdx/api': minor +--- + +feat: provision tile alerts from dashboard files + +The dashboard file provisioner stored a tile's `config.alert` inside the tile +but never created the alert, so a provisioned alert passed validation and never +fired. It now creates or updates each declared tile alert on every sync, +validated the same way the dashboards API validates them, and flags it +`provisioned: true`. + +A provisioned alert is deleted once its tile, or its file entry, no longer +declares it. An alert that fails validation is skipped with a warning and keeps +its last valid version. Alerts created in the app on a provisioned dashboard are +not touched unless the file declares an alert on the same tile. diff --git a/packages/api/docs/auto_provision/AUTO_PROVISION.md b/packages/api/docs/auto_provision/AUTO_PROVISION.md index deab07ab19..b1262259df 100644 --- a/packages/api/docs/auto_provision/AUTO_PROVISION.md +++ b/packages/api/docs/auto_provision/AUTO_PROVISION.md @@ -220,6 +220,43 @@ with at minimum a `name` and `tiles` array: - Files are validated against the `DashboardWithoutIdSchema` Zod schema; invalid files are skipped with a warning +### Tile Alerts + +A tile can declare an alert in `config.alert`, in the same shape the dashboards +API accepts. The provisioner creates or updates that tile alert on every sync: + +```json +{ + "id": "error-rate", + "x": 0, + "y": 0, + "w": 12, + "h": 4, + "config": { + "name": "Error rate", + "source": "Logs", + "displayType": "line", + "select": [{ "aggFn": "count", "where": "SeverityText:error" }], + "alert": { + "interval": "5m", + "threshold": 100, + "thresholdType": "above", + "channel": { "type": "webhook", "webhookId": "" } + } + } +} +``` + +- Provisioned alerts are flagged with `provisioned: true`. Removing an alert + from its tile, or the tile from the file, deletes it on the next sync +- An alert that fails validation (for example an unknown webhook, or a raw SQL + tile whose display type or query does not support alerts) is skipped with a + warning and keeps its last valid version +- Alerts created in the app on a provisioned dashboard are left alone, unless + the file declares an alert on the same tile, which then replaces it +- Webhooks belong to a team, so with `DASHBOARD_PROVISIONER_ALL_TEAMS=true` an + alert is only created for the team that owns its webhook + ## Note on Security diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index 0e8762e48c..7c12e170e1 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -419,6 +419,7 @@ export const createOrUpdateDashboardAlerts = async ( teamId: ObjectId, alertsByTile: Record, userId?: ObjectId, + { provisioned = false }: { provisioned?: boolean } = {}, ) => { const dashboardId = dashboard._id; const result = await Promise.all( @@ -440,6 +441,9 @@ export const createOrUpdateDashboardAlerts = async ( oldAlert && oldAlert.createdBy ? makeAlertUpdate(alertInput, undefined, { dashboard }) : makeAlertUpdate(alertInput, userId, { dashboard }); + if (provisioned) { + alertUpdate.$set.provisioned = true; + } return await Alert.findOneAndUpdate(filter, alertUpdate, { new: true, diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index 3e62d19eab..1a158e1d63 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -127,6 +127,9 @@ export interface IAlert { // Multi-window alerting: fire only after N violations in M consecutive windows numConsecutiveWindows?: number | null; + // Created by the dashboard file provisioner, which keeps it in sync with its file + provisioned?: boolean; + // Errors recorded during the most recent execution executionErrors?: IAlertError[]; createdAt: Date; @@ -256,6 +259,10 @@ const AlertSchema = new Schema( required: false, min: 1, }, + provisioned: { + type: Boolean, + required: false, + }, silenced: { required: false, type: { diff --git a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts index 3e17594813..905b22abea 100644 --- a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts +++ b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts @@ -1,4 +1,5 @@ /* eslint-disable security/detect-non-literal-fs-filename */ +import { AlertThresholdType } from '@hyperdx/common-utils/dist/types'; import fs from 'fs'; import mongoose from 'mongoose'; import os from 'os'; @@ -6,8 +7,10 @@ import path from 'path'; import { createTeam } from '@/controllers/team'; import { clearDBCollections, closeDB, connectDB, makeTile } from '@/fixtures'; +import Alert, { AlertSource } from '@/models/alert'; import Dashboard from '@/models/dashboard'; import Team from '@/models/team'; +import Webhook from '@/models/webhook'; import { readDashboardFiles, syncDashboards, @@ -314,6 +317,150 @@ describe('provisionDashboards', () => { }); }); + describe('syncDashboards tile alerts', () => { + const tileAlert = (webhookId: string, threshold = 10) => + ({ + interval: '5m', + threshold, + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookId }, + }) as any; + + const writeDashboard = (tiles: ReturnType[]) => + fs.writeFileSync( + path.join(tmpDir, 'alerting.json'), + JSON.stringify({ name: 'Alerting', tiles, tags: [] }), + ); + + const setup = async () => { + const team = await createTeam({ name: 'My Team' }); + const webhook = await new Webhook({ + team: team._id, + service: 'generic', + url: 'https://example.com/hook', + name: 'Hook', + }).save(); + return { team, webhookId: webhook._id.toString() }; + }; + + const provisionedDashboard = (teamId: unknown) => + Dashboard.findOne({ name: 'Alerting', team: teamId, provisioned: true }); + + it('creates a provisioned alert for a tile that declares one', async () => { + const { team, webhookId } = await setup(); + writeDashboard([ + makeTile({ id: 'with-alert', alert: tileAlert(webhookId) }), + makeTile({ id: 'without-alert' }), + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + const dashboard = await provisionedDashboard(team._id); + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0].source).toBe(AlertSource.TILE); + expect(alerts[0].dashboard?.toString()).toBe(dashboard?._id.toString()); + expect(alerts[0].tileId).toBe('with-alert'); + expect(alerts[0].threshold).toBe(10); + expect(alerts[0].channel).toEqual({ type: 'webhook', webhookId }); + expect(alerts[0].provisioned).toBe(true); + }); + + it('updates the alert in place on later syncs', async () => { + const { team, webhookId } = await setup(); + writeDashboard([makeTile({ id: 'tile', alert: tileAlert(webhookId) })]); + await syncDashboards(team._id.toString(), tmpDir); + const [created] = await Alert.find({ team: team._id }); + + writeDashboard([ + makeTile({ id: 'tile', alert: tileAlert(webhookId, 50) }), + ]); + await syncDashboards(team._id.toString(), tmpDir); + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0]._id.toString()).toBe(created._id.toString()); + expect(alerts[0].threshold).toBe(50); + }); + + it('removes a provisioned alert once its tile no longer declares one', async () => { + const { team, webhookId } = await setup(); + writeDashboard([ + makeTile({ id: 'kept', alert: tileAlert(webhookId) }), + makeTile({ id: 'dropped', alert: tileAlert(webhookId) }), + makeTile({ id: 'removed', alert: tileAlert(webhookId) }), + ]); + await syncDashboards(team._id.toString(), tmpDir); + expect(await Alert.countDocuments({ team: team._id })).toBe(3); + + writeDashboard([ + makeTile({ id: 'kept', alert: tileAlert(webhookId) }), + makeTile({ id: 'dropped' }), + ]); + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts.map(a => a.tileId)).toEqual(['kept']); + }); + + it('skips an alert whose webhook does not exist', async () => { + const { team } = await setup(); + writeDashboard([ + makeTile({ + id: 'tile', + alert: tileAlert(new mongoose.Types.ObjectId().toString()), + }), + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await provisionedDashboard(team._id)).toBeTruthy(); + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('keeps the last valid version when a declared alert becomes invalid', async () => { + const { team, webhookId } = await setup(); + writeDashboard([makeTile({ id: 'tile', alert: tileAlert(webhookId) })]); + await syncDashboards(team._id.toString(), tmpDir); + + writeDashboard([ + makeTile({ + id: 'tile', + alert: tileAlert(new mongoose.Types.ObjectId().toString(), 99), + }), + ]); + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0].threshold).toBe(10); + expect(alerts[0].channel).toEqual({ type: 'webhook', webhookId }); + }); + + it('leaves alerts created in the app alone', async () => { + const { team, webhookId } = await setup(); + writeDashboard([makeTile({ id: 'tile' })]); + await syncDashboards(team._id.toString(), tmpDir); + const dashboard = await provisionedDashboard(team._id); + await new Alert({ + team: team._id, + source: AlertSource.TILE, + dashboard: dashboard?._id, + tileId: 'tile', + interval: '5m', + threshold: 1, + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookId }, + createdBy: new mongoose.Types.ObjectId(), + }).save(); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await Alert.countDocuments({ team: team._id })).toBe(1); + }); + }); + describe('ProvisionDashboardsTask', () => { const originalEnv = process.env; diff --git a/packages/api/src/tasks/provisionDashboards/index.ts b/packages/api/src/tasks/provisionDashboards/index.ts index cf10427c72..cc804e9c7f 100644 --- a/packages/api/src/tasks/provisionDashboards/index.ts +++ b/packages/api/src/tasks/provisionDashboards/index.ts @@ -3,13 +3,21 @@ import { DashboardWithoutId, DashboardWithoutIdSchema, resolveChartPaletteToken, + Tile, walkRawDashboardTileColors, } from '@hyperdx/common-utils/dist/types'; import fs from 'fs'; +import { Types } from 'mongoose'; import path from 'path'; +import { + AlertInput, + createOrUpdateDashboardAlerts, + validateAlertInput, +} from '@/controllers/alerts'; import { connectDB, mongooseConnection } from '@/models'; -import Dashboard from '@/models/dashboard'; +import Alert, { AlertSource } from '@/models/alert'; +import Dashboard, { IDashboard } from '@/models/dashboard'; import Team from '@/models/team'; import type { HdxTask } from '@/tasks/types'; import { ProvisionDashboardsTaskArgs } from '@/tasks/types'; @@ -68,6 +76,61 @@ export function readDashboardFiles(dir: string): DashboardWithoutId[] { return dashboards; } +// Turns the `config.alert` of each provisioned tile into a tile alert, the way +// saving a dashboard through the API does. An alert that fails validation is +// skipped and keeps its last valid version; a provisioned alert whose tile no +// longer declares one is removed. Alerts created in the app are left alone. +export async function syncProvisionedAlerts( + dashboard: Pick, + teamId: string, + tiles: Tile[], +) { + const team = new Types.ObjectId(teamId); + const declared = tiles.filter(tile => tile.config.alert != null); + + const alertsByTile: Record = {}; + for (const tile of declared) { + const alert = tile.config.alert as AlertInput; + try { + await validateAlertInput(team, { + source: AlertSource.TILE, + dashboardId: dashboard._id.toString(), + tileId: tile.id, + channel: alert.channel, + channels: alert.channels, + }); + alertsByTile[tile.id] = alert; + } catch (err) { + logger.warn( + { + name: dashboard.name, + tileId: tile.id, + err: err instanceof Error ? err.message : err, + }, + 'Skipping invalid provisioned tile alert', + ); + } + } + + if (Object.keys(alertsByTile).length > 0) { + await createOrUpdateDashboardAlerts( + dashboard, + team, + alertsByTile, + undefined, + { provisioned: true }, + ); + } + + await Alert.deleteMany({ + dashboard: dashboard._id, + team, + source: AlertSource.TILE, + provisioned: true, + tileId: { $nin: declared.map(tile => tile.id) }, + }); +} + export async function syncDashboards(teamId: string, dir: string) { const dashboards = readDashboardFiles(dir); if (dashboards.length === 0) return; @@ -110,6 +173,15 @@ export async function syncDashboards(teamId: string, dir: string) { if (result === null) { logger.info({ name: dashboard.name }, 'Created provisioned dashboard'); } + + const saved = await Dashboard.findOne({ + name: dashboard.name, + team: teamId, + provisioned: true, + }); + if (saved) { + await syncProvisionedAlerts(saved, teamId, dashboard.tiles || []); + } } catch (err) { logger.error( { err, name: dashboard.name }, From f1fcbc13d0d5ae65e429daf9f832583e0550a26d Mon Sep 17 00:00:00 2001 From: Jorik Date: Mon, 28 Sep 2026 09:03:30 +0200 Subject: [PATCH 2/3] feat(api): resolve provisioned alert webhooks by name, clean up orphaned alerts A provisioned tile alert can name its webhook with `webhookName`, resolved among the team's webhooks on every sync, so a dashboard file no longer carries a Mongo id that only exists in one install or team. A name that matches no webhook, or webhooks of several services, fails validation and keeps the alert's last valid version. Removing a dashboard's file, or renaming the dashboard, now deletes the provisioned alerts of the dashboard left behind, which would otherwise keep notifying. The dashboard itself is still kept. Cleanup is skipped whenever a file could not be read or validated, or the directory has no valid files. Co-Authored-By: Claude Opus 5.5 --- .changeset/provision-tile-alerts.md | 10 +- .../api/docs/auto_provision/AUTO_PROVISION.md | 15 +- .../__tests__/provisionDashboards.int.test.ts | 211 +++++++++++++++++- .../src/tasks/provisionDashboards/index.ts | 195 +++++++++++++++- 4 files changed, 410 insertions(+), 21 deletions(-) diff --git a/.changeset/provision-tile-alerts.md b/.changeset/provision-tile-alerts.md index c29637ba57..702387474f 100644 --- a/.changeset/provision-tile-alerts.md +++ b/.changeset/provision-tile-alerts.md @@ -10,7 +10,11 @@ fired. It now creates or updates each declared tile alert on every sync, validated the same way the dashboards API validates them, and flags it `provisioned: true`. +Channels can name their webhook with `webhookName`, resolved per team on each +sync, instead of a `webhookId` that only exists in one install. + A provisioned alert is deleted once its tile, or its file entry, no longer -declares it. An alert that fails validation is skipped with a warning and keeps -its last valid version. Alerts created in the app on a provisioned dashboard are -not touched unless the file declares an alert on the same tile. +declares it, and when its dashboard's file is removed or the dashboard is +renamed. An alert that fails validation is skipped with a warning and keeps its +last valid version. Alerts created in the app on a provisioned dashboard are not +touched unless the file declares an alert on the same tile. diff --git a/packages/api/docs/auto_provision/AUTO_PROVISION.md b/packages/api/docs/auto_provision/AUTO_PROVISION.md index b1262259df..54b428c53c 100644 --- a/packages/api/docs/auto_provision/AUTO_PROVISION.md +++ b/packages/api/docs/auto_provision/AUTO_PROVISION.md @@ -241,22 +241,29 @@ API accepts. The provisioner creates or updates that tile alert on every sync: "interval": "5m", "threshold": 100, "thresholdType": "above", - "channel": { "type": "webhook", "webhookId": "" } + "channel": { "type": "webhook", "webhookName": "On-call" } } } } ``` +- A channel names its webhook with `webhookName`, which is looked up among the + team's webhooks on every sync, so the same file works across installs and, + with `DASHBOARD_PROVISIONER_ALL_TEAMS=true`, across teams that each have a + webhook of that name. `webhookId` still works too. A name that matches no + webhook, or webhooks of more than one service, fails validation - Provisioned alerts are flagged with `provisioned: true`. Removing an alert from its tile, or the tile from the file, deletes it on the next sync +- Removing a dashboard's file, or renaming the dashboard, deletes the + provisioned alerts of the dashboard it leaves behind. The dashboard itself is + kept, as above. This cleanup only runs when every file in the directory was + read and validated, and never when the directory has no valid files, so a + broken file or an empty mount does not remove alerts - An alert that fails validation (for example an unknown webhook, or a raw SQL tile whose display type or query does not support alerts) is skipped with a warning and keeps its last valid version - Alerts created in the app on a provisioned dashboard are left alone, unless the file declares an alert on the same tile, which then replaces it -- Webhooks belong to a team, so with `DASHBOARD_PROVISIONER_ALL_TEAMS=true` an - alert is only created for the team that owns its webhook - ## Note on Security diff --git a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts index 905b22abea..9e0e1c4c3b 100644 --- a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts +++ b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts @@ -332,8 +332,8 @@ describe('provisionDashboards', () => { JSON.stringify({ name: 'Alerting', tiles, tags: [] }), ); - const setup = async () => { - const team = await createTeam({ name: 'My Team' }); + const setup = async (makeTeam = () => createTeam({ name: 'My Team' })) => { + const team = await makeTeam(); const webhook = await new Webhook({ team: team._id, service: 'generic', @@ -459,6 +459,213 @@ describe('provisionDashboards', () => { expect(await Alert.countDocuments({ team: team._id })).toBe(1); }); + + describe('webhooks by name', () => { + const namedAlert = (webhookName: string, threshold = 10) => + ({ + interval: '5m', + threshold, + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookName }, + }) as any; + + it('resolves a webhook name to the team webhook', async () => { + const { team, webhookId } = await setup(); + writeDashboard([makeTile({ id: 'tile', alert: namedAlert('Hook') })]); + + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0].channel).toEqual({ type: 'webhook', webhookId }); + expect(alerts[0].channels).toEqual([{ type: 'webhook', webhookId }]); + const dashboard = await provisionedDashboard(team._id); + expect(dashboard?.tiles[0].config.alert?.channel).toEqual({ + type: 'webhook', + webhookId, + }); + }); + + it('resolves names in `channels` too', async () => { + const { team, webhookId } = await setup(); + const other = await new Webhook({ + team: team._id, + service: 'generic', + url: 'https://example.com/other', + name: 'Other', + }).save(); + writeDashboard([ + makeTile({ + id: 'tile', + alert: { + ...namedAlert('Hook'), + channel: undefined, + channels: [ + { type: 'webhook', webhookName: 'Hook' }, + { type: 'webhook', webhookName: 'Other' }, + ], + }, + }), + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + const [alert] = await Alert.find({ team: team._id }); + expect(alert.channels).toEqual([ + { type: 'webhook', webhookId }, + { type: 'webhook', webhookId: other._id.toString() }, + ]); + }); + + it('resolves the name separately for each team', async () => { + const a = await setup(); + const b = await setup(() => new Team({ name: 'Team B' }).save()); + writeDashboard([makeTile({ id: 'tile', alert: namedAlert('Hook') })]); + + await syncDashboards(a.team._id.toString(), tmpDir); + await syncDashboards(b.team._id.toString(), tmpDir); + + const [alertA] = await Alert.find({ team: a.team._id }); + const [alertB] = await Alert.find({ team: b.team._id }); + expect(alertA.channel).toEqual({ + type: 'webhook', + webhookId: a.webhookId, + }); + expect(alertB.channel).toEqual({ + type: 'webhook', + webhookId: b.webhookId, + }); + }); + + it('keeps the last valid version when the name matches no webhook', async () => { + const { team, webhookId } = await setup(); + writeDashboard([makeTile({ id: 'tile', alert: namedAlert('Hook') })]); + await syncDashboards(team._id.toString(), tmpDir); + + writeDashboard([ + makeTile({ id: 'tile', alert: namedAlert('Missing', 99) }), + ]); + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0].threshold).toBe(10); + expect(alerts[0].channel).toEqual({ type: 'webhook', webhookId }); + // The unresolved reference is not stored on the tile. + const dashboard = await provisionedDashboard(team._id); + expect(dashboard?.tiles[0].config.alert).toBeUndefined(); + }); + + it('skips a name that matches webhooks of several services', async () => { + const { team } = await setup(); + await new Webhook({ + team: team._id, + service: 'slack', + url: 'https://hooks.slack.com/services/x', + name: 'Hook', + }).save(); + writeDashboard([makeTile({ id: 'tile', alert: namedAlert('Hook') })]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + }); + + describe('orphaned alerts', () => { + const writeFile = (file: string, name: string, tiles: unknown[]) => + fs.writeFileSync( + path.join(tmpDir, file), + JSON.stringify({ name, tiles, tags: [] }), + ); + + const setupTwoDashboards = async () => { + const { team, webhookId } = await setup(); + writeFile('a.json', 'A', [ + makeTile({ id: 'tile', alert: tileAlert(webhookId) }), + ]); + writeFile('b.json', 'B', [ + makeTile({ id: 'tile', alert: tileAlert(webhookId) }), + ]); + await syncDashboards(team._id.toString(), tmpDir); + expect(await Alert.countDocuments({ team: team._id })).toBe(2); + return { team, webhookId }; + }; + + const alertDashboardNames = async (teamId: unknown) => { + const alerts = await Alert.find({ team: teamId }).populate<{ + dashboard: { name: string }; + }>('dashboard', 'name'); + return alerts.map(a => a.dashboard.name).sort(); + }; + + it('deletes the alerts of a dashboard whose file was removed, and keeps the dashboard', async () => { + const { team } = await setupTwoDashboards(); + + fs.rmSync(path.join(tmpDir, 'b.json')); + await syncDashboards(team._id.toString(), tmpDir); + + expect(await alertDashboardNames(team._id)).toEqual(['A']); + expect( + await Dashboard.countDocuments({ team: team._id, provisioned: true }), + ).toBe(2); + }); + + it('moves the alerts along when a dashboard is renamed', async () => { + const { team, webhookId } = await setupTwoDashboards(); + + writeFile('b.json', 'B renamed', [ + makeTile({ id: 'tile', alert: tileAlert(webhookId) }), + ]); + await syncDashboards(team._id.toString(), tmpDir); + + expect(await alertDashboardNames(team._id)).toEqual(['A', 'B renamed']); + }); + + it('keeps all alerts while any file is invalid', async () => { + const { team } = await setupTwoDashboards(); + + fs.writeFileSync(path.join(tmpDir, 'b.json'), '{ not json'); + await syncDashboards(team._id.toString(), tmpDir); + + expect(await alertDashboardNames(team._id)).toEqual(['A', 'B']); + }); + + it('keeps all alerts when the directory is empty', async () => { + const { team } = await setupTwoDashboards(); + + fs.rmSync(path.join(tmpDir, 'a.json')); + fs.rmSync(path.join(tmpDir, 'b.json')); + await syncDashboards(team._id.toString(), tmpDir); + + expect(await alertDashboardNames(team._id)).toEqual(['A', 'B']); + }); + + it('leaves app-created alerts on an orphaned dashboard alone', async () => { + const { team, webhookId } = await setupTwoDashboards(); + const b = await Dashboard.findOne({ team: team._id, name: 'B' }); + await new Alert({ + team: team._id, + source: AlertSource.TILE, + dashboard: b?._id, + tileId: 'other', + interval: '5m', + threshold: 1, + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookId }, + createdBy: new mongoose.Types.ObjectId(), + }).save(); + + fs.rmSync(path.join(tmpDir, 'b.json')); + await syncDashboards(team._id.toString(), tmpDir); + + const remaining = await Alert.find({ + team: team._id, + dashboard: b?._id, + }); + expect(remaining.map(a => a.tileId)).toEqual(['other']); + }); + }); }); describe('ProvisionDashboardsTask', () => { diff --git a/packages/api/src/tasks/provisionDashboards/index.ts b/packages/api/src/tasks/provisionDashboards/index.ts index cc804e9c7f..d9fa8476f8 100644 --- a/packages/api/src/tasks/provisionDashboards/index.ts +++ b/packages/api/src/tasks/provisionDashboards/index.ts @@ -19,6 +19,7 @@ import { connectDB, mongooseConnection } from '@/models'; import Alert, { AlertSource } from '@/models/alert'; import Dashboard, { IDashboard } from '@/models/dashboard'; import Team from '@/models/team'; +import Webhook from '@/models/webhook'; import type { HdxTask } from '@/tasks/types'; import { ProvisionDashboardsTaskArgs } from '@/tasks/types'; import logger from '@/utils/logger'; @@ -42,20 +43,64 @@ const provisionedDashboardSchema = DashboardWithoutIdSchema.superRefine( validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx), ); -export function readDashboardFiles(dir: string): DashboardWithoutId[] { +// A file names a webhook with `{ "type": "webhook", "webhookName": "..." }`, +// since webhook ids differ per team and per install. Names only resolve per +// team, after the file is parsed, so until then the name rides in `webhookId` +// behind this prefix. It never reaches Mongo: resolveWebhookNames swaps it for +// the real id, or drops the alert from the stored tile. +const WEBHOOK_NAME_REF = 'webhook-name:'; + +const isRecord = (value: unknown): value is Record => + value != null && typeof value === 'object' && !Array.isArray(value); + +function encodeWebhookName(channel: unknown): unknown { + if ( + !isRecord(channel) || + typeof channel.webhookName !== 'string' || + 'webhookId' in channel + ) { + return channel; + } + const { webhookName, ...rest } = channel; + return { ...rest, webhookId: `${WEBHOOK_NAME_REF}${webhookName}` }; +} + +function encodeWebhookNames(raw: unknown): unknown { + if (!isRecord(raw) || !Array.isArray(raw.tiles)) return raw; + for (const tile of raw.tiles) { + const alert = + isRecord(tile) && isRecord(tile.config) ? tile.config.alert : undefined; + if (!isRecord(alert)) continue; + if ('channel' in alert) alert.channel = encodeWebhookName(alert.channel); + if (Array.isArray(alert.channels)) { + alert.channels = alert.channels.map(encodeWebhookName); + } + } + return raw; +} + +// `complete` is false when the directory or any file in it could not be read +// or validated, so the caller cannot tell which dashboards the files declare. +function readDashboardDir(dir: string): { + dashboards: DashboardWithoutId[]; + complete: boolean; +} { let files: string[]; try { files = fs.readdirSync(dir).filter(f => f.endsWith('.json')); } catch (err) { logger.error({ err, dir }, 'Failed to read dashboard directory'); - return []; + return { dashboards: [], complete: false }; } const dashboards: DashboardWithoutId[] = []; + let complete = true; for (const file of files) { try { - const raw = migrateLegacyDashboardTileColorsRaw( - JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')), + const raw = encodeWebhookNames( + migrateLegacyDashboardTileColorsRaw( + JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')), + ), ) as Record | null | undefined; const parsed = provisionedDashboardSchema.safeParse({ tags: [], @@ -66,28 +111,94 @@ export function readDashboardFiles(dir: string): DashboardWithoutId[] { { file, errors: parsed.error.issues }, 'Skipping invalid dashboard file', ); + complete = false; continue; } dashboards.push(parsed.data); } catch (err) { logger.error({ err, file }, 'Failed to parse dashboard file'); + complete = false; } } - return dashboards; + return { dashboards, complete }; +} + +export function readDashboardFiles(dir: string): DashboardWithoutId[] { + return readDashboardDir(dir).dashboards; +} + +type AlertChannelInput = { type: 'webhook'; webhookId: string }; + +// Swaps each `webhookName` reference for the id of the team's webhook with +// that name. A tile whose alert names a missing or ambiguous webhook loses its +// alert, and is reported in `unresolved` so its last valid version is kept. +export function resolveWebhookNames( + tiles: Tile[], + webhookIdsByName: Map, +): { tiles: Tile[]; unresolved: Map } { + const unresolved = new Map(); + const resolveChannel = (channel: AlertChannelInput): AlertChannelInput => { + if (!channel.webhookId.startsWith(WEBHOOK_NAME_REF)) return channel; + const name = channel.webhookId.slice(WEBHOOK_NAME_REF.length); + const ids = webhookIdsByName.get(name) ?? []; + if (ids.length === 0) { + throw new Error(`Webhook named "${name}" not found`); + } + if (ids.length > 1) { + throw new Error( + `Webhook name "${name}" matches ${ids.length} webhooks; use webhookId`, + ); + } + return { ...channel, webhookId: ids[0] }; + }; + + const resolved = tiles.map((tile): Tile => { + const alert = tile.config.alert; + if (alert == null) return tile; + try { + return { + ...tile, + config: { + ...tile.config, + alert: { + ...alert, + ...(alert.channel && { channel: resolveChannel(alert.channel) }), + ...(alert.channels && { + channels: alert.channels.map(resolveChannel), + }), + }, + }, + }; + } catch (err) { + unresolved.set(tile.id, err instanceof Error ? err.message : String(err)); + const { alert: _alert, ...config } = tile.config; + return { ...tile, config }; + } + }); + return { tiles: resolved, unresolved }; } // Turns the `config.alert` of each provisioned tile into a tile alert, the way -// saving a dashboard through the API does. An alert that fails validation is -// skipped and keeps its last valid version; a provisioned alert whose tile no -// longer declares one is removed. Alerts created in the app are left alone. +// saving a dashboard through the API does. An alert that fails validation, or +// is listed in `unresolved`, is skipped and keeps its last valid version; a +// provisioned alert whose tile no longer declares one is removed. Alerts +// created in the app are left alone. export async function syncProvisionedAlerts( dashboard: Pick, teamId: string, tiles: Tile[], + unresolved: Map = new Map(), ) { const team = new Types.ObjectId(teamId); const declared = tiles.filter(tile => tile.config.alert != null); + for (const [tileId, reason] of unresolved) { + logger.warn( + { name: dashboard.name, tileId, err: reason }, + 'Skipping invalid provisioned tile alert', + ); + } + const alertsByTile: Record = {}; for (const tile of declared) { const alert = tile.config.alert as AlertInput; @@ -127,14 +238,54 @@ export async function syncProvisionedAlerts( team, source: AlertSource.TILE, provisioned: true, - tileId: { $nin: declared.map(tile => tile.id) }, + tileId: { + $nin: [...declared.map(tile => tile.id), ...unresolved.keys()], + }, + }); +} + +// Provisioned dashboards outlive their files, but their provisioned alerts +// must not: a removed or renamed file would otherwise keep notifying. Only +// called once every file in the directory was read, so a file that is briefly +// invalid does not lose its alerts. +export async function deleteOrphanedProvisionedAlerts( + teamId: string, + declaredNames: string[], +) { + const orphaned = await Dashboard.find( + { team: teamId, provisioned: true, name: { $nin: declaredNames } }, + { _id: 1, name: 1 }, + ).lean(); + if (orphaned.length === 0) return; + + const { deletedCount } = await Alert.deleteMany({ + team: teamId, + source: AlertSource.TILE, + provisioned: true, + dashboard: { $in: orphaned.map(d => d._id) }, }); + if (deletedCount > 0) { + logger.info( + { teamId, deletedCount, dashboards: orphaned.map(d => d.name) }, + 'Deleted provisioned alerts of dashboards no longer in the provisioner directory', + ); + } } export async function syncDashboards(teamId: string, dir: string) { - const dashboards = readDashboardFiles(dir); + const { dashboards, complete } = readDashboardDir(dir); + // An empty directory is more likely a failed mount than an intent to drop + // every provisioned alert, so orphan cleanup needs at least one file. if (dashboards.length === 0) return; + const webhookIdsByName = new Map(); + const webhooks = await Webhook.find({ team: teamId }, { name: 1 }).lean(); + for (const webhook of webhooks) { + const ids = webhookIdsByName.get(webhook.name) ?? []; + ids.push(webhook._id.toString()); + webhookIdsByName.set(webhook.name, ids); + } + for (const dashboard of dashboards) { try { const userDashboard = await Dashboard.exists({ @@ -149,11 +300,16 @@ export async function syncDashboards(teamId: string, dir: string) { ); } + const { tiles, unresolved } = resolveWebhookNames( + dashboard.tiles || [], + webhookIdsByName, + ); + const result = await Dashboard.findOneAndUpdate( { name: dashboard.name, team: teamId, provisioned: true }, { $set: { - tiles: dashboard.tiles || [], + tiles, tags: dashboard.tags || [], filters: dashboard.filters || [], savedQuery: dashboard.savedQuery ?? null, @@ -180,7 +336,7 @@ export async function syncDashboards(teamId: string, dir: string) { provisioned: true, }); if (saved) { - await syncProvisionedAlerts(saved, teamId, dashboard.tiles || []); + await syncProvisionedAlerts(saved, teamId, tiles, unresolved); } } catch (err) { logger.error( @@ -189,6 +345,21 @@ export async function syncDashboards(teamId: string, dir: string) { ); } } + + if (!complete) { + logger.warn( + 'Some dashboard files could not be read, skipping cleanup of orphaned provisioned alerts', + ); + return; + } + try { + await deleteOrphanedProvisionedAlerts( + teamId, + dashboards.map(d => d.name), + ); + } catch (err) { + logger.error({ err }, 'Failed to clean up orphaned provisioned alerts'); + } } export default class ProvisionDashboardsTask implements HdxTask { From ed73826dc3ba79f27ea7b2994e31ccdb30e671c2 Mon Sep 17 00:00:00 2001 From: Jorik Date: Tue, 29 Sep 2026 13:37:26 +0200 Subject: [PATCH 3/3] fix(api): address review of provisioned tile alerts - Validate each declared alert like the alerts API, plus what the alert task needs to evaluate it: alertable display type, no PromQL, and a source, connection and webhooks in the team; apply the threshold and schedule rules. - Validate before the dashboard upsert. A tile whose alert is invalid keeps its last synced version, so an old alert never evaluates a new query. - Never take over an app-created alert: skip the file's alert on that tile. - A unique partial index keeps one provisioned alert per tile. - A failed dashboard or alert sync, or files sharing a dashboard name, skip orphan cleanup; duplicate names sync only the first file. - Read webhooks only when a file names one; a failing team no longer stops later teams. - Move alert and webhook-name logic out of index.ts; add business context and a skipped-alert counter; type the test helpers. --- .changeset/provision-tile-alerts.md | 13 +- .../api/docs/auto_provision/AUTO_PROVISION.md | 72 +-- packages/api/src/controllers/alerts.ts | 1 + packages/api/src/models/alert.ts | 12 +- .../__tests__/provisionDashboards.int.test.ts | 415 +++++++++++++----- .../src/tasks/provisionDashboards/alerts.ts | 204 +++++++++ .../src/tasks/provisionDashboards/index.ts | 267 +++-------- .../tasks/provisionDashboards/webhookNames.ts | 118 +++++ 8 files changed, 735 insertions(+), 367 deletions(-) create mode 100644 packages/api/src/tasks/provisionDashboards/alerts.ts create mode 100644 packages/api/src/tasks/provisionDashboards/webhookNames.ts diff --git a/.changeset/provision-tile-alerts.md b/.changeset/provision-tile-alerts.md index 702387474f..8d1a89b638 100644 --- a/.changeset/provision-tile-alerts.md +++ b/.changeset/provision-tile-alerts.md @@ -6,15 +6,16 @@ feat: provision tile alerts from dashboard files The dashboard file provisioner stored a tile's `config.alert` inside the tile but never created the alert, so a provisioned alert passed validation and never -fired. It now creates or updates each declared tile alert on every sync, -validated the same way the dashboards API validates them, and flags it -`provisioned: true`. +fired. It now creates or updates each declared tile alert on every sync, flagged +`provisioned: true`, after the checks the alerts API applies plus the ones the +alert task needs to evaluate it (display type, no PromQL, and a source, +connection and webhooks in the team). Channels can name their webhook with `webhookName`, resolved per team on each sync, instead of a `webhookId` that only exists in one install. A provisioned alert is deleted once its tile, or its file entry, no longer declares it, and when its dashboard's file is removed or the dashboard is -renamed. An alert that fails validation is skipped with a warning and keeps its -last valid version. Alerts created in the app on a provisioned dashboard are not -touched unless the file declares an alert on the same tile. +renamed. An alert that fails validation is skipped with a warning, and its tile +keeps its last synced version. Alerts created in the app are never changed or +deleted. diff --git a/packages/api/docs/auto_provision/AUTO_PROVISION.md b/packages/api/docs/auto_provision/AUTO_PROVISION.md index 54b428c53c..4ecb4b9732 100644 --- a/packages/api/docs/auto_provision/AUTO_PROVISION.md +++ b/packages/api/docs/auto_provision/AUTO_PROVISION.md @@ -166,22 +166,22 @@ to manage these values. ## Dashboard Provisioning HyperDX supports file-based dashboard provisioning, similar to Grafana's -provisioning system. A scheduled task reads `.json` files from a directory -and upserts dashboards into MongoDB, matched by name for idempotency. -The task runs on the same schedule as other HyperDX tasks (every minute -when using the built-in scheduler, or on your own schedule when running -tasks externally). +provisioning system. A scheduled task reads `.json` files from a directory and +upserts dashboards into MongoDB, matched by name for idempotency. The task runs +on the same schedule as other HyperDX tasks (every minute when using the +built-in scheduler, or on your own schedule when running tasks externally). ### Environment Variables -| Variable | Required | Default | Description | -| ----------------------------------- | -------- | ------- | --------------------------------------------------------------- | -| `DASHBOARD_PROVISIONER_DIR` | Yes | | Directory to watch for `.json` dashboard files | -| `DASHBOARD_PROVISIONER_TEAM_ID` | No\* | | Scope provisioning to a specific team ID | -| `DASHBOARD_PROVISIONER_ALL_TEAMS` | No\* | `false` | Set to `true` to provision dashboards to all teams | +| Variable | Required | Default | Description | +| --------------------------------- | -------- | ------- | -------------------------------------------------- | +| `DASHBOARD_PROVISIONER_DIR` | Yes | | Directory to watch for `.json` dashboard files | +| `DASHBOARD_PROVISIONER_TEAM_ID` | No\* | | Scope provisioning to a specific team ID | +| `DASHBOARD_PROVISIONER_ALL_TEAMS` | No\* | `false` | Set to `true` to provision dashboards to all teams | -\*One of `DASHBOARD_PROVISIONER_TEAM_ID` or `DASHBOARD_PROVISIONER_ALL_TEAMS=true` -is required when `DASHBOARD_PROVISIONER_DIR` is set. +\*One of `DASHBOARD_PROVISIONER_TEAM_ID` or +`DASHBOARD_PROVISIONER_ALL_TEAMS=true` is required when +`DASHBOARD_PROVISIONER_DIR` is set. ### Dashboard JSON Format @@ -234,9 +234,18 @@ API accepts. The provisioner creates or updates that tile alert on every sync: "h": 4, "config": { "name": "Error rate", - "source": "Logs", + "source": "", "displayType": "line", - "select": [{ "aggFn": "count", "where": "SeverityText:error" }], + "select": [ + { + "aggFn": "count", + "aggCondition": "SeverityText:error", + "aggConditionLanguage": "lucene", + "valueExpression": "" + } + ], + "where": "", + "whereLanguage": "lucene", "alert": { "interval": "5m", "threshold": 100, @@ -247,23 +256,32 @@ API accepts. The provisioner creates or updates that tile alert on every sync: } ``` -- A channel names its webhook with `webhookName`, which is looked up among the - team's webhooks on every sync, so the same file works across installs and, - with `DASHBOARD_PROVISIONER_ALL_TEAMS=true`, across teams that each have a - webhook of that name. `webhookId` still works too. A name that matches no - webhook, or webhooks of more than one service, fails validation +- Each alert gets the checks the alerts API applies, plus the ones the alert + task needs to evaluate it: a Line, Stacked Bar or Number display type, no + PromQL, a valid raw SQL alert query, and a source, connection and webhooks + that exist in the team. `source` is the source's ObjectId, so with + `DASHBOARD_PROVISIONER_ALL_TEAMS=true` an alert is only created in the team + that owns its source +- A channel can name its webhook with `webhookName`, looked up among the team's + webhooks on every sync, instead of a `webhookId` that only exists in one + install. A name that matches no webhook, or more than one, fails validation - Provisioned alerts are flagged with `provisioned: true`. Removing an alert - from its tile, or the tile from the file, deletes it on the next sync + from its tile, or the tile from the file, deletes it on the next sync. Keep + tile ids stable: changing one replaces the alert and loses its history - Removing a dashboard's file, or renaming the dashboard, deletes the provisioned alerts of the dashboard it leaves behind. The dashboard itself is kept, as above. This cleanup only runs when every file in the directory was - read and validated, and never when the directory has no valid files, so a - broken file or an empty mount does not remove alerts -- An alert that fails validation (for example an unknown webhook, or a raw SQL - tile whose display type or query does not support alerts) is skipped with a - warning and keeps its last valid version -- Alerts created in the app on a provisioned dashboard are left alone, unless - the file declares an alert on the same tile, which then replaces it + read, validated and synced, and never when the directory has no valid files, + so a broken file, a failed sync or an empty mount does not remove alerts +- An alert that fails validation is skipped with a warning. Its tile keeps the + version from the last sync, query and alert together, so an old alert never + evaluates a new query; a new tile is added without its alert +- Alerts created in the app are never changed or deleted. A file that declares + an alert on a tile that already has one from the app is skipped with a warning +- Like the dashboard, a provisioned alert edited or deleted in the app is + restored from its file on the next sync +- Files that share a dashboard name are skipped with a warning, except the first + in file name order ## Note on Security diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index 7c12e170e1..ee6b3286b6 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -429,6 +429,7 @@ export const createOrUpdateDashboardAlerts = async ( tileId, source: AlertSource.TILE, team: teamId, + ...(provisioned && { provisioned: true }), }; const alertInput = { ...alert, diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index 1a158e1d63..8a9fb52bb6 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -261,7 +261,7 @@ const AlertSchema = new Schema( }, provisioned: { type: Boolean, - required: false, + default: false, }, silenced: { required: false, @@ -313,4 +313,14 @@ AlertSchema.index({ team: 1, _id: 1 }); // tie-break, and walk that order with a keyset cursor. AlertSchema.index({ team: 1, displayName: 1, _id: 1 }); +// One alert per provisioned tile, so concurrent provisioner runs upsert the +// same document instead of inserting a duplicate that no sync would remove. +AlertSchema.index( + { team: 1, dashboard: 1, tileId: 1 }, + { + unique: true, + partialFilterExpression: { provisioned: true, source: AlertSource.TILE }, + }, +); + export default mongoose.model('Alert', AlertSchema); diff --git a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts index 9e0e1c4c3b..6bd1e2e8f2 100644 --- a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts +++ b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts @@ -1,5 +1,9 @@ /* eslint-disable security/detect-non-literal-fs-filename */ -import { AlertThresholdType } from '@hyperdx/common-utils/dist/types'; +import { + AlertThresholdType, + BuilderSavedChartConfig, + SourceKind, +} from '@hyperdx/common-utils/dist/types'; import fs from 'fs'; import mongoose from 'mongoose'; import os from 'os'; @@ -9,6 +13,7 @@ import { createTeam } from '@/controllers/team'; import { clearDBCollections, closeDB, connectDB, makeTile } from '@/fixtures'; import Alert, { AlertSource } from '@/models/alert'; import Dashboard from '@/models/dashboard'; +import { Source } from '@/models/source'; import Team from '@/models/team'; import Webhook from '@/models/webhook'; import { @@ -318,20 +323,24 @@ describe('provisionDashboards', () => { }); describe('syncDashboards tile alerts', () => { - const tileAlert = (webhookId: string, threshold = 10) => - ({ - interval: '5m', - threshold, - thresholdType: AlertThresholdType.ABOVE, - channel: { type: 'webhook', webhookId }, - }) as any; + type TileAlert = NonNullable; - const writeDashboard = (tiles: ReturnType[]) => + const tileAlert = (webhookId: string, threshold = 10): TileAlert => ({ + interval: '5m', + threshold, + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookId }, + }); + + const writeFile = (file: string, name: string, tiles: unknown[]) => fs.writeFileSync( - path.join(tmpDir, 'alerting.json'), - JSON.stringify({ name: 'Alerting', tiles, tags: [] }), + path.join(tmpDir, file), + JSON.stringify({ name, tiles, tags: [] }), ); + const writeDashboard = (tiles: unknown[]) => + writeFile('alerting.json', 'Alerting', tiles); + const setup = async (makeTeam = () => createTeam({ name: 'My Team' })) => { const team = await makeTeam(); const webhook = await new Webhook({ @@ -340,17 +349,46 @@ describe('provisionDashboards', () => { url: 'https://example.com/hook', name: 'Hook', }).save(); - return { team, webhookId: webhook._id.toString() }; + const source = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + connection: new mongoose.Types.ObjectId(), + name: 'Logs', + }); + const sourceId = source._id.toString(); + // A tile on this team's source; `alert` is raw JSON, as a file holds it. + const tile = (id: string, alert?: unknown, where?: string) => { + const base = makeTile({ id, sourceId, where }); + return alert == null + ? base + : { ...base, config: { ...base.config, alert } }; + }; + return { team, webhookId: webhook._id.toString(), sourceId, tile }; }; const provisionedDashboard = (teamId: unknown) => Dashboard.findOne({ name: 'Alerting', team: teamId, provisioned: true }); + const appAlert = (teamId: unknown, dashboardId: unknown, tileId: string) => + new Alert({ + team: teamId, + source: AlertSource.TILE, + dashboard: dashboardId, + tileId, + interval: '5m', + threshold: 1, + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookId: new mongoose.Types.ObjectId() }, + createdBy: new mongoose.Types.ObjectId(), + }).save(); + it('creates a provisioned alert for a tile that declares one', async () => { - const { team, webhookId } = await setup(); + const { team, webhookId, tile } = await setup(); writeDashboard([ - makeTile({ id: 'with-alert', alert: tileAlert(webhookId) }), - makeTile({ id: 'without-alert' }), + tile('with-alert', tileAlert(webhookId)), + tile('without-alert'), ]); await syncDashboards(team._id.toString(), tmpDir); @@ -367,14 +405,12 @@ describe('provisionDashboards', () => { }); it('updates the alert in place on later syncs', async () => { - const { team, webhookId } = await setup(); - writeDashboard([makeTile({ id: 'tile', alert: tileAlert(webhookId) })]); + const { team, webhookId, tile } = await setup(); + writeDashboard([tile('tile', tileAlert(webhookId))]); await syncDashboards(team._id.toString(), tmpDir); const [created] = await Alert.find({ team: team._id }); - writeDashboard([ - makeTile({ id: 'tile', alert: tileAlert(webhookId, 50) }), - ]); + writeDashboard([tile('tile', tileAlert(webhookId, 50))]); await syncDashboards(team._id.toString(), tmpDir); await syncDashboards(team._id.toString(), tmpDir); @@ -385,19 +421,16 @@ describe('provisionDashboards', () => { }); it('removes a provisioned alert once its tile no longer declares one', async () => { - const { team, webhookId } = await setup(); + const { team, webhookId, tile } = await setup(); writeDashboard([ - makeTile({ id: 'kept', alert: tileAlert(webhookId) }), - makeTile({ id: 'dropped', alert: tileAlert(webhookId) }), - makeTile({ id: 'removed', alert: tileAlert(webhookId) }), + tile('kept', tileAlert(webhookId)), + tile('dropped', tileAlert(webhookId)), + tile('removed', tileAlert(webhookId)), ]); await syncDashboards(team._id.toString(), tmpDir); expect(await Alert.countDocuments({ team: team._id })).toBe(3); - writeDashboard([ - makeTile({ id: 'kept', alert: tileAlert(webhookId) }), - makeTile({ id: 'dropped' }), - ]); + writeDashboard([tile('kept', tileAlert(webhookId)), tile('dropped')]); await syncDashboards(team._id.toString(), tmpDir); const alerts = await Alert.find({ team: team._id }); @@ -405,73 +438,175 @@ describe('provisionDashboards', () => { }); it('skips an alert whose webhook does not exist', async () => { - const { team } = await setup(); + const { team, tile } = await setup(); writeDashboard([ - makeTile({ - id: 'tile', - alert: tileAlert(new mongoose.Types.ObjectId().toString()), - }), + tile('tile', tileAlert(new mongoose.Types.ObjectId().toString())), ]); await syncDashboards(team._id.toString(), tmpDir); - expect(await provisionedDashboard(team._id)).toBeTruthy(); + const dashboard = await provisionedDashboard(team._id); + expect(dashboard?.tiles[0].config.alert).toBeUndefined(); expect(await Alert.countDocuments({ team: team._id })).toBe(0); }); - it('keeps the last valid version when a declared alert becomes invalid', async () => { - const { team, webhookId } = await setup(); - writeDashboard([makeTile({ id: 'tile', alert: tileAlert(webhookId) })]); + it('skips an alert on a display type the alert task cannot evaluate', async () => { + const { team, webhookId, tile } = await setup(); + const table = tile('tile', tileAlert(webhookId)); + writeDashboard([ + { ...table, config: { ...table.config, displayType: 'table' } }, + ]); + await syncDashboards(team._id.toString(), tmpDir); + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('skips an alert on a raw SQL tile whose display type does not support alerts', async () => { + const { team, webhookId, tile } = await setup(); + const { config: _config, ...base } = tile('tile'); + writeDashboard([ + { + ...base, + config: { + configType: 'sql', + name: 'Raw', + displayType: 'table', + connection: new mongoose.Types.ObjectId().toString(), + sqlTemplate: 'SELECT 1', + alert: tileAlert(webhookId), + }, + }, + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await provisionedDashboard(team._id)).toBeTruthy(); + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it("skips an alert whose tile's source is not in the team", async () => { + const { team, webhookId } = await setup(); writeDashboard([ makeTile({ id: 'tile', - alert: tileAlert(new mongoose.Types.ObjectId().toString(), 99), + sourceId: new mongoose.Types.ObjectId().toString(), + alert: tileAlert(webhookId), }), ]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('applies the alerts API threshold rules', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([ + tile('tile', { + ...tileAlert(webhookId), + thresholdType: AlertThresholdType.BETWEEN, + }), + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('keeps the last synced tile, query and alert together, when a declared alert becomes invalid', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([ + tile('tile', tileAlert(webhookId), 'level:error'), + tile('other'), + ]); + await syncDashboards(team._id.toString(), tmpDir); + + writeDashboard([ + tile( + 'tile', + tileAlert(new mongoose.Types.ObjectId().toString(), 99), + 'level:warn', + ), + tile('other', undefined, 'updated'), + ]); await syncDashboards(team._id.toString(), tmpDir); const alerts = await Alert.find({ team: team._id }); expect(alerts).toHaveLength(1); expect(alerts[0].threshold).toBe(10); expect(alerts[0].channel).toEqual({ type: 'webhook', webhookId }); + const dashboard = await provisionedDashboard(team._id); + const [kept, other] = dashboard?.tiles ?? []; + expect(kept.config).toMatchObject({ where: 'level:error' }); + expect(other.config).toMatchObject({ where: 'updated' }); + }); + + it('skips alerts on tiles whose id is not unique', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([ + tile('tile', tileAlert(webhookId)), + tile('tile', tileAlert(webhookId, 20)), + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); }); it('leaves alerts created in the app alone', async () => { - const { team, webhookId } = await setup(); - writeDashboard([makeTile({ id: 'tile' })]); + const { team, tile } = await setup(); + writeDashboard([tile('tile')]); await syncDashboards(team._id.toString(), tmpDir); const dashboard = await provisionedDashboard(team._id); - await new Alert({ - team: team._id, - source: AlertSource.TILE, - dashboard: dashboard?._id, - tileId: 'tile', - interval: '5m', - threshold: 1, - thresholdType: AlertThresholdType.ABOVE, - channel: { type: 'webhook', webhookId }, - createdBy: new mongoose.Types.ObjectId(), - }).save(); + await appAlert(team._id, dashboard?._id, 'tile'); await syncDashboards(team._id.toString(), tmpDir); expect(await Alert.countDocuments({ team: team._id })).toBe(1); }); + it('does not take over an app-created alert on a tile the file declares one on', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([tile('tile')]); + await syncDashboards(team._id.toString(), tmpDir); + const dashboard = await provisionedDashboard(team._id); + const userAlert = await appAlert(team._id, dashboard?._id, 'tile'); + + writeDashboard([tile('tile', tileAlert(webhookId, 50))]); + await syncDashboards(team._id.toString(), tmpDir); + writeDashboard([tile('tile')]); + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0]._id.toString()).toBe(userAlert._id.toString()); + expect(alerts[0].threshold).toBe(1); + expect(alerts[0].provisioned).toBe(false); + }); + + it('allows only one provisioned alert per tile', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([tile('tile', tileAlert(webhookId))]); + await syncDashboards(team._id.toString(), tmpDir); + await Alert.init(); + const [alert] = await Alert.find({ team: team._id }).lean(); + const { _id, ...duplicate } = alert; + + await expect(Alert.create(duplicate)).rejects.toThrow(/duplicate key/); + }); + describe('webhooks by name', () => { - const namedAlert = (webhookName: string, threshold = 10) => - ({ - interval: '5m', - threshold, - thresholdType: AlertThresholdType.ABOVE, - channel: { type: 'webhook', webhookName }, - }) as any; + const namedAlert = (webhookName: string, threshold = 10) => ({ + interval: '5m', + threshold, + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookName }, + }); it('resolves a webhook name to the team webhook', async () => { - const { team, webhookId } = await setup(); - writeDashboard([makeTile({ id: 'tile', alert: namedAlert('Hook') })]); + const { team, webhookId, tile } = await setup(); + writeDashboard([tile('tile', namedAlert('Hook'))]); await syncDashboards(team._id.toString(), tmpDir); @@ -487,109 +622,104 @@ describe('provisionDashboards', () => { }); it('resolves names in `channels` too', async () => { - const { team, webhookId } = await setup(); + const { team, webhookId, tile } = await setup(); const other = await new Webhook({ team: team._id, service: 'generic', url: 'https://example.com/other', name: 'Other', }).save(); + const { channel: _channel, ...alert } = namedAlert('Hook'); writeDashboard([ - makeTile({ - id: 'tile', - alert: { - ...namedAlert('Hook'), - channel: undefined, - channels: [ - { type: 'webhook', webhookName: 'Hook' }, - { type: 'webhook', webhookName: 'Other' }, - ], - }, + tile('tile', { + ...alert, + channels: [ + { type: 'webhook', webhookName: 'Hook' }, + { type: 'webhook', webhookName: 'Other' }, + ], }), ]); await syncDashboards(team._id.toString(), tmpDir); - const [alert] = await Alert.find({ team: team._id }); - expect(alert.channels).toEqual([ + const [created] = await Alert.find({ team: team._id }); + expect(created.channels).toEqual([ { type: 'webhook', webhookId }, { type: 'webhook', webhookId: other._id.toString() }, ]); }); - it('resolves the name separately for each team', async () => { + it('only creates the alert in the team that owns the tile source', async () => { const a = await setup(); const b = await setup(() => new Team({ name: 'Team B' }).save()); - writeDashboard([makeTile({ id: 'tile', alert: namedAlert('Hook') })]); + writeDashboard([a.tile('tile', namedAlert('Hook'))]); await syncDashboards(a.team._id.toString(), tmpDir); await syncDashboards(b.team._id.toString(), tmpDir); const [alertA] = await Alert.find({ team: a.team._id }); - const [alertB] = await Alert.find({ team: b.team._id }); expect(alertA.channel).toEqual({ type: 'webhook', webhookId: a.webhookId, }); - expect(alertB.channel).toEqual({ - type: 'webhook', - webhookId: b.webhookId, - }); + expect(await Alert.countDocuments({ team: b.team._id })).toBe(0); + expect(await provisionedDashboard(b.team._id)).toBeTruthy(); }); it('keeps the last valid version when the name matches no webhook', async () => { - const { team, webhookId } = await setup(); - writeDashboard([makeTile({ id: 'tile', alert: namedAlert('Hook') })]); + const { team, webhookId, tile } = await setup(); + writeDashboard([tile('tile', namedAlert('Hook'))]); await syncDashboards(team._id.toString(), tmpDir); - writeDashboard([ - makeTile({ id: 'tile', alert: namedAlert('Missing', 99) }), - ]); + writeDashboard([tile('tile', namedAlert('Missing', 99))]); await syncDashboards(team._id.toString(), tmpDir); const alerts = await Alert.find({ team: team._id }); expect(alerts).toHaveLength(1); expect(alerts[0].threshold).toBe(10); expect(alerts[0].channel).toEqual({ type: 'webhook', webhookId }); - // The unresolved reference is not stored on the tile. const dashboard = await provisionedDashboard(team._id); - expect(dashboard?.tiles[0].config.alert).toBeUndefined(); + expect(dashboard?.tiles[0].config.alert?.channel).toEqual({ + type: 'webhook', + webhookId, + }); }); it('skips a name that matches webhooks of several services', async () => { - const { team } = await setup(); + const { team, tile } = await setup(); await new Webhook({ team: team._id, service: 'slack', url: 'https://hooks.slack.com/services/x', name: 'Hook', }).save(); - writeDashboard([makeTile({ id: 'tile', alert: namedAlert('Hook') })]); + writeDashboard([tile('tile', namedAlert('Hook'))]); await syncDashboards(team._id.toString(), tmpDir); expect(await Alert.countDocuments({ team: team._id })).toBe(0); }); + + it('does not read webhooks when no file names one', async () => { + const { team, webhookId, tile } = await setup(); + const find = jest.spyOn(Webhook, 'find'); + writeDashboard([tile('tile', tileAlert(webhookId))]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(find).not.toHaveBeenCalled(); + find.mockRestore(); + }); }); describe('orphaned alerts', () => { - const writeFile = (file: string, name: string, tiles: unknown[]) => - fs.writeFileSync( - path.join(tmpDir, file), - JSON.stringify({ name, tiles, tags: [] }), - ); - const setupTwoDashboards = async () => { - const { team, webhookId } = await setup(); - writeFile('a.json', 'A', [ - makeTile({ id: 'tile', alert: tileAlert(webhookId) }), - ]); - writeFile('b.json', 'B', [ - makeTile({ id: 'tile', alert: tileAlert(webhookId) }), - ]); + const { team, webhookId, tile } = await setup(); + writeFile('a.json', 'A', [tile('tile', tileAlert(webhookId))]); + writeFile('b.json', 'B', [tile('tile', tileAlert(webhookId))]); await syncDashboards(team._id.toString(), tmpDir); expect(await Alert.countDocuments({ team: team._id })).toBe(2); - return { team, webhookId }; + return { team, webhookId, tile }; }; const alertDashboardNames = async (teamId: unknown) => { @@ -612,16 +742,33 @@ describe('provisionDashboards', () => { }); it('moves the alerts along when a dashboard is renamed', async () => { - const { team, webhookId } = await setupTwoDashboards(); + const { team, webhookId, tile } = await setupTwoDashboards(); - writeFile('b.json', 'B renamed', [ - makeTile({ id: 'tile', alert: tileAlert(webhookId) }), - ]); + writeFile('b.json', 'B renamed', [tile('tile', tileAlert(webhookId))]); await syncDashboards(team._id.toString(), tmpDir); expect(await alertDashboardNames(team._id)).toEqual(['A', 'B renamed']); }); + it('keeps the old alerts when the renamed dashboard fails to sync', async () => { + const { team, webhookId, tile } = await setupTwoDashboards(); + const upsert = Dashboard.findOneAndUpdate.bind(Dashboard); + const spy = jest + .spyOn(Dashboard, 'findOneAndUpdate') + .mockImplementation((filter, ...rest) => { + if (filter?.name === 'B renamed') { + throw new Error('write failed'); + } + return upsert(filter, ...rest); + }); + + writeFile('b.json', 'B renamed', [tile('tile', tileAlert(webhookId))]); + await syncDashboards(team._id.toString(), tmpDir); + spy.mockRestore(); + + expect(await alertDashboardNames(team._id)).toEqual(['A', 'B']); + }); + it('keeps all alerts while any file is invalid', async () => { const { team } = await setupTwoDashboards(); @@ -641,20 +788,25 @@ describe('provisionDashboards', () => { expect(await alertDashboardNames(team._id)).toEqual(['A', 'B']); }); + it('syncs only the first of two files with the same name, and keeps all alerts', async () => { + const { team, webhookId, tile } = await setupTwoDashboards(); + const [created] = await Alert.find({ team: team._id, tileId: 'tile' }) + .populate<{ dashboard: { name: string } }>('dashboard', 'name') + .then(alerts => alerts.filter(a => a.dashboard.name === 'A')); + + writeFile('c.json', 'A', [tile('tile', tileAlert(webhookId, 99))]); + await syncDashboards(team._id.toString(), tmpDir); + await syncDashboards(team._id.toString(), tmpDir); + + expect(await alertDashboardNames(team._id)).toEqual(['A', 'B']); + const same = await Alert.findById(created._id); + expect(same?.threshold).toBe(10); + }); + it('leaves app-created alerts on an orphaned dashboard alone', async () => { - const { team, webhookId } = await setupTwoDashboards(); + const { team } = await setupTwoDashboards(); const b = await Dashboard.findOne({ team: team._id, name: 'B' }); - await new Alert({ - team: team._id, - source: AlertSource.TILE, - dashboard: b?._id, - tileId: 'other', - interval: '5m', - threshold: 1, - thresholdType: AlertThresholdType.ABOVE, - channel: { type: 'webhook', webhookId }, - createdBy: new mongoose.Types.ObjectId(), - }).save(); + await appAlert(team._id, b?._id, 'other'); fs.rmSync(path.join(tmpDir, 'b.json')); await syncDashboards(team._id.toString(), tmpDir); @@ -666,6 +818,31 @@ describe('provisionDashboards', () => { expect(remaining.map(a => a.tileId)).toEqual(['other']); }); }); + + it('creates alerts only in the team owning the webhook when provisioning all teams', async () => { + const a = await setup(); + const b = await setup(() => new Team({ name: 'Team B' }).save()); + writeDashboard([a.tile('tile', tileAlert(a.webhookId))]); + const env = process.env; + process.env = { + ...env, + DASHBOARD_PROVISIONER_DIR: tmpDir, + DASHBOARD_PROVISIONER_ALL_TEAMS: 'true', + DASHBOARD_PROVISIONER_TEAM_ID: '', + }; + + try { + await new ProvisionDashboardsTask({ + taskName: TaskName.PROVISION_DASHBOARDS, + }).execute(); + } finally { + process.env = env; + } + + expect(await Alert.countDocuments({ team: a.team._id })).toBe(1); + expect(await Alert.countDocuments({ team: b.team._id })).toBe(0); + expect(await provisionedDashboard(b.team._id)).toBeTruthy(); + }); }); describe('ProvisionDashboardsTask', () => { diff --git a/packages/api/src/tasks/provisionDashboards/alerts.ts b/packages/api/src/tasks/provisionDashboards/alerts.ts new file mode 100644 index 0000000000..3491a9214a --- /dev/null +++ b/packages/api/src/tasks/provisionDashboards/alerts.ts @@ -0,0 +1,204 @@ +import { isPromqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { Tile } from '@hyperdx/common-utils/dist/types'; +import { Types } from 'mongoose'; + +import { + AlertInput, + createOrUpdateDashboardAlerts, + validateAlertInput, +} from '@/controllers/alerts'; +import Alert, { AlertSource } from '@/models/alert'; +import Dashboard, { IDashboard } from '@/models/dashboard'; +import { + hasWebhookNames, + loadWebhookIdsByName, + resolveWebhookNames, + withoutAlert, +} from '@/tasks/provisionDashboards/webhookNames'; +import { getCounter } from '@/utils/instrumentation'; +import logger from '@/utils/logger'; +import { internalAlertSchema } from '@/utils/zod'; + +const skippedAlerts = getCounter('hyperdx.provisioner.tile_alert_skipped', { + description: + 'Tile alerts declared in a provisioned dashboard file that were not created or updated because they failed validation.', +}); + +// Runs the checks the alerts API applies, against the tile as the file +// declares it, plus the ones the alert task relies on to evaluate it at all: +// alertable display type, no PromQL, and a source, connection and webhooks +// that exist in this team. +async function validateTileAlert( + team: Types.ObjectId, + dashboardId: string, + tile: Tile, +): Promise { + const parsed = internalAlertSchema.safeParse({ + ...tile.config.alert, + source: AlertSource.TILE, + dashboardId, + tileId: tile.id, + }); + if (!parsed.success) { + throw new Error(parsed.error.issues.map(i => i.message).join(', ')); + } + if (isPromqlSavedChartConfig(tile.config)) { + throw new Error('Alerts are not supported on PromQL tiles'); + } + const { alert: _alert, ...chartConfig } = tile.config; + await validateAlertInput(team, { + source: AlertSource.INLINE, + chartConfig, + channel: parsed.data.channel, + channels: parsed.data.channels, + }); + return parsed.data; +} + +type ProvisionedTiles = { + tiles: Tile[]; + alertsByTile: Record; +}; + +// Decides what each declared alert becomes on this sync: +// - valid: stored and created or updated; +// - invalid: the tile keeps its last synced version, query and alert together, +// so an old alert never evaluates a new query; a new tile is stored without +// its alert; +// - on a tile that already carries an alert created in the app: skipped, so a +// file never takes over, or later deletes, a user's alert. +export async function prepareProvisionedTiles( + teamId: string, + dashboardId: Types.ObjectId, + declaredTiles: Tile[], + storedTiles: Tile[], +): Promise { + const team = new Types.ObjectId(teamId); + const { tiles, unresolved } = hasWebhookNames(declaredTiles) + ? resolveWebhookNames(declaredTiles, await loadWebhookIdsByName(teamId)) + : { tiles: declaredTiles, unresolved: new Map() }; + + const tileIdCounts = new Map(); + for (const tile of tiles) { + tileIdCounts.set(tile.id, (tileIdCounts.get(tile.id) ?? 0) + 1); + } + const userAlertTileIds = new Set( + ( + await Alert.find( + { + team, + dashboard: dashboardId, + source: AlertSource.TILE, + provisioned: { $ne: true }, + }, + { tileId: 1 }, + ).lean() + ).map(a => a.tileId), + ); + + const skip = (tile: Tile, reason: string) => { + skippedAlerts.add(1); + logger.warn( + { teamId, dashboardId, tileId: tile.id, err: reason }, + 'Skipping invalid provisioned tile alert', + ); + }; + const lastSynced = (tile: Tile): Tile => + storedTiles.find(stored => stored.id === tile.id) ?? withoutAlert(tile); + + const alertsByTile: Record = {}; + const result: Tile[] = []; + for (const tile of tiles) { + const reason = unresolved.get(tile.id); + if (reason != null) { + skip(tile, reason); + result.push(lastSynced(tile)); + continue; + } + if (tile.config.alert == null) { + result.push(tile); + continue; + } + if ((tileIdCounts.get(tile.id) ?? 0) > 1) { + skip(tile, 'Tile id is not unique in the dashboard'); + result.push(withoutAlert(tile)); + continue; + } + if (userAlertTileIds.has(tile.id)) { + skip(tile, 'Tile already has an alert created in the app'); + result.push(withoutAlert(tile)); + continue; + } + try { + alertsByTile[tile.id] = await validateTileAlert( + team, + dashboardId.toString(), + tile, + ); + result.push(tile); + } catch (err) { + skip(tile, err instanceof Error ? err.message : String(err)); + result.push(lastSynced(tile)); + } + } + return { tiles: result, alertsByTile }; +} + +// Creates or updates the dashboard's valid provisioned alerts, and deletes +// provisioned alerts whose stored tile no longer declares one. +export async function syncProvisionedAlerts( + dashboard: Pick, + teamId: string, + alertsByTile: Record, +) { + const team = new Types.ObjectId(teamId); + if (Object.keys(alertsByTile).length > 0) { + await createOrUpdateDashboardAlerts( + dashboard, + team, + alertsByTile, + undefined, + { provisioned: true }, + ); + } + + await Alert.deleteMany({ + dashboard: dashboard._id, + team, + source: AlertSource.TILE, + provisioned: true, + tileId: { + $nin: dashboard.tiles + .filter(tile => tile.config.alert != null) + .map(tile => tile.id), + }, + }); +} + +// Provisioned dashboards outlive their files, but their provisioned alerts +// must not: a removed or renamed file would otherwise keep notifying. Only +// called once every file in the directory was read and synced, so a file that +// is briefly invalid, or a dashboard that failed to sync, keeps its alerts. +export async function deleteOrphanedProvisionedAlerts( + teamId: string, + declaredNames: string[], +) { + const orphaned = await Dashboard.find( + { team: teamId, provisioned: true, name: { $nin: declaredNames } }, + { _id: 1, name: 1 }, + ).lean(); + if (orphaned.length === 0) return; + + const { deletedCount } = await Alert.deleteMany({ + team: teamId, + source: AlertSource.TILE, + provisioned: true, + dashboard: { $in: orphaned.map(d => d._id) }, + }); + if (deletedCount > 0) { + logger.info( + { teamId, deletedCount, dashboards: orphaned.map(d => d.name) }, + 'Deleted provisioned alerts of dashboards no longer in the provisioner directory', + ); + } +} diff --git a/packages/api/src/tasks/provisionDashboards/index.ts b/packages/api/src/tasks/provisionDashboards/index.ts index d9fa8476f8..485db22093 100644 --- a/packages/api/src/tasks/provisionDashboards/index.ts +++ b/packages/api/src/tasks/provisionDashboards/index.ts @@ -3,25 +3,24 @@ import { DashboardWithoutId, DashboardWithoutIdSchema, resolveChartPaletteToken, - Tile, walkRawDashboardTileColors, } from '@hyperdx/common-utils/dist/types'; import fs from 'fs'; import { Types } from 'mongoose'; import path from 'path'; -import { - AlertInput, - createOrUpdateDashboardAlerts, - validateAlertInput, -} from '@/controllers/alerts'; import { connectDB, mongooseConnection } from '@/models'; -import Alert, { AlertSource } from '@/models/alert'; -import Dashboard, { IDashboard } from '@/models/dashboard'; +import Dashboard from '@/models/dashboard'; import Team from '@/models/team'; -import Webhook from '@/models/webhook'; +import { + deleteOrphanedProvisionedAlerts, + prepareProvisionedTiles, + syncProvisionedAlerts, +} from '@/tasks/provisionDashboards/alerts'; +import { encodeWebhookNames } from '@/tasks/provisionDashboards/webhookNames'; import type { HdxTask } from '@/tasks/types'; import { ProvisionDashboardsTaskArgs } from '@/tasks/types'; +import { setBusinessContext } from '@/utils/instrumentation'; import logger from '@/utils/logger'; // Heal legacy `chart-1`..`chart-10` tile colors from #2265 before the @@ -43,42 +42,6 @@ const provisionedDashboardSchema = DashboardWithoutIdSchema.superRefine( validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx), ); -// A file names a webhook with `{ "type": "webhook", "webhookName": "..." }`, -// since webhook ids differ per team and per install. Names only resolve per -// team, after the file is parsed, so until then the name rides in `webhookId` -// behind this prefix. It never reaches Mongo: resolveWebhookNames swaps it for -// the real id, or drops the alert from the stored tile. -const WEBHOOK_NAME_REF = 'webhook-name:'; - -const isRecord = (value: unknown): value is Record => - value != null && typeof value === 'object' && !Array.isArray(value); - -function encodeWebhookName(channel: unknown): unknown { - if ( - !isRecord(channel) || - typeof channel.webhookName !== 'string' || - 'webhookId' in channel - ) { - return channel; - } - const { webhookName, ...rest } = channel; - return { ...rest, webhookId: `${WEBHOOK_NAME_REF}${webhookName}` }; -} - -function encodeWebhookNames(raw: unknown): unknown { - if (!isRecord(raw) || !Array.isArray(raw.tiles)) return raw; - for (const tile of raw.tiles) { - const alert = - isRecord(tile) && isRecord(tile.config) ? tile.config.alert : undefined; - if (!isRecord(alert)) continue; - if ('channel' in alert) alert.channel = encodeWebhookName(alert.channel); - if (Array.isArray(alert.channels)) { - alert.channels = alert.channels.map(encodeWebhookName); - } - } - return raw; -} - // `complete` is false when the directory or any file in it could not be read // or validated, so the caller cannot tell which dashboards the files declare. function readDashboardDir(dir: string): { @@ -87,13 +50,17 @@ function readDashboardDir(dir: string): { } { let files: string[]; try { - files = fs.readdirSync(dir).filter(f => f.endsWith('.json')); + files = fs + .readdirSync(dir) + .filter(f => f.endsWith('.json')) + .sort(); } catch (err) { logger.error({ err, dir }, 'Failed to read dashboard directory'); return { dashboards: [], complete: false }; } const dashboards: DashboardWithoutId[] = []; + const names = new Set(); let complete = true; for (const file of files) { try { @@ -114,6 +81,15 @@ function readDashboardDir(dir: string): { complete = false; continue; } + if (names.has(parsed.data.name)) { + logger.warn( + { file, name: parsed.data.name }, + 'Skipping dashboard file whose name another file already uses', + ); + complete = false; + continue; + } + names.add(parsed.data.name); dashboards.push(parsed.data); } catch (err) { logger.error({ err, file }, 'Failed to parse dashboard file'); @@ -127,165 +103,14 @@ export function readDashboardFiles(dir: string): DashboardWithoutId[] { return readDashboardDir(dir).dashboards; } -type AlertChannelInput = { type: 'webhook'; webhookId: string }; - -// Swaps each `webhookName` reference for the id of the team's webhook with -// that name. A tile whose alert names a missing or ambiguous webhook loses its -// alert, and is reported in `unresolved` so its last valid version is kept. -export function resolveWebhookNames( - tiles: Tile[], - webhookIdsByName: Map, -): { tiles: Tile[]; unresolved: Map } { - const unresolved = new Map(); - const resolveChannel = (channel: AlertChannelInput): AlertChannelInput => { - if (!channel.webhookId.startsWith(WEBHOOK_NAME_REF)) return channel; - const name = channel.webhookId.slice(WEBHOOK_NAME_REF.length); - const ids = webhookIdsByName.get(name) ?? []; - if (ids.length === 0) { - throw new Error(`Webhook named "${name}" not found`); - } - if (ids.length > 1) { - throw new Error( - `Webhook name "${name}" matches ${ids.length} webhooks; use webhookId`, - ); - } - return { ...channel, webhookId: ids[0] }; - }; - - const resolved = tiles.map((tile): Tile => { - const alert = tile.config.alert; - if (alert == null) return tile; - try { - return { - ...tile, - config: { - ...tile.config, - alert: { - ...alert, - ...(alert.channel && { channel: resolveChannel(alert.channel) }), - ...(alert.channels && { - channels: alert.channels.map(resolveChannel), - }), - }, - }, - }; - } catch (err) { - unresolved.set(tile.id, err instanceof Error ? err.message : String(err)); - const { alert: _alert, ...config } = tile.config; - return { ...tile, config }; - } - }); - return { tiles: resolved, unresolved }; -} - -// Turns the `config.alert` of each provisioned tile into a tile alert, the way -// saving a dashboard through the API does. An alert that fails validation, or -// is listed in `unresolved`, is skipped and keeps its last valid version; a -// provisioned alert whose tile no longer declares one is removed. Alerts -// created in the app are left alone. -export async function syncProvisionedAlerts( - dashboard: Pick, - teamId: string, - tiles: Tile[], - unresolved: Map = new Map(), -) { - const team = new Types.ObjectId(teamId); - const declared = tiles.filter(tile => tile.config.alert != null); - - for (const [tileId, reason] of unresolved) { - logger.warn( - { name: dashboard.name, tileId, err: reason }, - 'Skipping invalid provisioned tile alert', - ); - } - - const alertsByTile: Record = {}; - for (const tile of declared) { - const alert = tile.config.alert as AlertInput; - try { - await validateAlertInput(team, { - source: AlertSource.TILE, - dashboardId: dashboard._id.toString(), - tileId: tile.id, - channel: alert.channel, - channels: alert.channels, - }); - alertsByTile[tile.id] = alert; - } catch (err) { - logger.warn( - { - name: dashboard.name, - tileId: tile.id, - err: err instanceof Error ? err.message : err, - }, - 'Skipping invalid provisioned tile alert', - ); - } - } - - if (Object.keys(alertsByTile).length > 0) { - await createOrUpdateDashboardAlerts( - dashboard, - team, - alertsByTile, - undefined, - { provisioned: true }, - ); - } - - await Alert.deleteMany({ - dashboard: dashboard._id, - team, - source: AlertSource.TILE, - provisioned: true, - tileId: { - $nin: [...declared.map(tile => tile.id), ...unresolved.keys()], - }, - }); -} - -// Provisioned dashboards outlive their files, but their provisioned alerts -// must not: a removed or renamed file would otherwise keep notifying. Only -// called once every file in the directory was read, so a file that is briefly -// invalid does not lose its alerts. -export async function deleteOrphanedProvisionedAlerts( - teamId: string, - declaredNames: string[], -) { - const orphaned = await Dashboard.find( - { team: teamId, provisioned: true, name: { $nin: declaredNames } }, - { _id: 1, name: 1 }, - ).lean(); - if (orphaned.length === 0) return; - - const { deletedCount } = await Alert.deleteMany({ - team: teamId, - source: AlertSource.TILE, - provisioned: true, - dashboard: { $in: orphaned.map(d => d._id) }, - }); - if (deletedCount > 0) { - logger.info( - { teamId, deletedCount, dashboards: orphaned.map(d => d.name) }, - 'Deleted provisioned alerts of dashboards no longer in the provisioner directory', - ); - } -} - export async function syncDashboards(teamId: string, dir: string) { - const { dashboards, complete } = readDashboardDir(dir); + setBusinessContext({ teamId }); + const { dashboards, complete: allRead } = readDashboardDir(dir); + let complete = allRead; // An empty directory is more likely a failed mount than an intent to drop // every provisioned alert, so orphan cleanup needs at least one file. if (dashboards.length === 0) return; - const webhookIdsByName = new Map(); - const webhooks = await Webhook.find({ team: teamId }, { name: 1 }).lean(); - for (const webhook of webhooks) { - const ids = webhookIdsByName.get(webhook.name) ?? []; - ids.push(webhook._id.toString()); - webhookIdsByName.set(webhook.name, ids); - } - for (const dashboard of dashboards) { try { const userDashboard = await Dashboard.exists({ @@ -300,12 +125,19 @@ export async function syncDashboards(teamId: string, dir: string) { ); } - const { tiles, unresolved } = resolveWebhookNames( + const existing = await Dashboard.findOne( + { name: dashboard.name, team: teamId, provisioned: true }, + { _id: 1, tiles: 1 }, + ).lean(); + const dashboardId = existing?._id ?? new Types.ObjectId(); + const { tiles, alertsByTile } = await prepareProvisionedTiles( + teamId, + dashboardId, dashboard.tiles || [], - webhookIdsByName, + existing?.tiles ?? [], ); - const result = await Dashboard.findOneAndUpdate( + const saved = await Dashboard.findOneAndUpdate( { name: dashboard.name, team: teamId, provisioned: true }, { $set: { @@ -318,27 +150,30 @@ export async function syncDashboards(teamId: string, dir: string) { containers: dashboard.containers || [], }, $setOnInsert: { + _id: dashboardId, name: dashboard.name, team: teamId, provisioned: true, }, }, - { upsert: true, new: false }, + { upsert: true, new: true }, ); - if (result === null) { + if (existing == null) { logger.info({ name: dashboard.name }, 'Created provisioned dashboard'); } - const saved = await Dashboard.findOne({ - name: dashboard.name, - team: teamId, - provisioned: true, - }); - if (saved) { - await syncProvisionedAlerts(saved, teamId, tiles, unresolved); + try { + await syncProvisionedAlerts(saved, teamId, alertsByTile); + } catch (err) { + complete = false; + logger.error( + { err, name: dashboard.name }, + 'Failed to provision dashboard alerts', + ); } } catch (err) { + complete = false; logger.error( { err, name: dashboard.name }, 'Failed to provision dashboard', @@ -348,7 +183,7 @@ export async function syncDashboards(teamId: string, dir: string) { if (!complete) { logger.warn( - 'Some dashboard files could not be read, skipping cleanup of orphaned provisioned alerts', + 'Some dashboard files could not be read or synced, skipping cleanup of orphaned provisioned alerts', ); return; } @@ -423,7 +258,11 @@ export default class ProvisionDashboardsTask implements HdxTask { } for (const id of teamIds) { - await syncDashboards(id, dir); + try { + await syncDashboards(id, dir); + } catch (err) { + logger.error({ err, teamId: id }, 'Failed to provision dashboards'); + } } } diff --git a/packages/api/src/tasks/provisionDashboards/webhookNames.ts b/packages/api/src/tasks/provisionDashboards/webhookNames.ts new file mode 100644 index 0000000000..17f2a6d9a9 --- /dev/null +++ b/packages/api/src/tasks/provisionDashboards/webhookNames.ts @@ -0,0 +1,118 @@ +import { Tile } from '@hyperdx/common-utils/dist/types'; + +import Webhook from '@/models/webhook'; + +// A file names a webhook with `{ "type": "webhook", "webhookName": "..." }`, +// since webhook ids differ per team and per install. Names only resolve per +// team, after the file is parsed, so until then the name rides in `webhookId` +// behind this prefix. It never reaches Mongo: resolveWebhookNames swaps it for +// the real id, or drops the alert from the stored tile. +const WEBHOOK_NAME_REF = 'webhook-name:'; + +const isRecord = (value: unknown): value is Record => + value != null && typeof value === 'object' && !Array.isArray(value); + +function encodeWebhookName(channel: unknown): unknown { + if ( + !isRecord(channel) || + typeof channel.webhookName !== 'string' || + 'webhookId' in channel + ) { + return channel; + } + const { webhookName, ...rest } = channel; + return { ...rest, webhookId: `${WEBHOOK_NAME_REF}${webhookName}` }; +} + +export function encodeWebhookNames(raw: unknown): unknown { + if (!isRecord(raw) || !Array.isArray(raw.tiles)) return raw; + for (const tile of raw.tiles) { + const alert = + isRecord(tile) && isRecord(tile.config) ? tile.config.alert : undefined; + if (!isRecord(alert)) continue; + if ('channel' in alert) alert.channel = encodeWebhookName(alert.channel); + if (Array.isArray(alert.channels)) { + alert.channels = alert.channels.map(encodeWebhookName); + } + } + return raw; +} + +type AlertChannelInput = { type: 'webhook'; webhookId: string }; + +const alertChannels = (tile: Tile): AlertChannelInput[] => { + const alert = tile.config.alert; + return [ + ...(alert?.channel ? [alert.channel] : []), + ...(alert?.channels ?? []), + ]; +}; + +export const hasWebhookNames = (tiles: Tile[]) => + tiles.some(tile => + alertChannels(tile).some(c => c.webhookId.startsWith(WEBHOOK_NAME_REF)), + ); + +export async function loadWebhookIdsByName(teamId: string) { + const webhookIdsByName = new Map(); + const webhooks = await Webhook.find({ team: teamId }, { name: 1 }).lean(); + for (const webhook of webhooks) { + const ids = webhookIdsByName.get(webhook.name) ?? []; + ids.push(webhook._id.toString()); + webhookIdsByName.set(webhook.name, ids); + } + return webhookIdsByName; +} + +// Swaps each `webhookName` reference for the id of the team's webhook with +// that name. A tile whose alert names a missing or ambiguous webhook loses its +// alert, and is reported in `unresolved`. +export function resolveWebhookNames( + tiles: Tile[], + webhookIdsByName: Map, +): { tiles: Tile[]; unresolved: Map } { + const unresolved = new Map(); + const resolveChannel = (channel: AlertChannelInput): AlertChannelInput => { + if (!channel.webhookId.startsWith(WEBHOOK_NAME_REF)) return channel; + const name = channel.webhookId.slice(WEBHOOK_NAME_REF.length); + const ids = webhookIdsByName.get(name) ?? []; + if (ids.length === 0) { + throw new Error(`Webhook named "${name}" not found`); + } + if (ids.length > 1) { + throw new Error( + `Webhook name "${name}" matches ${ids.length} webhooks; use webhookId`, + ); + } + return { ...channel, webhookId: ids[0] }; + }; + + const resolved = tiles.map((tile): Tile => { + const alert = tile.config.alert; + if (alert == null) return tile; + try { + return { + ...tile, + config: { + ...tile.config, + alert: { + ...alert, + ...(alert.channel && { channel: resolveChannel(alert.channel) }), + ...(alert.channels && { + channels: alert.channels.map(resolveChannel), + }), + }, + }, + }; + } catch (err) { + unresolved.set(tile.id, err instanceof Error ? err.message : String(err)); + return withoutAlert(tile); + } + }); + return { tiles: resolved, unresolved }; +} + +export const withoutAlert = (tile: Tile): Tile => { + const { alert: _alert, ...config } = tile.config; + return { ...tile, config }; +};