diff --git a/.changeset/provision-tile-alerts.md b/.changeset/provision-tile-alerts.md new file mode 100644 index 0000000000..8d1a89b638 --- /dev/null +++ b/.changeset/provision-tile-alerts.md @@ -0,0 +1,21 @@ +--- +'@hyperdx/api': minor +--- + +feat: provision tile alerts from dashboard files + +The dashboard file provisioner stored a tile's `config.alert` inside the tile +but never created the alert, so a provisioned alert passed validation and never +fired. It now creates or updates each declared tile alert on every sync, flagged +`provisioned: true`, after the checks the alerts API applies plus the ones the +alert task needs to evaluate it (display type, no PromQL, and a source, +connection and webhooks in the team). + +Channels can name their webhook with `webhookName`, resolved per team on each +sync, instead of a `webhookId` that only exists in one install. + +A provisioned alert is deleted once its tile, or its file entry, no longer +declares it, and when its dashboard's file is removed or the dashboard is +renamed. An alert that fails validation is skipped with a warning, and its tile +keeps its last synced version. Alerts created in the app are never changed or +deleted. diff --git a/packages/api/docs/auto_provision/AUTO_PROVISION.md b/packages/api/docs/auto_provision/AUTO_PROVISION.md index deab07ab19..4ecb4b9732 100644 --- a/packages/api/docs/auto_provision/AUTO_PROVISION.md +++ b/packages/api/docs/auto_provision/AUTO_PROVISION.md @@ -166,22 +166,22 @@ to manage these values. ## Dashboard Provisioning HyperDX supports file-based dashboard provisioning, similar to Grafana's -provisioning system. A scheduled task reads `.json` files from a directory -and upserts dashboards into MongoDB, matched by name for idempotency. -The task runs on the same schedule as other HyperDX tasks (every minute -when using the built-in scheduler, or on your own schedule when running -tasks externally). +provisioning system. A scheduled task reads `.json` files from a directory and +upserts dashboards into MongoDB, matched by name for idempotency. The task runs +on the same schedule as other HyperDX tasks (every minute when using the +built-in scheduler, or on your own schedule when running tasks externally). ### Environment Variables -| Variable | Required | Default | Description | -| ----------------------------------- | -------- | ------- | --------------------------------------------------------------- | -| `DASHBOARD_PROVISIONER_DIR` | Yes | | Directory to watch for `.json` dashboard files | -| `DASHBOARD_PROVISIONER_TEAM_ID` | No\* | | Scope provisioning to a specific team ID | -| `DASHBOARD_PROVISIONER_ALL_TEAMS` | No\* | `false` | Set to `true` to provision dashboards to all teams | +| Variable | Required | Default | Description | +| --------------------------------- | -------- | ------- | -------------------------------------------------- | +| `DASHBOARD_PROVISIONER_DIR` | Yes | | Directory to watch for `.json` dashboard files | +| `DASHBOARD_PROVISIONER_TEAM_ID` | No\* | | Scope provisioning to a specific team ID | +| `DASHBOARD_PROVISIONER_ALL_TEAMS` | No\* | `false` | Set to `true` to provision dashboards to all teams | -\*One of `DASHBOARD_PROVISIONER_TEAM_ID` or `DASHBOARD_PROVISIONER_ALL_TEAMS=true` -is required when `DASHBOARD_PROVISIONER_DIR` is set. +\*One of `DASHBOARD_PROVISIONER_TEAM_ID` or +`DASHBOARD_PROVISIONER_ALL_TEAMS=true` is required when +`DASHBOARD_PROVISIONER_DIR` is set. ### Dashboard JSON Format @@ -220,6 +220,68 @@ with at minimum a `name` and `tiles` array: - Files are validated against the `DashboardWithoutIdSchema` Zod schema; invalid files are skipped with a warning +### Tile Alerts + +A tile can declare an alert in `config.alert`, in the same shape the dashboards +API accepts. The provisioner creates or updates that tile alert on every sync: + +```json +{ + "id": "error-rate", + "x": 0, + "y": 0, + "w": 12, + "h": 4, + "config": { + "name": "Error rate", + "source": "", + "displayType": "line", + "select": [ + { + "aggFn": "count", + "aggCondition": "SeverityText:error", + "aggConditionLanguage": "lucene", + "valueExpression": "" + } + ], + "where": "", + "whereLanguage": "lucene", + "alert": { + "interval": "5m", + "threshold": 100, + "thresholdType": "above", + "channel": { "type": "webhook", "webhookName": "On-call" } + } + } +} +``` + +- Each alert gets the checks the alerts API applies, plus the ones the alert + task needs to evaluate it: a Line, Stacked Bar or Number display type, no + PromQL, a valid raw SQL alert query, and a source, connection and webhooks + that exist in the team. `source` is the source's ObjectId, so with + `DASHBOARD_PROVISIONER_ALL_TEAMS=true` an alert is only created in the team + that owns its source +- A channel can name its webhook with `webhookName`, looked up among the team's + webhooks on every sync, instead of a `webhookId` that only exists in one + install. A name that matches no webhook, or more than one, fails validation +- Provisioned alerts are flagged with `provisioned: true`. Removing an alert + from its tile, or the tile from the file, deletes it on the next sync. Keep + tile ids stable: changing one replaces the alert and loses its history +- Removing a dashboard's file, or renaming the dashboard, deletes the + provisioned alerts of the dashboard it leaves behind. The dashboard itself is + kept, as above. This cleanup only runs when every file in the directory was + read, validated and synced, and never when the directory has no valid files, + so a broken file, a failed sync or an empty mount does not remove alerts +- An alert that fails validation is skipped with a warning. Its tile keeps the + version from the last sync, query and alert together, so an old alert never + evaluates a new query; a new tile is added without its alert +- Alerts created in the app are never changed or deleted. A file that declares + an alert on a tile that already has one from the app is skipped with a warning +- Like the dashboard, a provisioned alert edited or deleted in the app is + restored from its file on the next sync +- Files that share a dashboard name are skipped with a warning, except the first + in file name order ## Note on Security diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index 0e8762e48c..ee6b3286b6 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -419,6 +419,7 @@ export const createOrUpdateDashboardAlerts = async ( teamId: ObjectId, alertsByTile: Record, userId?: ObjectId, + { provisioned = false }: { provisioned?: boolean } = {}, ) => { const dashboardId = dashboard._id; const result = await Promise.all( @@ -428,6 +429,7 @@ export const createOrUpdateDashboardAlerts = async ( tileId, source: AlertSource.TILE, team: teamId, + ...(provisioned && { provisioned: true }), }; const alertInput = { ...alert, @@ -440,6 +442,9 @@ export const createOrUpdateDashboardAlerts = async ( oldAlert && oldAlert.createdBy ? makeAlertUpdate(alertInput, undefined, { dashboard }) : makeAlertUpdate(alertInput, userId, { dashboard }); + if (provisioned) { + alertUpdate.$set.provisioned = true; + } return await Alert.findOneAndUpdate(filter, alertUpdate, { new: true, diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index 3e62d19eab..8a9fb52bb6 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -127,6 +127,9 @@ export interface IAlert { // Multi-window alerting: fire only after N violations in M consecutive windows numConsecutiveWindows?: number | null; + // Created by the dashboard file provisioner, which keeps it in sync with its file + provisioned?: boolean; + // Errors recorded during the most recent execution executionErrors?: IAlertError[]; createdAt: Date; @@ -256,6 +259,10 @@ const AlertSchema = new Schema( required: false, min: 1, }, + provisioned: { + type: Boolean, + default: false, + }, silenced: { required: false, type: { @@ -306,4 +313,14 @@ AlertSchema.index({ team: 1, _id: 1 }); // tie-break, and walk that order with a keyset cursor. AlertSchema.index({ team: 1, displayName: 1, _id: 1 }); +// One alert per provisioned tile, so concurrent provisioner runs upsert the +// same document instead of inserting a duplicate that no sync would remove. +AlertSchema.index( + { team: 1, dashboard: 1, tileId: 1 }, + { + unique: true, + partialFilterExpression: { provisioned: true, source: AlertSource.TILE }, + }, +); + export default mongoose.model('Alert', AlertSchema); diff --git a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts index 3e17594813..6bd1e2e8f2 100644 --- a/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts +++ b/packages/api/src/tasks/provisionDashboards/__tests__/provisionDashboards.int.test.ts @@ -1,4 +1,9 @@ /* eslint-disable security/detect-non-literal-fs-filename */ +import { + AlertThresholdType, + BuilderSavedChartConfig, + SourceKind, +} from '@hyperdx/common-utils/dist/types'; import fs from 'fs'; import mongoose from 'mongoose'; import os from 'os'; @@ -6,8 +11,11 @@ import path from 'path'; import { createTeam } from '@/controllers/team'; import { clearDBCollections, closeDB, connectDB, makeTile } from '@/fixtures'; +import Alert, { AlertSource } from '@/models/alert'; import Dashboard from '@/models/dashboard'; +import { Source } from '@/models/source'; import Team from '@/models/team'; +import Webhook from '@/models/webhook'; import { readDashboardFiles, syncDashboards, @@ -314,6 +322,529 @@ describe('provisionDashboards', () => { }); }); + describe('syncDashboards tile alerts', () => { + type TileAlert = NonNullable; + + const tileAlert = (webhookId: string, threshold = 10): TileAlert => ({ + interval: '5m', + threshold, + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookId }, + }); + + const writeFile = (file: string, name: string, tiles: unknown[]) => + fs.writeFileSync( + path.join(tmpDir, file), + JSON.stringify({ name, tiles, tags: [] }), + ); + + const writeDashboard = (tiles: unknown[]) => + writeFile('alerting.json', 'Alerting', tiles); + + const setup = async (makeTeam = () => createTeam({ name: 'My Team' })) => { + const team = await makeTeam(); + const webhook = await new Webhook({ + team: team._id, + service: 'generic', + url: 'https://example.com/hook', + name: 'Hook', + }).save(); + const source = await Source.create({ + kind: SourceKind.Log, + team: team._id, + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + connection: new mongoose.Types.ObjectId(), + name: 'Logs', + }); + const sourceId = source._id.toString(); + // A tile on this team's source; `alert` is raw JSON, as a file holds it. + const tile = (id: string, alert?: unknown, where?: string) => { + const base = makeTile({ id, sourceId, where }); + return alert == null + ? base + : { ...base, config: { ...base.config, alert } }; + }; + return { team, webhookId: webhook._id.toString(), sourceId, tile }; + }; + + const provisionedDashboard = (teamId: unknown) => + Dashboard.findOne({ name: 'Alerting', team: teamId, provisioned: true }); + + const appAlert = (teamId: unknown, dashboardId: unknown, tileId: string) => + new Alert({ + team: teamId, + source: AlertSource.TILE, + dashboard: dashboardId, + tileId, + interval: '5m', + threshold: 1, + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookId: new mongoose.Types.ObjectId() }, + createdBy: new mongoose.Types.ObjectId(), + }).save(); + + it('creates a provisioned alert for a tile that declares one', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([ + tile('with-alert', tileAlert(webhookId)), + tile('without-alert'), + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + const dashboard = await provisionedDashboard(team._id); + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0].source).toBe(AlertSource.TILE); + expect(alerts[0].dashboard?.toString()).toBe(dashboard?._id.toString()); + expect(alerts[0].tileId).toBe('with-alert'); + expect(alerts[0].threshold).toBe(10); + expect(alerts[0].channel).toEqual({ type: 'webhook', webhookId }); + expect(alerts[0].provisioned).toBe(true); + }); + + it('updates the alert in place on later syncs', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([tile('tile', tileAlert(webhookId))]); + await syncDashboards(team._id.toString(), tmpDir); + const [created] = await Alert.find({ team: team._id }); + + writeDashboard([tile('tile', tileAlert(webhookId, 50))]); + await syncDashboards(team._id.toString(), tmpDir); + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0]._id.toString()).toBe(created._id.toString()); + expect(alerts[0].threshold).toBe(50); + }); + + it('removes a provisioned alert once its tile no longer declares one', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([ + tile('kept', tileAlert(webhookId)), + tile('dropped', tileAlert(webhookId)), + tile('removed', tileAlert(webhookId)), + ]); + await syncDashboards(team._id.toString(), tmpDir); + expect(await Alert.countDocuments({ team: team._id })).toBe(3); + + writeDashboard([tile('kept', tileAlert(webhookId)), tile('dropped')]); + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts.map(a => a.tileId)).toEqual(['kept']); + }); + + it('skips an alert whose webhook does not exist', async () => { + const { team, tile } = await setup(); + writeDashboard([ + tile('tile', tileAlert(new mongoose.Types.ObjectId().toString())), + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + const dashboard = await provisionedDashboard(team._id); + expect(dashboard?.tiles[0].config.alert).toBeUndefined(); + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('skips an alert on a display type the alert task cannot evaluate', async () => { + const { team, webhookId, tile } = await setup(); + const table = tile('tile', tileAlert(webhookId)); + writeDashboard([ + { ...table, config: { ...table.config, displayType: 'table' } }, + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('skips an alert on a raw SQL tile whose display type does not support alerts', async () => { + const { team, webhookId, tile } = await setup(); + const { config: _config, ...base } = tile('tile'); + writeDashboard([ + { + ...base, + config: { + configType: 'sql', + name: 'Raw', + displayType: 'table', + connection: new mongoose.Types.ObjectId().toString(), + sqlTemplate: 'SELECT 1', + alert: tileAlert(webhookId), + }, + }, + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await provisionedDashboard(team._id)).toBeTruthy(); + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it("skips an alert whose tile's source is not in the team", async () => { + const { team, webhookId } = await setup(); + writeDashboard([ + makeTile({ + id: 'tile', + sourceId: new mongoose.Types.ObjectId().toString(), + alert: tileAlert(webhookId), + }), + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('applies the alerts API threshold rules', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([ + tile('tile', { + ...tileAlert(webhookId), + thresholdType: AlertThresholdType.BETWEEN, + }), + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('keeps the last synced tile, query and alert together, when a declared alert becomes invalid', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([ + tile('tile', tileAlert(webhookId), 'level:error'), + tile('other'), + ]); + await syncDashboards(team._id.toString(), tmpDir); + + writeDashboard([ + tile( + 'tile', + tileAlert(new mongoose.Types.ObjectId().toString(), 99), + 'level:warn', + ), + tile('other', undefined, 'updated'), + ]); + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0].threshold).toBe(10); + expect(alerts[0].channel).toEqual({ type: 'webhook', webhookId }); + const dashboard = await provisionedDashboard(team._id); + const [kept, other] = dashboard?.tiles ?? []; + expect(kept.config).toMatchObject({ where: 'level:error' }); + expect(other.config).toMatchObject({ where: 'updated' }); + }); + + it('skips alerts on tiles whose id is not unique', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([ + tile('tile', tileAlert(webhookId)), + tile('tile', tileAlert(webhookId, 20)), + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('leaves alerts created in the app alone', async () => { + const { team, tile } = await setup(); + writeDashboard([tile('tile')]); + await syncDashboards(team._id.toString(), tmpDir); + const dashboard = await provisionedDashboard(team._id); + await appAlert(team._id, dashboard?._id, 'tile'); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await Alert.countDocuments({ team: team._id })).toBe(1); + }); + + it('does not take over an app-created alert on a tile the file declares one on', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([tile('tile')]); + await syncDashboards(team._id.toString(), tmpDir); + const dashboard = await provisionedDashboard(team._id); + const userAlert = await appAlert(team._id, dashboard?._id, 'tile'); + + writeDashboard([tile('tile', tileAlert(webhookId, 50))]); + await syncDashboards(team._id.toString(), tmpDir); + writeDashboard([tile('tile')]); + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0]._id.toString()).toBe(userAlert._id.toString()); + expect(alerts[0].threshold).toBe(1); + expect(alerts[0].provisioned).toBe(false); + }); + + it('allows only one provisioned alert per tile', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([tile('tile', tileAlert(webhookId))]); + await syncDashboards(team._id.toString(), tmpDir); + await Alert.init(); + const [alert] = await Alert.find({ team: team._id }).lean(); + const { _id, ...duplicate } = alert; + + await expect(Alert.create(duplicate)).rejects.toThrow(/duplicate key/); + }); + + describe('webhooks by name', () => { + const namedAlert = (webhookName: string, threshold = 10) => ({ + interval: '5m', + threshold, + thresholdType: AlertThresholdType.ABOVE, + channel: { type: 'webhook', webhookName }, + }); + + it('resolves a webhook name to the team webhook', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([tile('tile', namedAlert('Hook'))]); + + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0].channel).toEqual({ type: 'webhook', webhookId }); + expect(alerts[0].channels).toEqual([{ type: 'webhook', webhookId }]); + const dashboard = await provisionedDashboard(team._id); + expect(dashboard?.tiles[0].config.alert?.channel).toEqual({ + type: 'webhook', + webhookId, + }); + }); + + it('resolves names in `channels` too', async () => { + const { team, webhookId, tile } = await setup(); + const other = await new Webhook({ + team: team._id, + service: 'generic', + url: 'https://example.com/other', + name: 'Other', + }).save(); + const { channel: _channel, ...alert } = namedAlert('Hook'); + writeDashboard([ + tile('tile', { + ...alert, + channels: [ + { type: 'webhook', webhookName: 'Hook' }, + { type: 'webhook', webhookName: 'Other' }, + ], + }), + ]); + + await syncDashboards(team._id.toString(), tmpDir); + + const [created] = await Alert.find({ team: team._id }); + expect(created.channels).toEqual([ + { type: 'webhook', webhookId }, + { type: 'webhook', webhookId: other._id.toString() }, + ]); + }); + + it('only creates the alert in the team that owns the tile source', async () => { + const a = await setup(); + const b = await setup(() => new Team({ name: 'Team B' }).save()); + writeDashboard([a.tile('tile', namedAlert('Hook'))]); + + await syncDashboards(a.team._id.toString(), tmpDir); + await syncDashboards(b.team._id.toString(), tmpDir); + + const [alertA] = await Alert.find({ team: a.team._id }); + expect(alertA.channel).toEqual({ + type: 'webhook', + webhookId: a.webhookId, + }); + expect(await Alert.countDocuments({ team: b.team._id })).toBe(0); + expect(await provisionedDashboard(b.team._id)).toBeTruthy(); + }); + + it('keeps the last valid version when the name matches no webhook', async () => { + const { team, webhookId, tile } = await setup(); + writeDashboard([tile('tile', namedAlert('Hook'))]); + await syncDashboards(team._id.toString(), tmpDir); + + writeDashboard([tile('tile', namedAlert('Missing', 99))]); + await syncDashboards(team._id.toString(), tmpDir); + + const alerts = await Alert.find({ team: team._id }); + expect(alerts).toHaveLength(1); + expect(alerts[0].threshold).toBe(10); + expect(alerts[0].channel).toEqual({ type: 'webhook', webhookId }); + const dashboard = await provisionedDashboard(team._id); + expect(dashboard?.tiles[0].config.alert?.channel).toEqual({ + type: 'webhook', + webhookId, + }); + }); + + it('skips a name that matches webhooks of several services', async () => { + const { team, tile } = await setup(); + await new Webhook({ + team: team._id, + service: 'slack', + url: 'https://hooks.slack.com/services/x', + name: 'Hook', + }).save(); + writeDashboard([tile('tile', namedAlert('Hook'))]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(await Alert.countDocuments({ team: team._id })).toBe(0); + }); + + it('does not read webhooks when no file names one', async () => { + const { team, webhookId, tile } = await setup(); + const find = jest.spyOn(Webhook, 'find'); + writeDashboard([tile('tile', tileAlert(webhookId))]); + + await syncDashboards(team._id.toString(), tmpDir); + + expect(find).not.toHaveBeenCalled(); + find.mockRestore(); + }); + }); + + describe('orphaned alerts', () => { + const setupTwoDashboards = async () => { + const { team, webhookId, tile } = await setup(); + writeFile('a.json', 'A', [tile('tile', tileAlert(webhookId))]); + writeFile('b.json', 'B', [tile('tile', tileAlert(webhookId))]); + await syncDashboards(team._id.toString(), tmpDir); + expect(await Alert.countDocuments({ team: team._id })).toBe(2); + return { team, webhookId, tile }; + }; + + const alertDashboardNames = async (teamId: unknown) => { + const alerts = await Alert.find({ team: teamId }).populate<{ + dashboard: { name: string }; + }>('dashboard', 'name'); + return alerts.map(a => a.dashboard.name).sort(); + }; + + it('deletes the alerts of a dashboard whose file was removed, and keeps the dashboard', async () => { + const { team } = await setupTwoDashboards(); + + fs.rmSync(path.join(tmpDir, 'b.json')); + await syncDashboards(team._id.toString(), tmpDir); + + expect(await alertDashboardNames(team._id)).toEqual(['A']); + expect( + await Dashboard.countDocuments({ team: team._id, provisioned: true }), + ).toBe(2); + }); + + it('moves the alerts along when a dashboard is renamed', async () => { + const { team, webhookId, tile } = await setupTwoDashboards(); + + writeFile('b.json', 'B renamed', [tile('tile', tileAlert(webhookId))]); + await syncDashboards(team._id.toString(), tmpDir); + + expect(await alertDashboardNames(team._id)).toEqual(['A', 'B renamed']); + }); + + it('keeps the old alerts when the renamed dashboard fails to sync', async () => { + const { team, webhookId, tile } = await setupTwoDashboards(); + const upsert = Dashboard.findOneAndUpdate.bind(Dashboard); + const spy = jest + .spyOn(Dashboard, 'findOneAndUpdate') + .mockImplementation((filter, ...rest) => { + if (filter?.name === 'B renamed') { + throw new Error('write failed'); + } + return upsert(filter, ...rest); + }); + + writeFile('b.json', 'B renamed', [tile('tile', tileAlert(webhookId))]); + await syncDashboards(team._id.toString(), tmpDir); + spy.mockRestore(); + + expect(await alertDashboardNames(team._id)).toEqual(['A', 'B']); + }); + + it('keeps all alerts while any file is invalid', async () => { + const { team } = await setupTwoDashboards(); + + fs.writeFileSync(path.join(tmpDir, 'b.json'), '{ not json'); + await syncDashboards(team._id.toString(), tmpDir); + + expect(await alertDashboardNames(team._id)).toEqual(['A', 'B']); + }); + + it('keeps all alerts when the directory is empty', async () => { + const { team } = await setupTwoDashboards(); + + fs.rmSync(path.join(tmpDir, 'a.json')); + fs.rmSync(path.join(tmpDir, 'b.json')); + await syncDashboards(team._id.toString(), tmpDir); + + expect(await alertDashboardNames(team._id)).toEqual(['A', 'B']); + }); + + it('syncs only the first of two files with the same name, and keeps all alerts', async () => { + const { team, webhookId, tile } = await setupTwoDashboards(); + const [created] = await Alert.find({ team: team._id, tileId: 'tile' }) + .populate<{ dashboard: { name: string } }>('dashboard', 'name') + .then(alerts => alerts.filter(a => a.dashboard.name === 'A')); + + writeFile('c.json', 'A', [tile('tile', tileAlert(webhookId, 99))]); + await syncDashboards(team._id.toString(), tmpDir); + await syncDashboards(team._id.toString(), tmpDir); + + expect(await alertDashboardNames(team._id)).toEqual(['A', 'B']); + const same = await Alert.findById(created._id); + expect(same?.threshold).toBe(10); + }); + + it('leaves app-created alerts on an orphaned dashboard alone', async () => { + const { team } = await setupTwoDashboards(); + const b = await Dashboard.findOne({ team: team._id, name: 'B' }); + await appAlert(team._id, b?._id, 'other'); + + fs.rmSync(path.join(tmpDir, 'b.json')); + await syncDashboards(team._id.toString(), tmpDir); + + const remaining = await Alert.find({ + team: team._id, + dashboard: b?._id, + }); + expect(remaining.map(a => a.tileId)).toEqual(['other']); + }); + }); + + it('creates alerts only in the team owning the webhook when provisioning all teams', async () => { + const a = await setup(); + const b = await setup(() => new Team({ name: 'Team B' }).save()); + writeDashboard([a.tile('tile', tileAlert(a.webhookId))]); + const env = process.env; + process.env = { + ...env, + DASHBOARD_PROVISIONER_DIR: tmpDir, + DASHBOARD_PROVISIONER_ALL_TEAMS: 'true', + DASHBOARD_PROVISIONER_TEAM_ID: '', + }; + + try { + await new ProvisionDashboardsTask({ + taskName: TaskName.PROVISION_DASHBOARDS, + }).execute(); + } finally { + process.env = env; + } + + expect(await Alert.countDocuments({ team: a.team._id })).toBe(1); + expect(await Alert.countDocuments({ team: b.team._id })).toBe(0); + expect(await provisionedDashboard(b.team._id)).toBeTruthy(); + }); + }); + describe('ProvisionDashboardsTask', () => { const originalEnv = process.env; diff --git a/packages/api/src/tasks/provisionDashboards/alerts.ts b/packages/api/src/tasks/provisionDashboards/alerts.ts new file mode 100644 index 0000000000..3491a9214a --- /dev/null +++ b/packages/api/src/tasks/provisionDashboards/alerts.ts @@ -0,0 +1,204 @@ +import { isPromqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { Tile } from '@hyperdx/common-utils/dist/types'; +import { Types } from 'mongoose'; + +import { + AlertInput, + createOrUpdateDashboardAlerts, + validateAlertInput, +} from '@/controllers/alerts'; +import Alert, { AlertSource } from '@/models/alert'; +import Dashboard, { IDashboard } from '@/models/dashboard'; +import { + hasWebhookNames, + loadWebhookIdsByName, + resolveWebhookNames, + withoutAlert, +} from '@/tasks/provisionDashboards/webhookNames'; +import { getCounter } from '@/utils/instrumentation'; +import logger from '@/utils/logger'; +import { internalAlertSchema } from '@/utils/zod'; + +const skippedAlerts = getCounter('hyperdx.provisioner.tile_alert_skipped', { + description: + 'Tile alerts declared in a provisioned dashboard file that were not created or updated because they failed validation.', +}); + +// Runs the checks the alerts API applies, against the tile as the file +// declares it, plus the ones the alert task relies on to evaluate it at all: +// alertable display type, no PromQL, and a source, connection and webhooks +// that exist in this team. +async function validateTileAlert( + team: Types.ObjectId, + dashboardId: string, + tile: Tile, +): Promise { + const parsed = internalAlertSchema.safeParse({ + ...tile.config.alert, + source: AlertSource.TILE, + dashboardId, + tileId: tile.id, + }); + if (!parsed.success) { + throw new Error(parsed.error.issues.map(i => i.message).join(', ')); + } + if (isPromqlSavedChartConfig(tile.config)) { + throw new Error('Alerts are not supported on PromQL tiles'); + } + const { alert: _alert, ...chartConfig } = tile.config; + await validateAlertInput(team, { + source: AlertSource.INLINE, + chartConfig, + channel: parsed.data.channel, + channels: parsed.data.channels, + }); + return parsed.data; +} + +type ProvisionedTiles = { + tiles: Tile[]; + alertsByTile: Record; +}; + +// Decides what each declared alert becomes on this sync: +// - valid: stored and created or updated; +// - invalid: the tile keeps its last synced version, query and alert together, +// so an old alert never evaluates a new query; a new tile is stored without +// its alert; +// - on a tile that already carries an alert created in the app: skipped, so a +// file never takes over, or later deletes, a user's alert. +export async function prepareProvisionedTiles( + teamId: string, + dashboardId: Types.ObjectId, + declaredTiles: Tile[], + storedTiles: Tile[], +): Promise { + const team = new Types.ObjectId(teamId); + const { tiles, unresolved } = hasWebhookNames(declaredTiles) + ? resolveWebhookNames(declaredTiles, await loadWebhookIdsByName(teamId)) + : { tiles: declaredTiles, unresolved: new Map() }; + + const tileIdCounts = new Map(); + for (const tile of tiles) { + tileIdCounts.set(tile.id, (tileIdCounts.get(tile.id) ?? 0) + 1); + } + const userAlertTileIds = new Set( + ( + await Alert.find( + { + team, + dashboard: dashboardId, + source: AlertSource.TILE, + provisioned: { $ne: true }, + }, + { tileId: 1 }, + ).lean() + ).map(a => a.tileId), + ); + + const skip = (tile: Tile, reason: string) => { + skippedAlerts.add(1); + logger.warn( + { teamId, dashboardId, tileId: tile.id, err: reason }, + 'Skipping invalid provisioned tile alert', + ); + }; + const lastSynced = (tile: Tile): Tile => + storedTiles.find(stored => stored.id === tile.id) ?? withoutAlert(tile); + + const alertsByTile: Record = {}; + const result: Tile[] = []; + for (const tile of tiles) { + const reason = unresolved.get(tile.id); + if (reason != null) { + skip(tile, reason); + result.push(lastSynced(tile)); + continue; + } + if (tile.config.alert == null) { + result.push(tile); + continue; + } + if ((tileIdCounts.get(tile.id) ?? 0) > 1) { + skip(tile, 'Tile id is not unique in the dashboard'); + result.push(withoutAlert(tile)); + continue; + } + if (userAlertTileIds.has(tile.id)) { + skip(tile, 'Tile already has an alert created in the app'); + result.push(withoutAlert(tile)); + continue; + } + try { + alertsByTile[tile.id] = await validateTileAlert( + team, + dashboardId.toString(), + tile, + ); + result.push(tile); + } catch (err) { + skip(tile, err instanceof Error ? err.message : String(err)); + result.push(lastSynced(tile)); + } + } + return { tiles: result, alertsByTile }; +} + +// Creates or updates the dashboard's valid provisioned alerts, and deletes +// provisioned alerts whose stored tile no longer declares one. +export async function syncProvisionedAlerts( + dashboard: Pick, + teamId: string, + alertsByTile: Record, +) { + const team = new Types.ObjectId(teamId); + if (Object.keys(alertsByTile).length > 0) { + await createOrUpdateDashboardAlerts( + dashboard, + team, + alertsByTile, + undefined, + { provisioned: true }, + ); + } + + await Alert.deleteMany({ + dashboard: dashboard._id, + team, + source: AlertSource.TILE, + provisioned: true, + tileId: { + $nin: dashboard.tiles + .filter(tile => tile.config.alert != null) + .map(tile => tile.id), + }, + }); +} + +// Provisioned dashboards outlive their files, but their provisioned alerts +// must not: a removed or renamed file would otherwise keep notifying. Only +// called once every file in the directory was read and synced, so a file that +// is briefly invalid, or a dashboard that failed to sync, keeps its alerts. +export async function deleteOrphanedProvisionedAlerts( + teamId: string, + declaredNames: string[], +) { + const orphaned = await Dashboard.find( + { team: teamId, provisioned: true, name: { $nin: declaredNames } }, + { _id: 1, name: 1 }, + ).lean(); + if (orphaned.length === 0) return; + + const { deletedCount } = await Alert.deleteMany({ + team: teamId, + source: AlertSource.TILE, + provisioned: true, + dashboard: { $in: orphaned.map(d => d._id) }, + }); + if (deletedCount > 0) { + logger.info( + { teamId, deletedCount, dashboards: orphaned.map(d => d.name) }, + 'Deleted provisioned alerts of dashboards no longer in the provisioner directory', + ); + } +} diff --git a/packages/api/src/tasks/provisionDashboards/index.ts b/packages/api/src/tasks/provisionDashboards/index.ts index cf10427c72..485db22093 100644 --- a/packages/api/src/tasks/provisionDashboards/index.ts +++ b/packages/api/src/tasks/provisionDashboards/index.ts @@ -6,13 +6,21 @@ import { walkRawDashboardTileColors, } from '@hyperdx/common-utils/dist/types'; import fs from 'fs'; +import { Types } from 'mongoose'; import path from 'path'; import { connectDB, mongooseConnection } from '@/models'; import Dashboard from '@/models/dashboard'; import Team from '@/models/team'; +import { + deleteOrphanedProvisionedAlerts, + prepareProvisionedTiles, + syncProvisionedAlerts, +} from '@/tasks/provisionDashboards/alerts'; +import { encodeWebhookNames } from '@/tasks/provisionDashboards/webhookNames'; import type { HdxTask } from '@/tasks/types'; import { ProvisionDashboardsTaskArgs } from '@/tasks/types'; +import { setBusinessContext } from '@/utils/instrumentation'; import logger from '@/utils/logger'; // Heal legacy `chart-1`..`chart-10` tile colors from #2265 before the @@ -34,20 +42,32 @@ const provisionedDashboardSchema = DashboardWithoutIdSchema.superRefine( validateDashboardFilterOptionUniqueness(data.filters ?? [], ctx), ); -export function readDashboardFiles(dir: string): DashboardWithoutId[] { +// `complete` is false when the directory or any file in it could not be read +// or validated, so the caller cannot tell which dashboards the files declare. +function readDashboardDir(dir: string): { + dashboards: DashboardWithoutId[]; + complete: boolean; +} { let files: string[]; try { - files = fs.readdirSync(dir).filter(f => f.endsWith('.json')); + files = fs + .readdirSync(dir) + .filter(f => f.endsWith('.json')) + .sort(); } catch (err) { logger.error({ err, dir }, 'Failed to read dashboard directory'); - return []; + return { dashboards: [], complete: false }; } const dashboards: DashboardWithoutId[] = []; + const names = new Set(); + let complete = true; for (const file of files) { try { - const raw = migrateLegacyDashboardTileColorsRaw( - JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')), + const raw = encodeWebhookNames( + migrateLegacyDashboardTileColorsRaw( + JSON.parse(fs.readFileSync(path.join(dir, file), 'utf8')), + ), ) as Record | null | undefined; const parsed = provisionedDashboardSchema.safeParse({ tags: [], @@ -58,18 +78,37 @@ export function readDashboardFiles(dir: string): DashboardWithoutId[] { { file, errors: parsed.error.issues }, 'Skipping invalid dashboard file', ); + complete = false; + continue; + } + if (names.has(parsed.data.name)) { + logger.warn( + { file, name: parsed.data.name }, + 'Skipping dashboard file whose name another file already uses', + ); + complete = false; continue; } + names.add(parsed.data.name); dashboards.push(parsed.data); } catch (err) { logger.error({ err, file }, 'Failed to parse dashboard file'); + complete = false; } } - return dashboards; + return { dashboards, complete }; +} + +export function readDashboardFiles(dir: string): DashboardWithoutId[] { + return readDashboardDir(dir).dashboards; } export async function syncDashboards(teamId: string, dir: string) { - const dashboards = readDashboardFiles(dir); + setBusinessContext({ teamId }); + const { dashboards, complete: allRead } = readDashboardDir(dir); + let complete = allRead; + // An empty directory is more likely a failed mount than an intent to drop + // every provisioned alert, so orphan cleanup needs at least one file. if (dashboards.length === 0) return; for (const dashboard of dashboards) { @@ -86,11 +125,23 @@ export async function syncDashboards(teamId: string, dir: string) { ); } - const result = await Dashboard.findOneAndUpdate( + const existing = await Dashboard.findOne( + { name: dashboard.name, team: teamId, provisioned: true }, + { _id: 1, tiles: 1 }, + ).lean(); + const dashboardId = existing?._id ?? new Types.ObjectId(); + const { tiles, alertsByTile } = await prepareProvisionedTiles( + teamId, + dashboardId, + dashboard.tiles || [], + existing?.tiles ?? [], + ); + + const saved = await Dashboard.findOneAndUpdate( { name: dashboard.name, team: teamId, provisioned: true }, { $set: { - tiles: dashboard.tiles || [], + tiles, tags: dashboard.tags || [], filters: dashboard.filters || [], savedQuery: dashboard.savedQuery ?? null, @@ -99,24 +150,51 @@ export async function syncDashboards(teamId: string, dir: string) { containers: dashboard.containers || [], }, $setOnInsert: { + _id: dashboardId, name: dashboard.name, team: teamId, provisioned: true, }, }, - { upsert: true, new: false }, + { upsert: true, new: true }, ); - if (result === null) { + if (existing == null) { logger.info({ name: dashboard.name }, 'Created provisioned dashboard'); } + + try { + await syncProvisionedAlerts(saved, teamId, alertsByTile); + } catch (err) { + complete = false; + logger.error( + { err, name: dashboard.name }, + 'Failed to provision dashboard alerts', + ); + } } catch (err) { + complete = false; logger.error( { err, name: dashboard.name }, 'Failed to provision dashboard', ); } } + + if (!complete) { + logger.warn( + 'Some dashboard files could not be read or synced, skipping cleanup of orphaned provisioned alerts', + ); + return; + } + try { + await deleteOrphanedProvisionedAlerts( + teamId, + dashboards.map(d => d.name), + ); + } catch (err) { + logger.error({ err }, 'Failed to clean up orphaned provisioned alerts'); + } } export default class ProvisionDashboardsTask implements HdxTask { @@ -180,7 +258,11 @@ export default class ProvisionDashboardsTask implements HdxTask { } for (const id of teamIds) { - await syncDashboards(id, dir); + try { + await syncDashboards(id, dir); + } catch (err) { + logger.error({ err, teamId: id }, 'Failed to provision dashboards'); + } } } diff --git a/packages/api/src/tasks/provisionDashboards/webhookNames.ts b/packages/api/src/tasks/provisionDashboards/webhookNames.ts new file mode 100644 index 0000000000..17f2a6d9a9 --- /dev/null +++ b/packages/api/src/tasks/provisionDashboards/webhookNames.ts @@ -0,0 +1,118 @@ +import { Tile } from '@hyperdx/common-utils/dist/types'; + +import Webhook from '@/models/webhook'; + +// A file names a webhook with `{ "type": "webhook", "webhookName": "..." }`, +// since webhook ids differ per team and per install. Names only resolve per +// team, after the file is parsed, so until then the name rides in `webhookId` +// behind this prefix. It never reaches Mongo: resolveWebhookNames swaps it for +// the real id, or drops the alert from the stored tile. +const WEBHOOK_NAME_REF = 'webhook-name:'; + +const isRecord = (value: unknown): value is Record => + value != null && typeof value === 'object' && !Array.isArray(value); + +function encodeWebhookName(channel: unknown): unknown { + if ( + !isRecord(channel) || + typeof channel.webhookName !== 'string' || + 'webhookId' in channel + ) { + return channel; + } + const { webhookName, ...rest } = channel; + return { ...rest, webhookId: `${WEBHOOK_NAME_REF}${webhookName}` }; +} + +export function encodeWebhookNames(raw: unknown): unknown { + if (!isRecord(raw) || !Array.isArray(raw.tiles)) return raw; + for (const tile of raw.tiles) { + const alert = + isRecord(tile) && isRecord(tile.config) ? tile.config.alert : undefined; + if (!isRecord(alert)) continue; + if ('channel' in alert) alert.channel = encodeWebhookName(alert.channel); + if (Array.isArray(alert.channels)) { + alert.channels = alert.channels.map(encodeWebhookName); + } + } + return raw; +} + +type AlertChannelInput = { type: 'webhook'; webhookId: string }; + +const alertChannels = (tile: Tile): AlertChannelInput[] => { + const alert = tile.config.alert; + return [ + ...(alert?.channel ? [alert.channel] : []), + ...(alert?.channels ?? []), + ]; +}; + +export const hasWebhookNames = (tiles: Tile[]) => + tiles.some(tile => + alertChannels(tile).some(c => c.webhookId.startsWith(WEBHOOK_NAME_REF)), + ); + +export async function loadWebhookIdsByName(teamId: string) { + const webhookIdsByName = new Map(); + const webhooks = await Webhook.find({ team: teamId }, { name: 1 }).lean(); + for (const webhook of webhooks) { + const ids = webhookIdsByName.get(webhook.name) ?? []; + ids.push(webhook._id.toString()); + webhookIdsByName.set(webhook.name, ids); + } + return webhookIdsByName; +} + +// Swaps each `webhookName` reference for the id of the team's webhook with +// that name. A tile whose alert names a missing or ambiguous webhook loses its +// alert, and is reported in `unresolved`. +export function resolveWebhookNames( + tiles: Tile[], + webhookIdsByName: Map, +): { tiles: Tile[]; unresolved: Map } { + const unresolved = new Map(); + const resolveChannel = (channel: AlertChannelInput): AlertChannelInput => { + if (!channel.webhookId.startsWith(WEBHOOK_NAME_REF)) return channel; + const name = channel.webhookId.slice(WEBHOOK_NAME_REF.length); + const ids = webhookIdsByName.get(name) ?? []; + if (ids.length === 0) { + throw new Error(`Webhook named "${name}" not found`); + } + if (ids.length > 1) { + throw new Error( + `Webhook name "${name}" matches ${ids.length} webhooks; use webhookId`, + ); + } + return { ...channel, webhookId: ids[0] }; + }; + + const resolved = tiles.map((tile): Tile => { + const alert = tile.config.alert; + if (alert == null) return tile; + try { + return { + ...tile, + config: { + ...tile.config, + alert: { + ...alert, + ...(alert.channel && { channel: resolveChannel(alert.channel) }), + ...(alert.channels && { + channels: alert.channels.map(resolveChannel), + }), + }, + }, + }; + } catch (err) { + unresolved.set(tile.id, err instanceof Error ? err.message : String(err)); + return withoutAlert(tile); + } + }); + return { tiles: resolved, unresolved }; +} + +export const withoutAlert = (tile: Tile): Tile => { + const { alert: _alert, ...config } = tile.config; + return { ...tile, config }; +};