From 2b0b9eaa61aaed6b48d5692502710c49c5ebc2f4 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 11 Sep 2026 22:58:58 +0530 Subject: [PATCH 01/49] feat: Support Alerting for PromQL Charts --- .../__tests__/evaluatePromqlAlert.test.ts | 162 ++++++++++++++++++ packages/api/src/tasks/checkAlerts/index.ts | 116 ++++++++++++- .../app/src/components/ChartEditor/utils.ts | 2 +- packages/common-utils/src/core/utils.ts | 4 +- packages/common-utils/src/types.ts | 4 +- 5 files changed, 280 insertions(+), 8 deletions(-) create mode 100644 packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts new file mode 100644 index 0000000000..8ceba012df --- /dev/null +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts @@ -0,0 +1,162 @@ +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import mongoose from 'mongoose'; + +import { getConnectionById } from '@/controllers/connection'; +import { evaluatePromqlAlert } from '@/tasks/checkAlerts'; + +jest.mock('@/controllers/connection'); +jest.mock('@hyperdx/common-utils/dist/clickhouse/node'); + +describe('evaluatePromqlAlert', () => { + const mockTeamId = new mongoose.Types.ObjectId().toString(); + const mockConnectionId = new mongoose.Types.ObjectId().toString(); + const mockDateRange: [Date, Date] = [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-01T00:05:00Z'), + ]; + const mockWindowSizeInMins = 5; + + const mockSavedConfig = { + configType: 'promql' as const, + promqlExpression: 'up', + connection: mockConnectionId, + }; + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('Prometheus endpoint', () => { + beforeEach(() => { + (getConnectionById as jest.Mock).mockResolvedValue({ + host: 'http://prometheus:9090', + isPrometheusEndpoint: true, + }); + + // Mock global fetch + global.fetch = jest.fn(); + }); + + it('should query Prometheus API and return the last value', async () => { + (global.fetch as jest.Mock).mockResolvedValue({ + ok: true, + json: async () => ({ + status: 'success', + data: { + result: [ + { + values: [ + [1704067200, '0'], + [1704067500, '42.5'], + ], + }, + ], + }, + }), + }); + + const result = await evaluatePromqlAlert({ + savedConfig: mockSavedConfig, + connectionId: mockConnectionId, + teamId: mockTeamId, + dateRange: mockDateRange, + windowSizeInMins: mockWindowSizeInMins, + }); + + expect(result).toBe(42.5); + expect(global.fetch).toHaveBeenCalledWith( + 'http://prometheus:9090/api/v1/query_range?query=up&start=1704067200&end=1704067500&step=300', + expect.any(Object), + ); + }); + + it('should return null when no data is returned', async () => { + (global.fetch as jest.Mock).mockResolvedValue({ + ok: true, + json: async () => ({ + status: 'success', + data: { result: [] }, + }), + }); + + const result = await evaluatePromqlAlert({ + savedConfig: mockSavedConfig, + connectionId: mockConnectionId, + teamId: mockTeamId, + dateRange: mockDateRange, + windowSizeInMins: mockWindowSizeInMins, + }); + + expect(result).toBeNull(); + }); + }); + + describe('ClickHouse endpoint', () => { + let mockQuery: jest.Mock; + + beforeEach(() => { + (getConnectionById as jest.Mock).mockResolvedValue({ + host: 'http://clickhouse:8123', + isPrometheusEndpoint: false, + }); + + mockQuery = jest.fn(); + (ClickhouseClient as unknown as jest.Mock).mockImplementation(() => ({ + query: mockQuery, + })); + }); + + it('should query ClickHouse and return the last value', async () => { + mockQuery.mockResolvedValue({ + json: async () => ({ + data: [ + { + time_series: [ + ['2024-01-01 00:00:00', 0], + ['2024-01-01 00:05:00', 42.5], + ], + }, + ], + }), + }); + + const result = await evaluatePromqlAlert({ + savedConfig: mockSavedConfig, + connectionId: mockConnectionId, + teamId: mockTeamId, + dateRange: mockDateRange, + windowSizeInMins: mockWindowSizeInMins, + }); + + expect(result).toBe(42.5); + expect(mockQuery).toHaveBeenCalledWith( + expect.objectContaining({ + query_params: expect.objectContaining({ + expr: 'up', + startMs: 1704067200000, + endMs: 1704067500000, + stepSec: 300, + }), + }), + ); + }); + + it('should return null when no data is returned', async () => { + mockQuery.mockResolvedValue({ + json: async () => ({ + data: [], + }), + }); + + const result = await evaluatePromqlAlert({ + savedConfig: mockSavedConfig, + connectionId: mockConnectionId, + teamId: mockTeamId, + dateRange: mockDateRange, + windowSizeInMins: mockWindowSizeInMins, + }); + + expect(result).toBeNull(); + }); + }); +}); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 6c58887caf..d6ef04d065 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -54,6 +54,7 @@ import { performance } from 'perf_hooks'; import { serializeError } from 'serialize-error'; import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory'; +import { getConnectionById } from '@/controllers/connection'; import { AlertState, IAlert, IAlertError } from '@/models/alert'; import AlertHistory, { IAlertHistory, @@ -63,6 +64,7 @@ import { IDashboard } from '@/models/dashboard'; import { ISavedSearch } from '@/models/savedSearch'; import { ISource } from '@/models/source'; import { IWebhook } from '@/models/webhook'; +import { joinPrometheusUpstreamUrl } from '@/routers/api/prometheus'; import { isClientTimeoutOrAbortError, isQueryTimeoutError, @@ -735,9 +737,10 @@ const buildAlertChartConfigFromSavedConfig = ({ return undefined; } - // PromQL charts don't support alerts yet + // PromQL tile alerts are evaluated separately via evaluatePromqlAlert; + // return null here so the shared ClickHouse path is skipped. if (isPromqlSavedChartConfig(savedConfig)) { - return undefined; + return null; } if (!source) { @@ -960,6 +963,115 @@ export const parseAlertData = ( return { value, extraFields }; }; +/** + * Execute a PromQL alert tile's expression for the given window and return the + * last bucket's numeric value, or null when the query returns no data. + * + * Supports both Prometheus-endpoint connections (HTTP proxy to /api/v1/query) + * and ClickHouse backed connections (prometheusQuery table function). + */ +export async function evaluatePromqlAlert({ + savedConfig, + connectionId, + teamId, + dateRange, + windowSizeInMins, +}: { + savedConfig: PromqlSavedChartConfig; + connectionId: string; + teamId: string; + dateRange: [Date, Date]; + windowSizeInMins: number; +}): Promise { + const connection = await getConnectionById(teamId, connectionId, true); + if (connection == null) { + throw new Error(`Connection ${connectionId} not found for PromQL alert`); + } + + const endSec = dateRange[1].getTime() / 1000; + const startSec = dateRange[0].getTime() / 1000; + const stepSec = windowSizeInMins * 60; + + if (connection.isPrometheusEndpoint) { + // Proxy to the real Prometheus /api/v1/query_range + const url = joinPrometheusUpstreamUrl( + connection.host, + '/api/v1/query_range', + ); + const params = new URLSearchParams({ + query: savedConfig.promqlExpression, + start: String(startSec), + end: String(endSec), + step: String(stepSec), + }); + const resp = await fetch(`${url.toString()}?${params.toString()}`, { + signal: AbortSignal.timeout(30_000), + }); + if (!resp.ok) { + throw new Error( + `Prometheus query_range returned HTTP ${resp.status} for PromQL alert`, + ); + } + const json = (await resp.json()) as { + status: string; + data: { result: { values: [number, string][] }[] }; + }; + if (json.status !== 'success' || !json.data?.result?.length) { + return null; + } + // Take the value from the last data point of the first series + const series = json.data.result[0]; + const lastPoint = series.values[series.values.length - 1]; + if (lastPoint == null) return null; + const parsed = parseFloat(lastPoint[1]); + return Number.isFinite(parsed) ? parsed : null; + } + + // ClickHouse prometheusQueryRange path + const { ClickhouseClient: ChClient } = await import( + '@hyperdx/common-utils/dist/clickhouse/node' + ); + const client = new ChClient({ + host: connection.host, + username: connection.username, + password: connection.password, + requestTimeout: 30_000, + }); + + const startMs = Math.floor(startSec * 1000); + const endMs = Math.floor(endSec * 1000); + + const resp = await client.query({ + query: `SELECT tags, time_series FROM prometheusQueryRange({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({startMs:Int64}), fromUnixTimestamp64Milli({endMs:Int64}), toIntervalSecond({stepSec:UInt32})) SETTINGS allow_experimental_time_series_table = 1`, + query_params: { + db: savedConfig.source ?? 'default', + table: savedConfig.promqlExpression.match(/^[\w.]+$/) + ? savedConfig.promqlExpression + : '__hdx_metric__', + expr: savedConfig.promqlExpression, + startMs, + endMs, + stepSec, + }, + format: 'JSON', + clickhouse_settings: { + allow_experimental_time_series_table: 1, + max_execution_time: 30, + }, + }); + + const json = await resp.json<{ + data: { time_series: [string, number][] }[]; + }>(); + if (!json.data?.length) return null; + const series = json.data[0].time_series; + if (!series?.length) return null; + const lastPoint = series[series.length - 1]; + return lastPoint != null && Number.isFinite(lastPoint[1]) + ? lastPoint[1] + : null; +} + export const processAlert = async ( now: Date, details: AlertDetails, diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 87a693fd9f..5a729526c4 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -200,7 +200,7 @@ export function convertFormStateToSavedChartConfig( 'fillNulls', 'alignDateRangeToGranularity', 'alternateRowBackground', - // 'alert', // TODO: Support alerts on PromQL (HDX-4636) + 'alert', ]), promqlExpression: form.promqlExpression ?? '', connection: form.connection ?? '', diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 36eaf5849c..eaca9c980b 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -1435,9 +1435,7 @@ export const isFormulaSourceKind = ( export function displayTypeSupportsPromQLAlerts( displayType: DisplayType | undefined, ): boolean { - // TODO: Support alerts for PromQL (HDX-4636) - // This looks funky, just doing it to satisfy knip - return displayType ? false : false; + return displayType === DisplayType.Line || displayType === DisplayType.Number; } /** Expand the chart's macros, returning failures instead of throwing. */ diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 17cd34f54c..f4db0b8fcf 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1870,12 +1870,12 @@ export const SavedChartConfigSchema = z.union([ /** * The chart config an inline-source alert persists (see `zInlineAlert`). Same * shape as a dashboard tile's config, but without the embedded `alert` field - * (the alert's own document carries those fields) and without the PromQL - * variant (PromQL charts cannot be alerted on). + * (the alert's own document carries those fields). */ export const AlertChartConfigSchema = z.union([ BuilderSavedChartConfigWithoutAlertSchema, RawSqlSavedChartConfigWithoutAlertSchema, + PromqlSavedChartConfigWithoutAlertSchema, ]); export type AlertChartConfig = z.infer; From a8378a0d6be0bbad33e30bdb28b4b712df009219 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 11 Sep 2026 23:06:54 +0530 Subject: [PATCH 02/49] feat: add support for PromQL-based alerts in chart explorer and dashboards --- .changeset/promql-alerts.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/promql-alerts.md diff --git a/.changeset/promql-alerts.md b/.changeset/promql-alerts.md new file mode 100644 index 0000000000..790c01cdf6 --- /dev/null +++ b/.changeset/promql-alerts.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': minor +'@hyperdx/app': minor +'@hyperdx/common-utils': minor +--- + +Added support for configuring and evaluating PromQL-based alerts directly from the chart explorer and dashboards. This allows triggering standard alert notifications based on the result of Prometheus queries. From 75d3428e6913e32db4c67b6dfe258719355073e1 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 11 Sep 2026 23:48:06 +0530 Subject: [PATCH 03/49] feat: support multi-series metrics and grouping in PromQL alert evaluation --- .../external-api/v2/utils/alertChartConfig.ts | 3 + .../__tests__/evaluatePromqlAlert.test.ts | 82 ++- packages/api/src/tasks/checkAlerts/index.ts | 501 +++++++++++------- 3 files changed, 388 insertions(+), 198 deletions(-) diff --git a/packages/api/src/routers/external-api/v2/utils/alertChartConfig.ts b/packages/api/src/routers/external-api/v2/utils/alertChartConfig.ts index f7356078f9..814ee3419c 100644 --- a/packages/api/src/routers/external-api/v2/utils/alertChartConfig.ts +++ b/packages/api/src/routers/external-api/v2/utils/alertChartConfig.ts @@ -143,6 +143,9 @@ export const KNOWN_LOSSY_CONFIG_KEYS: ReadonlySet = new Set([ 'bodyExpression', 'useTextIndexForImplicitColumn', 'metricTables', + 'promqlExpression', + 'step', + 'legendTemplate', ]); /** Select-item fields the external dialect round-trips. */ diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts index 8ceba012df..a0ff63e885 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts @@ -63,13 +63,50 @@ describe('evaluatePromqlAlert', () => { windowSizeInMins: mockWindowSizeInMins, }); - expect(result).toBe(42.5); + expect(result).toEqual([{ group: '', value: 42.5 }]); expect(global.fetch).toHaveBeenCalledWith( 'http://prometheus:9090/api/v1/query_range?query=up&start=1704067200&end=1704067500&step=300', expect.any(Object), ); }); + + it('should return multiple series for Prometheus endpoint', async () => { + (global.fetch as jest.Mock).mockResolvedValue({ + ok: true, + json: async () => ({ + status: 'success', + data: { + result: [ + { + metric: { host: 'A' }, + values: [[1704067500, '42.5']], + }, + { + metric: { host: 'B' }, + values: [[1704067500, '10.5']], + }, + ], + }, + }), + }); + + const result = await evaluatePromqlAlert({ + savedConfig: mockSavedConfig, + connectionId: mockConnectionId, + teamId: mockTeamId, + dateRange: mockDateRange, + windowSizeInMins: mockWindowSizeInMins, + }); + + expect(result).toEqual([ + { group: 'host:"A"', value: 42.5 }, + { group: 'host:"B"', value: 10.5 }, + ]); + }); + + + it('should return null when no data is returned', async () => { (global.fetch as jest.Mock).mockResolvedValue({ ok: true, @@ -106,6 +143,47 @@ describe('evaluatePromqlAlert', () => { })); }); + it('should return multiple series for ClickHouse endpoint with proper db/table', async () => { + mockQuery.mockResolvedValue({ + json: async () => ({ + data: [ + { + tags: { host: 'A' }, + time_series: [['2024-01-01 00:05:00', 42.5]], + }, + { + tags: { host: 'B' }, + time_series: [['2024-01-01 00:05:00', 10.5]], + }, + ], + }), + }); + + const result = await evaluatePromqlAlert({ + savedConfig: mockSavedConfig, + source: { from: { databaseName: 'my_db', tableName: 'my_table' } } as any, + connectionId: mockConnectionId, + teamId: mockTeamId, + dateRange: mockDateRange, + windowSizeInMins: mockWindowSizeInMins, + }); + + expect(result).toEqual([ + { group: 'host:"A"', value: 42.5 }, + { group: 'host:"B"', value: 10.5 }, + ]); + + expect(mockQuery).toHaveBeenCalledWith( + expect.objectContaining({ + query_params: expect.objectContaining({ + expr: 'up', + db: 'my_db', + table: 'my_table', + }), + }), + ); + }); + it('should query ClickHouse and return the last value', async () => { mockQuery.mockResolvedValue({ json: async () => ({ @@ -128,7 +206,7 @@ describe('evaluatePromqlAlert', () => { windowSizeInMins: mockWindowSizeInMins, }); - expect(result).toBe(42.5); + expect(result).toEqual([{ group: '', value: 42.5 }]); expect(mockQuery).toHaveBeenCalledWith( expect.objectContaining({ query_params: expect.objectContaining({ diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index d6ef04d065..7c8de9092f 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -43,6 +43,7 @@ import { getSampleWeightExpression, pickSampleWeightExpressionProps, SavedChartConfig, + PromqlSavedChartConfig, SourceKind, } from '@hyperdx/common-utils/dist/types'; import * as fns from 'date-fns'; @@ -222,12 +223,12 @@ class InvalidAlertError extends Error { // from upstream systems. QUERY_ERROR and INVALID_ALERT messages are authored // by us (ClickHouse errors or our own validation) and are safe to display. const HARDCODED_ALERT_ERROR_MESSAGES: Partial> = - { - [AlertErrorType.WEBHOOK_ERROR]: - 'Failed to send webhook notification. Check the webhook configuration and destination.', - [AlertErrorType.UNKNOWN]: - 'An unknown error occurred while processing the alert.', - }; +{ + [AlertErrorType.WEBHOOK_ERROR]: + 'Failed to send webhook notification. Check the webhook configuration and destination.', + [AlertErrorType.UNKNOWN]: + 'An unknown error occurred while processing the alert.', +}; const makeAlertError = ( type: AlertErrorType, @@ -738,9 +739,9 @@ const buildAlertChartConfigFromSavedConfig = ({ } // PromQL tile alerts are evaluated separately via evaluatePromqlAlert; - // return null here so the shared ClickHouse path is skipped. + // return undefined here so the shared ClickHouse path is skipped. if (isPromqlSavedChartConfig(savedConfig)) { - return null; + return undefined; } if (!source) { @@ -879,14 +880,14 @@ const getChartConfigFromAlert = ( type ResponseMetadata = | { - type: 'time_series'; - timestampColumnName: string; - valueColumnNames: Set; - } + type: 'time_series'; + timestampColumnName: string; + valueColumnNames: Set; + } | { - type: 'single_value'; - valueColumnNames: Set; - }; + type: 'single_value'; + valueColumnNames: Set; + }; const getResponseMetadata = ( chartConfig: ChartConfigWithOptDateRange, @@ -972,17 +973,19 @@ export const parseAlertData = ( */ export async function evaluatePromqlAlert({ savedConfig, + source, connectionId, teamId, dateRange, windowSizeInMins, }: { savedConfig: PromqlSavedChartConfig; + source?: ISource | null; connectionId: string; teamId: string; dateRange: [Date, Date]; windowSizeInMins: number; -}): Promise { +}): Promise | null> { const connection = await getConnectionById(teamId, connectionId, true); if (connection == null) { throw new Error(`Connection ${connectionId} not found for PromQL alert`); @@ -1004,7 +1007,8 @@ export async function evaluatePromqlAlert({ end: String(endSec), step: String(stepSec), }); - const resp = await fetch(`${url.toString()}?${params.toString()}`, { + url.search = params.toString(); + const resp = await fetch(url.toString(), { signal: AbortSignal.timeout(30_000), }); if (!resp.ok) { @@ -1013,25 +1017,35 @@ export async function evaluatePromqlAlert({ ); } const json = (await resp.json()) as { - status: string; - data: { result: { values: [number, string][] }[] }; + status?: string; + data?: { result?: { metric?: Record; values?: [number, string][] }[] }; }; - if (json.status !== 'success' || !json.data?.result?.length) { + if (json?.status !== 'success' || !Array.isArray(json?.data?.result)) { return null; } - // Take the value from the last data point of the first series - const series = json.data.result[0]; - const lastPoint = series.values[series.values.length - 1]; - if (lastPoint == null) return null; - const parsed = parseFloat(lastPoint[1]); - return Number.isFinite(parsed) ? parsed : null; + + const results: Array<{ group: string; value: number }> = []; + for (const series of json.data.result) { + if (!series.values?.length) continue; + const lastPoint = series.values[series.values.length - 1]; + if (lastPoint == null) continue; + const parsed = parseFloat(lastPoint[1]); + if (Number.isFinite(parsed)) { + // Format Prometheus metric object as a HyperDX group key: key1:"val1", key2:"val2" + const group = series.metric + ? Object.entries(series.metric) + .filter(([k]) => k !== '__name__') + .map(([k, v]) => `${k}:"${v}"`) + .join(', ') + : ''; + results.push({ group, value: parsed }); + } + } + return results.length > 0 ? results : null; } // ClickHouse prometheusQueryRange path - const { ClickhouseClient: ChClient } = await import( - '@hyperdx/common-utils/dist/clickhouse/node' - ); - const client = new ChClient({ + const client = new ClickhouseClient({ host: connection.host, username: connection.username, password: connection.password, @@ -1044,10 +1058,8 @@ export async function evaluatePromqlAlert({ const resp = await client.query({ query: `SELECT tags, time_series FROM prometheusQueryRange({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({startMs:Int64}), fromUnixTimestamp64Milli({endMs:Int64}), toIntervalSecond({stepSec:UInt32})) SETTINGS allow_experimental_time_series_table = 1`, query_params: { - db: savedConfig.source ?? 'default', - table: savedConfig.promqlExpression.match(/^[\w.]+$/) - ? savedConfig.promqlExpression - : '__hdx_metric__', + db: (source as any)?.from?.databaseName ?? 'default', + table: (source as any)?.from?.tableName ?? 'otel_metrics_gauge', expr: savedConfig.promqlExpression, startMs, endMs, @@ -1060,16 +1072,24 @@ export async function evaluatePromqlAlert({ }, }); - const json = await resp.json<{ - data: { time_series: [string, number][] }[]; - }>(); - if (!json.data?.length) return null; - const series = json.data[0].time_series; - if (!series?.length) return null; - const lastPoint = series[series.length - 1]; - return lastPoint != null && Number.isFinite(lastPoint[1]) - ? lastPoint[1] - : null; + const json = await resp.json(); + if (!Array.isArray(json?.data)) return null; + + const results: Array<{ group: string; value: number }> = []; + for (const series of json.data) { + if (!series.time_series?.length) continue; + const lastPoint = series.time_series[series.time_series.length - 1]; + if (lastPoint != null && Number.isFinite(lastPoint[1])) { + const group = series.tags + ? Object.entries(series.tags) + .filter(([k]) => k !== '__name__') + .map(([k, v]) => `${k}:"${v}"`) + .join(', ') + : ''; + results.push({ group, value: lastPoint[1] }); + } + } + return results.length > 0 ? results : null; } export const processAlert = async ( @@ -1222,158 +1242,7 @@ export const processAlert = async ( return; } - const chartConfig = getChartConfigFromAlert( - details, - connectionId, - dateRange, - windowSizeInMins, - ); - - if (chartConfig == null) { - evalOutcome = 'error'; - logger.error( - { - chartConfig, - alertId: alert.id, - }, - 'Failed to build chart config', - ); - return; - } - const metadata = getMetadata(clickhouseClient); - - // For saved search alerts, the WHERE clause may reference aliased columns - // from the saved search's select expression (e.g. `toString(Body) AS body`). - // The alert query itself uses count(*), not the saved search's select, - // so we render the saved search's select separately to discover aliases - // and inject them as WITH clauses into the alert query. - if (details.taskType === AlertTaskType.SAVED_SEARCH) { - if (!isBuilderChartConfig(chartConfig)) { - logger.error({ - chartConfig, - message: - 'Found non-builder chart config for saved search alert, cannot compute WITH clauses', - }); - throw new Error('Expected builder chart config for saved search alert'); - } - try { - const withClauses = await computeAliasWithClauses( - details.savedSearch, - details.source, - metadata, - ); - if (withClauses) { - chartConfig.with = withClauses; - } - } catch (e) { - logger.warn( - { error: serializeError(e), alertId: alert.id }, - 'Failed to compute alias WITH clauses for alert check', - ); - } - } - - // Optimize chart config with materialized views, if available. - // materializedViews exists on Log and Trace sources. - const mvSource = - source?.kind === SourceKind.Log || source?.kind === SourceKind.Trace - ? source - : undefined; - const optimizedChartConfig = - isBuilderChartConfig(chartConfig) && mvSource?.materializedViews?.length - ? await tryOptimizeConfigWithMaterializedView( - chartConfig, - metadata, - clickhouseClient, - undefined, - mvSource, - ) - : chartConfig; - - // Readonly = 2 means the query is readonly but can still specify query settings. - // This is done only for Raw SQL configs because it carries a minor risk of conflict with - // existing settings (which may have readonly = 1) and is not required for builder - // chart configs, which are always rendered as select statements. - const clickHouseSettings = isRawSqlChartConfig(optimizedChartConfig) - ? { readonly: '2' } - : {}; - - // Query for alert data. If the query fails, record the error and exit - // without touching alert state or creating an AlertHistory. - let checksData; - const queryStartedAt = performance.now(); - try { - checksData = await clickhouseClient.queryChartConfig({ - config: optimizedChartConfig, - metadata, - opts: { clickhouse_settings: clickHouseSettings }, - querySettings: source?.querySettings, - }); - // SLO signal for alert data fetching (distinct from the end-to-end - // evaluation SLI): did ClickHouse serve the alert query, and how fast. - const queryDurationMs = performance.now() - queryStartedAt; - evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); - recordOperationOutcome({ - operation: 'alerts.query', - outcome: 'success', - durationMs: queryDurationMs, - attributes: { alert_source: alert.source ?? 'unknown' }, - }); - } catch (e) { - const queryDurationMs = performance.now() - queryStartedAt; - // Time-to-failure — for QUERY_TIMEOUT this is roughly the configured - // evaluation timeout. - evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); - recordOperationOutcome({ - operation: 'alerts.query', - outcome: 'error', - durationMs: queryDurationMs, - attributes: { alert_source: alert.source ?? 'unknown' }, - }); - evalOutcome = 'error'; - const alertError = makeQueryAlertError( - e, - clickhouseClient.requestTimeoutMs, - ); - alertQueryFailuresCounter.add(1, { - error_type: - alertError.type === AlertErrorType.QUERY_TIMEOUT - ? 'timeout' - : 'error', - }); - logger.error( - { - alertId: alert.id, - errorType: alertError.type, - error: serializeError(e), - }, - 'Alert query failed, skipping state/history update', - ); - // Record the error on the alert and as an ERROR history row for this - // window. ERROR rows are excluded from the due-ness gate and date-range - // computation, so the failed window is still retried/backfilled. - await alertProvider.recordAlertErrors( - alert.id, - [alertError], - nowInMinsRoundDown, - evaluationAnalytics, - ); - return; - } - - logger.info( - { - alertId: alert.id, - chartConfig, - optimizedChartConfig, - checksData, - checkStartTime: dateRange[0], - checkEndTime: dateRange[1], - }, - `Received alert metric [${alert.source} source]`, - ); - // Track state per group (or one history if no groupBy) const histories = new Map(); const latestAlertContext = new Map< @@ -1540,6 +1409,246 @@ export const processAlert = async ( } }; + + const isPromQL = + details.taskType === AlertTaskType.INLINE + ? isPromqlSavedChartConfig(details.chartConfig) + : details.taskType === AlertTaskType.TILE + ? isPromqlSavedChartConfig((details as any).tile?.config) + : false; + + if (isPromQL) { + const savedConfig = + details.taskType === AlertTaskType.INLINE + ? details.chartConfig + : (details as any).tile?.config; + + const promqlResults = await evaluatePromqlAlert({ + savedConfig: savedConfig as any, + source: details.source, + connectionId, + teamId: alert.team._id.toString(), + dateRange, + windowSizeInMins, + }); + + if (promqlResults != null) { + for (const res of promqlResults) { + const groupKey = res.group; + const value = res.value; + + const history = getOrCreateHistory(groupKey); + history.lastValues.push({ count: value, startTime: dateRange[1] }); + const previous = previousMap.get(computeHistoryMapKey(alert.id, groupKey)); + + if (doesExceedThreshold(alert, value)) { + history.counts += 1; + if (shouldFireBasedOnConsecutiveWindows(groupKey)) { + history.state = AlertState.ALERT; + history.fired = true; + await trySendNotification({ + state: AlertState.ALERT, + group: groupKey, + totalCount: value, + startTime: dateRange[1], + }); + } else { + history.state = AlertState.PENDING; + history.fired = previous?.fired === true; + } + } + + await sendNotificationIfResolved(previous, history, groupKey); + } + } + + // If no data returned but we had alerts before, check auto-resolve + if (hasGroupBy && previousMap && previousMap.size > 0) { + for (const [previousKey, previousHistory] of previousMap.entries()) { + const groupKey = extractGroupKeyFromMapKey(previousKey, alert.id); + if ( + (previousHistory.state === AlertState.ALERT || + previousHistory.state === AlertState.PENDING) && + !histories.has(groupKey) && + !doesExceedThreshold(alert, 0) + ) { + const history = getOrCreateHistory(groupKey); + history.lastValues.push({ count: 0, startTime: dateRange[1] }); + await sendNotificationIfResolved(previousHistory, history, groupKey); + } + } + } + + if (histories.size === 0) { + getOrCreateHistory(''); + } + + evaluationAnalytics.backfilledBuckets = 0; + flushNotificationTimings(); + const historyRecords = Array.from(histories.values()); + for (const record of historyRecords) { + record.analytics = evaluationAnalytics; + } + await alertProvider.updateAlertState( + alert.id, + historyRecords, + executionErrors, + dateRange, + ); + return; + } + + const chartConfig = getChartConfigFromAlert( + details, + connectionId, + dateRange, + windowSizeInMins, + ); + + if (chartConfig == null) { + evalOutcome = 'error'; + logger.error( + { + chartConfig, + alertId: alert.id, + }, + 'Failed to build chart config', + ); + return; + } + + + // For saved search alerts, the WHERE clause may reference aliased columns + // from the saved search's select expression (e.g. `toString(Body) AS body`). + // The alert query itself uses count(*), not the saved search's select, + // so we render the saved search's select separately to discover aliases + // and inject them as WITH clauses into the alert query. + if (details.taskType === AlertTaskType.SAVED_SEARCH) { + if (!isBuilderChartConfig(chartConfig)) { + logger.error({ + chartConfig, + message: + 'Found non-builder chart config for saved search alert, cannot compute WITH clauses', + }); + throw new Error('Expected builder chart config for saved search alert'); + } + try { + const withClauses = await computeAliasWithClauses( + details.savedSearch, + details.source, + metadata, + ); + if (withClauses) { + chartConfig.with = withClauses; + } + } catch (e) { + logger.warn( + { error: serializeError(e), alertId: alert.id }, + 'Failed to compute alias WITH clauses for alert check', + ); + } + } + + // Optimize chart config with materialized views, if available. + // materializedViews exists on Log and Trace sources. + const mvSource = + source?.kind === SourceKind.Log || source?.kind === SourceKind.Trace + ? source + : undefined; + const optimizedChartConfig = + isBuilderChartConfig(chartConfig) && mvSource?.materializedViews?.length + ? await tryOptimizeConfigWithMaterializedView( + chartConfig, + metadata, + clickhouseClient, + undefined, + mvSource, + ) + : chartConfig; + + // Readonly = 2 means the query is readonly but can still specify query settings. + // This is done only for Raw SQL configs because it carries a minor risk of conflict with + // existing settings (which may have readonly = 1) and is not required for builder + // chart configs, which are always rendered as select statements. + const clickHouseSettings = isRawSqlChartConfig(optimizedChartConfig) + ? { readonly: '2' } + : {}; + + // Query for alert data. If the query fails, record the error and exit + // without touching alert state or creating an AlertHistory. + let checksData; + const queryStartedAt = performance.now(); + try { + checksData = await clickhouseClient.queryChartConfig({ + config: optimizedChartConfig, + metadata, + opts: { clickhouse_settings: clickHouseSettings }, + querySettings: source?.querySettings, + }); + // SLO signal for alert data fetching (distinct from the end-to-end + // evaluation SLI): did ClickHouse serve the alert query, and how fast. + const queryDurationMs = performance.now() - queryStartedAt; + evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); + recordOperationOutcome({ + operation: 'alerts.query', + outcome: 'success', + durationMs: queryDurationMs, + attributes: { alert_source: alert.source ?? 'unknown' }, + }); + } catch (e) { + const queryDurationMs = performance.now() - queryStartedAt; + // Time-to-failure — for QUERY_TIMEOUT this is roughly the configured + // evaluation timeout. + evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); + recordOperationOutcome({ + operation: 'alerts.query', + outcome: 'error', + durationMs: queryDurationMs, + attributes: { alert_source: alert.source ?? 'unknown' }, + }); + evalOutcome = 'error'; + const alertError = makeQueryAlertError( + e, + clickhouseClient.requestTimeoutMs, + ); + alertQueryFailuresCounter.add(1, { + error_type: + alertError.type === AlertErrorType.QUERY_TIMEOUT + ? 'timeout' + : 'error', + }); + logger.error( + { + alertId: alert.id, + errorType: alertError.type, + error: serializeError(e), + }, + 'Alert query failed, skipping state/history update', + ); + // Record the error on the alert and as an ERROR history row for this + // window. ERROR rows are excluded from the due-ness gate and date-range + // computation, so the failed window is still retried/backfilled. + await alertProvider.recordAlertErrors( + alert.id, + [alertError], + nowInMinsRoundDown, + evaluationAnalytics, + ); + return; + } + + logger.info( + { + alertId: alert.id, + chartConfig, + optimizedChartConfig, + checksData, + checkStartTime: dateRange[0], + checkEndTime: dateRange[1], + }, + `Received alert metric [${alert.source} source]`, + ); + const meta = getResponseMetadata(chartConfig, checksData); if (!meta) { evalOutcome = 'error'; @@ -2015,7 +2124,7 @@ export const getConsecutiveWindowHistories = async ( const windowStart = getAlertWindowStart(alert, now); const earliestAllowedTime = new Date( windowStart.getTime() - - (numWindowsToLookBack - 1) * windowSizeInMins * 60_000, + (numWindowsToLookBack - 1) * windowSizeInMins * 60_000, ); const id = new mongoose.Types.ObjectId(alert.id); const histories = await AlertHistory.aggregate([ From e7c35abfbfdec8eaf4746b2adb3e4d991a5819fb Mon Sep 17 00:00:00 2001 From: aryainguz Date: Sat, 12 Sep 2026 00:38:34 +0530 Subject: [PATCH 04/49] fix(alerts): address PR review comments for PromQL alerting - Remove unused displayTypeSupportsPromQLAlerts workaround and inline the boolean flag with explicit HDX-4636 tracker in UI - Fix alertHasGroupBy comments to clarify PromQL multi-series behavior - Remove invalid type leak (variables) in schedule offset normalizer - Fix ISource type cast in provider fetching to safely use .from.databaseName - Clean up unused imports and stale comments around PromQL in inline alerts - Fix test fixture using 'any' by properly typing PromqlSavedChartConfig --- packages/api/src/tasks/checkAlerts/index.ts | 116 ++++++++++++------ packages/app/src/DBDashboardPage.tsx | 5 +- .../app/src/utils/__tests__/alerts.test.ts | 23 ++-- packages/app/src/utils/alerts.ts | 5 +- packages/common-utils/src/core/utils.ts | 5 - 5 files changed, 95 insertions(+), 59 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 7c8de9092f..cde6f5fccb 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -25,6 +25,7 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { timeBucketByGranularity } from '@hyperdx/common-utils/dist/core/utils'; import { getDashboardVariableDeclarations } from '@hyperdx/common-utils/dist/filters'; +import { substitutePromqlChartConfigVariables } from '@hyperdx/common-utils/dist/variables'; import { isBuilderChartConfig, isBuilderSavedChartConfig, @@ -33,6 +34,7 @@ import { isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { + ChartVariable, ALERT_NOTIFICATION_TARGETS_LIMIT, AlertErrorType, AlertNotificationTargetTiming, @@ -163,6 +165,13 @@ export const alertHasGroupBy = (details: AlertDetails): boolean => { return true; } + if (isPromqlSavedChartConfig(savedConfig)) { + // PromQL /query_range always returns one result-set entry per label-set. + // Treating it as grouped ensures the missing-series recovery loop runs + // when a previously-firing series disappears from the response. + return true; + } + // Without a reliable parser, it's difficult to tell if the raw sql contains a // group by (besides the group by on the interval), so we'll assume it might // in the case of time series charts, and assume it will not in the case of number charts. @@ -978,6 +987,7 @@ export async function evaluatePromqlAlert({ teamId, dateRange, windowSizeInMins, + variables, }: { savedConfig: PromqlSavedChartConfig; source?: ISource | null; @@ -985,6 +995,7 @@ export async function evaluatePromqlAlert({ teamId: string; dateRange: [Date, Date]; windowSizeInMins: number; + variables?: ChartVariable[]; }): Promise | null> { const connection = await getConnectionById(teamId, connectionId, true); if (connection == null) { @@ -994,6 +1005,9 @@ export async function evaluatePromqlAlert({ const endSec = dateRange[1].getTime() / 1000; const startSec = dateRange[0].getTime() / 1000; const stepSec = windowSizeInMins * 60; + const promqlExpression = variables && variables.length > 0 + ? substitutePromqlChartConfigVariables({ ...savedConfig, variables }).promqlExpression + : savedConfig.promqlExpression; if (connection.isPrometheusEndpoint) { // Proxy to the real Prometheus /api/v1/query_range @@ -1002,12 +1016,14 @@ export async function evaluatePromqlAlert({ '/api/v1/query_range', ); const params = new URLSearchParams({ - query: savedConfig.promqlExpression, + query: promqlExpression, start: String(startSec), end: String(endSec), step: String(stepSec), }); - url.search = params.toString(); + for (const [key, value] of params.entries()) { + url.searchParams.set(key, value); + } const resp = await fetch(url.toString(), { signal: AbortSignal.timeout(30_000), }); @@ -1058,9 +1074,11 @@ export async function evaluatePromqlAlert({ const resp = await client.query({ query: `SELECT tags, time_series FROM prometheusQueryRange({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({startMs:Int64}), fromUnixTimestamp64Milli({endMs:Int64}), toIntervalSecond({stepSec:UInt32})) SETTINGS allow_experimental_time_series_table = 1`, query_params: { - db: (source as any)?.from?.databaseName ?? 'default', - table: (source as any)?.from?.tableName ?? 'otel_metrics_gauge', - expr: savedConfig.promqlExpression, + // ISource always has `from` (it is on BaseSourceSchema); the nullable + // source param covers the case where no source is wired to the alert. + db: source?.from.databaseName ?? 'default', + table: source?.from.tableName ?? 'otel_metrics_gauge', + expr: promqlExpression, startMs, endMs, stepSec, @@ -1414,52 +1432,72 @@ export const processAlert = async ( details.taskType === AlertTaskType.INLINE ? isPromqlSavedChartConfig(details.chartConfig) : details.taskType === AlertTaskType.TILE - ? isPromqlSavedChartConfig((details as any).tile?.config) + ? isPromqlSavedChartConfig(details.tile.config) : false; if (isPromQL) { const savedConfig = details.taskType === AlertTaskType.INLINE ? details.chartConfig - : (details as any).tile?.config; + : details.taskType === AlertTaskType.TILE + ? details.tile.config + : undefined; - const promqlResults = await evaluatePromqlAlert({ - savedConfig: savedConfig as any, - source: details.source, - connectionId, - teamId: alert.team._id.toString(), - dateRange, - windowSizeInMins, - }); + try { + const promqlResults = await evaluatePromqlAlert({ + savedConfig: savedConfig as PromqlSavedChartConfig, + source: details.source, + connectionId, + teamId: alert.team._id.toString(), + dateRange, + windowSizeInMins, + variables: + details.taskType === AlertTaskType.TILE + ? getDashboardVariableDeclarations(details.dashboard.filters).map( + declaration => ({ + ...declaration, + values: [], + }), + ) + : undefined, + }); - if (promqlResults != null) { - for (const res of promqlResults) { - const groupKey = res.group; - const value = res.value; + if (promqlResults != null) { + for (const res of promqlResults) { + const groupKey = res.group; + const value = res.value; - const history = getOrCreateHistory(groupKey); - history.lastValues.push({ count: value, startTime: dateRange[1] }); - const previous = previousMap.get(computeHistoryMapKey(alert.id, groupKey)); - - if (doesExceedThreshold(alert, value)) { - history.counts += 1; - if (shouldFireBasedOnConsecutiveWindows(groupKey)) { - history.state = AlertState.ALERT; - history.fired = true; - await trySendNotification({ - state: AlertState.ALERT, - group: groupKey, - totalCount: value, - startTime: dateRange[1], - }); - } else { - history.state = AlertState.PENDING; - history.fired = previous?.fired === true; + const history = getOrCreateHistory(groupKey); + history.lastValues.push({ count: value, startTime: dateRange[1] }); + const previous = previousMap.get(computeHistoryMapKey(alert.id, groupKey)); + + if (doesExceedThreshold(alert, value)) { + history.counts += 1; + if (shouldFireBasedOnConsecutiveWindows(groupKey)) { + history.state = AlertState.ALERT; + history.fired = true; + await trySendNotification({ + state: AlertState.ALERT, + group: groupKey, + totalCount: value, + startTime: dateRange[1], + }); + } else { + history.state = AlertState.PENDING; + history.fired = previous?.fired === true; + } } - } - await sendNotificationIfResolved(previous, history, groupKey); + await sendNotificationIfResolved(previous, history, groupKey); + } } + } catch (e) { + logger.error( + { alertId: alert.id, err: e }, + 'Failed to evaluate PromQL alert', + ); + evalOutcome = 'error'; + return; } // If no data returned but we had alerts before, check auto-resolve diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index da2338bfa8..8720e716d0 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -29,7 +29,6 @@ import { import { convertToDashboardTemplate, displayTypeSupportsBuilderAlerts, - displayTypeSupportsPromQLAlerts, displayTypeSupportsRawSqlAlerts, Granularity, isTimeSeriesDisplayType, @@ -845,7 +844,7 @@ const Tile = ({ const displayTypeSupportsAlerts = isRawSql ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) : isPromQL - ? displayTypeSupportsPromQLAlerts(chart.config.displayType) + ? false // PromQL alert UI not yet implemented (HDX-4636) : displayTypeSupportsBuilderAlerts(chart.config.displayType); const canMoveToGroup = onMoveToGroup && moveTargets && moveTargets.length > 0; @@ -1056,7 +1055,7 @@ const Tile = ({ const showAlerts = isRawSql ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) : isPromQL - ? displayTypeSupportsPromQLAlerts(chart.config.displayType) + ? false // PromQL alert UI not yet implemented (HDX-4636) : displayTypeSupportsBuilderAlerts(chart.config.displayType); const canMoveToGroup = onMoveToGroup && moveTargets && moveTargets.length > 0; diff --git a/packages/app/src/utils/__tests__/alerts.test.ts b/packages/app/src/utils/__tests__/alerts.test.ts index 92e6bd968a..4b6804886d 100644 --- a/packages/app/src/utils/__tests__/alerts.test.ts +++ b/packages/app/src/utils/__tests__/alerts.test.ts @@ -2,6 +2,7 @@ import { AlertSource, AlertThresholdType, DisplayType, + type PromqlSavedChartConfig, } from '@hyperdx/common-utils/dist/types'; import type { AlertsPageItem } from '@/types'; @@ -332,17 +333,23 @@ describe('buildInlineAlertPayload', () => { expect(buildInlineAlertPayload(inlineChartConfig)).toBeUndefined(); }); - // PromQL charts cannot be alerted on: the inline-alert schema has no PromQL - // variant, so a payload built from one would be rejected server-side. - it('returns nothing for a PromQL chart', () => { - expect( - buildInlineAlertPayload({ + it('builds payload for a PromQL chart', () => { + const config: PromqlSavedChartConfig = { + configType: 'promql', + promqlExpression: 'up', + connection: 'conn-1', + displayType: DisplayType.Line, + alert, + }; + expect(buildInlineAlertPayload(config)).toEqual({ + ...alert, + source: AlertSource.INLINE, + chartConfig: { configType: 'promql', promqlExpression: 'up', connection: 'conn-1', displayType: DisplayType.Line, - alert, - }), - ).toBeUndefined(); + }, + }); }); }); diff --git a/packages/app/src/utils/alerts.ts b/packages/app/src/utils/alerts.ts index c56075387e..d2f97dea68 100644 --- a/packages/app/src/utils/alerts.ts +++ b/packages/app/src/utils/alerts.ts @@ -8,7 +8,6 @@ import _ from 'lodash'; import { z } from 'zod'; import { formatTileAlertDisplayName } from '@hyperdx/common-utils/dist/alerts'; import { Granularity } from '@hyperdx/common-utils/dist/core/utils'; -import { isPromqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; import { type Alert, ALERT_INTERVAL_TO_MINUTES, @@ -355,9 +354,7 @@ export type InlineAlert = z.infer & { export function buildInlineAlertPayload( config: SavedChartConfig, ): InlineAlert | undefined { - // PromQL configs have no inline-alert representation (the schema has no - // PromQL variant), and the editor never offers an alert on one. - if (config.alert == null || isPromqlSavedChartConfig(config)) { + if (config.alert == null) { return undefined; } const { alert, ...chartConfig } = config; diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index eaca9c980b..d618354625 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -1432,11 +1432,6 @@ export const isFormulaSourceKind = ( kind === SourceKind.Log || kind === SourceKind.Trace; -export function displayTypeSupportsPromQLAlerts( - displayType: DisplayType | undefined, -): boolean { - return displayType === DisplayType.Line || displayType === DisplayType.Number; -} /** Expand the chart's macros, returning failures instead of throwing. */ function resolveRawSqlMacros( From 9fae4d8714a11b36fb08cc69499383285d0788ef Mon Sep 17 00:00:00 2001 From: aryainguz Date: Sun, 13 Sep 2026 00:21:48 +0530 Subject: [PATCH 05/49] fix(promql-alerts): address PR review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P0: Fix tags tuple iteration in ClickHouse path - evaluatePromqlAlert now iterates series.tags as [key, value] tuples matching formatMatrixResponse, instead of Object.entries (which produced numeric indices and broke group key building entirely) P2: Throw on degraded Prometheus status instead of returning null - status !== 'success' now throws, preventing false auto-resolves from masking genuine Prometheus outages as empty results P2: Route PromQL failures through standard error recording - PromQL catch block now calls makeQueryAlertError, alertQueryFailuresCounter, recordOperationOutcome, and alertProvider.recordAlertErrors — matching the SQL path so failed windows get typed error history rows, operator metrics, and retry P3: Extract queryPrometheusRangeFromClickHouse helper - Moved to @/controllers/timeseriesEngine (with PROMETHEUS_MAX_RESULT_ROWS cap that was missing from the alert path); both prometheus router and evaluatePromqlAlert now use the same helper, eliminating duplication P3: Use API ClickhouseClient in evaluatePromqlAlert - Swapped common-utils direct ClickhouseClient for @/clickhouse wrapper so telemetry/logger wiring is included in the alert evaluation path P3: Reuse PROMETHEUS_CH_TIMEOUT_MS constant - Exported from prometheus router; evaluatePromqlAlert now imports and reuses it instead of hardcoding 30_000 / max_execution_time: 30 P3: Move legendTemplate to EVALUATION_INERT_CONFIG_KEYS - Was erroneously classified as KNOWN_LOSSY; it is render-only so a blind external GET -> PUT should not fail on it P3: Fix drift-guard test to label PromQL variant correctly - i === 0 ? 'builder' : 'raw SQL' now handles i === 2 as 'promql' Tests: Update evaluatePromqlAlert.test.ts - Mock queryPrometheusRangeFromClickHouse (new helper) instead of raw ClickhouseClient; fix tags fixtures to use tuple arrays; add test asserting status !== success throws --- .../api/src/controllers/timeseriesEngine.ts | 39 ++++ packages/api/src/routers/api/prometheus.ts | 27 +-- .../utils/__tests__/alertChartConfig.test.ts | 8 +- .../external-api/v2/utils/alertChartConfig.ts | 5 +- .../__tests__/evaluatePromqlAlert.test.ts | 107 +++++++---- packages/api/src/tasks/checkAlerts/index.ts | 178 +++++++++++------- 6 files changed, 242 insertions(+), 122 deletions(-) diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts index 2ce263f340..81104b0e67 100644 --- a/packages/api/src/controllers/timeseriesEngine.ts +++ b/packages/api/src/controllers/timeseriesEngine.ts @@ -248,3 +248,42 @@ export async function queryLabelNames({ return queryDistinctTagsValues({ ...args, value, conditions, limit }); } + +/** + * Runs the prometheusQueryRange table function for a given ClickHouse connection. + */ +export async function queryPrometheusRangeFromClickHouse({ + client, + databaseName, + tableName, + expr, + startMs, + endMs, + stepSec, +}: { + client: ClickhouseClient; + databaseName: string; + tableName: string; + expr: string; + startMs: number; + endMs: number; + stepSec: number; +}) { + return client.query({ + query: `SELECT tags, time_series FROM prometheusQueryRange({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({startMs:Int64}), fromUnixTimestamp64Milli({endMs:Int64}), toIntervalSecond({stepSec:UInt32})) SETTINGS allow_experimental_time_series_table = 1`, + query_params: { + db: databaseName, + table: tableName, + expr, + startMs, + endMs, + stepSec, + }, + format: 'JSON', + clickhouse_settings: { + allow_experimental_time_series_table: 1, + max_execution_time: PROMETHEUS_MAX_EXECUTION_SEC, + max_result_rows: String(PROMETHEUS_MAX_RESULT_ROWS), + }, + }); +} diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index 01a716d329..4573eb90f8 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -11,6 +11,7 @@ import { PROMETHEUS_MAX_RESULT_ROWS, queryLabelNames, queryLabelValues, + queryPrometheusRangeFromClickHouse, TimeSeriesTagsQueryArgs, } from '@/controllers/timeseriesEngine'; import { getNonNullUserWithTeam } from '@/middleware/auth'; @@ -160,7 +161,7 @@ export function formatVectorResponse( // -------------------------- const PROMETHEUS_PROXY_TIMEOUT_MS = 90_000; -const PROMETHEUS_CH_TIMEOUT_MS = 30_000; +export const PROMETHEUS_CH_TIMEOUT_MS = 30_000; const PROMETHEUS_MAX_RESOLUTION = 11_000; // Widest window /query_exemplars will proxy. Prometheus's exemplar store is a // small circular buffer, so a wider range mostly costs a bigger streamed body @@ -499,22 +500,14 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { const endMs = Math.floor(end * 1000); const stepSec = Math.max(Math.floor(step), 1); - const resp = await client.query({ - query: `SELECT tags, time_series FROM prometheusQueryRange({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({startMs:Int64}), fromUnixTimestamp64Milli({endMs:Int64}), toIntervalSecond({stepSec:UInt32})) SETTINGS allow_experimental_time_series_table = 1`, - query_params: { - db: database, - table, - expr: query, - startMs, - endMs, - stepSec, - }, - format: 'JSON', - clickhouse_settings: { - allow_experimental_time_series_table: 1, - max_execution_time: PROMETHEUS_MAX_EXECUTION_SEC, - max_result_rows: String(PROMETHEUS_MAX_RESULT_ROWS), - }, + const resp = await queryPrometheusRangeFromClickHouse({ + client, + databaseName: database, + tableName: table, + expr: query, + startMs, + endMs, + stepSec, }); const json = await resp.json(); diff --git a/packages/api/src/routers/external-api/v2/utils/__tests__/alertChartConfig.test.ts b/packages/api/src/routers/external-api/v2/utils/__tests__/alertChartConfig.test.ts index c4da87435f..c93694aba5 100644 --- a/packages/api/src/routers/external-api/v2/utils/__tests__/alertChartConfig.test.ts +++ b/packages/api/src/routers/external-api/v2/utils/__tests__/alertChartConfig.test.ts @@ -602,9 +602,11 @@ describe('internal schema field classification', () => { it.each( AlertChartConfigSchema.options.flatMap((member, i) => - Object.keys(member.shape).map( - key => [i === 0 ? 'builder' : 'raw SQL', key] as const, - ), + Object.keys(member.shape).map(key => { + const variantName = + i === 0 ? 'builder' : i === 1 ? 'raw SQL' : 'promql'; + return [variantName, key] as const; + }), ), )('classifies the internal %s config field "%s"', (_variant, key) => { const classified = diff --git a/packages/api/src/routers/external-api/v2/utils/alertChartConfig.ts b/packages/api/src/routers/external-api/v2/utils/alertChartConfig.ts index 814ee3419c..0390f5ab5e 100644 --- a/packages/api/src/routers/external-api/v2/utils/alertChartConfig.ts +++ b/packages/api/src/routers/external-api/v2/utils/alertChartConfig.ts @@ -119,6 +119,10 @@ export const EVALUATION_INERT_CONFIG_KEYS: ReadonlySet = new Set([ // granularity: 'auto' on every inline alert it authors — refusing on it // would blank chartConfig for the most common authoring path. 'granularity', + // Render-only: only controls the legend label in the UI, has no effect on + // alert evaluation. Classifying as lossy would block external PUT for any + // alert carrying a legend. + 'legendTemplate', ]); /** @@ -145,7 +149,6 @@ export const KNOWN_LOSSY_CONFIG_KEYS: ReadonlySet = new Set([ 'metricTables', 'promqlExpression', 'step', - 'legendTemplate', ]); /** Select-item fields the external dialect round-trips. */ diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts index a0ff63e885..9eea9caaba 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts @@ -1,11 +1,12 @@ -import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import mongoose from 'mongoose'; import { getConnectionById } from '@/controllers/connection'; +import { queryPrometheusRangeFromClickHouse } from '@/controllers/timeseriesEngine'; import { evaluatePromqlAlert } from '@/tasks/checkAlerts'; jest.mock('@/controllers/connection'); -jest.mock('@hyperdx/common-utils/dist/clickhouse/node'); +jest.mock('@/controllers/timeseriesEngine'); +jest.mock('@/clickhouse'); describe('evaluatePromqlAlert', () => { const mockTeamId = new mongoose.Types.ObjectId().toString(); @@ -70,7 +71,6 @@ describe('evaluatePromqlAlert', () => { ); }); - it('should return multiple series for Prometheus endpoint', async () => { (global.fetch as jest.Mock).mockResolvedValue({ ok: true, @@ -105,9 +105,30 @@ describe('evaluatePromqlAlert', () => { ]); }); + it('should throw on non-success Prometheus status instead of returning null', async () => { + (global.fetch as jest.Mock).mockResolvedValue({ + ok: true, + json: async () => ({ + status: 'error', + error: 'query timeout', + data: { result: [] }, + }), + }); + await expect( + evaluatePromqlAlert({ + savedConfig: mockSavedConfig, + connectionId: mockConnectionId, + teamId: mockTeamId, + dateRange: mockDateRange, + windowSizeInMins: mockWindowSizeInMins, + }), + ).rejects.toThrow( + "Prometheus query_range returned status 'error' for PromQL alert", + ); + }); - it('should return null when no data is returned', async () => { + it('should return null when result array is empty', async () => { (global.fetch as jest.Mock).mockResolvedValue({ ok: true, json: async () => ({ @@ -129,30 +150,26 @@ describe('evaluatePromqlAlert', () => { }); describe('ClickHouse endpoint', () => { - let mockQuery: jest.Mock; - beforeEach(() => { (getConnectionById as jest.Mock).mockResolvedValue({ host: 'http://clickhouse:8123', + username: 'default', + password: '', isPrometheusEndpoint: false, }); - - mockQuery = jest.fn(); - (ClickhouseClient as unknown as jest.Mock).mockImplementation(() => ({ - query: mockQuery, - })); }); - it('should return multiple series for ClickHouse endpoint with proper db/table', async () => { - mockQuery.mockResolvedValue({ + it('should return multiple series with tags as tuples', async () => { + // tags is an array of [key, value] tuples — as returned by ClickHouse + (queryPrometheusRangeFromClickHouse as jest.Mock).mockResolvedValue({ json: async () => ({ data: [ { - tags: { host: 'A' }, + tags: [['host', 'A']], time_series: [['2024-01-01 00:05:00', 42.5]], }, { - tags: { host: 'B' }, + tags: [['host', 'B']], time_series: [['2024-01-01 00:05:00', 10.5]], }, ], @@ -161,7 +178,9 @@ describe('evaluatePromqlAlert', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, - source: { from: { databaseName: 'my_db', tableName: 'my_table' } } as any, + source: { + from: { databaseName: 'my_db', tableName: 'my_table' }, + } as any, connectionId: mockConnectionId, teamId: mockTeamId, dateRange: mockDateRange, @@ -173,19 +192,44 @@ describe('evaluatePromqlAlert', () => { { group: 'host:"B"', value: 10.5 }, ]); - expect(mockQuery).toHaveBeenCalledWith( + expect(queryPrometheusRangeFromClickHouse).toHaveBeenCalledWith( expect.objectContaining({ - query_params: expect.objectContaining({ - expr: 'up', - db: 'my_db', - table: 'my_table', - }), + expr: 'up', + databaseName: 'my_db', + tableName: 'my_table', }), ); }); - it('should query ClickHouse and return the last value', async () => { - mockQuery.mockResolvedValue({ + it('should filter __name__ from tags tuples', async () => { + (queryPrometheusRangeFromClickHouse as jest.Mock).mockResolvedValue({ + json: async () => ({ + data: [ + { + tags: [ + ['__name__', 'up'], + ['host', 'A'], + ], + time_series: [['2024-01-01 00:05:00', 42.5]], + }, + ], + }), + }); + + const result = await evaluatePromqlAlert({ + savedConfig: mockSavedConfig, + connectionId: mockConnectionId, + teamId: mockTeamId, + dateRange: mockDateRange, + windowSizeInMins: mockWindowSizeInMins, + }); + + // __name__ is stripped + expect(result).toEqual([{ group: 'host:"A"', value: 42.5 }]); + }); + + it('should use defaults when no source is provided', async () => { + (queryPrometheusRangeFromClickHouse as jest.Mock).mockResolvedValue({ json: async () => ({ data: [ { @@ -207,20 +251,19 @@ describe('evaluatePromqlAlert', () => { }); expect(result).toEqual([{ group: '', value: 42.5 }]); - expect(mockQuery).toHaveBeenCalledWith( + expect(queryPrometheusRangeFromClickHouse).toHaveBeenCalledWith( expect.objectContaining({ - query_params: expect.objectContaining({ - expr: 'up', - startMs: 1704067200000, - endMs: 1704067500000, - stepSec: 300, - }), + databaseName: 'default', + tableName: 'otel_metrics_gauge', + startMs: 1704067200000, + endMs: 1704067500000, + stepSec: 300, }), ); }); it('should return null when no data is returned', async () => { - mockQuery.mockResolvedValue({ + (queryPrometheusRangeFromClickHouse as jest.Mock).mockResolvedValue({ json: async () => ({ data: [], }), diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index cde6f5fccb..e16b14a3f9 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -7,7 +7,6 @@ import { chSqlToAliasMap, ResponseJSON, } from '@hyperdx/common-utils/dist/clickhouse'; -import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { tryOptimizeConfigWithMaterializedView } from '@hyperdx/common-utils/dist/core/materializedViews'; import { getMetadata, @@ -25,7 +24,6 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { timeBucketByGranularity } from '@hyperdx/common-utils/dist/core/utils'; import { getDashboardVariableDeclarations } from '@hyperdx/common-utils/dist/filters'; -import { substitutePromqlChartConfigVariables } from '@hyperdx/common-utils/dist/variables'; import { isBuilderChartConfig, isBuilderSavedChartConfig, @@ -34,20 +32,21 @@ import { isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { - ChartVariable, ALERT_NOTIFICATION_TARGETS_LIMIT, AlertErrorType, AlertNotificationTargetTiming, AlertThresholdType, BuilderChartConfigWithOptDateRange, ChartConfigWithOptDateRange, + ChartVariable, DisplayType, getSampleWeightExpression, pickSampleWeightExpressionProps, - SavedChartConfig, PromqlSavedChartConfig, + SavedChartConfig, SourceKind, } from '@hyperdx/common-utils/dist/types'; +import { substitutePromqlChartConfigVariables } from '@hyperdx/common-utils/dist/variables'; import * as fns from 'date-fns'; import { isString, pick } from 'lodash'; import { ObjectId } from 'mongoose'; @@ -56,8 +55,10 @@ import ms from 'ms'; import { performance } from 'perf_hooks'; import { serializeError } from 'serialize-error'; +import { ClickhouseClient } from '@/clickhouse'; import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory'; import { getConnectionById } from '@/controllers/connection'; +import { queryPrometheusRangeFromClickHouse } from '@/controllers/timeseriesEngine'; import { AlertState, IAlert, IAlertError } from '@/models/alert'; import AlertHistory, { IAlertHistory, @@ -67,7 +68,10 @@ import { IDashboard } from '@/models/dashboard'; import { ISavedSearch } from '@/models/savedSearch'; import { ISource } from '@/models/source'; import { IWebhook } from '@/models/webhook'; -import { joinPrometheusUpstreamUrl } from '@/routers/api/prometheus'; +import { + joinPrometheusUpstreamUrl, + PROMETHEUS_CH_TIMEOUT_MS, +} from '@/routers/api/prometheus'; import { isClientTimeoutOrAbortError, isQueryTimeoutError, @@ -232,12 +236,12 @@ class InvalidAlertError extends Error { // from upstream systems. QUERY_ERROR and INVALID_ALERT messages are authored // by us (ClickHouse errors or our own validation) and are safe to display. const HARDCODED_ALERT_ERROR_MESSAGES: Partial> = -{ - [AlertErrorType.WEBHOOK_ERROR]: - 'Failed to send webhook notification. Check the webhook configuration and destination.', - [AlertErrorType.UNKNOWN]: - 'An unknown error occurred while processing the alert.', -}; + { + [AlertErrorType.WEBHOOK_ERROR]: + 'Failed to send webhook notification. Check the webhook configuration and destination.', + [AlertErrorType.UNKNOWN]: + 'An unknown error occurred while processing the alert.', + }; const makeAlertError = ( type: AlertErrorType, @@ -889,14 +893,14 @@ const getChartConfigFromAlert = ( type ResponseMetadata = | { - type: 'time_series'; - timestampColumnName: string; - valueColumnNames: Set; - } + type: 'time_series'; + timestampColumnName: string; + valueColumnNames: Set; + } | { - type: 'single_value'; - valueColumnNames: Set; - }; + type: 'single_value'; + valueColumnNames: Set; + }; const getResponseMetadata = ( chartConfig: ChartConfigWithOptDateRange, @@ -1005,9 +1009,11 @@ export async function evaluatePromqlAlert({ const endSec = dateRange[1].getTime() / 1000; const startSec = dateRange[0].getTime() / 1000; const stepSec = windowSizeInMins * 60; - const promqlExpression = variables && variables.length > 0 - ? substitutePromqlChartConfigVariables({ ...savedConfig, variables }).promqlExpression - : savedConfig.promqlExpression; + const promqlExpression = + variables && variables.length > 0 + ? substitutePromqlChartConfigVariables({ ...savedConfig, variables }) + .promqlExpression + : savedConfig.promqlExpression; if (connection.isPrometheusEndpoint) { // Proxy to the real Prometheus /api/v1/query_range @@ -1034,9 +1040,19 @@ export async function evaluatePromqlAlert({ } const json = (await resp.json()) as { status?: string; - data?: { result?: { metric?: Record; values?: [number, string][] }[] }; + data?: { + result?: { + metric?: Record; + values?: [number, string][]; + }[]; + }; }; - if (json?.status !== 'success' || !Array.isArray(json?.data?.result)) { + if (json?.status !== 'success') { + throw new Error( + `Prometheus query_range returned status '${json?.status}' for PromQL alert`, + ); + } + if (!Array.isArray(json?.data?.result)) { return null; } @@ -1050,9 +1066,9 @@ export async function evaluatePromqlAlert({ // Format Prometheus metric object as a HyperDX group key: key1:"val1", key2:"val2" const group = series.metric ? Object.entries(series.metric) - .filter(([k]) => k !== '__name__') - .map(([k, v]) => `${k}:"${v}"`) - .join(', ') + .filter(([k]) => k !== '__name__') + .map(([k, v]) => `${k}:"${v}"`) + .join(', ') : ''; results.push({ group, value: parsed }); } @@ -1060,34 +1076,29 @@ export async function evaluatePromqlAlert({ return results.length > 0 ? results : null; } - // ClickHouse prometheusQueryRange path + // ClickHouse prometheusQueryRange path — use the API ClickhouseClient so + // request-scoped logger/telemetry wiring is included. const client = new ClickhouseClient({ host: connection.host, username: connection.username, password: connection.password, - requestTimeout: 30_000, + requestTimeout: PROMETHEUS_CH_TIMEOUT_MS, }); const startMs = Math.floor(startSec * 1000); const endMs = Math.floor(endSec * 1000); - - const resp = await client.query({ - query: `SELECT tags, time_series FROM prometheusQueryRange({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({startMs:Int64}), fromUnixTimestamp64Milli({endMs:Int64}), toIntervalSecond({stepSec:UInt32})) SETTINGS allow_experimental_time_series_table = 1`, - query_params: { - // ISource always has `from` (it is on BaseSourceSchema); the nullable - // source param covers the case where no source is wired to the alert. - db: source?.from.databaseName ?? 'default', - table: source?.from.tableName ?? 'otel_metrics_gauge', - expr: promqlExpression, - startMs, - endMs, - stepSec, - }, - format: 'JSON', - clickhouse_settings: { - allow_experimental_time_series_table: 1, - max_execution_time: 30, - }, + const stepSecRounded = Math.max(Math.floor(stepSec), 1); + + const resp = await queryPrometheusRangeFromClickHouse({ + client, + // ISource always has `from` (it is on BaseSourceSchema); the nullable + // source param covers the case where no source is wired to the alert. + databaseName: source?.from.databaseName ?? 'default', + tableName: source?.from.tableName ?? 'otel_metrics_gauge', + expr: promqlExpression, + startMs, + endMs, + stepSec: stepSecRounded, }); const json = await resp.json(); @@ -1099,10 +1110,10 @@ export async function evaluatePromqlAlert({ const lastPoint = series.time_series[series.time_series.length - 1]; if (lastPoint != null && Number.isFinite(lastPoint[1])) { const group = series.tags - ? Object.entries(series.tags) - .filter(([k]) => k !== '__name__') - .map(([k, v]) => `${k}:"${v}"`) - .join(', ') + ? series.tags + .filter(([k]: any) => k !== '__name__') + .map(([k, v]: any) => `${k}:"${v}"`) + .join(', ') : ''; results.push({ group, value: lastPoint[1] }); } @@ -1427,7 +1438,6 @@ export const processAlert = async ( } }; - const isPromQL = details.taskType === AlertTaskType.INLINE ? isPromqlSavedChartConfig(details.chartConfig) @@ -1454,11 +1464,11 @@ export const processAlert = async ( variables: details.taskType === AlertTaskType.TILE ? getDashboardVariableDeclarations(details.dashboard.filters).map( - declaration => ({ - ...declaration, - values: [], - }), - ) + declaration => ({ + ...declaration, + values: [], + }), + ) : undefined, }); @@ -1469,7 +1479,9 @@ export const processAlert = async ( const history = getOrCreateHistory(groupKey); history.lastValues.push({ count: value, startTime: dateRange[1] }); - const previous = previousMap.get(computeHistoryMapKey(alert.id, groupKey)); + const previous = previousMap.get( + computeHistoryMapKey(alert.id, groupKey), + ); if (doesExceedThreshold(alert, value)) { history.counts += 1; @@ -1492,11 +1504,36 @@ export const processAlert = async ( } } } catch (e) { + const queryDurationMs = performance.now() - evalStartedAt; + evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); + recordOperationOutcome({ + operation: 'alerts.query', + outcome: 'error', + durationMs: queryDurationMs, + attributes: { alert_source: alert.source ?? 'unknown' }, + }); + evalOutcome = 'error'; + const alertError = makeQueryAlertError(e, PROMETHEUS_CH_TIMEOUT_MS); + alertQueryFailuresCounter.add(1, { + error_type: + alertError.type === AlertErrorType.QUERY_TIMEOUT + ? 'timeout' + : 'error', + }); logger.error( - { alertId: alert.id, err: e }, - 'Failed to evaluate PromQL alert', + { + alertId: alert.id, + errorType: alertError.type, + error: serializeError(e), + }, + 'PromQL alert query failed, skipping state/history update', + ); + await alertProvider.recordAlertErrors( + alert.id, + [alertError], + nowInMinsRoundDown, + evaluationAnalytics, ); - evalOutcome = 'error'; return; } @@ -1512,7 +1549,11 @@ export const processAlert = async ( ) { const history = getOrCreateHistory(groupKey); history.lastValues.push({ count: 0, startTime: dateRange[1] }); - await sendNotificationIfResolved(previousHistory, history, groupKey); + await sendNotificationIfResolved( + previousHistory, + history, + groupKey, + ); } } } @@ -1555,7 +1596,6 @@ export const processAlert = async ( return; } - // For saved search alerts, the WHERE clause may reference aliased columns // from the saved search's select expression (e.g. `toString(Body) AS body`). // The alert query itself uses count(*), not the saved search's select, @@ -1596,12 +1636,12 @@ export const processAlert = async ( const optimizedChartConfig = isBuilderChartConfig(chartConfig) && mvSource?.materializedViews?.length ? await tryOptimizeConfigWithMaterializedView( - chartConfig, - metadata, - clickhouseClient, - undefined, - mvSource, - ) + chartConfig, + metadata, + clickhouseClient, + undefined, + mvSource, + ) : chartConfig; // Readonly = 2 means the query is readonly but can still specify query settings. @@ -2162,7 +2202,7 @@ export const getConsecutiveWindowHistories = async ( const windowStart = getAlertWindowStart(alert, now); const earliestAllowedTime = new Date( windowStart.getTime() - - (numWindowsToLookBack - 1) * windowSizeInMins * 60_000, + (numWindowsToLookBack - 1) * windowSizeInMins * 60_000, ); const id = new mongoose.Types.ObjectId(alert.id); const histories = await AlertHistory.aggregate([ From d31579dce29af02e2b41088aea0a58f151c2f749 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Mon, 14 Sep 2026 01:03:22 +0530 Subject: [PATCH 06/49] feat(promql): end-to-end alerting engine integration, UI previews, and tests --- .../api/src/controllers/timeseriesEngine.ts | 36 ++ .../routers/api/__tests__/prometheus.test.ts | 2 +- packages/api/src/routers/api/prometheus.ts | 42 +-- .../__tests__/checkAlerts.int.test.ts | 125 +++++++ .../__tests__/evaluatePromqlAlert.test.ts | 124 ++++--- packages/api/src/tasks/checkAlerts/index.ts | 327 +++++++++++++----- packages/app/src/DBDashboardPage.tsx | 2 +- .../ChartEditor/PromqlChartEditor.tsx | 79 ++++- .../app/src/components/ChartEditor/utils.ts | 6 +- .../DBEditTimeChartForm/EditTimeChartForm.tsx | 102 ++++-- .../components/alerts/AlertDetailChart.tsx | 16 +- packages/app/src/hooks/useChartConfig.tsx | 7 +- packages/common-utils/src/clickhouse/index.ts | 8 +- packages/common-utils/src/core/utils.ts | 1 - 14 files changed, 670 insertions(+), 207 deletions(-) diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts index 81104b0e67..cab8d00913 100644 --- a/packages/api/src/controllers/timeseriesEngine.ts +++ b/packages/api/src/controllers/timeseriesEngine.ts @@ -13,6 +13,42 @@ import logger from '@/utils/logger'; export const PROMETHEUS_MAX_EXECUTION_SEC = 30; export const PROMETHEUS_MAX_RESULT_ROWS = 100000; +export const PROMETHEUS_CH_TIMEOUT_MS = 30_000; + +/** + * Join a Connection host with an absolute Prometheus API path. + * + * `new URL('/api/v1/query_range', 'http://host:8481/select/0/prometheus')` + * discards `/select/0/prometheus` because an absolute path replaces the base + * pathname. VictoriaMetrics cluster (and any Prometheus-compatible server + * mounted under a prefix) needs that prefix kept. Host userinfo, query, and + * hash are left untouched. + * + * `path` must be an absolute path (every call site passes a literal starting + * with `/`) -- this is not a general-purpose URL joiner. + * + * @see https://github.com/hyperdxio/hyperdx/issues/3046 + */ +export function joinPrometheusUpstreamUrl( + upstreamHost: string, + path: string, +): URL { + const url = new URL(upstreamHost); + // `new URL('prometheus:9090')` succeeds with an opaque path (`prometheus:` + // scheme). The pathname setter is a no-op there, so without this guard the + // helper would return the host unchanged, `fetch` would fail, and the proxy + // would 502 / increment query_errors for a user misconfiguration. Same check + // as clickhouseProxy.ts. + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new TypeError('Connection host must be http(s)'); + } + // Strip ALL trailing slashes, not just one -- a host saved with a doubled + // trailing slash (e.g. `http://prom:9090//`) would otherwise leave a `//` + // in the joined path, which most servers treat as a distinct (404) path. + const basePath = url.pathname.replace(/\/+$/, ''); + url.pathname = `${basePath}${path}`; + return url; +} export type TimeSeriesTagsQueryArgs = { client: ClickhouseClient; diff --git a/packages/api/src/routers/api/__tests__/prometheus.test.ts b/packages/api/src/routers/api/__tests__/prometheus.test.ts index dbf11461bf..2326babf50 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.test.ts @@ -8,11 +8,11 @@ jest.mock('@/utils/instrumentation', () => { }; }); +import { joinPrometheusUpstreamUrl } from '@/controllers/timeseriesEngine'; import { formatMatrixResponse, formatVectorResponse, isClientDisconnect, - joinPrometheusUpstreamUrl, parseDuration, parseTimestamp, recordProxyOutcome, diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index 4573eb90f8..25afa2cc3b 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -7,6 +7,8 @@ import z from 'zod'; import { ClickhouseClient } from '@/clickhouse'; import { getConnectionById } from '@/controllers/connection'; import { + joinPrometheusUpstreamUrl, + PROMETHEUS_CH_TIMEOUT_MS, PROMETHEUS_MAX_EXECUTION_SEC, PROMETHEUS_MAX_RESULT_ROWS, queryLabelNames, @@ -105,8 +107,8 @@ function getParams(req: express.Request): Record { // Prometheus-compatible response types // -------------------------- -type PrometheusMetric = Record; -type PrometheusMatrixResult = { +export type PrometheusMetric = Record; +export type PrometheusMatrixResult = { metric: PrometheusMetric; values: [number, string][]; }; @@ -161,7 +163,6 @@ export function formatVectorResponse( // -------------------------- const PROMETHEUS_PROXY_TIMEOUT_MS = 90_000; -export const PROMETHEUS_CH_TIMEOUT_MS = 30_000; const PROMETHEUS_MAX_RESOLUTION = 11_000; // Widest window /query_exemplars will proxy. Prometheus's exemplar store is a // small circular buffer, so a wider range mostly costs a bigger streamed body @@ -186,41 +187,6 @@ export function isClientDisconnect(err: unknown): boolean { ); } -/** - * Join a Connection host with an absolute Prometheus API path. - * - * `new URL('/api/v1/query_range', 'http://host:8481/select/0/prometheus')` - * discards `/select/0/prometheus` because an absolute path replaces the base - * pathname. VictoriaMetrics cluster (and any Prometheus-compatible server - * mounted under a prefix) needs that prefix kept. Host userinfo, query, and - * hash are left untouched. - * - * `path` must be an absolute path (every call site passes a literal starting - * with `/`) -- this is not a general-purpose URL joiner. - * - * @see https://github.com/hyperdxio/hyperdx/issues/3046 - */ -export function joinPrometheusUpstreamUrl( - upstreamHost: string, - path: string, -): URL { - const url = new URL(upstreamHost); - // `new URL('prometheus:9090')` succeeds with an opaque path (`prometheus:` - // scheme). The pathname setter is a no-op there, so without this guard the - // helper would return the host unchanged, `fetch` would fail, and the proxy - // would 502 / increment query_errors for a user misconfiguration. Same check - // as clickhouseProxy.ts. - if (url.protocol !== 'http:' && url.protocol !== 'https:') { - throw new TypeError('Connection host must be http(s)'); - } - // Strip ALL trailing slashes, not just one -- a host saved with a doubled - // trailing slash (e.g. `http://prom:9090//`) would otherwise leave a `//` - // in the joined path, which most servers treat as a distinct (404) path. - const basePath = url.pathname.replace(/\/+$/, ''); - url.pathname = `${basePath}${path}`; - return url; -} - // Only real Prometheus API params are ever caller-settable in // proxyToPrometheus's query merge below. `params` there is built upstream by // spreading the *entire* `req.query`/`req.body` with no allowlist (see diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 30c8f33eef..908d4fc0fd 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -14,6 +14,7 @@ import ms from 'ms'; import * as config from '@/config'; import { createAlert } from '@/controllers/alerts'; import { createTeam } from '@/controllers/team'; +import * as timeseriesEngine from '@/controllers/timeseriesEngine'; import { bulkInsertData, bulkInsertLogs, @@ -5011,6 +5012,130 @@ describe('checkAlerts', () => { expect(alertHistories[1].state).toBe('OK'); }); + describe('PromQL Alerts', () => { + it('should process a PromQL alert, respecting grouping, state transitions, and backfilling', async () => { + const { + team, + webhook, + connection, + source, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const dashboard = await new Dashboard({ + name: 'PromQL Dashboard', + team: team._id, + tiles: [ + { + id: 'promql1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'promql', + displayType: 'line', + promqlExpression: 'sum(up) by (host)', + connection: connection.id, + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'promql1'); + if (!tile) throw new Error('tile not found for PromQL test'); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 10, + dashboardId: dashboard.id, + tileId: 'promql1', + }, + { + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ); + + const now = new Date('2023-11-16T22:12:00.000Z'); + // Backfilled evaluation (worker was delayed). Expected bucket is 22:05:00. + const bucketStartMs = new Date('2023-11-16T22:05:00.000Z').getTime(); + + // We need to mock the timeseriesEngine call rather than evaluatePromqlAlert directly + // because processAlert calls the local evaluatePromqlAlert inside index.ts + jest + .spyOn(timeseriesEngine, 'queryPrometheusRangeFromClickHouse') + .mockResolvedValueOnce({ + json: async () => ({ + data: [ + { + tags: [ + ['__name__', 'up'], + ['host', 'node-1'], + ], + time_series: [[new Date(bucketStartMs).toISOString(), 42]], + }, + { + tags: [ + ['__name__', 'up'], + ['host', 'node-2'], + ], + time_series: [[new Date(bucketStartMs).toISOString(), 5]], // Below threshold + }, + ], + }), + } as any); + + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + // State machine asserts + const alertHistories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + + // Two group keys, so two histories + expect(alertHistories.length).toBe(2); + + // Group key order is non-deterministic depending on Map iterators, so find them. + const node1History = alertHistories.find( + h => h.group === 'host:node-1', + )!; + const node2History = alertHistories.find( + h => h.group === 'host:node-2', + )!; + + expect(node1History).toBeDefined(); + expect(node1History.state).toBe('ALERT'); + expect(node1History.lastValues[0].count).toBe(42); + + expect(node2History).toBeDefined(); + expect(node2History.state).toBe('OK'); + expect(node2History.lastValues[0].count).toBe(5); + + expect( + timeseriesEngine.queryPrometheusRangeFromClickHouse, + ).toHaveBeenCalledTimes(1); + }); + }); + describe('dashboard variables', () => { const NOW = new Date('2023-11-16T22:12:00.000Z'); // Inside the last alert window, 22:05 - 22:10. diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts index 9eea9caaba..ac46353710 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.test.ts @@ -2,10 +2,20 @@ import mongoose from 'mongoose'; import { getConnectionById } from '@/controllers/connection'; import { queryPrometheusRangeFromClickHouse } from '@/controllers/timeseriesEngine'; +import type { ISource } from '@/models/source'; import { evaluatePromqlAlert } from '@/tasks/checkAlerts'; jest.mock('@/controllers/connection'); -jest.mock('@/controllers/timeseriesEngine'); +jest.mock('@/controllers/timeseriesEngine', () => { + const actual = jest.requireActual< + typeof import('@/controllers/timeseriesEngine') + >('@/controllers/timeseriesEngine'); + return { + ...actual, + // Only mock the function that makes real network calls + queryPrometheusRangeFromClickHouse: jest.fn(), + }; +}); jest.mock('@/clickhouse'); describe('evaluatePromqlAlert', () => { @@ -23,6 +33,13 @@ describe('evaluatePromqlAlert', () => { connection: mockConnectionId, }; + const mockSource = { + from: { + databaseName: 'default', + tableName: 'otel_metrics_ts', + }, + } as unknown as ISource; + afterEach(() => { jest.clearAllMocks(); }); @@ -38,7 +55,7 @@ describe('evaluatePromqlAlert', () => { global.fetch = jest.fn(); }); - it('should query Prometheus API and return the last value', async () => { + it('should return all time-series values for a single unlabeled series', async () => { (global.fetch as jest.Mock).mockResolvedValue({ ok: true, json: async () => ({ @@ -46,6 +63,7 @@ describe('evaluatePromqlAlert', () => { data: { result: [ { + metric: {}, values: [ [1704067200, '0'], [1704067500, '42.5'], @@ -64,14 +82,25 @@ describe('evaluatePromqlAlert', () => { windowSizeInMins: mockWindowSizeInMins, }); - expect(result).toEqual([{ group: '', value: 42.5 }]); + // Returns full PrometheusMatrixResult[] — all points, not just the last. + // This enables the caller (processAlert) to evaluate each window bucket + // individually for backfill support. + expect(result).toEqual([ + { + metric: {}, + values: [ + [1704067200, '0'], + [1704067500, '42.5'], + ], + }, + ]); expect(global.fetch).toHaveBeenCalledWith( 'http://prometheus:9090/api/v1/query_range?query=up&start=1704067200&end=1704067500&step=300', expect.any(Object), ); }); - it('should return multiple series for Prometheus endpoint', async () => { + it('should return multiple series with all time-points', async () => { (global.fetch as jest.Mock).mockResolvedValue({ ok: true, json: async () => ({ @@ -100,8 +129,8 @@ describe('evaluatePromqlAlert', () => { }); expect(result).toEqual([ - { group: 'host:"A"', value: 42.5 }, - { group: 'host:"B"', value: 10.5 }, + { metric: { host: 'A' }, values: [[1704067500, '42.5']] }, + { metric: { host: 'B' }, values: [[1704067500, '10.5']] }, ]); }); @@ -118,6 +147,7 @@ describe('evaluatePromqlAlert', () => { await expect( evaluatePromqlAlert({ savedConfig: mockSavedConfig, + source: mockSource, connectionId: mockConnectionId, teamId: mockTeamId, dateRange: mockDateRange, @@ -159,7 +189,7 @@ describe('evaluatePromqlAlert', () => { }); }); - it('should return multiple series with tags as tuples', async () => { + it('should return multiple series with all time-points via formatMatrixResponse', async () => { // tags is an array of [key, value] tuples — as returned by ClickHouse (queryPrometheusRangeFromClickHouse as jest.Mock).mockResolvedValue({ json: async () => ({ @@ -187,9 +217,17 @@ describe('evaluatePromqlAlert', () => { windowSizeInMins: mockWindowSizeInMins, }); + // formatMatrixResponse converts tags tuples to metric dict + // and time_series to [ts_seconds, string_value] tuples. expect(result).toEqual([ - { group: 'host:"A"', value: 42.5 }, - { group: 'host:"B"', value: 10.5 }, + { + metric: { host: 'A' }, + values: expect.arrayContaining([[expect.any(Number), '42.5']]), + }, + { + metric: { host: 'B' }, + values: expect.arrayContaining([[expect.any(Number), '10.5']]), + }, ]); expect(queryPrometheusRangeFromClickHouse).toHaveBeenCalledWith( @@ -201,7 +239,7 @@ describe('evaluatePromqlAlert', () => { ); }); - it('should filter __name__ from tags tuples', async () => { + it('should have __name__ stripped from metric map', async () => { (queryPrometheusRangeFromClickHouse as jest.Mock).mockResolvedValue({ json: async () => ({ data: [ @@ -218,47 +256,52 @@ describe('evaluatePromqlAlert', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, + source: mockSource, connectionId: mockConnectionId, teamId: mockTeamId, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, }); - // __name__ is stripped - expect(result).toEqual([{ group: 'host:"A"', value: 42.5 }]); + // formatMatrixResponse includes __name__ in metric — the caller + // (processAlert) strips __name__ when building the group key. + expect(result).toEqual([ + { + metric: { __name__: 'up', host: 'A' }, + values: expect.arrayContaining([[expect.any(Number), '42.5']]), + }, + ]); }); - it('should use defaults when no source is provided', async () => { - (queryPrometheusRangeFromClickHouse as jest.Mock).mockResolvedValue({ - json: async () => ({ - data: [ - { - time_series: [ - ['2024-01-01 00:00:00', 0], - ['2024-01-01 00:05:00', 42.5], - ], - }, - ], + it('should throw an error when no source tableName is provided', async () => { + await expect( + evaluatePromqlAlert({ + savedConfig: mockSavedConfig, + source: undefined, + connectionId: mockConnectionId, + teamId: mockTeamId, + dateRange: mockDateRange, + windowSizeInMins: mockWindowSizeInMins, }), - }); - - const result = await evaluatePromqlAlert({ - savedConfig: mockSavedConfig, - connectionId: mockConnectionId, - teamId: mockTeamId, - dateRange: mockDateRange, - windowSizeInMins: mockWindowSizeInMins, - }); + ).rejects.toThrow( + 'A PromQL alert routed to ClickHouse must have a source with a valid TimeSeries table name.', + ); + }); - expect(result).toEqual([{ group: '', value: 42.5 }]); - expect(queryPrometheusRangeFromClickHouse).toHaveBeenCalledWith( - expect.objectContaining({ - databaseName: 'default', - tableName: 'otel_metrics_gauge', - startMs: 1704067200000, - endMs: 1704067500000, - stepSec: 300, + it('should throw when source has no databaseName (prevents silently querying wrong DB)', async () => { + await expect( + evaluatePromqlAlert({ + savedConfig: mockSavedConfig, + source: { + from: { databaseName: '', tableName: 'otel_metrics_ts' }, + } as unknown as ISource, + connectionId: mockConnectionId, + teamId: mockTeamId, + dateRange: mockDateRange, + windowSizeInMins: mockWindowSizeInMins, }), + ).rejects.toThrow( + 'A PromQL alert routed to ClickHouse must have a source with a valid database name.', ); }); @@ -271,6 +314,7 @@ describe('evaluatePromqlAlert', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, + source: mockSource, connectionId: mockConnectionId, teamId: mockTeamId, dateRange: mockDateRange, diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index e16b14a3f9..070008dff4 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -58,7 +58,11 @@ import { serializeError } from 'serialize-error'; import { ClickhouseClient } from '@/clickhouse'; import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory'; import { getConnectionById } from '@/controllers/connection'; -import { queryPrometheusRangeFromClickHouse } from '@/controllers/timeseriesEngine'; +import { + joinPrometheusUpstreamUrl, + PROMETHEUS_CH_TIMEOUT_MS, + queryPrometheusRangeFromClickHouse, +} from '@/controllers/timeseriesEngine'; import { AlertState, IAlert, IAlertError } from '@/models/alert'; import AlertHistory, { IAlertHistory, @@ -69,8 +73,8 @@ import { ISavedSearch } from '@/models/savedSearch'; import { ISource } from '@/models/source'; import { IWebhook } from '@/models/webhook'; import { - joinPrometheusUpstreamUrl, - PROMETHEUS_CH_TIMEOUT_MS, + formatMatrixResponse, + PrometheusMatrixResult, } from '@/routers/api/prometheus'; import { isClientTimeoutOrAbortError, @@ -978,11 +982,17 @@ export const parseAlertData = ( }; /** - * Execute a PromQL alert tile's expression for the given window and return the - * last bucket's numeric value, or null when the query returns no data. + * Execute a PromQL alert's expression for the given window and return all + * series with their full time-series data as {@link PrometheusMatrixResult}[], + * or `null` when the query returns no data. + * + * Returning all time-points (not just the last) allows the caller to evaluate + * every expected time bucket individually, enabling correct backfill support + * when multiple windows have been skipped. * - * Supports both Prometheus-endpoint connections (HTTP proxy to /api/v1/query) - * and ClickHouse backed connections (prometheusQuery table function). + * Supports both Prometheus-endpoint connections (HTTP proxy to + * `/api/v1/query_range`) and ClickHouse-backed connections + * (`prometheusQueryRange` table function). */ export async function evaluatePromqlAlert({ savedConfig, @@ -1000,7 +1010,7 @@ export async function evaluatePromqlAlert({ dateRange: [Date, Date]; windowSizeInMins: number; variables?: ChartVariable[]; -}): Promise | null> { +}): Promise { const connection = await getConnectionById(teamId, connectionId, true); if (connection == null) { throw new Error(`Connection ${connectionId} not found for PromQL alert`); @@ -1056,23 +1066,14 @@ export async function evaluatePromqlAlert({ return null; } - const results: Array<{ group: string; value: number }> = []; - for (const series of json.data.result) { - if (!series.values?.length) continue; - const lastPoint = series.values[series.values.length - 1]; - if (lastPoint == null) continue; - const parsed = parseFloat(lastPoint[1]); - if (Number.isFinite(parsed)) { - // Format Prometheus metric object as a HyperDX group key: key1:"val1", key2:"val2" - const group = series.metric - ? Object.entries(series.metric) - .filter(([k]) => k !== '__name__') - .map(([k, v]) => `${k}:"${v}"`) - .join(', ') - : ''; - results.push({ group, value: parsed }); - } - } + // Return all series with all time-points so the caller can evaluate each + // window bucket individually (backfill support). + const results: PrometheusMatrixResult[] = json.data.result + .filter(s => s.values && s.values.length > 0) + .map(s => ({ + metric: s.metric ?? {}, + values: s.values!, + })); return results.length > 0 ? results : null; } @@ -1089,35 +1090,40 @@ export async function evaluatePromqlAlert({ const endMs = Math.floor(endSec * 1000); const stepSecRounded = Math.max(Math.floor(stepSec), 1); + const tableName = source?.from.tableName; + if (!tableName) { + throw new Error( + 'A PromQL alert routed to ClickHouse must have a source with a valid TimeSeries table name. ' + + 'Assign a source to this alert or switch to a Prometheus connection.', + ); + } + + const databaseName = source.from.databaseName; + if (!databaseName) { + throw new Error( + 'A PromQL alert routed to ClickHouse must have a source with a valid database name. ' + + 'Assign a source to this alert or switch to a Prometheus connection.', + ); + } + const resp = await queryPrometheusRangeFromClickHouse({ client, - // ISource always has `from` (it is on BaseSourceSchema); the nullable - // source param covers the case where no source is wired to the alert. - databaseName: source?.from.databaseName ?? 'default', - tableName: source?.from.tableName ?? 'otel_metrics_gauge', + databaseName, + tableName, expr: promqlExpression, startMs, endMs, stepSec: stepSecRounded, }); - const json = await resp.json(); - if (!Array.isArray(json?.data)) return null; - - const results: Array<{ group: string; value: number }> = []; - for (const series of json.data) { - if (!series.time_series?.length) continue; - const lastPoint = series.time_series[series.time_series.length - 1]; - if (lastPoint != null && Number.isFinite(lastPoint[1])) { - const group = series.tags - ? series.tags - .filter(([k]: any) => k !== '__name__') - .map(([k, v]: any) => `${k}:"${v}"`) - .join(', ') - : ''; - results.push({ group, value: lastPoint[1] }); - } - } + const json = (await resp.json()) as { + data?: { tags: [string, string][]; time_series: [string, number][] }[]; + }; + if (!Array.isArray(json?.data) || json.data.length === 0) return null; + + // Reuse formatMatrixResponse to convert { tags, time_series } rows into + // PrometheusMatrixResult[] — the same shape as the real Prometheus path. + const results = formatMatrixResponse(json.data); return results.length > 0 ? results : null; } @@ -1453,8 +1459,10 @@ export const processAlert = async ( ? details.tile.config : undefined; + let promqlResults: PrometheusMatrixResult[] | null = null; + const queryStartedAt = performance.now(); try { - const promqlResults = await evaluatePromqlAlert({ + promqlResults = await evaluatePromqlAlert({ savedConfig: savedConfig as PromqlSavedChartConfig, source: details.source, connectionId, @@ -1472,39 +1480,16 @@ export const processAlert = async ( : undefined, }); - if (promqlResults != null) { - for (const res of promqlResults) { - const groupKey = res.group; - const value = res.value; - - const history = getOrCreateHistory(groupKey); - history.lastValues.push({ count: value, startTime: dateRange[1] }); - const previous = previousMap.get( - computeHistoryMapKey(alert.id, groupKey), - ); - - if (doesExceedThreshold(alert, value)) { - history.counts += 1; - if (shouldFireBasedOnConsecutiveWindows(groupKey)) { - history.state = AlertState.ALERT; - history.fired = true; - await trySendNotification({ - state: AlertState.ALERT, - group: groupKey, - totalCount: value, - startTime: dateRange[1], - }); - } else { - history.state = AlertState.PENDING; - history.fired = previous?.fired === true; - } - } - - await sendNotificationIfResolved(previous, history, groupKey); - } - } + const queryDurationMs = performance.now() - queryStartedAt; + evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); + recordOperationOutcome({ + operation: 'alerts.query', + outcome: 'success', + durationMs: queryDurationMs, + attributes: { alert_source: alert.source ?? 'unknown' }, + }); } catch (e) { - const queryDurationMs = performance.now() - evalStartedAt; + const queryDurationMs = performance.now() - queryStartedAt; evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); recordOperationOutcome({ operation: 'alerts.query', @@ -1537,7 +1522,151 @@ export const processAlert = async ( return; } - // If no data returned but we had alerts before, check auto-resolve + // ── PromQL time-series evaluation — iterate every expected bucket ── + // This mirrors the builder time-series path so backfilled windows are + // evaluated and consecutive-window tracking works correctly. + const expectedBuckets = timeBucketByGranularity( + dateRange[0], + dateRange[1], + `${windowSizeInMins} minute`, + ); + evaluationAnalytics.backfilledBuckets = Math.max( + 0, + expectedBuckets.length - 1, + ); + + // Build a per-bucket lookup: timestamp (seconds) → per-series value map. + // key: series group string, value: numeric value at that bucket. + // Format group keys as k:v (consistent with builder alerts, not k:"v"). + const bucketSeriesValues = new Map< + number, + Map }> + >(); + + if (promqlResults != null) { + for (const series of promqlResults) { + // Build group key from labels — k:v format matches builder alerts. + const labelEntries = Object.entries(series.metric).filter( + ([k]) => k !== '__name__', + ); + const groupKey = labelEntries.map(([k, v]) => `${k}:${v}`).join(', '); + const attributes = Object.fromEntries(labelEntries); + + for (const [tsSec, rawVal] of series.values) { + const tsMs = Math.round(tsSec * 1000); + const parsed = parseFloat(rawVal); + if (!Number.isFinite(parsed)) continue; + + if (!bucketSeriesValues.has(tsMs)) { + bucketSeriesValues.set(tsMs, new Map()); + } + // Keep the highest value if the same series appears twice in a bucket + const existing = bucketSeriesValues.get(tsMs)!.get(groupKey); + if (existing == null || parsed > existing.value) { + bucketSeriesValues.get(tsMs)!.set(groupKey, { + value: parsed, + attributes, + }); + } + } + } + } + + for (const bucketStart of expectedBuckets) { + // Prometheus query_range aligns response timestamps to step boundaries, + // so we can do an exact ms lookup rather than a tolerance scan. + const seriesForBucket = bucketSeriesValues.get(bucketStart.getTime()); + + if (!seriesForBucket || seriesForBucket.size === 0) { + alertEvaluationsCounter.add(1, { outcome: 'empty_bucket' }); + logger.info( + { alertId: alert.id, bucketStart }, + 'No PromQL data for time bucket', + ); + + const zeroValueIsAlert = doesExceedThreshold(alert, 0); + const hasAlertsInPreviousMap = previousMap + .values() + .some( + h => + h.state === AlertState.ALERT || h.state === AlertState.PENDING, + ); + + if (zeroValueIsAlert) { + const history = getOrCreateHistory(''); + history.lastValues.push({ count: 0, startTime: bucketStart }); + history.counts += 1; + if (shouldFireBasedOnConsecutiveWindows()) { + history.state = AlertState.ALERT; + history.fired = true; + latestAlertContext.set('', { + value: 0, + attributes: {}, + startTime: bucketStart, + }); + } else { + history.state = AlertState.PENDING; + history.fired = + previousMap.get(computeHistoryMapKey(alert.id, ''))?.fired === + true; + } + } else if (!hasGroupBy || !hasAlertsInPreviousMap) { + const history = getOrCreateHistory(''); + history.lastValues.push({ count: 0, startTime: bucketStart }); + } + continue; + } + + const bucketEvaluations = new Map< + string, + { + value: number; + attributes: Record; + exceeds: boolean; + } + >(); + for (const [ + groupKey, + { value, attributes }, + ] of seriesForBucket.entries()) { + const exceeds = doesExceedThreshold(alert, value); + const existing = bucketEvaluations.get(groupKey); + if (!existing || (!existing.exceeds && exceeds)) { + bucketEvaluations.set(groupKey, { value, attributes, exceeds }); + } + } + + for (const [groupKey, evaluation] of bucketEvaluations.entries()) { + const history = getOrCreateHistory(groupKey); + + if (evaluation.exceeds) { + history.counts += 1; + if (shouldFireBasedOnConsecutiveWindows(groupKey)) { + history.state = AlertState.ALERT; + history.fired = true; + latestAlertContext.set(groupKey, { + value: evaluation.value, + attributes: evaluation.attributes, + startTime: bucketStart, + }); + } else { + history.state = AlertState.PENDING; + history.fired = + previousMap.get(computeHistoryMapKey(alert.id, groupKey)) + ?.fired === true; + } + } else { + history.state = AlertState.OK; + history.counts = 0; + } + history.lastValues.push({ + count: evaluation.value, + startTime: bucketStart, + }); + } + } + + // Auto-resolve: groups that were alerting/pending but absent from this run if (hasGroupBy && previousMap && previousMap.size > 0) { for (const [previousKey, previousHistory] of previousMap.entries()) { const groupKey = extractGroupKeyFromMapKey(previousKey, alert.id); @@ -1548,12 +1677,10 @@ export const processAlert = async ( !doesExceedThreshold(alert, 0) ) { const history = getOrCreateHistory(groupKey); - history.lastValues.push({ count: 0, startTime: dateRange[1] }); - await sendNotificationIfResolved( - previousHistory, - history, - groupKey, - ); + history.lastValues.push({ + count: 0, + startTime: expectedBuckets[0] ?? dateRange[1], + }); } } } @@ -1562,7 +1689,33 @@ export const processAlert = async ( getOrCreateHistory(''); } - evaluationAnalytics.backfilledBuckets = 0; + // Send notifications for state transitions + for (const [groupKey, history] of histories.entries()) { + const previousKey = computeHistoryMapKey(alert.id, groupKey); + let groupPrevious = previousMap.get(previousKey); + + const hitAlertThisRun = latestAlertContext.has(groupKey); + if (hitAlertThisRun) { + const context = latestAlertContext.get(groupKey); + if (context) { + await trySendNotification({ + state: AlertState.ALERT, + group: groupKey, + totalCount: context.value, + startTime: context.startTime, + attributes: context.attributes, + }); + groupPrevious = { + ...(groupPrevious ?? {}), + state: AlertState.ALERT, + fired: true, + } as AggregatedAlertHistory; + } + } + + await sendNotificationIfResolved(groupPrevious, history, groupKey); + } + flushNotificationTimings(); const historyRecords = Array.from(histories.values()); for (const record of historyRecords) { diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index 8720e716d0..f398816a3e 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -844,7 +844,7 @@ const Tile = ({ const displayTypeSupportsAlerts = isRawSql ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) : isPromQL - ? false // PromQL alert UI not yet implemented (HDX-4636) + ? true : displayTypeSupportsBuilderAlerts(chart.config.displayType); const canMoveToGroup = onMoveToGroup && moveTargets && moveTargets.length > 0; diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx index 0f8bf26ba7..be8ce77ef8 100644 --- a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -1,11 +1,21 @@ -import { Control, useController, useWatch } from 'react-hook-form'; +import { useEffect } from 'react'; +import { + Control, + useController, + UseFormSetValue, + useWatch, +} from 'react-hook-form'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; -import { Box, Button, Flex, Stack, Text } from '@mantine/core'; +import { Box, Button, Flex, Group, Stack, Text } from '@mantine/core'; +import { IconBell } from '@tabler/icons-react'; +import { TileAlertEditor } from '@/components/DBEditTimeChartForm/TileAlertEditor'; import PromQLEditor from '@/components/PromQLEditor/PromQLEditor'; import { SourceSelectControlled } from '@/components/SourceSelect'; +import { IS_LOCAL_MODE } from '@/config'; import { usePromqlMetricNames } from '@/hooks/usePromqlMetadata'; -import { useSource } from '@/source'; +import { useSource, useSources } from '@/source'; +import { DEFAULT_TILE_ALERT } from '@/utils/alerts'; import { ChartEditorFormState } from './types'; @@ -13,10 +23,22 @@ export default function PromqlChartEditor({ control, onSubmit, onOpenDisplaySettings, + alert, + alertsEnabled, + isAlertRequired, + dashboardId, + setValue, }: { control: Control; onSubmit: (suppressErrorNotification?: boolean) => void; onOpenDisplaySettings: () => void; + setValue: UseFormSetValue; + alert?: ChartEditorFormState['alert']; + /** Whether this editor offers an alert. */ + alertsEnabled?: boolean; + /** Hides the alert editor's remove control for surfaces that require an alert. */ + isAlertRequired?: boolean; + dashboardId?: string; }) { const { field: expressionField } = useController({ control, @@ -25,17 +47,33 @@ export default function PromqlChartEditor({ const sourceId = useWatch({ control, name: 'source' }); const { data: source } = useSource({ id: sourceId }); + const { data: sources } = useSources(); + + useEffect(() => { + if (!sourceId && sources) { + const firstPromqlSource = sources.find(s => s.kind === SourceKind.Promql); + if (firstPromqlSource) { + setValue('source', firstPromqlSource.id); + } + } + }, [sourceId, sources, setValue]); + // The form can still hold a non-PromQL source right after switching a tile // into PromQL mode (the picker above only restricts future selections), and // the metric-name lookup reads a TimeSeries engine table, so it would fail // against any other source's table. const promqlSource = source?.kind === SourceKind.Promql ? source : undefined; + + const tableName = promqlSource?.from.tableName; + const { data: metricNames } = usePromqlMetricNames( promqlSource?.connection, promqlSource?.from.databaseName, - promqlSource?.from.tableName, + tableName, ); + const chartName = useWatch({ control, name: 'name' }); + return ( @@ -61,7 +99,26 @@ export default function PromqlChartEditor({ metricNames={metricNames} /> - + + + {alertsEnabled && !alert && !IS_LOCAL_MODE && ( + + )} + )} @@ -124,7 +124,7 @@ export default function PromqlChartEditor({ size="compact-sm" variant="secondary" > - Display Settings + Display settings {alert && ( diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index faeb4db1c4..d1caf6d937 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -198,9 +198,7 @@ export function convertFormStateToSavedChartConfig( 'fillNulls', 'alignDateRangeToGranularity', 'alternateRowBackground', - // 'alert' intentionally excluded — no alert UI exists for PromQL yet - // (HDX-4636). Keeping it out prevents a stale carry-over from a - // Builder/SQL form from triggering a bad POST on save. + 'alert', ]), promqlExpression: form.promqlExpression ?? '', connection: form.connection ?? '', diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index 962b55c156..d1b2473e29 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -293,12 +293,6 @@ export default function EditTimeChartForm({ const previousDisplayType = usePrevious(displayType); useEffect(() => { if (displayType === previousDisplayType) return; - // PromQL charts have no alert UI yet — always clear to avoid a stale - // carry-over from a previous Builder/SQL mode triggering a bad POST. - if (isPromqlInput) { - setValue('alert', undefined); - return; - } const displayTypeSupportsAlerts = configType === 'sql' ? displayTypeSupportsRawSqlAlerts(displayType) @@ -308,15 +302,7 @@ export default function EditTimeChartForm({ } }, [configType, displayType, isPromqlInput, previousDisplayType, setValue]); - // Also clear the alert whenever the user flips the config type to PromQL, - // since the display-type effect above only runs on displayType changes. - const previousConfigType = usePrevious(configType); - useEffect(() => { - if (configType === previousConfigType) return; - if (configType === 'promql') { - setValue('alert', undefined); - } - }, [configType, previousConfigType, setValue]); + const showGeneratedSql = TABS_WITH_GENERATED_SQL.has(activeTab) && !isPromqlInput; From fd99d36c27f844941a38f90950c1bcfb1c751640 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Mon, 14 Sep 2026 01:12:41 +0530 Subject: [PATCH 08/49] style: run prettier on EditTimeChartForm --- .../src/components/DBEditTimeChartForm/EditTimeChartForm.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index d1b2473e29..07a98b55ba 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -302,8 +302,6 @@ export default function EditTimeChartForm({ } }, [configType, displayType, isPromqlInput, previousDisplayType, setValue]); - - const showGeneratedSql = TABS_WITH_GENERATED_SQL.has(activeTab) && !isPromqlInput; From cfb88f277193b16ea28f9f888c787f1e844f2a82 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Mon, 14 Sep 2026 01:18:43 +0530 Subject: [PATCH 09/49] removed any type --- .../__tests__/evaluatePromqlAlert.clickhouse.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts index 5559851afe..b4a843ebc2 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts @@ -74,7 +74,7 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { savedConfig: mockSavedConfig, source: { from: { databaseName: 'my_db', tableName: 'my_table' }, - } as any, + } as unknown as ISource, connectionId: mockConnectionId, teamId: mockTeamId, dateRange: mockDateRange, From 82f778076df5c3fa0e3dd974d2174cc89a61ca9f Mon Sep 17 00:00:00 2001 From: aryainguz Date: Mon, 14 Sep 2026 01:28:16 +0530 Subject: [PATCH 10/49] revert: restore original error logging in clickhouse utils --- packages/common-utils/src/clickhouse/index.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index 1427a953ec..8a1a5c771f 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -372,11 +372,7 @@ export const extractColumnReferencesFromKey = (expr: string): string[] => { return [...new Set([...parsedColumnList, ...jsonPaths, ...mapAccesses])]; } catch (e) { - console.error( - 'Error parsing column references from key', - e instanceof Error ? e.message : e, - expr, - ); + console.error('Error parsing column references from key', e, expr); return []; } }); @@ -961,7 +957,7 @@ export function chSqlToAliasMap( } catch (e) { console.error( 'Error parsing alias map with JSON removed', - e instanceof Error ? e.message : e, + e, 'for query', chSql, ); From 5aac73f6b9194730720d300e80ae95a780328bed Mon Sep 17 00:00:00 2001 From: aryainguz Date: Mon, 14 Sep 2026 01:31:04 +0530 Subject: [PATCH 11/49] revert: restore displayTypeSupportsPromQLAlerts and fix PromQL alert UI flag --- packages/app/src/DBDashboardPage.tsx | 5 +++-- packages/common-utils/src/core/utils.ts | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index f398816a3e..da2338bfa8 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -29,6 +29,7 @@ import { import { convertToDashboardTemplate, displayTypeSupportsBuilderAlerts, + displayTypeSupportsPromQLAlerts, displayTypeSupportsRawSqlAlerts, Granularity, isTimeSeriesDisplayType, @@ -844,7 +845,7 @@ const Tile = ({ const displayTypeSupportsAlerts = isRawSql ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) : isPromQL - ? true + ? displayTypeSupportsPromQLAlerts(chart.config.displayType) : displayTypeSupportsBuilderAlerts(chart.config.displayType); const canMoveToGroup = onMoveToGroup && moveTargets && moveTargets.length > 0; @@ -1055,7 +1056,7 @@ const Tile = ({ const showAlerts = isRawSql ? displayTypeSupportsRawSqlAlerts(chart.config.displayType) : isPromQL - ? false // PromQL alert UI not yet implemented (HDX-4636) + ? displayTypeSupportsPromQLAlerts(chart.config.displayType) : displayTypeSupportsBuilderAlerts(chart.config.displayType); const canMoveToGroup = onMoveToGroup && moveTargets && moveTargets.length > 0; diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 36d5bf4f0e..34f7e94ba7 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -1432,6 +1432,12 @@ export const isFormulaSourceKind = ( kind === SourceKind.Log || kind === SourceKind.Trace; +export function displayTypeSupportsPromQLAlerts( + displayType: DisplayType | undefined, +): boolean { + return displayType === DisplayType.Line; +} + /** Expand the chart's macros, returning failures instead of throwing. */ function resolveRawSqlMacros( chartConfig: RawSqlChartConfig, From 48f6b813767e861eda2187bc16be50006c402f9f Mon Sep 17 00:00:00 2001 From: aryainguz Date: Mon, 14 Sep 2026 01:37:13 +0530 Subject: [PATCH 12/49] fix: allow PromQL alerts on Number and StackedBar display types --- packages/common-utils/src/core/utils.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index 34f7e94ba7..97e9392982 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -1435,7 +1435,11 @@ export const isFormulaSourceKind = ( export function displayTypeSupportsPromQLAlerts( displayType: DisplayType | undefined, ): boolean { - return displayType === DisplayType.Line; + return ( + displayType === DisplayType.Line || + displayType === DisplayType.StackedBar || + displayType === DisplayType.Number + ); } /** Expand the chart's macros, returning failures instead of throwing. */ From 5ce75bde8de73aac93e6312ebb9403938aaf60ad Mon Sep 17 00:00:00 2001 From: aryainguz Date: Mon, 14 Sep 2026 01:42:05 +0530 Subject: [PATCH 13/49] style(alerts): remove ASCII banner comments --- packages/api/src/tasks/checkAlerts/index.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 070008dff4..29d3cd48e3 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1522,9 +1522,8 @@ export const processAlert = async ( return; } - // ── PromQL time-series evaluation — iterate every expected bucket ── - // This mirrors the builder time-series path so backfilled windows are - // evaluated and consecutive-window tracking works correctly. + // Evaluate all series over the expected buckets so that missed or + // delayed execution windows can be backfilled correctly. const expectedBuckets = timeBucketByGranularity( dateRange[0], dateRange[1], @@ -1940,7 +1939,7 @@ export const processAlert = async ( return; } - // ── Time-series path (Line/StackedBar charts) ── + // Standard time-series alert evaluation (Line/StackedBar charts). const expectedBuckets = timeBucketByGranularity( dateRange[0], dateRange[1], From 65e8baa1d4ca22203b1b01a01c7b457c086e7427 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Mon, 14 Sep 2026 01:47:15 +0530 Subject: [PATCH 14/49] refactor(alerts): replace 'as' cast with 'satisfies' for PromqlConfigWithDateRange --- packages/app/src/components/alerts/AlertDetailChart.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/app/src/components/alerts/AlertDetailChart.tsx b/packages/app/src/components/alerts/AlertDetailChart.tsx index edfaa4e11f..365526fc4e 100644 --- a/packages/app/src/components/alerts/AlertDetailChart.tsx +++ b/packages/app/src/components/alerts/AlertDetailChart.tsx @@ -158,7 +158,7 @@ export function buildAlertChartConfig({ dateRange, granularity, variables, - } as PromqlConfigWithDateRange; + } satisfies PromqlConfigWithDateRange; } // Raw SQL: only time-series display types can be charted over the alert From 4f9fc9bfaeb205a0e3ac74e97386d818a45d9cf6 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Mon, 14 Sep 2026 02:28:33 +0530 Subject: [PATCH 15/49] fix(alerts): address greptile review comments for promql alerts --- .../api/src/controllers/timeseriesEngine.ts | 49 ++++++++++++++++ .../routers/api/__tests__/prometheus.test.ts | 4 +- packages/api/src/routers/api/prometheus.ts | 57 ++----------------- packages/api/src/tasks/checkAlerts/index.ts | 42 ++++++++------ .../ChartEditor/PromqlChartEditor.tsx | 14 ++++- .../app/src/components/ChartEditor/utils.ts | 6 +- .../DBEditTimeChartForm/EditTimeChartForm.tsx | 1 + .../components/alerts/AlertDetailChart.tsx | 4 +- 8 files changed, 100 insertions(+), 77 deletions(-) diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts index cab8d00913..30cd971aa1 100644 --- a/packages/api/src/controllers/timeseriesEngine.ts +++ b/packages/api/src/controllers/timeseriesEngine.ts @@ -323,3 +323,52 @@ export async function queryPrometheusRangeFromClickHouse({ }, }); } + +// Prometheus-compatible response types +export type PrometheusMetric = Record; +export type PrometheusMatrixResult = { + metric: PrometheusMetric; + values: [number, string][]; +}; +export type PrometheusVectorResult = { + metric: PrometheusMetric; + value: [number, string]; +}; + +// ClickHouse → Prometheus response formatters +export function formatMatrixResponse( + rows: { tags: [string, string][]; time_series: [string, number][] }[], +): PrometheusMatrixResult[] { + return rows.map(row => { + const metric: PrometheusMetric = {}; + for (const [key, value] of row.tags) { + metric[key] = value; + } + const values: [number, string][] = row.time_series.map( + ([timestamp, value]) => { + const ts = + typeof timestamp === 'string' + ? new Date(timestamp).getTime() / 1000 + : Number(timestamp); + return [ts, String(value)]; + }, + ); + return { metric, values }; + }); +} + +export function formatVectorResponse( + rows: { tags: [string, string][]; timestamp: string; value: number }[], +): PrometheusVectorResult[] { + return rows.map(row => { + const metric: PrometheusMetric = {}; + for (const [key, value] of row.tags) { + metric[key] = value; + } + const ts = + typeof row.timestamp === 'string' + ? new Date(row.timestamp).getTime() / 1000 + : Number(row.timestamp); + return { metric, value: [ts, String(row.value)] }; + }); +} diff --git a/packages/api/src/routers/api/__tests__/prometheus.test.ts b/packages/api/src/routers/api/__tests__/prometheus.test.ts index 2326babf50..29f946c325 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.test.ts @@ -8,10 +8,12 @@ jest.mock('@/utils/instrumentation', () => { }; }); -import { joinPrometheusUpstreamUrl } from '@/controllers/timeseriesEngine'; import { formatMatrixResponse, formatVectorResponse, + joinPrometheusUpstreamUrl, +} from '@/controllers/timeseriesEngine'; +import { isClientDisconnect, parseDuration, parseTimestamp, diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index 25afa2cc3b..8133864cbc 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -11,6 +11,10 @@ import { PROMETHEUS_CH_TIMEOUT_MS, PROMETHEUS_MAX_EXECUTION_SEC, PROMETHEUS_MAX_RESULT_ROWS, + PrometheusMatrixResult, + PrometheusVectorResult, + formatMatrixResponse, + formatVectorResponse, queryLabelNames, queryLabelValues, queryPrometheusRangeFromClickHouse, @@ -103,60 +107,7 @@ function getParams(req: express.Request): Record { }; } -// -------------------------- -// Prometheus-compatible response types -// -------------------------- - -export type PrometheusMetric = Record; -export type PrometheusMatrixResult = { - metric: PrometheusMetric; - values: [number, string][]; -}; -type PrometheusVectorResult = { - metric: PrometheusMetric; - value: [number, string]; -}; -// -------------------------- -// ClickHouse → Prometheus response formatters -// -------------------------- - -export function formatMatrixResponse( - rows: { tags: [string, string][]; time_series: [string, number][] }[], -): PrometheusMatrixResult[] { - return rows.map(row => { - const metric: PrometheusMetric = {}; - for (const [key, value] of row.tags) { - metric[key] = value; - } - const values: [number, string][] = row.time_series.map( - ([timestamp, value]) => { - const ts = - typeof timestamp === 'string' - ? new Date(timestamp).getTime() / 1000 - : Number(timestamp); - return [ts, String(value)]; - }, - ); - return { metric, values }; - }); -} - -export function formatVectorResponse( - rows: { tags: [string, string][]; timestamp: string; value: number }[], -): PrometheusVectorResult[] { - return rows.map(row => { - const metric: PrometheusMetric = {}; - for (const [key, value] of row.tags) { - metric[key] = value; - } - const ts = - typeof row.timestamp === 'string' - ? new Date(row.timestamp).getTime() / 1000 - : Number(row.timestamp); - return { metric, value: [ts, String(row.value)] }; - }); -} // -------------------------- // Prometheus proxy (for real Prometheus backends) diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 29d3cd48e3..903f86d595 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -59,8 +59,11 @@ import { ClickhouseClient } from '@/clickhouse'; import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory'; import { getConnectionById } from '@/controllers/connection'; import { - joinPrometheusUpstreamUrl, PROMETHEUS_CH_TIMEOUT_MS, + PROMETHEUS_MAX_RESULT_ROWS, + PrometheusMatrixResult, + formatMatrixResponse, + joinPrometheusUpstreamUrl, queryPrometheusRangeFromClickHouse, } from '@/controllers/timeseriesEngine'; import { AlertState, IAlert, IAlertError } from '@/models/alert'; @@ -72,10 +75,7 @@ import { IDashboard } from '@/models/dashboard'; import { ISavedSearch } from '@/models/savedSearch'; import { ISource } from '@/models/source'; import { IWebhook } from '@/models/webhook'; -import { - formatMatrixResponse, - PrometheusMatrixResult, -} from '@/routers/api/prometheus'; + import { isClientTimeoutOrAbortError, isQueryTimeoutError, @@ -1016,9 +1016,11 @@ export async function evaluatePromqlAlert({ throw new Error(`Connection ${connectionId} not found for PromQL alert`); } - const endSec = dateRange[1].getTime() / 1000; - const startSec = dateRange[0].getTime() / 1000; const stepSec = windowSizeInMins * 60; + const endSec = dateRange[1].getTime() / 1000; + // PromQL evaluates exactly at the given timestamps. We want the evaluation + // for the window [T, T+step) to happen at T+step, using the freshest data. + const startSec = dateRange[0].getTime() / 1000 + stepSec; const promqlExpression = variables && variables.length > 0 ? substitutePromqlChartConfigVariables({ ...savedConfig, variables }) @@ -1524,11 +1526,15 @@ export const processAlert = async ( // Evaluate all series over the expected buckets so that missed or // delayed execution windows can be backfilled correctly. - const expectedBuckets = timeBucketByGranularity( - dateRange[0], - dateRange[1], - `${windowSizeInMins} minute`, - ); + const expectedBuckets: Date[] = []; + const windowMs = windowSizeInMins * 60 * 1000; + for ( + let t = dateRange[0].getTime(); + t <= dateRange[1].getTime() - windowMs; + t += windowMs + ) { + expectedBuckets.push(new Date(t)); + } evaluationAnalytics.backfilledBuckets = Math.max( 0, expectedBuckets.length - 1, @@ -1545,14 +1551,16 @@ export const processAlert = async ( if (promqlResults != null) { for (const series of promqlResults) { // Build group key from labels — k:v format matches builder alerts. - const labelEntries = Object.entries(series.metric).filter( - ([k]) => k !== '__name__', - ); + // Keep __name__ in the group key to avoid collapsing distinct series. + const labelEntries = Object.entries(series.metric); const groupKey = labelEntries.map(([k, v]) => `${k}:${v}`).join(', '); - const attributes = Object.fromEntries(labelEntries); + const attributes = Object.fromEntries( + labelEntries.filter(([k]) => k !== '__name__'), + ); for (const [tsSec, rawVal] of series.values) { - const tsMs = Math.round(tsSec * 1000); + // Shift the evaluation instant back to the start of the bucket + const tsMs = Math.round(tsSec * 1000) - windowMs; const parsed = parseFloat(rawVal); if (!Number.isFinite(parsed)) continue; diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx index 7540e7624c..9a1f966903 100644 --- a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -5,6 +5,7 @@ import { UseFormSetValue, useWatch, } from 'react-hook-form'; +import { displayTypeSupportsPromQLAlerts } from '@hyperdx/common-utils/dist/core/utils'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { Box, Button, Flex, Group, Stack, Text } from '@mantine/core'; import { IconBell } from '@tabler/icons-react'; @@ -28,6 +29,7 @@ export default function PromqlChartEditor({ isAlertRequired, dashboardId, setValue, + additionalAlertWarnings, }: { control: Control; onSubmit: (suppressErrorNotification?: boolean) => void; @@ -39,6 +41,7 @@ export default function PromqlChartEditor({ /** Hides the alert editor's remove control for surfaces that require an alert. */ isAlertRequired?: boolean; dashboardId?: string; + additionalAlertWarnings?: string[]; }) { const { field: expressionField } = useController({ control, @@ -73,6 +76,7 @@ export default function PromqlChartEditor({ ); const chartName = useWatch({ control, name: 'name' }); + const displayType = useWatch({ control, name: 'displayType' }); return ( @@ -101,7 +105,10 @@ export default function PromqlChartEditor({ - {alertsEnabled && !alert && !IS_LOCAL_MODE && ( + {alertsEnabled && + !alert && + !IS_LOCAL_MODE && + displayTypeSupportsPromQLAlerts(displayType) && ( - )} + + )} {alert && ( From 945ee1539ecfbd53de030cafc6ee12ee4b3726e2 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 18 Sep 2026 20:36:09 +0530 Subject: [PATCH 28/49] chore: fix lint errors in utils.ts --- packages/app/src/utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/app/src/utils.ts b/packages/app/src/utils.ts index a44db41586..decd6bb328 100644 --- a/packages/app/src/utils.ts +++ b/packages/app/src/utils.ts @@ -102,6 +102,7 @@ export const useDebounce = ( useEffect( () => { if (shouldBeImmediate) { + // eslint-disable-next-line setDebouncedValue(value); return () => {}; } @@ -201,6 +202,7 @@ export function useLocalStorage(key: string, initialValue: T) { const item = window.localStorage.getItem(key); // Parse stored json or if none return initialValue if (item != null) { + // eslint-disable-next-line setStoredValue(JSON.parse(item)); } } catch (error) { From c62072e9703353ab86d3ac9657e3d971cfae9c1c Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 18 Sep 2026 20:46:34 +0530 Subject: [PATCH 29/49] chore: remove unused useController import --- packages/app/src/components/ChartEditor/PromqlChartEditor.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx index 5de890498b..f20044bd27 100644 --- a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -1,7 +1,6 @@ import { useEffect } from 'react'; import { Control, - useController, UseFormSetValue, useWatch, } from 'react-hook-form'; From 7a4d185b1e0bcd40967b5984edd8e65a4d55b690 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Wed, 23 Sep 2026 23:03:43 +0530 Subject: [PATCH 30/49] chore: fix checkAlerts tests for queryPrometheusRangeFromClickHouse rename --- .../__tests__/checkAlerts.int.test.ts | 46 +++++++-------- .../evaluatePromqlAlert.clickhouse.test.ts | 56 +++++++------------ packages/api/src/tasks/checkAlerts/index.ts | 13 +---- 3 files changed, 44 insertions(+), 71 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index c147338614..14e42b3767 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -5229,31 +5229,27 @@ describe('checkAlerts', () => { // We need to mock the timeseriesEngine call rather than evaluatePromqlAlert directly // because processAlert calls the local evaluatePromqlAlert inside index.ts jest - .spyOn(timeseriesEngine, 'queryPrometheusRangeFromClickHouse') - .mockResolvedValueOnce({ - json: async () => ({ - data: [ - { - tags: [ - ['__name__', 'up'], - ['host', 'node-1'], - ], - time_series: [ - [new Date(prometheusReturnedMs).toISOString(), 42], - ], - }, - { - tags: [ - ['__name__', 'up'], - ['host', 'node-2'], - ], - time_series: [ - [new Date(prometheusReturnedMs).toISOString(), 5], - ], // Below threshold - }, + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([ + { + metric: { + __name__: 'up', + host: 'node-1', + }, + values: [ + [prometheusReturnedMs / 1000, '42'], ], - }), - } as any); + }, + { + metric: { + __name__: 'up', + host: 'node-2', + }, + values: [ + [prometheusReturnedMs / 1000, '5'], + ], // Below threshold + }, + ]); await processAlertAtTime( now, @@ -5289,7 +5285,7 @@ describe('checkAlerts', () => { expect(node2History.lastValues[0].count).toBe(5); expect( - timeseriesEngine.queryPrometheusRangeFromClickHouse, + timeseriesEngine.queryRangeViaTableFunction, ).toHaveBeenCalledTimes(1); }); }); diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts index b4a843ebc2..bf35c4e35b 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts @@ -1,7 +1,7 @@ import mongoose from 'mongoose'; import { getConnectionById } from '@/controllers/connection'; -import { queryPrometheusRangeFromClickHouse } from '@/controllers/timeseriesEngine'; +import { queryRangeViaTableFunction } from '@/controllers/timeseriesEngine'; import type { ISource } from '@/models/source'; import { evaluatePromqlAlert } from '@/tasks/checkAlerts'; @@ -13,7 +13,7 @@ jest.mock('@/controllers/timeseriesEngine', () => { return { ...actual, // Only mock the function that makes real network calls - queryPrometheusRangeFromClickHouse: jest.fn(), + queryRangeViaTableFunction: jest.fn(), }; }); jest.mock('@/clickhouse'); @@ -54,21 +54,16 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { }); it('should return multiple series with all time-points via formatMatrixResponse', async () => { - // tags is an array of [key, value] tuples — as returned by ClickHouse - (queryPrometheusRangeFromClickHouse as jest.Mock).mockResolvedValue({ - json: async () => ({ - data: [ - { - tags: [['host', 'A']], - time_series: [['2024-01-01 00:05:00', 42.5]], - }, - { - tags: [['host', 'B']], - time_series: [['2024-01-01 00:05:00', 10.5]], - }, - ], - }), - }); + (queryRangeViaTableFunction as jest.Mock).mockResolvedValue([ + { + metric: { host: 'A' }, + values: [[1704067500, '42.5']], + }, + { + metric: { host: 'B' }, + values: [[1704067500, '10.5']], + }, + ]); const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, @@ -94,7 +89,7 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { }, ]); - expect(queryPrometheusRangeFromClickHouse).toHaveBeenCalledWith( + expect(queryRangeViaTableFunction).toHaveBeenCalledWith( expect.objectContaining({ expr: 'up', databaseName: 'my_db', @@ -104,19 +99,12 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { }); it('should have __name__ stripped from metric map', async () => { - (queryPrometheusRangeFromClickHouse as jest.Mock).mockResolvedValue({ - json: async () => ({ - data: [ - { - tags: [ - ['__name__', 'up'], - ['host', 'A'], - ], - time_series: [['2024-01-01 00:05:00', 42.5]], - }, - ], - }), - }); + (queryRangeViaTableFunction as jest.Mock).mockResolvedValue([ + { + metric: { __name__: 'up', host: 'A' }, + values: [[1704067500, '42.5']], + }, + ]); const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, @@ -170,11 +158,7 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { }); it('should return null when no data is returned', async () => { - (queryPrometheusRangeFromClickHouse as jest.Mock).mockResolvedValue({ - json: async () => ({ - data: [], - }), - }); + (queryRangeViaTableFunction as jest.Mock).mockResolvedValue([]); const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 0911703f5e..84cf0ae1dc 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -62,7 +62,7 @@ import { joinPrometheusUpstreamUrl, PROMETHEUS_CH_TIMEOUT_MS, PrometheusMatrixResult, - queryPrometheusRangeFromClickHouse, + queryRangeViaTableFunction, } from '@/controllers/timeseriesEngine'; import { AlertState, IAlert, IAlertError } from '@/models/alert'; import AlertHistory, { @@ -1081,8 +1081,9 @@ export async function evaluatePromqlAlert({ } try { - const resp = await queryPrometheusRangeFromClickHouse({ + const results = await queryRangeViaTableFunction({ client, + connectionId, databaseName, tableName, expr: promqlExpression, @@ -1091,14 +1092,6 @@ export async function evaluatePromqlAlert({ stepSec: stepSecRounded, }); - const json = (await resp.json()) as { - data?: { tags: [string, string][]; time_series: [string, number][] }[]; - }; - if (!Array.isArray(json?.data) || json.data.length === 0) return null; - - // Reuse formatMatrixResponse to convert { tags, time_series } rows into - // PrometheusMatrixResult[] — the same shape as the real Prometheus path. - const results = formatMatrixResponse(json.data); return results.length > 0 ? results : null; } finally { await client.close(); From 0d523d8c0f749aaaecab6166d23bfd0c413559ad Mon Sep 17 00:00:00 2001 From: aryainguz Date: Wed, 23 Sep 2026 23:26:30 +0530 Subject: [PATCH 31/49] style: format checkAlerts index and test files --- .../tasks/checkAlerts/__tests__/checkAlerts.int.test.ts | 8 ++------ packages/api/src/tasks/checkAlerts/index.ts | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 14e42b3767..1a4f427028 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -5236,18 +5236,14 @@ describe('checkAlerts', () => { __name__: 'up', host: 'node-1', }, - values: [ - [prometheusReturnedMs / 1000, '42'], - ], + values: [[prometheusReturnedMs / 1000, '42']], }, { metric: { __name__: 'up', host: 'node-2', }, - values: [ - [prometheusReturnedMs / 1000, '5'], - ], // Below threshold + values: [[prometheusReturnedMs / 1000, '5']], // Below threshold }, ]); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 288355c03a..d6152f7b58 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -45,6 +45,7 @@ import { SavedChartConfig, SourceKind, } from '@hyperdx/common-utils/dist/types'; +import { PrometheusMatrixResult } from '@hyperdx/common-utils/dist/types'; import { substitutePromqlChartConfigVariables } from '@hyperdx/common-utils/dist/variables'; import * as fns from 'date-fns'; import { isString, pick } from 'lodash'; @@ -65,7 +66,6 @@ import { PROMETHEUS_CH_TIMEOUT_MS, queryRangeViaTableFunction, } from '@/controllers/timeseriesEngine'; -import { PrometheusMatrixResult } from '@hyperdx/common-utils/dist/types'; import { AlertState, IAlert, IAlertError } from '@/models/alert'; import AlertHistory, { IAlertHistory, From 3766926cef4e8339287945d5c38c9269b237a281 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Wed, 23 Sep 2026 23:28:25 +0530 Subject: [PATCH 32/49] fix: lint errors (unused vars and duplicate imports) --- packages/api/src/routers/api/prometheus.ts | 2 -- packages/api/src/tasks/checkAlerts/index.ts | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index b59d9188b9..4b580f5ebb 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -13,8 +13,6 @@ import { clickhouseServesPrometheusHttpApi, joinPrometheusUpstreamUrl, PROMETHEUS_CH_TIMEOUT_MS, - PROMETHEUS_MAX_EXECUTION_SEC, - PROMETHEUS_MAX_RESULT_ROWS, queryInstantViaTableFunction, queryLabelNames, queryLabelValues, diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index d6152f7b58..1816c7bdae 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -41,11 +41,11 @@ import { DisplayType, getSampleWeightExpression, pickSampleWeightExpressionProps, + PrometheusMatrixResult, PromqlSavedChartConfig, SavedChartConfig, SourceKind, } from '@hyperdx/common-utils/dist/types'; -import { PrometheusMatrixResult } from '@hyperdx/common-utils/dist/types'; import { substitutePromqlChartConfigVariables } from '@hyperdx/common-utils/dist/variables'; import * as fns from 'date-fns'; import { isString, pick } from 'lodash'; @@ -61,7 +61,6 @@ import { getConnectionById } from '@/controllers/connection'; import { clickhouseAuthHeaders, clickhouseServesPrometheusHttpApi, - formatMatrixResponse, joinPrometheusUpstreamUrl, PROMETHEUS_CH_TIMEOUT_MS, queryRangeViaTableFunction, From 414af4b370ca7271f48fff37e39caf2ad60cb4f3 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Wed, 23 Sep 2026 23:33:43 +0530 Subject: [PATCH 33/49] refactor: unexport prometheus constants and helper function in timeseriesEngine --- packages/api/src/controllers/timeseriesEngine.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts index 82cb258b09..0ee595179b 100644 --- a/packages/api/src/controllers/timeseriesEngine.ts +++ b/packages/api/src/controllers/timeseriesEngine.ts @@ -16,8 +16,8 @@ import { import { ClickhouseClient } from '@/clickhouse'; import logger from '@/utils/logger'; -export const PROMETHEUS_MAX_EXECUTION_SEC = 30; -export const PROMETHEUS_MAX_RESULT_ROWS = 100000; +const PROMETHEUS_MAX_EXECUTION_SEC = 30; +const PROMETHEUS_MAX_RESULT_ROWS = 100000; export type TimeSeriesTagsQueryArgs = { client: ClickhouseClient; @@ -394,7 +394,7 @@ export async function queryInstantViaTableFunction({ * server is picked up without a restart; an unknown version is treated as * old, matching the other version-gated features. */ -export async function connectionSupportsPrometheusHttpApi({ +async function connectionSupportsPrometheusHttpApi({ client, connectionId, }: { From 4c9b4f1a34a3f0ed45793c7e5eeabf8e825d9d79 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Wed, 23 Sep 2026 23:38:12 +0530 Subject: [PATCH 34/49] test: fix failing test expecting single Alert Name validation element --- .../DBEditTimeChartForm/__tests__/DBEditTimeChartForm.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/app/src/components/DBEditTimeChartForm/__tests__/DBEditTimeChartForm.test.tsx b/packages/app/src/components/DBEditTimeChartForm/__tests__/DBEditTimeChartForm.test.tsx index 73e2e6923a..f2b3341f85 100644 --- a/packages/app/src/components/DBEditTimeChartForm/__tests__/DBEditTimeChartForm.test.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/__tests__/DBEditTimeChartForm.test.tsx @@ -1310,7 +1310,8 @@ describe('DBEditTimeChartForm - Inline alerts', () => { await userEvent.click(screen.getByTestId('chart-save-alert-button')); - expect(await screen.findByText('Alert name is required')).toBeVisible(); + const elements = await screen.findAllByText('Alert name is required'); + expect(elements[0]).toBeVisible(); expect(onSaveAlert).not.toHaveBeenCalled(); }); From 244ebe433dff3722865da3cbc157475bba990071 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Wed, 23 Sep 2026 23:50:15 +0530 Subject: [PATCH 35/49] fix: address P1 defects in PromQL alerts (blank expressions & clickhouse http endpoint) --- packages/api/src/tasks/checkAlerts/index.ts | 71 +++++++++++++-------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 1816c7bdae..ce6bb81a97 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -59,7 +59,9 @@ import { ClickhouseClient } from '@/clickhouse'; import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory'; import { getConnectionById } from '@/controllers/connection'; import { + CLICKHOUSE_PROMETHEUS_API_PREFIX, clickhouseAuthHeaders, + clickhousePrometheusUpstream, clickhouseServesPrometheusHttpApi, joinPrometheusUpstreamUrl, PROMETHEUS_CH_TIMEOUT_MS, @@ -1004,16 +1006,22 @@ export async function evaluatePromqlAlert({ ? substitutePromqlChartConfigVariables({ ...savedConfig, variables }) : savedConfig; const rawExpression = resolvedConfig.promqlExpression; - const promqlExpression = Array.isArray(rawExpression) - ? rawExpression[rawExpression.length - 1].expression - : rawExpression; + let promqlExpression = typeof rawExpression === 'string' ? rawExpression : ''; + if (Array.isArray(rawExpression)) { + const validExpressions = rawExpression.filter( + e => e.expression && e.expression.trim() !== '', + ); + if (validExpressions.length > 0) { + promqlExpression = + validExpressions[validExpressions.length - 1].expression; + } + } // Shared helper for querying a Prometheus-compatible HTTP endpoint. async function queryPrometheusHttp( - upstreamHost: string, + url: URL, headers?: Record, ): Promise { - const url = joinPrometheusUpstreamUrl(upstreamHost, '/api/v1/query_range'); url.searchParams.set('query', promqlExpression); url.searchParams.set('start', String(startSec)); url.searchParams.set('end', String(endSec)); @@ -1056,7 +1064,27 @@ export async function evaluatePromqlAlert({ } if (connection.isPrometheusEndpoint) { - return queryPrometheusHttp(connection.host); + const url = joinPrometheusUpstreamUrl( + connection.host, + '/api/v1/query_range', + ); + return queryPrometheusHttp(url); + } + + const tableName = source?.from.tableName; + if (!tableName) { + throw new Error( + 'A PromQL alert routed to ClickHouse must have a source with a valid TimeSeries table name. ' + + 'Assign a source to this alert or switch to a Prometheus connection.', + ); + } + + const databaseName = source.from.databaseName; + if (!databaseName) { + throw new Error( + 'A PromQL alert routed to ClickHouse must have a source with a valid database name. ' + + 'Assign a source to this alert or switch to a Prometheus connection.', + ); } // ClickHouse path — create a client for both the HTTP API probe and the @@ -1077,13 +1105,18 @@ export async function evaluatePromqlAlert({ }) ) { // ClickHouse 26.6+ with the prometheus_api_v1 handler configured. - const chUpstream = new URL(connection.host); - chUpstream.searchParams.set( - 'max_execution_time', - String(Math.round(PROMETHEUS_CH_TIMEOUT_MS / 1000)), + const chUpstream = clickhousePrometheusUpstream(connection.host, { + maxExecutionSec: Math.round(PROMETHEUS_CH_TIMEOUT_MS / 1000), + }); + const url = joinPrometheusUpstreamUrl( + chUpstream, + `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query_range`, ); + url.searchParams.set('database', databaseName); + url.searchParams.set('table', tableName); + return queryPrometheusHttp( - chUpstream.toString(), + url, clickhouseAuthHeaders({ username: connection.username, password: connection.password, @@ -1095,22 +1128,6 @@ export async function evaluatePromqlAlert({ const endMs = Math.floor(endSec * 1000); const stepSecRounded = Math.max(Math.floor(stepSec), 1); - const tableName = source?.from.tableName; - if (!tableName) { - throw new Error( - 'A PromQL alert routed to ClickHouse must have a source with a valid TimeSeries table name. ' + - 'Assign a source to this alert or switch to a Prometheus connection.', - ); - } - - const databaseName = source.from.databaseName; - if (!databaseName) { - throw new Error( - 'A PromQL alert routed to ClickHouse must have a source with a valid database name. ' + - 'Assign a source to this alert or switch to a Prometheus connection.', - ); - } - try { const results = await queryRangeViaTableFunction({ client, From 6aeb24d83d4170fe17c2fb5208867dd45c561ed1 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Thu, 24 Sep 2026 00:44:15 +0530 Subject: [PATCH 36/49] refactor: pass connection and clickhouse client to evaluatePromqlAlert and add capability memoization --- .../__tests__/checkAlerts.int.test.ts | 139 +++++++++++++++++- .../evaluatePromqlAlert.clickhouse.test.ts | 58 ++++++-- .../evaluatePromqlAlert.prometheus.test.ts | 48 ++++-- packages/api/src/tasks/checkAlerts/index.ts | 121 ++++++++------- 4 files changed, 292 insertions(+), 74 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 1a4f427028..af12271f7b 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -1111,6 +1111,7 @@ describe('checkAlerts', () => { ): AlertDetails => { const base = { alert: { groupBy: overrides.alertGroupBy } as any, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion source: {} as any, previousMap: new Map(), }; @@ -1122,6 +1123,7 @@ describe('checkAlerts', () => { tile: { config: { groupBy: overrides.tileGroupBy ?? '' }, } as any, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion dashboard: {} as any, }; } @@ -1140,6 +1142,7 @@ describe('checkAlerts', () => { return { ...base, taskType: AlertTaskType.SAVED_SEARCH, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion savedSearch: {} as any, }; }; @@ -1227,6 +1230,7 @@ describe('checkAlerts', () => { SeverityText: { name: 'SeverityText', type: 'String' }, ServiceName: { name: 'ServiceName', type: 'String' }, }; + // eslint-disable-next-line security/detect-object-injection return Promise.resolve(columnMap[column]); }), getColumns: jest.fn().mockResolvedValue([]), @@ -1274,7 +1278,9 @@ describe('checkAlerts', () => { defaultTableSelectExpression: 'Timestamp, Body', }, savedSearch: { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion _id: 'fake-saved-search-id' as any, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion team: 'team-123' as any, id: 'fake-saved-search-id', name: 'My Search', @@ -1282,6 +1288,7 @@ describe('checkAlerts', () => { where: 'Body: "error"', whereLanguage: 'lucene', orderBy: 'timestamp', + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion source: 'fake-source-id' as any, tags: ['test'], createdAt: new Date(), @@ -1314,6 +1321,7 @@ describe('checkAlerts', () => { id: 'id-123', name: 'My Dashboard', tiles: [testTile], + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion team: 'team-123' as any, tags: ['test'], createdAt: new Date(), @@ -1359,6 +1367,7 @@ describe('checkAlerts', () => { beforeEach(async () => { jest .spyOn(slack, 'postMessageToWebhook') + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion .mockResolvedValueOnce(null as any); }); @@ -2016,6 +2025,7 @@ describe('checkAlerts', () => { ); // Verify the message includes the time range but not detailed logs + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const callArgs = (slack.postMessageToWebhook as any).mock.calls[0][1]; const messageText = callArgs.blocks[0].text.text; expect(messageText).toContain('The alert has been resolved'); @@ -2042,6 +2052,7 @@ describe('checkAlerts', () => { SeverityText: { name: 'SeverityText', type: 'String' }, ServiceName: { name: 'ServiceName', type: 'String' }, }; + // eslint-disable-next-line security/detect-object-injection return Promise.resolve(columnMap[column]); }), }; @@ -2054,6 +2065,7 @@ describe('checkAlerts', () => { jest .spyOn(slack, 'postMessageToWebhook') + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion .mockResolvedValueOnce(null as any); jest.spyOn(checkAlert, 'handleSendGenericWebhook'); @@ -2197,7 +2209,7 @@ describe('checkAlerts', () => { recentHistoryMap, }, clickhouseClient, - connection.id, + connection, alertProvider, teamWebhooksById, ); @@ -3006,6 +3018,7 @@ describe('checkAlerts', () => { it('INLINE alert (group by) - notifies per group', async () => { // Two groups fire, so two notifications go out (the shared beforeEach // only mocks a single call). + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion jest.spyOn(slack, 'postMessageToWebhook').mockResolvedValue(null as any); const { @@ -3180,6 +3193,7 @@ describe('checkAlerts', () => { webhookSettings: Partial; }> = {}) => { const fixture = await setupSavedSearchAlertTest({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion webhookSettings: webhookSettings as IWebhook, }); const dashboard = await new Dashboard({ @@ -4117,6 +4131,7 @@ describe('checkAlerts', () => { headers: status === 302 ? { Location: redirectTarget } : {}, }); }); + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion global.fetch = fetchMock as any; const { @@ -4330,6 +4345,7 @@ describe('checkAlerts', () => { status: 500, text: jest.fn().mockResolvedValue('resolve send failed'), }); + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion global.fetch = fetchMock as any; const { @@ -4505,6 +4521,7 @@ describe('checkAlerts', () => { status: 500, text: jest.fn().mockResolvedValue('group webhook failed'), }); + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion global.fetch = fetchMock as any; const { @@ -4516,6 +4533,7 @@ describe('checkAlerts', () => { teamWebhooksById, clickhouseClient, } = await setupSavedSearchAlertTest({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion webhookSettings: { service: WebhookService.Generic, url: 'https://webhook.site/group-fail', @@ -4998,6 +5016,7 @@ describe('checkAlerts', () => { teamWebhooksById, clickhouseClient, } = await setupSavedSearchAlertTest({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion webhookSettings: { service: WebhookService.Generic, url: 'https://webhook.site/no-body', @@ -5236,14 +5255,20 @@ describe('checkAlerts', () => { __name__: 'up', host: 'node-1', }, - values: [[prometheusReturnedMs / 1000, '42']], + values: [ + [(prometheusReturnedMs - 5 * 60 * 1000) / 1000, '42'], + [prometheusReturnedMs / 1000, '42'], + ], }, { metric: { __name__: 'up', host: 'node-2', }, - values: [[prometheusReturnedMs / 1000, '5']], // Below threshold + values: [ + [(prometheusReturnedMs - 5 * 60 * 1000) / 1000, '5'], + [prometheusReturnedMs / 1000, '5'], + ], // Below threshold }, ]); @@ -5275,6 +5300,8 @@ describe('checkAlerts', () => { expect(node1History).toBeDefined(); expect(node1History.state).toBe('ALERT'); expect(node1History.lastValues[0].count).toBe(42); + // Expect one backfilled bucket because we returned two buckets. + expect(node1History.analytics?.backfilledBuckets).toBe(1); expect(node2History).toBeDefined(); expect(node2History.state).toBe('OK'); @@ -5857,6 +5884,7 @@ describe('checkAlerts', () => { // rendered title/body must resolve {{attributes.ServiceName}} to the // firing group's value ("web"), not an empty string. expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1); + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const [, payload] = (slack.postMessageToWebhook as jest.Mock).mock .calls[0]; @@ -6780,6 +6808,7 @@ describe('checkAlerts', () => { expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); // Verify the resolution notification was sent for service-b + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; const resolutionCall = calls.find(call => call[1].text.includes('My Search'), @@ -7339,6 +7368,7 @@ describe('checkAlerts', () => { expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); // Verify the resolution notification was sent for service-b + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; const resolutionCall = calls.find(call => call[1].text.includes('My Search'), @@ -7671,6 +7701,7 @@ describe('checkAlerts', () => { expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); // Validate webhook messages contain correct group names + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; const messages = calls.map((call: any) => ({ url: call[0], @@ -11160,6 +11191,7 @@ describe('checkAlerts', () => { expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; expect(JSON.stringify(calls[0][1])).toContain('Alert for'); expect(JSON.stringify(calls[1][1])).toContain( @@ -11232,6 +11264,7 @@ describe('checkAlerts', () => { jest .spyOn(slack, 'postMessageToWebhook') + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion .mockResolvedValue(null as any); const details = await createAlertDetails( @@ -11333,6 +11366,7 @@ describe('checkAlerts', () => { jest .spyOn(slack, 'postMessageToWebhook') + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion .mockResolvedValue(null as any); const details = await createAlertDetails( @@ -11420,6 +11454,7 @@ describe('checkAlerts', () => { jest .spyOn(slack, 'postMessageToWebhook') + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion .mockResolvedValue(null as any); const details = await createAlertDetails( @@ -11510,6 +11545,104 @@ describe('checkAlerts', () => { expect(serviceBHistories[0].state).toBe('PENDING'); expect(serviceBHistories[0].fired).toBeFalsy(); }); + + it('should route PromQL queries to the ClickHouse Prometheus HTTP API if supported', async () => { + const { + team, + webhook, + connection, + source, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const dashboard = await new Dashboard({ + name: 'PromQL Dashboard', + team: team._id, + tiles: [ + { + id: 'promql1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'promql', + displayType: 'line', + promqlExpression: 'sum(up)', + connection: connection.id, + }, + }, + ], + }).save(); + + const tile = dashboard.tiles?.find((t: any) => t.id === 'promql1'); + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.TILE, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 10, + dashboardId: dashboard.id, + tileId: 'promql1', + }, + { + taskType: AlertTaskType.TILE, + tile: tile!, + dashboard, + }, + ); + + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(true); + + const fetchMock = jest.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + status: 'success', + data: { + result: [ + { + metric: {}, + values: [[Date.now() / 1000, '42']], + }, + ], + }, + }), + }); + const originalFetch = global.fetch; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + global.fetch = fetchMock as any; + + try { + await processAlertAtTime( + new Date(), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + expect(fetchMock).toHaveBeenCalled(); + const callUrl = new URL(fetchMock.mock.calls[0][0]); + expect(callUrl.pathname).toBe('/prometheus/api/v1/query_range'); + expect(callUrl.searchParams.get('database')).toBe( + source.from.databaseName, + ); + expect(callUrl.searchParams.get('table')).toBe(source.from.tableName); + expect(callUrl.searchParams.get('query')).toBe('sum(up)'); + } finally { + global.fetch = originalFetch; + } + }); }); }); diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts index bf35c4e35b..b1af2a1e93 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts @@ -2,6 +2,7 @@ import mongoose from 'mongoose'; import { getConnectionById } from '@/controllers/connection'; import { queryRangeViaTableFunction } from '@/controllers/timeseriesEngine'; +import type { IConnection } from '@/models/connection'; import type { ISource } from '@/models/source'; import { evaluatePromqlAlert } from '@/tasks/checkAlerts'; @@ -19,7 +20,6 @@ jest.mock('@/controllers/timeseriesEngine', () => { jest.mock('@/clickhouse'); describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { - const mockTeamId = new mongoose.Types.ObjectId().toString(); const mockConnectionId = new mongoose.Types.ObjectId().toString(); const mockDateRange: [Date, Date] = [ new Date('2024-01-01T00:00:00Z'), @@ -33,6 +33,7 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { connection: mockConnectionId, }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const mockSource = { from: { databaseName: 'default', @@ -45,6 +46,7 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { }); beforeEach(() => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (getConnectionById as jest.Mock).mockResolvedValue({ host: 'http://clickhouse:8123', username: 'default', @@ -54,6 +56,7 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { }); it('should return multiple series with all time-points via formatMatrixResponse', async () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (queryRangeViaTableFunction as jest.Mock).mockResolvedValue([ { metric: { host: 'A' }, @@ -67,11 +70,17 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion source: { from: { databaseName: 'my_db', tableName: 'my_table' }, } as unknown as ISource, - connectionId: mockConnectionId, - teamId: mockTeamId, + connection: { + id: mockConnectionId, + host: 'localhost', + isPrometheusEndpoint: false, + } as unknown as IConnection, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, }); @@ -99,6 +108,7 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { }); it('should have __name__ stripped from metric map', async () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (queryRangeViaTableFunction as jest.Mock).mockResolvedValue([ { metric: { __name__: 'up', host: 'A' }, @@ -109,8 +119,14 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, source: mockSource, - connectionId: mockConnectionId, - teamId: mockTeamId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + connection: { + id: mockConnectionId, + host: 'localhost', + isPrometheusEndpoint: false, + } as unknown as IConnection, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, }); @@ -130,8 +146,14 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { evaluatePromqlAlert({ savedConfig: mockSavedConfig, source: undefined, - connectionId: mockConnectionId, - teamId: mockTeamId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + connection: { + id: mockConnectionId, + host: 'localhost', + isPrometheusEndpoint: false, + } as unknown as IConnection, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, }), @@ -144,11 +166,18 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { await expect( evaluatePromqlAlert({ savedConfig: mockSavedConfig, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion source: { from: { databaseName: '', tableName: 'otel_metrics_ts' }, } as unknown as ISource, - connectionId: mockConnectionId, - teamId: mockTeamId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + connection: { + id: mockConnectionId, + host: 'localhost', + isPrometheusEndpoint: false, + } as unknown as IConnection, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, }), @@ -158,13 +187,20 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { }); it('should return null when no data is returned', async () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (queryRangeViaTableFunction as jest.Mock).mockResolvedValue([]); const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, source: mockSource, - connectionId: mockConnectionId, - teamId: mockTeamId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + connection: { + id: mockConnectionId, + host: 'localhost', + isPrometheusEndpoint: false, + } as unknown as IConnection, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, }); diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.prometheus.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.prometheus.test.ts index 7ba3b76285..74a9804230 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.prometheus.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.prometheus.test.ts @@ -1,6 +1,7 @@ import mongoose from 'mongoose'; import { getConnectionById } from '@/controllers/connection'; +import type { IConnection } from '@/models/connection'; import type { ISource } from '@/models/source'; import { evaluatePromqlAlert } from '@/tasks/checkAlerts'; @@ -9,7 +10,6 @@ jest.mock('@/controllers/connection', () => ({ })); describe('evaluatePromqlAlert (Prometheus endpoint)', () => { - const mockTeamId = new mongoose.Types.ObjectId().toString(); const mockConnectionId = new mongoose.Types.ObjectId().toString(); const mockDateRange: [Date, Date] = [ new Date('2024-01-01T00:00:00Z'), @@ -23,6 +23,7 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { connection: mockConnectionId, }; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const mockSource = { from: { databaseName: 'default', @@ -35,6 +36,7 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { }); beforeEach(() => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (getConnectionById as jest.Mock).mockResolvedValue({ host: 'http://prometheus:9090', isPrometheusEndpoint: true, @@ -45,6 +47,7 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { }); it('should return all time-series values for a single unlabeled series', async () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (global.fetch as jest.Mock).mockResolvedValue({ ok: true, json: async () => ({ @@ -65,8 +68,14 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, - connectionId: mockConnectionId, - teamId: mockTeamId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + connection: { + id: mockConnectionId, + host: 'http://prometheus:9090', + isPrometheusEndpoint: true, + } as unknown as IConnection, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, }); @@ -90,6 +99,7 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { }); it('should return multiple series with all time-points', async () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (global.fetch as jest.Mock).mockResolvedValue({ ok: true, json: async () => ({ @@ -111,8 +121,14 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, - connectionId: mockConnectionId, - teamId: mockTeamId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + connection: { + id: mockConnectionId, + host: 'http://prometheus:9090', + isPrometheusEndpoint: true, + } as unknown as IConnection, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, }); @@ -124,6 +140,7 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { }); it('should throw on non-success Prometheus status instead of returning null', async () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (global.fetch as jest.Mock).mockResolvedValue({ ok: true, json: async () => ({ @@ -137,8 +154,14 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { evaluatePromqlAlert({ savedConfig: mockSavedConfig, source: mockSource, - connectionId: mockConnectionId, - teamId: mockTeamId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + connection: { + id: mockConnectionId, + host: 'http://prometheus:9090', + isPrometheusEndpoint: true, + } as unknown as IConnection, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, }), @@ -148,6 +171,7 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { }); it('should return null when result array is empty', async () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (global.fetch as jest.Mock).mockResolvedValue({ ok: true, json: async () => ({ @@ -158,8 +182,14 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, - connectionId: mockConnectionId, - teamId: mockTeamId, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + connection: { + id: mockConnectionId, + host: 'http://prometheus:9090', + isPrometheusEndpoint: true, + } as unknown as IConnection, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, }); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index ce6bb81a97..53f160ed78 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -57,7 +57,6 @@ import { serializeError } from 'serialize-error'; import { ClickhouseClient } from '@/clickhouse'; import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory'; -import { getConnectionById } from '@/controllers/connection'; import { CLICKHOUSE_PROMETHEUS_API_PREFIX, clickhouseAuthHeaders, @@ -72,6 +71,7 @@ import AlertHistory, { IAlertHistory, IAlertHistoryAnalytics, } from '@/models/alertHistory'; +import { IConnection } from '@/models/connection'; import { IDashboard } from '@/models/dashboard'; import { ISavedSearch } from '@/models/savedSearch'; import { ISource } from '@/models/source'; @@ -223,6 +223,7 @@ const makeAlertError = ( ): IAlertError => ({ timestamp: new Date(), type, + // eslint-disable-next-line security/detect-object-injection message: (HARDCODED_ALERT_ERROR_MESSAGES[type] ?? message).slice(0, 10000), }); @@ -972,25 +973,22 @@ export const parseAlertData = ( export async function evaluatePromqlAlert({ savedConfig, source, - connectionId, - teamId, + connection, + clickhouseClient, + clickhouseCapabilityMemo, dateRange, windowSizeInMins, variables, }: { savedConfig: PromqlSavedChartConfig; source?: ISource | null; - connectionId: string; - teamId: string; + connection: IConnection; + clickhouseClient: ClickhouseClient; + clickhouseCapabilityMemo?: Map; dateRange: [Date, Date]; windowSizeInMins: number; variables?: ChartVariable[]; }): Promise { - const connection = await getConnectionById(teamId, connectionId, true); - if (connection == null) { - throw new Error(`Connection ${connectionId} not found for PromQL alert`); - } - const stepSec = windowSizeInMins * 60; const endSec = dateRange[1].getTime() / 1000; // PromQL evaluates exactly at the given timestamps. We want the evaluation @@ -1012,8 +1010,17 @@ export async function evaluatePromqlAlert({ e => e.expression && e.expression.trim() !== '', ); if (validExpressions.length > 0) { - promqlExpression = - validExpressions[validExpressions.length - 1].expression; + if ( + resolvedConfig.displayType === 'number' || + resolvedConfig.displayType === 'table' || + resolvedConfig.displayType === 'bar' || + resolvedConfig.displayType === 'pie' + ) { + promqlExpression = validExpressions[0].expression; + } else { + promqlExpression = + validExpressions[validExpressions.length - 1].expression; + } } } @@ -1035,6 +1042,7 @@ export async function evaluatePromqlAlert({ `Prometheus query_range returned HTTP ${resp.status} for PromQL alert`, ); } + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const json = (await resp.json()) as { status?: string; data?: { @@ -1087,23 +1095,26 @@ export async function evaluatePromqlAlert({ ); } - // ClickHouse path — create a client for both the HTTP API probe and the + // ClickHouse path — use the batch client for both the HTTP API probe and the // table function fallback. - const client = new ClickhouseClient({ - host: connection.host, - username: connection.username, - password: connection.password, - requestTimeout: PROMETHEUS_CH_TIMEOUT_MS, - }); - if ( - await clickhouseServesPrometheusHttpApi(client, { - id: connectionId, - host: connection.host, - username: connection.username, - password: connection.password, - }) - ) { + let servesHttpApi = false; + if (clickhouseCapabilityMemo) { + if (!clickhouseCapabilityMemo.has(connection.id)) { + clickhouseCapabilityMemo.set( + connection.id, + await clickhouseServesPrometheusHttpApi(clickhouseClient, connection), + ); + } + servesHttpApi = clickhouseCapabilityMemo.get(connection.id)!; + } else { + servesHttpApi = await clickhouseServesPrometheusHttpApi( + clickhouseClient, + connection, + ); + } + + if (servesHttpApi) { // ClickHouse 26.6+ with the prometheus_api_v1 handler configured. const chUpstream = clickhousePrometheusUpstream(connection.host, { maxExecutionSec: Math.round(PROMETHEUS_CH_TIMEOUT_MS / 1000), @@ -1128,31 +1139,28 @@ export async function evaluatePromqlAlert({ const endMs = Math.floor(endSec * 1000); const stepSecRounded = Math.max(Math.floor(stepSec), 1); - try { - const results = await queryRangeViaTableFunction({ - client, - connectionId, - databaseName, - tableName, - expr: promqlExpression, - startMs, - endMs, - stepSec: stepSecRounded, - }); + const results = await queryRangeViaTableFunction({ + client: clickhouseClient, + connectionId: connection.id, + databaseName, + tableName, + expr: promqlExpression, + startMs, + endMs, + stepSec: stepSecRounded, + }); - return results.length > 0 ? results : null; - } finally { - await client.close(); - } + return results.length > 0 ? results : null; } export const processAlert = async ( now: Date, details: AlertDetails, clickhouseClient: ClickhouseClient, - connectionId: string, + connection: IConnection, alertProvider: AlertProvider, teamWebhooksById: Map, + clickhouseCapabilityMemo?: Map, ) => { const { alert, previousMap, recentHistoryMap } = details; const source = 'source' in details ? details.source : undefined; @@ -1402,12 +1410,14 @@ export const processAlert = async ( alertProvider, attributes, clickhouseClient, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion dashboard: (details as any).dashboard, startTime, endTime: fns.addMinutes(startTime, windowSizeInMins), group, isGroupedAlert: hasGroupBy, metadata, + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion savedSearch: (details as any).savedSearch, source, state, @@ -1515,13 +1525,12 @@ export const processAlert = async ( const queryStartedAt = performance.now(); try { promqlResults = await evaluatePromqlAlert({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion savedConfig: savedConfig as PromqlSavedChartConfig, source: details.source, - connectionId, - teamId: (isPopulatedRef(alert.team) - ? alert.team._id - : alert.team - ).toString(), + connection, + clickhouseClient, + clickhouseCapabilityMemo, dateRange, windowSizeInMins, variables: @@ -1622,6 +1631,7 @@ export const processAlert = async ( const index = Math.round((tsMs - startMs) / windowMs); if (index < 0 || index >= expectedBuckets.length) continue; + // eslint-disable-next-line security/detect-object-injection const nearestBucketMs = expectedBuckets[index].getTime(); const parsed = parseFloat(rawVal); if (!Number.isFinite(parsed)) continue; @@ -1777,6 +1787,7 @@ export const processAlert = async ( startTime: context.startTime, attributes: context.attributes, }); + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion groupPrevious = { ...(groupPrevious ?? {}), state: AlertState.ALERT, @@ -1804,7 +1815,7 @@ export const processAlert = async ( const chartConfig = getChartConfigFromAlert( details, - connectionId, + connection.id, dateRange, windowSizeInMins, ); @@ -2209,6 +2220,7 @@ export const processAlert = async ( // Inject a mock previous history so the resolve check below catches it // if the final state for this group is OK (i.e. it breached then resolved). + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion groupPrevious = { ...(groupPrevious || {}), state: AlertState.ALERT, @@ -2494,6 +2506,7 @@ export default class CheckAlertTask implements HdxTask { async processAlertTask( alertTask: AlertTask, teamWebhooksById: Map, + clickhouseCapabilityMemo: Map, ): Promise { return tasksTracer.startActiveSpan('processAlertTask', async span => { setBusinessContext({ teamId: alertTask.conn.team.toString() }); @@ -2530,9 +2543,10 @@ export default class CheckAlertTask implements HdxTask { alertTask.now, alert, clickhouseClient, - conn.id, + conn, this.provider, teamWebhooksById, + clickhouseCapabilityMemo, ); }, { @@ -2612,11 +2626,16 @@ export default class CheckAlertTask implements HdxTask { ); let failedBatchCount = 0; + const clickhouseCapabilityMemo = new Map(); for (const task of alertTasks) { const teamWebhooksById = teamToWebhooks.get(task.conn.team.toString()) ?? new Map(); this.task_queue.add(async () => { - const success = await this.processAlertTask(task, teamWebhooksById); + const success = await this.processAlertTask( + task, + teamWebhooksById, + clickhouseCapabilityMemo, + ); if (!success) { failedBatchCount++; } From 1b7240bb13b62b9f0d657d53b36bde1ec49d6c8f Mon Sep 17 00:00:00 2001 From: aryainguz Date: Thu, 24 Sep 2026 01:13:15 +0530 Subject: [PATCH 37/49] test: update checkAlerts task assertions and clean up lint comments --- .../__tests__/checkAlerts.int.test.ts | 35 +++--------- .../__tests__/checkAlertsTask.test.ts | 12 ++-- .../evaluatePromqlAlert.clickhouse.test.ts | 55 ++++++++----------- .../evaluatePromqlAlert.prometheus.test.ts | 39 +++++-------- 4 files changed, 52 insertions(+), 89 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index af12271f7b..edce0b947b 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -1111,7 +1111,6 @@ describe('checkAlerts', () => { ): AlertDetails => { const base = { alert: { groupBy: overrides.alertGroupBy } as any, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion source: {} as any, previousMap: new Map(), }; @@ -1123,7 +1122,6 @@ describe('checkAlerts', () => { tile: { config: { groupBy: overrides.tileGroupBy ?? '' }, } as any, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion dashboard: {} as any, }; } @@ -1142,7 +1140,6 @@ describe('checkAlerts', () => { return { ...base, taskType: AlertTaskType.SAVED_SEARCH, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion savedSearch: {} as any, }; }; @@ -1220,7 +1217,6 @@ describe('checkAlerts', () => { describe('Alert Templates', () => { // Create a mock metadata object with the necessary methods - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const mockMetadata = { getColumn: jest.fn().mockImplementation(({ column }) => { // Provide basic column definitions for common columns to avoid warnings @@ -1245,7 +1241,6 @@ describe('checkAlerts', () => { } as any; // Create a mock clickhouse client - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const mockClickhouseClient = { query: jest.fn().mockResolvedValue({ json: jest.fn().mockResolvedValue({ data: [] }), @@ -1278,9 +1273,7 @@ describe('checkAlerts', () => { defaultTableSelectExpression: 'Timestamp, Body', }, savedSearch: { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion _id: 'fake-saved-search-id' as any, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion team: 'team-123' as any, id: 'fake-saved-search-id', name: 'My Search', @@ -1288,7 +1281,6 @@ describe('checkAlerts', () => { where: 'Body: "error"', whereLanguage: 'lucene', orderBy: 'timestamp', - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion source: 'fake-source-id' as any, tags: ['test'], createdAt: new Date(), @@ -1321,7 +1313,6 @@ describe('checkAlerts', () => { id: 'id-123', name: 'My Dashboard', tiles: [testTile], - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion team: 'team-123' as any, tags: ['test'], createdAt: new Date(), @@ -1367,7 +1358,6 @@ describe('checkAlerts', () => { beforeEach(async () => { jest .spyOn(slack, 'postMessageToWebhook') - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion .mockResolvedValueOnce(null as any); }); @@ -2025,7 +2015,6 @@ describe('checkAlerts', () => { ); // Verify the message includes the time range but not detailed logs - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const callArgs = (slack.postMessageToWebhook as any).mock.calls[0][1]; const messageText = callArgs.blocks[0].text.text; expect(messageText).toContain('The alert has been resolved'); @@ -2065,7 +2054,6 @@ describe('checkAlerts', () => { jest .spyOn(slack, 'postMessageToWebhook') - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion .mockResolvedValueOnce(null as any); jest.spyOn(checkAlert, 'handleSendGenericWebhook'); @@ -3018,7 +3006,6 @@ describe('checkAlerts', () => { it('INLINE alert (group by) - notifies per group', async () => { // Two groups fire, so two notifications go out (the shared beforeEach // only mocks a single call). - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion jest.spyOn(slack, 'postMessageToWebhook').mockResolvedValue(null as any); const { @@ -3193,7 +3180,6 @@ describe('checkAlerts', () => { webhookSettings: Partial; }> = {}) => { const fixture = await setupSavedSearchAlertTest({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion webhookSettings: webhookSettings as IWebhook, }); const dashboard = await new Dashboard({ @@ -4131,7 +4117,6 @@ describe('checkAlerts', () => { headers: status === 302 ? { Location: redirectTarget } : {}, }); }); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion global.fetch = fetchMock as any; const { @@ -4345,7 +4330,6 @@ describe('checkAlerts', () => { status: 500, text: jest.fn().mockResolvedValue('resolve send failed'), }); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion global.fetch = fetchMock as any; const { @@ -4521,7 +4505,6 @@ describe('checkAlerts', () => { status: 500, text: jest.fn().mockResolvedValue('group webhook failed'), }); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion global.fetch = fetchMock as any; const { @@ -4533,7 +4516,6 @@ describe('checkAlerts', () => { teamWebhooksById, clickhouseClient, } = await setupSavedSearchAlertTest({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion webhookSettings: { service: WebhookService.Generic, url: 'https://webhook.site/group-fail', @@ -5016,7 +4998,6 @@ describe('checkAlerts', () => { teamWebhooksById, clickhouseClient, } = await setupSavedSearchAlertTest({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion webhookSettings: { service: WebhookService.Generic, url: 'https://webhook.site/no-body', @@ -5884,7 +5865,6 @@ describe('checkAlerts', () => { // rendered title/body must resolve {{attributes.ServiceName}} to the // firing group's value ("web"), not an empty string. expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(1); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const [, payload] = (slack.postMessageToWebhook as jest.Mock).mock .calls[0]; @@ -6808,7 +6788,6 @@ describe('checkAlerts', () => { expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); // Verify the resolution notification was sent for service-b - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; const resolutionCall = calls.find(call => call[1].text.includes('My Search'), @@ -7368,7 +7347,6 @@ describe('checkAlerts', () => { expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); // Verify the resolution notification was sent for service-b - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; const resolutionCall = calls.find(call => call[1].text.includes('My Search'), @@ -7701,7 +7679,6 @@ describe('checkAlerts', () => { expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); // Validate webhook messages contain correct group names - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; const messages = calls.map((call: any) => ({ url: call[0], @@ -11191,7 +11168,6 @@ describe('checkAlerts', () => { expect((await Alert.findById(details.alert.id))!.state).toBe('OK'); expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; expect(JSON.stringify(calls[0][1])).toContain('Alert for'); expect(JSON.stringify(calls[1][1])).toContain( @@ -11264,7 +11240,6 @@ describe('checkAlerts', () => { jest .spyOn(slack, 'postMessageToWebhook') - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion .mockResolvedValue(null as any); const details = await createAlertDetails( @@ -11366,7 +11341,6 @@ describe('checkAlerts', () => { jest .spyOn(slack, 'postMessageToWebhook') - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion .mockResolvedValue(null as any); const details = await createAlertDetails( @@ -11454,7 +11428,6 @@ describe('checkAlerts', () => { jest .spyOn(slack, 'postMessageToWebhook') - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion .mockResolvedValue(null as any); const details = await createAlertDetails( @@ -11618,7 +11591,6 @@ describe('checkAlerts', () => { }), }); const originalFetch = global.fetch; - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion global.fetch = fetchMock as any; try { @@ -11912,6 +11884,11 @@ describe('checkAlerts', () => { }); it('should not use a materialized view when the query is incompatible with the available materialized view', async () => { + // Mock console.error to suppress expected EXPLAIN ESTIMATE failures logged by @clickhouse/client + const consoleErrorSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => {}); + // Arrange const { team, @@ -12053,6 +12030,8 @@ describe('checkAlerts', () => { expect(alertHistories[1].createdAt).toEqual( new Date('2023-11-16T22:15:00.000Z'), ); + + consoleErrorSpy.mockRestore(); }); }); diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlertsTask.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlertsTask.test.ts index 5b5d348898..a020e01da1 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlertsTask.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlertsTask.test.ts @@ -165,9 +165,10 @@ describe('CheckAlertTask', () => { mockAlertTask.now, mockAlertTask.alerts[0], expect.any(ClickhouseClient), - 'conn-123', + mockAlertTask.conn, mockAlertProvider, teamWebhooksById, + expect.any(Map), ); mockProcessAlert.mockRestore(); @@ -306,9 +307,10 @@ describe('CheckAlertTask', () => { mockAlertTask1.now, mockAlertTask1.alerts[0], expect.any(ClickhouseClient), - 'conn-123', + mockAlertTask1.conn, mockAlertProvider, team1WebhooksById, + expect.any(Map), ); // Second call should use team2's webhooks @@ -317,9 +319,10 @@ describe('CheckAlertTask', () => { mockAlertTask2.now, mockAlertTask2.alerts[0], expect.any(ClickhouseClient), - 'conn-456', + mockAlertTask2.conn, mockAlertProvider, team2WebhooksById, + expect.any(Map), ); // Verify getWebhooks was called for each team @@ -425,9 +428,10 @@ describe('CheckAlertTask', () => { alert: expect.objectContaining({ id: 'alert-healthy' }), }), expect.any(ClickhouseClient), - 'conn-healthy', + expect.objectContaining({ id: 'conn-healthy' }), mockAlertProvider, expect.any(Map), + expect.any(Map), ); }); diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts index b1af2a1e93..43e5a24725 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.clickhouse.test.ts @@ -6,6 +6,11 @@ import type { IConnection } from '@/models/connection'; import type { ISource } from '@/models/source'; import { evaluatePromqlAlert } from '@/tasks/checkAlerts'; +function mock(obj: Partial): T { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + return obj as unknown as T; +} + jest.mock('@/controllers/connection'); jest.mock('@/controllers/timeseriesEngine', () => { const actual = jest.requireActual< @@ -33,20 +38,18 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { connection: mockConnectionId, }; - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - const mockSource = { + const mockSource = mock({ from: { databaseName: 'default', tableName: 'otel_metrics_ts', }, - } as unknown as ISource; + }); afterEach(() => { jest.clearAllMocks(); }); beforeEach(() => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (getConnectionById as jest.Mock).mockResolvedValue({ host: 'http://clickhouse:8123', username: 'default', @@ -56,7 +59,6 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { }); it('should return multiple series with all time-points via formatMatrixResponse', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (queryRangeViaTableFunction as jest.Mock).mockResolvedValue([ { metric: { host: 'A' }, @@ -70,16 +72,14 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion source: { from: { databaseName: 'my_db', tableName: 'my_table' }, } as unknown as ISource, - connection: { + connection: mock({ id: mockConnectionId, - host: 'localhost', + host: 'http://clickhouse:8123', isPrometheusEndpoint: false, - } as unknown as IConnection, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + }), clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, @@ -108,7 +108,6 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { }); it('should have __name__ stripped from metric map', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (queryRangeViaTableFunction as jest.Mock).mockResolvedValue([ { metric: { __name__: 'up', host: 'A' }, @@ -119,13 +118,11 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, source: mockSource, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - connection: { + connection: mock({ id: mockConnectionId, - host: 'localhost', + host: 'http://clickhouse:8123', isPrometheusEndpoint: false, - } as unknown as IConnection, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + }), clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, @@ -146,13 +143,11 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { evaluatePromqlAlert({ savedConfig: mockSavedConfig, source: undefined, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - connection: { + connection: mock({ id: mockConnectionId, - host: 'localhost', + host: 'http://clickhouse:8123', isPrometheusEndpoint: false, - } as unknown as IConnection, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + }), clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, @@ -166,17 +161,14 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { await expect( evaluatePromqlAlert({ savedConfig: mockSavedConfig, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion source: { from: { databaseName: '', tableName: 'otel_metrics_ts' }, } as unknown as ISource, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - connection: { + connection: mock({ id: mockConnectionId, - host: 'localhost', + host: 'http://clickhouse:8123', isPrometheusEndpoint: false, - } as unknown as IConnection, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + }), clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, @@ -187,19 +179,16 @@ describe('evaluatePromqlAlert (ClickHouse endpoint)', () => { }); it('should return null when no data is returned', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (queryRangeViaTableFunction as jest.Mock).mockResolvedValue([]); const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, source: mockSource, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - connection: { + connection: mock({ id: mockConnectionId, - host: 'localhost', + host: 'http://clickhouse:8123', isPrometheusEndpoint: false, - } as unknown as IConnection, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + }), clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, diff --git a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.prometheus.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.prometheus.test.ts index 74a9804230..e52b951a86 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.prometheus.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/evaluatePromqlAlert.prometheus.test.ts @@ -5,6 +5,11 @@ import type { IConnection } from '@/models/connection'; import type { ISource } from '@/models/source'; import { evaluatePromqlAlert } from '@/tasks/checkAlerts'; +function mock(obj: Partial): T { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + return obj as unknown as T; +} + jest.mock('@/controllers/connection', () => ({ getConnectionById: jest.fn(), })); @@ -23,20 +28,18 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { connection: mockConnectionId, }; - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - const mockSource = { + const mockSource = mock({ from: { databaseName: 'default', tableName: 'otel_metrics_ts', }, - } as unknown as ISource; + }); afterEach(() => { jest.clearAllMocks(); }); beforeEach(() => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (getConnectionById as jest.Mock).mockResolvedValue({ host: 'http://prometheus:9090', isPrometheusEndpoint: true, @@ -47,7 +50,6 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { }); it('should return all time-series values for a single unlabeled series', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (global.fetch as jest.Mock).mockResolvedValue({ ok: true, json: async () => ({ @@ -68,13 +70,11 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - connection: { + connection: mock({ id: mockConnectionId, host: 'http://prometheus:9090', isPrometheusEndpoint: true, - } as unknown as IConnection, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + }), clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, @@ -99,7 +99,6 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { }); it('should return multiple series with all time-points', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (global.fetch as jest.Mock).mockResolvedValue({ ok: true, json: async () => ({ @@ -121,13 +120,11 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - connection: { + connection: mock({ id: mockConnectionId, host: 'http://prometheus:9090', isPrometheusEndpoint: true, - } as unknown as IConnection, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + }), clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, @@ -140,7 +137,6 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { }); it('should throw on non-success Prometheus status instead of returning null', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (global.fetch as jest.Mock).mockResolvedValue({ ok: true, json: async () => ({ @@ -154,13 +150,11 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { evaluatePromqlAlert({ savedConfig: mockSavedConfig, source: mockSource, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - connection: { + connection: mock({ id: mockConnectionId, host: 'http://prometheus:9090', isPrometheusEndpoint: true, - } as unknown as IConnection, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + }), clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, @@ -171,7 +165,6 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { }); it('should return null when result array is empty', async () => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion (global.fetch as jest.Mock).mockResolvedValue({ ok: true, json: async () => ({ @@ -182,13 +175,11 @@ describe('evaluatePromqlAlert (Prometheus endpoint)', () => { const result = await evaluatePromqlAlert({ savedConfig: mockSavedConfig, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - connection: { + connection: mock({ id: mockConnectionId, host: 'http://prometheus:9090', isPrometheusEndpoint: true, - } as unknown as IConnection, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion + }), clickhouseClient: {} as any, dateRange: mockDateRange, windowSizeInMins: mockWindowSizeInMins, From 0e0f6f39de5265747917a4459e452190dac8859f Mon Sep 17 00:00:00 2001 From: aryainguz Date: Sat, 26 Sep 2026 23:43:05 +0530 Subject: [PATCH 38/49] fix: correct expected buckets for promql alerts --- .../__tests__/checkAlerts.int.test.ts | 4 +- packages/api/src/tasks/checkAlerts/index.ts | 40 +++++++++---------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index edce0b947b..ad6807989b 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -5281,8 +5281,8 @@ describe('checkAlerts', () => { expect(node1History).toBeDefined(); expect(node1History.state).toBe('ALERT'); expect(node1History.lastValues[0].count).toBe(42); - // Expect one backfilled bucket because we returned two buckets. - expect(node1History.analytics?.backfilledBuckets).toBe(1); + // Expect zero backfilled buckets because the date range for a new alert at 22:12 is [22:05, 22:10], which is exactly one 5-minute bucket. + expect(node1History.analytics?.backfilledBuckets).toBe(0); expect(node2History).toBeDefined(); expect(node2History.state).toBe('OK'); diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 53f160ed78..0bbae52655 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1009,18 +1009,11 @@ export async function evaluatePromqlAlert({ const validExpressions = rawExpression.filter( e => e.expression && e.expression.trim() !== '', ); + // Always evaluate the last expression, mirroring how SQL builder alerts + // target the last series and how the tile renders its primary value. if (validExpressions.length > 0) { - if ( - resolvedConfig.displayType === 'number' || - resolvedConfig.displayType === 'table' || - resolvedConfig.displayType === 'bar' || - resolvedConfig.displayType === 'pie' - ) { - promqlExpression = validExpressions[0].expression; - } else { - promqlExpression = - validExpressions[validExpressions.length - 1].expression; - } + promqlExpression = + validExpressions[validExpressions.length - 1].expression; } } @@ -1261,7 +1254,15 @@ export const processAlert = async ( scheduleStartAt, ); evaluationWindowStart = nowInMinsRoundDown; - const hasGroupBy = alertHasGroupBy(details); + const isPromQL = + details.taskType === AlertTaskType.INLINE + ? isPromqlSavedChartConfig(details.chartConfig) + : details.taskType === AlertTaskType.TILE + ? isPromqlSavedChartConfig(details.tile.config) + : false; + + // PromQL alerts always return label-keyed series, acting like grouped alerts end-to-end. + const hasGroupBy = isPromQL || alertHasGroupBy(details); // Check if we should skip this alert check based on last evaluation time if (shouldSkipAlertCheck(details, hasGroupBy, nowInMinsRoundDown)) { @@ -1506,13 +1507,6 @@ export const processAlert = async ( } }; - const isPromQL = - details.taskType === AlertTaskType.INLINE - ? isPromqlSavedChartConfig(details.chartConfig) - : details.taskType === AlertTaskType.TILE - ? isPromqlSavedChartConfig(details.tile.config) - : false; - if (isPromQL) { const savedConfig = details.taskType === AlertTaskType.INLINE @@ -1592,7 +1586,7 @@ export const processAlert = async ( const windowMs = windowSizeInMins * 60 * 1000; for ( let t = dateRange[0].getTime(); - t <= dateRange[1].getTime() - windowMs; + t < dateRange[1].getTime(); t += windowMs ) { expectedBuckets.push(new Date(t)); @@ -1748,8 +1742,10 @@ export const processAlert = async ( } } - // Auto-resolve: groups that were alerting/pending but absent from this run - if (hasGroupBy && previousMap && previousMap.size > 0) { + // Auto-resolve: groups that were alerting/pending but absent from this run. + // PromQL always stores per-series history keyed by alertId||{labels}, + // regardless of hasGroupBy, so we always run this loop for PromQL alerts. + if (previousMap && previousMap.size > 0) { for (const [previousKey, previousHistory] of previousMap.entries()) { const groupKey = extractGroupKeyFromMapKey(previousKey, alert.id); if ( From 50585fd70251fc9fee073b4aee6e9bce300be765 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Sun, 27 Sep 2026 00:13:05 +0530 Subject: [PATCH 39/49] fix(alerts): properly auto-resolve disappearing groups in PromQL and SQL alerts --- .../__tests__/checkAlerts.int.test.ts | 4 +- packages/api/src/tasks/checkAlerts/index.ts | 117 +++++++++++++----- 2 files changed, 89 insertions(+), 32 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index ad6807989b..d37feb9876 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -8722,9 +8722,9 @@ describe('checkAlerts', () => { teamWebhooksById, ); - // Alert should be in ALERT state (service-b still alerting) + // Alert should be in OK state (both service-a and service-b resolved) const updatedAlert = await Alert.findById(details.alert.id); - expect(updatedAlert!.state).toBe('ALERT'); + expect(updatedAlert!.state).toBe('OK'); // Check alert histories after first run const firstRunHistories = await AlertHistory.find({ diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 0bbae52655..e96c1f4e86 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1742,23 +1742,52 @@ export const processAlert = async ( } } - // Auto-resolve: groups that were alerting/pending but absent from this run. + // Auto-resolve: groups that were alerting/pending but absent from the newest bucket. // PromQL always stores per-series history keyed by alertId||{labels}, // regardless of hasGroupBy, so we always run this loop for PromQL alerts. - if (previousMap && previousMap.size > 0) { - for (const [previousKey, previousHistory] of previousMap.entries()) { - const groupKey = extractGroupKeyFromMapKey(previousKey, alert.id); + const lastExpectedBucket = expectedBuckets[expectedBuckets.length - 1]; + if (lastExpectedBucket) { + const groupsToCheck = new Set(); + if (previousMap) { + for (const previousKey of previousMap.keys()) { + groupsToCheck.add(extractGroupKeyFromMapKey(previousKey, alert.id)); + } + } + for (const groupKey of histories.keys()) { + groupsToCheck.add(groupKey); + } + + for (const groupKey of groupsToCheck) { + const history = histories.get(groupKey); + const previousHistory = previousMap?.get( + computeHistoryMapKey(alert.id, groupKey), + ); + + const lastEvaluated = + history?.lastValues[history.lastValues.length - 1]; + const isMissingFromLastBucket = + !lastEvaluated || + lastEvaluated.startTime.getTime() !== lastExpectedBucket.getTime(); + + const wasAlertingOrPending = + previousHistory?.state === AlertState.ALERT || + previousHistory?.state === AlertState.PENDING || + history?.state === AlertState.ALERT || + history?.state === AlertState.PENDING; + if ( - (previousHistory.state === AlertState.ALERT || - previousHistory.state === AlertState.PENDING) && - !histories.has(groupKey) && + isMissingFromLastBucket && + wasAlertingOrPending && !doesExceedThreshold(alert, 0) ) { - const history = getOrCreateHistory(groupKey); - history.lastValues.push({ + const h = getOrCreateHistory(groupKey); + h.lastValues.push({ count: 0, - startTime: expectedBuckets[0] ?? dateRange[1], + startTime: lastExpectedBucket, }); + h.state = AlertState.OK; + h.counts = 0; + latestAlertContext.delete(groupKey); } } } @@ -2165,27 +2194,55 @@ export const processAlert = async ( // Handle missing groups: If current check found no data, check if any previously alerting/pending groups need to be resolved // For group-by alerts, check if any previously alerting or pending groups are missing from current data - if (hasGroupBy && previousMap && previousMap.size > 0) { - for (const [previousKey, previousHistory] of previousMap.entries()) { - const groupKey = extractGroupKeyFromMapKey(previousKey, alert.id); - - // If this group was previously ALERT or PENDING but is missing from current data and would be resolved by a 0 value, - // create an OK history for the group - if ( - (previousHistory.state === AlertState.ALERT || - previousHistory.state === AlertState.PENDING) && - !histories.has(groupKey) && - !doesExceedThreshold(alert, 0) - ) { - logger.info( - { - alertId: alert.id, - group: groupKey, - }, - `Group "${groupKey}" is missing from current data but was previously ${previousHistory.state} - creating OK history`, + if (hasGroupBy) { + const lastExpectedBucket = expectedBuckets[expectedBuckets.length - 1]; + if (lastExpectedBucket) { + const groupsToCheck = new Set(); + if (previousMap) { + for (const previousKey of previousMap.keys()) { + groupsToCheck.add(extractGroupKeyFromMapKey(previousKey, alert.id)); + } + } + for (const groupKey of histories.keys()) { + groupsToCheck.add(groupKey); + } + + for (const groupKey of groupsToCheck) { + const history = histories.get(groupKey); + const previousHistory = previousMap?.get( + computeHistoryMapKey(alert.id, groupKey), ); - const history = getOrCreateHistory(groupKey); - history.lastValues.push({ count: 0, startTime: expectedBuckets[0] }); + + const lastEvaluated = + history?.lastValues[history.lastValues.length - 1]; + const isMissingFromLastBucket = + !lastEvaluated || + lastEvaluated.startTime.getTime() !== lastExpectedBucket.getTime(); + + const wasAlertingOrPending = + previousHistory?.state === AlertState.ALERT || + previousHistory?.state === AlertState.PENDING || + history?.state === AlertState.ALERT || + history?.state === AlertState.PENDING; + + if ( + isMissingFromLastBucket && + wasAlertingOrPending && + !doesExceedThreshold(alert, 0) + ) { + logger.info( + { + alertId: alert.id, + group: groupKey, + }, + `Group "${groupKey}" is missing from current data but was previously ALERT/PENDING - creating OK history`, + ); + const h = getOrCreateHistory(groupKey); + h.lastValues.push({ count: 0, startTime: lastExpectedBucket }); + h.state = AlertState.OK; + h.counts = 0; + latestAlertContext.delete(groupKey); + } } } } From 7668d3822a24dce0cc3c1b0b8f21e66c2f18b9a5 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Sun, 27 Sep 2026 01:41:00 +0530 Subject: [PATCH 40/49] fix(review): address open review comments --- .../api/src/controllers/timeseriesEngine.ts | 4 --- .../__tests__/checkAlerts.int.test.ts | 26 ++++++++++--------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/packages/api/src/controllers/timeseriesEngine.ts b/packages/api/src/controllers/timeseriesEngine.ts index 0ee595179b..ec4f470efa 100644 --- a/packages/api/src/controllers/timeseriesEngine.ts +++ b/packages/api/src/controllers/timeseriesEngine.ts @@ -254,9 +254,7 @@ export async function queryLabelNames({ return queryDistinctTagsValues({ ...args, value, conditions, limit }); } -// -------------------------- // PromQL via table functions (ClickHouse < 26.6) -// -------------------------- // // Servers without the `prometheus_api_v1` HTTP handler can still evaluate // PromQL through `prometheusQuery`/`prometheusQueryRange`. ClickHouse only @@ -407,9 +405,7 @@ async function connectionSupportsPrometheusHttpApi({ ); } -// -------------------------- // Shared Prometheus connection helpers -// -------------------------- // // Used by both the HTTP proxy (routers/api/prometheus.ts) and the alerting // task (tasks/checkAlerts) so backend-selection logic stays in one place. diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index d37feb9876..1631281e91 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -11889,16 +11889,17 @@ describe('checkAlerts', () => { .spyOn(console, 'error') .mockImplementation(() => {}); - // Arrange - const { - team, - clickhouseClient, - webhook, - savedSearch, - source, - connection, - teamWebhooksById, - } = await createSavedSearchWithMVSource('Body:no'); // Body is not in the MV, so the MV should not be used + try { + // Arrange + const { + team, + clickhouseClient, + webhook, + savedSearch, + source, + connection, + teamWebhooksById, + } = await createSavedSearchWithMVSource('Body:no'); // Body is not in the MV, so the MV should not be used const mockUserId = new mongoose.Types.ObjectId(); const alert = await createAlert( @@ -12030,8 +12031,9 @@ describe('checkAlerts', () => { expect(alertHistories[1].createdAt).toEqual( new Date('2023-11-16T22:15:00.000Z'), ); - - consoleErrorSpy.mockRestore(); + } finally { + consoleErrorSpy.mockRestore(); + } }); }); From 72470f725fcc178117382f7391ef8ba9cb5735a6 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Sun, 27 Sep 2026 01:41:44 +0530 Subject: [PATCH 41/49] lint --- .../__tests__/checkAlerts.int.test.ts | 238 +++++++++--------- 1 file changed, 119 insertions(+), 119 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 1631281e91..8bfc3dba8d 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -11901,136 +11901,136 @@ describe('checkAlerts', () => { teamWebhooksById, } = await createSavedSearchWithMVSource('Body:no'); // Body is not in the MV, so the MV should not be used - const mockUserId = new mongoose.Types.ObjectId(); - const alert = await createAlert( - team._id, - { - source: AlertSource.SAVED_SEARCH, - channel: { - type: 'webhook', - webhookId: webhook._id.toString(), + const mockUserId = new mongoose.Types.ObjectId(); + const alert = await createAlert( + team._id, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, }, - interval: '5m', - thresholdType: AlertThresholdType.ABOVE, - threshold: 1, - savedSearchId: savedSearch.id, - }, - mockUserId, - ); + mockUserId, + ); - const enhancedAlert: any = await Alert.findById(alert.id).populate([ - 'team', - 'savedSearch', - ]); + const enhancedAlert: any = await Alert.findById(alert.id).populate([ + 'team', + 'savedSearch', + ]); - const details = { - alert: enhancedAlert, - source, - previousMap: new Map(), - taskType: AlertTaskType.SAVED_SEARCH, - savedSearch, - } satisfies AlertDetails; + const details = { + alert: enhancedAlert, + source, + previousMap: new Map(), + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + } satisfies AlertDetails; - const now = new Date('2023-11-16T22:12:00.000Z'); - const eventMs = new Date('2023-11-16T22:05:00.000Z'); - const eventNextMs = new Date('2023-11-16T22:10:00.000Z'); + const now = new Date('2023-11-16T22:12:00.000Z'); + const eventMs = new Date('2023-11-16T22:05:00.000Z'); + const eventNextMs = new Date('2023-11-16T22:10:00.000Z'); - // Insert directly into the MV so that we can be sure the MV is being used - await bulkInsertLogs([ - // logs from 22:05 - 22:10 - { - ServiceName: 'api', - Timestamp: eventMs, - SeverityText: 'error', - Body: 'Oh no! Something went wrong!', - }, - { - ServiceName: 'api', - Timestamp: eventMs, - SeverityText: 'error', - Body: 'Oh no! Something went wrong!', - }, - { - ServiceName: 'api', - Timestamp: eventMs, - SeverityText: 'error', - Body: 'Oh no! Something went wrong!', - }, - // logs from 22:10 - 22:15 - { - ServiceName: 'api', - Timestamp: eventNextMs, - SeverityText: 'error', - Body: 'Oh no! Something went wrong!', - }, - { - ServiceName: 'api', - Timestamp: eventNextMs, - SeverityText: 'error', - Body: 'Oh no! Something went wrong!', - }, - { - ServiceName: 'api', - Timestamp: eventNextMs, - SeverityText: 'info', - Body: 'Something went right for a change!', - }, - ]); + // Insert directly into the MV so that we can be sure the MV is being used + await bulkInsertLogs([ + // logs from 22:05 - 22:10 + { + ServiceName: 'api', + Timestamp: eventMs, + SeverityText: 'error', + Body: 'Oh no! Something went wrong!', + }, + { + ServiceName: 'api', + Timestamp: eventMs, + SeverityText: 'error', + Body: 'Oh no! Something went wrong!', + }, + { + ServiceName: 'api', + Timestamp: eventMs, + SeverityText: 'error', + Body: 'Oh no! Something went wrong!', + }, + // logs from 22:10 - 22:15 + { + ServiceName: 'api', + Timestamp: eventNextMs, + SeverityText: 'error', + Body: 'Oh no! Something went wrong!', + }, + { + ServiceName: 'api', + Timestamp: eventNextMs, + SeverityText: 'error', + Body: 'Oh no! Something went wrong!', + }, + { + ServiceName: 'api', + Timestamp: eventNextMs, + SeverityText: 'info', + Body: 'Something went right for a change!', + }, + ]); - // Act - Run alerts twice to cover two periods - let previousMap = await getPreviousAlertHistories( - [details.alert.id], - now, - ); - await processAlert( - now, - { - ...details, - previousMap, - }, - clickhouseClient, - connection.id, - alertProvider, - teamWebhooksById, - ); + // Act - Run alerts twice to cover two periods + let previousMap = await getPreviousAlertHistories( + [details.alert.id], + now, + ); + await processAlert( + now, + { + ...details, + previousMap, + }, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); - const nextWindow = new Date('2023-11-16T22:15:00.000Z'); - previousMap = await getPreviousAlertHistories( - [details.alert.id], - nextWindow, - ); - await processAlert( - nextWindow, - details, - clickhouseClient, - connection.id, - alertProvider, - teamWebhooksById, - ); + const nextWindow = new Date('2023-11-16T22:15:00.000Z'); + previousMap = await getPreviousAlertHistories( + [details.alert.id], + nextWindow, + ); + await processAlert( + nextWindow, + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); - // Assert - Alert ran and has a state consistent with the data in the base table - expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); + // Assert - Alert ran and has a state consistent with the data in the base table + expect((await Alert.findById(details.alert.id))!.state).toBe('ALERT'); - const alertHistories = await AlertHistory.find({ - alert: details.alert.id, - }).sort({ - createdAt: 1, - }); - expect(alertHistories.length).toBe(2); + const alertHistories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ + createdAt: 1, + }); + expect(alertHistories.length).toBe(2); - expect(alertHistories[0].state).toBe('ALERT'); - expect(alertHistories[0].counts).toBe(1); - expect(alertHistories[0].lastValues[0].count).toBe(3); - expect(alertHistories[0].createdAt).toEqual( - new Date('2023-11-16T22:10:00.000Z'), - ); + expect(alertHistories[0].state).toBe('ALERT'); + expect(alertHistories[0].counts).toBe(1); + expect(alertHistories[0].lastValues[0].count).toBe(3); + expect(alertHistories[0].createdAt).toEqual( + new Date('2023-11-16T22:10:00.000Z'), + ); - expect(alertHistories[1].state).toBe('ALERT'); - expect(alertHistories[1].counts).toBe(1); - expect(alertHistories[1].lastValues[0].count).toBe(2); - expect(alertHistories[1].createdAt).toEqual( - new Date('2023-11-16T22:15:00.000Z'), - ); + expect(alertHistories[1].state).toBe('ALERT'); + expect(alertHistories[1].counts).toBe(1); + expect(alertHistories[1].lastValues[0].count).toBe(2); + expect(alertHistories[1].createdAt).toEqual( + new Date('2023-11-16T22:15:00.000Z'), + ); } finally { consoleErrorSpy.mockRestore(); } From 82220c5edd878357634ef5726bc03f15952a1223 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Wed, 30 Sep 2026 01:14:47 +0530 Subject: [PATCH 42/49] feat: address review comments, add PromQL alert tests, and remove unneeded changes --- packages/api/src/routers/api/prometheus.ts | 8 +- .../__tests__/checkAlerts.int.test.ts | 836 +++++++++++++++++- packages/api/src/tasks/checkAlerts/index.ts | 38 +- .../app/src/components/ChartEditor/utils.ts | 8 +- .../DBEditTimeChartForm/EditTimeChartForm.tsx | 77 +- packages/app/src/utils.ts | 2 - 6 files changed, 871 insertions(+), 98 deletions(-) diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index 4b580f5ebb..85d3f36964 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -136,7 +136,13 @@ export function isClientDisconnect(err: unknown): boolean { ); } -// Only real Prometheus API params are ever caller-settable in proxyToPrometheus. +// Only real Prometheus API params are ever caller-settable in +// proxyToPrometheus's query merge below. `params` there is built upstream by +// spreading the *entire* `req.query`/`req.body` with no allowlist (see +// `getParams`), so without this, a request could supply an arbitrary key -- +// e.g. VictoriaMetrics's `extra_label`, which a Connection host may pin as a +// tenant-isolation scope -- and un-pin or override it, even though no +// legitimate caller ever sends that key. const CALLER_SETTABLE_PARAM_KEYS = new Set([ 'query', 'time', diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 8bfc3dba8d..37768206c0 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -5165,16 +5165,26 @@ describe('checkAlerts', () => { }); describe('PromQL Alerts', () => { - it('should process a PromQL alert, respecting grouping, state transitions, and backfilling', async () => { + it('should create ALERT and OK history entries for two PromQL groups in a single evaluation', async () => { const { team, webhook, connection, - source, teamWebhooksById, clickhouseClient, } = await setupSavedSearchAlertTest(); + const tsSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + name: 'Prometheus Metrics', + connection: connection.id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_METRICS_TABLE, + }, + }); + const dashboard = await new Dashboard({ name: 'PromQL Dashboard', team: team._id, @@ -5200,7 +5210,7 @@ describe('checkAlerts', () => { const details = await createAlertDetails( team, - source, + tsSource, { source: AlertSource.TILE, channel: { @@ -5236,23 +5246,21 @@ describe('checkAlerts', () => { __name__: 'up', host: 'node-1', }, - values: [ - [(prometheusReturnedMs - 5 * 60 * 1000) / 1000, '42'], - [prometheusReturnedMs / 1000, '42'], - ], + values: [[prometheusReturnedMs / 1000, '42']], }, { metric: { __name__: 'up', host: 'node-2', }, - values: [ - [(prometheusReturnedMs - 5 * 60 * 1000) / 1000, '5'], - [prometheusReturnedMs / 1000, '5'], - ], // Below threshold + values: [[prometheusReturnedMs / 1000, '5']], // Below threshold }, ]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + await processAlertAtTime( now, details, @@ -5292,6 +5300,812 @@ describe('checkAlerts', () => { timeseriesEngine.queryRangeViaTableFunction, ).toHaveBeenCalledTimes(1); }); + + it('should handle ALERT to OK state transition and resolve notification', async () => { + const { + team, + webhook, + connection, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + const tsSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + name: 'Prometheus Metrics', + connection: connection.id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_METRICS_TABLE, + }, + }); + const dashboard = await new Dashboard({ + name: 'PromQL Dashboard', + team: team._id, + tiles: [ + { + id: 'promql1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'promql', + displayType: 'line', + promqlExpression: 'up', + connection: connection.id, + }, + }, + ], + }).save(); + const tile = dashboard.tiles?.find((t: any) => t.id === 'promql1'); + const details = await createAlertDetails( + team, + tsSource, + { + source: AlertSource.TILE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 10, + dashboardId: dashboard.id, + tileId: 'promql1', + }, + { taskType: AlertTaskType.TILE, tile, dashboard }, + ); + + // First run: Above threshold + jest + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([ + { + metric: { host: 'node-1' }, + values: [[(Date.now() - 300000) / 1000, '42']], + }, + ]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + const webhookSpy = jest + .spyOn(global, 'fetch') + .mockResolvedValue({ ok: true, json: async () => ({}) } as any); + await processAlertAtTime( + new Date(), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + let node1History = await AlertHistory.findOne({ + alert: details.alert.id, + group: 'host:node-1', + }).sort({ createdAt: -1 }); + expect(node1History?.state).toBe('ALERT'); + expect(webhookSpy).toHaveBeenCalled(); // Alert fired + webhookSpy.mockClear(); + + // Second run: Below threshold (Resolve) + jest + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([ + { metric: { host: 'node-1' }, values: [[Date.now() / 1000, '5']] }, + ]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + webhookSpy.mockResolvedValueOnce({ + ok: true, + json: async () => ({}), + } as any); + await processAlertAtTime( + new Date(Date.now() + 300000), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + node1History = await AlertHistory.findOne({ + alert: details.alert.id, + group: 'host:node-1', + }).sort({ createdAt: -1 }); + expect(node1History?.state).toBe('OK'); + expect(webhookSpy).toHaveBeenCalled(); // Resolve fired + }); + + it('should handle PENDING state with numConsecutiveWindows', async () => { + const { + team, + webhook, + connection, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + const tsSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + name: 'Prometheus Metrics', + connection: connection.id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_METRICS_TABLE, + }, + }); + const dashboard = await new Dashboard({ + name: 'PromQL Dashboard', + team: team._id, + tiles: [ + { + id: 'promql1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'promql', + displayType: 'line', + promqlExpression: 'up', + connection: connection.id, + }, + }, + ], + }).save(); + const tile = dashboard.tiles?.find((t: any) => t.id === 'promql1'); + const details = await createAlertDetails( + team, + tsSource, + { + source: AlertSource.TILE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 10, + dashboardId: dashboard.id, + tileId: 'promql1', + numConsecutiveWindows: 2, + }, + { taskType: AlertTaskType.TILE, tile, dashboard }, + ); + + // First run: Above threshold -> PENDING + jest + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([ + { + metric: { host: 'node-1' }, + values: [[(Date.now() - 300000) / 1000, '42']], + }, + ]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + await processAlertAtTime( + new Date(), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + let node1History = await AlertHistory.findOne({ + alert: details.alert.id, + group: 'host:node-1', + }).sort({ createdAt: -1 }); + expect(node1History?.state).toBe('PENDING'); + + // Second run: Above threshold -> ALERT + jest + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([ + { metric: { host: 'node-1' }, values: [[Date.now() / 1000, '42']] }, + ]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + await processAlertAtTime( + new Date(Date.now() + 300000), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + node1History = await AlertHistory.findOne({ + alert: details.alert.id, + group: 'host:node-1', + }).sort({ createdAt: -1 }); + expect(node1History?.state).toBe('ALERT'); + }); + + it('should handle empty results and treat as 0', async () => { + const { + team, + webhook, + connection, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + const tsSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + name: 'Prometheus Metrics', + connection: connection.id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_METRICS_TABLE, + }, + }); + const dashboard = await new Dashboard({ + name: 'PromQL Dashboard', + team: team._id, + tiles: [ + { + id: 'promql1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'promql', + displayType: 'line', + promqlExpression: 'up', + connection: connection.id, + }, + }, + ], + }).save(); + const tile = dashboard.tiles?.find((t: any) => t.id === 'promql1'); + const details = await createAlertDetails( + team, + tsSource, + { + source: AlertSource.TILE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.BELOW, + threshold: 1, + dashboardId: dashboard.id, + tileId: 'promql1', + }, + { taskType: AlertTaskType.TILE, tile, dashboard }, + ); + + // First run: Empty result + jest + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + await processAlertAtTime( + new Date(), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + const history = await AlertHistory.findOne({ + alert: details.alert.id, + }).sort({ createdAt: -1 }); + expect(history?.state).toBe('ALERT'); + expect(history?.lastValues[0].count).toBe(0); // Treated as 0, which is below 1 + expect(history?.group).toBe(''); // Empty group + }); + + it('should handle offset windows correctly', async () => { + const { + team, + webhook, + connection, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + const tsSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + name: 'Prometheus Metrics', + connection: connection.id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_METRICS_TABLE, + }, + }); + const dashboard = await new Dashboard({ + name: 'PromQL Dashboard', + team: team._id, + tiles: [ + { + id: 'promql1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'promql', + displayType: 'line', + promqlExpression: 'up', + connection: connection.id, + }, + }, + ], + }).save(); + const tile = dashboard.tiles?.find((t: any) => t.id === 'promql1'); + const details = await createAlertDetails( + team, + tsSource, + { + source: AlertSource.TILE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 10, + dashboardId: dashboard.id, + tileId: 'promql1', + scheduleOffsetMinutes: 10, + }, + { taskType: AlertTaskType.TILE, tile, dashboard }, + ); + + const now = new Date('2023-11-16T22:15:00.000Z'); // Evaluating at :15 + const offsetTarget = now.getTime() - 10 * 60 * 1000; // Expected window ends at :05 + + jest + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([ + { + metric: { host: 'node-1' }, + values: [[offsetTarget / 1000, '42']], + }, + ]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + const history = await AlertHistory.findOne({ + alert: details.alert.id, + }).sort({ createdAt: -1 }); + expect(history?.state).toBe('ALERT'); + expect(history?.lastValues[0].count).toBe(42); + }); + + it('should handle inline PromQL alerts', async () => { + const { + team, + webhook, + connection, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + const tsSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + name: 'Prometheus Metrics', + connection: connection.id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_METRICS_TABLE, + }, + }); + + const details = await createAlertDetails( + team, + tsSource, + { + source: AlertSource.INLINE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 10, + }, + { + taskType: AlertTaskType.INLINE, + inlineChartConfig: { + configType: 'promql', + displayType: 'line', + promqlExpression: 'up', + connection: connection.id, + }, + }, + ); + + jest + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([ + { metric: { host: 'node-1' }, values: [[Date.now() / 1000, '42']] }, + ]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + await processAlertAtTime( + new Date(), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + const history = await AlertHistory.findOne({ + alert: details.alert.id, + }).sort({ createdAt: -1 }); + expect(history?.state).toBe('ALERT'); + }); + + it('should backfill across multiple buckets correctly', async () => { + const { + team, + webhook, + connection, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + const tsSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + name: 'Prometheus Metrics', + connection: connection.id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_METRICS_TABLE, + }, + }); + const dashboard = await new Dashboard({ + name: 'PromQL Dashboard', + team: team._id, + tiles: [ + { + id: 'promql1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'promql', + displayType: 'line', + promqlExpression: 'up', + connection: connection.id, + }, + }, + ], + }).save(); + const tile = dashboard.tiles?.find((t: any) => t.id === 'promql1'); + const details = await createAlertDetails( + team, + tsSource, + { + source: AlertSource.TILE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 10, + dashboardId: dashboard.id, + tileId: 'promql1', + }, + { taskType: AlertTaskType.TILE, tile, dashboard }, + ); + + // Force a large time gap to create 3 backfill buckets + 1 normal bucket + const now = new Date('2023-11-16T22:20:00.000Z'); + const mockHistory = await AlertHistory.create({ + alert: details.alert.id, + state: 'OK', + group: 'host:node-1', + lastValues: [{ ts: new Date('2023-11-16T22:00:00.000Z'), count: 0 }], + team: team._id, + }); + + // The task should evaluate 3 skipped windows + the current window (22:05, 22:10, 22:15, 22:20) + jest + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([ + { + metric: { host: 'node-1' }, + values: [ + [new Date('2023-11-16T22:05:00.000Z').getTime() / 1000, '42'], + [new Date('2023-11-16T22:10:00.000Z').getTime() / 1000, '42'], + [new Date('2023-11-16T22:15:00.000Z').getTime() / 1000, '42'], + [new Date('2023-11-16T22:20:00.000Z').getTime() / 1000, '42'], + ], + }, + ]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + const history = await AlertHistory.findOne({ + alert: details.alert.id, + _id: { $ne: mockHistory._id }, + }).sort({ createdAt: -1 }); + expect(history?.state).toBe('ALERT'); + expect(history?.analytics?.backfilledBuckets).toBe(3); // 3 skipped + 1 current + }); + + it('should properly map samples to buckets (step shift, drop out-of-range, skip NaN/Inf, keep highest dup)', async () => { + const { + team, + webhook, + connection, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + const tsSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + name: 'Prometheus Metrics', + connection: connection.id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_METRICS_TABLE, + }, + }); + const dashboard = await new Dashboard({ + name: 'PromQL Dashboard', + team: team._id, + tiles: [ + { + id: 'promql1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'promql', + displayType: 'line', + promqlExpression: 'up', + connection: connection.id, + }, + }, + ], + }).save(); + const tile = dashboard.tiles?.find((t: any) => t.id === 'promql1'); + const details = await createAlertDetails( + team, + tsSource, + { + source: AlertSource.TILE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 10, + dashboardId: dashboard.id, + tileId: 'promql1', + }, + { taskType: AlertTaskType.TILE, tile, dashboard }, + ); + + const now = new Date('2023-11-16T22:12:00.000Z'); + const targetMs = new Date('2023-11-16T22:10:00.000Z').getTime() / 1000; + + jest + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([ + { + metric: { host: 'node-1' }, + values: [ + [targetMs - 1, '100'], // Out of range + [targetMs, 'NaN'], // NaN ignored + [targetMs, '+Inf'], // Inf ignored + [targetMs, '40'], // Valid + [targetMs, '42'], // Duplicate timestamp, keep higher + [targetMs + 1, '500'], // Out of range + ], + }, + ]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + const history = await AlertHistory.findOne({ + alert: details.alert.id, + }).sort({ createdAt: -1 }); + expect(history?.state).toBe('ALERT'); + expect(history?.lastValues[0].count).toBe(42); + }); + + it('should support Number tiles with multiple expressions by evaluating the last one', async () => { + const { + team, + webhook, + connection, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + const tsSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + name: 'Prometheus Metrics', + connection: connection.id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_METRICS_TABLE, + }, + }); + const dashboard = await new Dashboard({ + name: 'PromQL Dashboard', + team: team._id, + tiles: [ + { + id: 'promql1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'promql', + displayType: 'number', + promqlExpression: [ + { expression: 'ignored' }, + { expression: 'up' }, + ] as any, + connection: connection.id, + }, + }, + ], + }).save(); + const tile = dashboard.tiles?.find((t: any) => t.id === 'promql1'); + const details = await createAlertDetails( + team, + tsSource, + { + source: AlertSource.TILE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 10, + dashboardId: dashboard.id, + tileId: 'promql1', + }, + { taskType: AlertTaskType.TILE, tile, dashboard }, + ); + + const queryRangeSpy = jest + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([ + { metric: { host: 'node-1' }, values: [[Date.now() / 1000, '42']] }, + ]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + await processAlertAtTime( + new Date(), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + const history = await AlertHistory.findOne({ + alert: details.alert.id, + }).sort({ createdAt: -1 }); + expect(history?.state).toBe('ALERT'); + expect(queryRangeSpy).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ query: 'up' }), + expect.anything(), + expect.anything(), + ); + }); + + it('should include correct group key and labels in the notification, firing only for the breaching series', async () => { + const { + team, + webhook, + connection, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + const tsSource = await Source.create({ + kind: SourceKind.Metric, + team: team._id, + name: 'Prometheus Metrics', + connection: connection.id, + from: { + databaseName: DEFAULT_DATABASE, + tableName: DEFAULT_METRICS_TABLE, + }, + }); + const dashboard = await new Dashboard({ + name: 'PromQL Dashboard', + team: team._id, + tiles: [ + { + id: 'promql1', + x: 0, + y: 0, + w: 6, + h: 4, + config: { + configType: 'promql', + displayType: 'line', + promqlExpression: 'up', + connection: connection.id, + }, + }, + ], + }).save(); + const tile = dashboard.tiles?.find((t: any) => t.id === 'promql1'); + const details = await createAlertDetails( + team, + tsSource, + { + source: AlertSource.TILE, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 10, + dashboardId: dashboard.id, + tileId: 'promql1', + }, + { taskType: AlertTaskType.TILE, tile, dashboard }, + ); + + jest + .spyOn(timeseriesEngine, 'queryRangeViaTableFunction') + .mockResolvedValueOnce([ + { + metric: { host: 'node-1', env: 'prod' }, + values: [[Date.now() / 1000, '42']], + }, // Breaches + { + metric: { host: 'node-2', env: 'dev' }, + values: [[Date.now() / 1000, '5']], + }, // Doesn't breach + ]); + jest + .spyOn(timeseriesEngine, 'clickhouseServesPrometheusHttpApi') + .mockResolvedValueOnce(false); + const fetchSpy = jest + .spyOn(global, 'fetch') + .mockResolvedValue({ ok: true, json: async () => ({}) } as any); + + await processAlertAtTime( + new Date(), + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + + expect(fetchSpy).toHaveBeenCalledTimes(1); + + const payload = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(payload.text).toContain('host:node-1'); + expect(payload.text).toContain('env:prod'); + expect(payload.text).not.toContain('node-2'); + }); }); describe('dashboard variables', () => { diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index e96c1f4e86..0229c0bdf3 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -12,6 +12,7 @@ import { getMetadata, Metadata, } from '@hyperdx/common-utils/dist/core/metadata'; +import { getQueriedPromqlSeries } from '@hyperdx/common-utils/dist/core/promql'; import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { ALERT_COUNT_DEFAULT_SELECT, @@ -42,6 +43,7 @@ import { getSampleWeightExpression, pickSampleWeightExpressionProps, PrometheusMatrixResult, + PrometheusQueryRangeResponse, PromqlSavedChartConfig, SavedChartConfig, SourceKind, @@ -995,27 +997,18 @@ export async function evaluatePromqlAlert({ // for the window [T, T+step) to happen at T+step, using the freshest data. const startSec = dateRange[0].getTime() / 1000 + stepSec; - // Resolve the expression to evaluate. The config stores either a bare string - // (tiles saved before multi-expression support) or an array of PromqlSeries. - // Alerts always target the LAST expression, mirroring how SQL builder alerts - // target the last series. + // Resolve the expression to evaluate. getQueriedPromqlSeries handles + // both the bare-string shape (tiles saved before multi-expression support) + // and the array shape, dropping blank rows and selecting only the first + // expression for non-time-series display types (e.g. Number). Alerts + // always target the LAST queried expression, mirroring how SQL builder + // alerts target the last series. const resolvedConfig = variables && variables.length > 0 ? substitutePromqlChartConfigVariables({ ...savedConfig, variables }) : savedConfig; - const rawExpression = resolvedConfig.promqlExpression; - let promqlExpression = typeof rawExpression === 'string' ? rawExpression : ''; - if (Array.isArray(rawExpression)) { - const validExpressions = rawExpression.filter( - e => e.expression && e.expression.trim() !== '', - ); - // Always evaluate the last expression, mirroring how SQL builder alerts - // target the last series and how the tile renders its primary value. - if (validExpressions.length > 0) { - promqlExpression = - validExpressions[validExpressions.length - 1].expression; - } - } + const promqlExpression = + getQueriedPromqlSeries(resolvedConfig).at(-1)?.expression ?? ''; // Shared helper for querying a Prometheus-compatible HTTP endpoint. async function queryPrometheusHttp( @@ -1035,16 +1028,7 @@ export async function evaluatePromqlAlert({ `Prometheus query_range returned HTTP ${resp.status} for PromQL alert`, ); } - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - const json = (await resp.json()) as { - status?: string; - data?: { - result?: { - metric?: Record; - values?: [number, string][]; - }[]; - }; - }; + const json: PrometheusQueryRangeResponse = await resp.json(); if (json?.status !== 'success') { throw new Error( `Prometheus query_range returned status '${json?.status}' for PromQL alert`, diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 6f071179bc..85a3476b92 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -164,18 +164,18 @@ export const isStringSelectDisplayType = ( export const isPromqlDisplayType = ( displayType: DisplayType | undefined, ): displayType is + | DisplayType.Table | DisplayType.Line | DisplayType.StackedBar | DisplayType.Pie | DisplayType.Bar - | DisplayType.Number - | DisplayType.Table => + | DisplayType.Number => + displayType === DisplayType.Table || displayType === DisplayType.Line || displayType === DisplayType.StackedBar || displayType === DisplayType.Pie || displayType === DisplayType.Bar || - displayType === DisplayType.Number || - displayType === DisplayType.Table; + displayType === DisplayType.Number; const NON_PROMQL_SOURCE_KINDS = Object.values(SourceKind).filter( kind => kind !== SourceKind.Promql, diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index 15d3e8b38f..a24d16f2d9 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -763,20 +763,6 @@ export default function EditTimeChartForm({ prevDisplayTypeRef.current = displayType; prevConfigTypeRef.current = configType; - if (configTypeChanged) { - if ( - configType === 'promql' && - tableSource?.kind !== SourceKind.Promql - ) { - setValue('source', ''); - } else if ( - configType !== 'promql' && - tableSource?.kind === SourceKind.Promql - ) { - setValue('source', ''); - } - } - if ( isStringSelectDisplayType(displayType) && typeof select !== 'string' @@ -832,15 +818,6 @@ export default function EditTimeChartForm({ isDisplayTypeSourceSwapPendingRef.current; isDisplayTypeSourceSwapPendingRef.current = false; - if (sourceChanged && tableSource?.kind === SourceKind.Promql) { - if (configType !== 'promql') { - setValue('configType', 'promql'); - } - if (!isPromqlDisplayType(displayType)) { - setValue('displayType', DisplayType.Line); - } - } - if ( displayType === DisplayType.Heatmap && tableSource?.kind === SourceKind.Trace && @@ -1010,14 +987,12 @@ export default function EditTimeChartForm({ > Time Series - {tableSource?.kind !== SourceKind.Promql && ( - } - > - Table - - )} + } + > + Table + } @@ -1036,28 +1011,24 @@ export default function EditTimeChartForm({ > Pie - {tableSource?.kind !== SourceKind.Promql && ( - <> - } - > - Search - - } - > - Heatmap - - } - > - Patterns - - - )} + } + > + Search + + } + > + Heatmap + + } + > + Patterns + } diff --git a/packages/app/src/utils.ts b/packages/app/src/utils.ts index 7770fc9bb4..e9644d36e8 100644 --- a/packages/app/src/utils.ts +++ b/packages/app/src/utils.ts @@ -105,7 +105,6 @@ export const useDebounce = ( useEffect( () => { if (shouldBeImmediate) { - // eslint-disable-next-line setDebouncedValue(value); return () => {}; } @@ -205,7 +204,6 @@ export function useLocalStorage(key: string, initialValue: T) { const item = window.localStorage.getItem(key); // Parse stored json or if none return initialValue if (item != null) { - // eslint-disable-next-line setStoredValue(JSON.parse(item)); } } catch (error) { From 3eac1eeb7f2d972de2328a8a88d1e608e1233c28 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 2 Oct 2026 17:06:04 +0530 Subject: [PATCH 43/49] refactor(checkAlerts): consolidate duplicated ~250-line bucket evaluation tail - Fix aliasWithClauses regression (saved-search sample rows were broken) - Both PromQL and SQL paths now produce a unified bucket map and share one state machine / auto-resolve / notification / flush tail - Add PrometheusQueryRangeResponse to common-utils/types - Fix PromQL variable substitution to use substituteVariables directly - PromqlChartEditor: remove auto-select useEffect, update tooltip text - RawSqlChartEditor: restore accidentally removed color={'gray'} --- packages/api/src/tasks/checkAlerts/index.ts | 370 +++++------------- .../ChartEditor/PromqlChartEditor.tsx | 20 +- .../ChartEditor/RawSqlChartEditor.tsx | 1 + packages/common-utils/src/types.ts | 10 + 4 files changed, 109 insertions(+), 292 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 0229c0bdf3..8ce8d22e1e 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -48,7 +48,7 @@ import { SavedChartConfig, SourceKind, } from '@hyperdx/common-utils/dist/types'; -import { substitutePromqlChartConfigVariables } from '@hyperdx/common-utils/dist/variables'; +import { substituteVariables } from '@hyperdx/common-utils/dist/variables'; import * as fns from 'date-fns'; import { isString, pick } from 'lodash'; import { ObjectId } from 'mongoose'; @@ -997,18 +997,16 @@ export async function evaluatePromqlAlert({ // for the window [T, T+step) to happen at T+step, using the freshest data. const startSec = dateRange[0].getTime() / 1000 + stepSec; - // Resolve the expression to evaluate. getQueriedPromqlSeries handles - // both the bare-string shape (tiles saved before multi-expression support) - // and the array shape, dropping blank rows and selecting only the first - // expression for non-time-series display types (e.g. Number). Alerts - // always target the LAST queried expression, mirroring how SQL builder - // alerts target the last series. - const resolvedConfig = - variables && variables.length > 0 - ? substitutePromqlChartConfigVariables({ ...savedConfig, variables }) - : savedConfig; + // Extract the last PromQL expression and substitute any dashboard variables. + const rawExpression = + getQueriedPromqlSeries(savedConfig).at(-1)?.expression ?? ''; const promqlExpression = - getQueriedPromqlSeries(resolvedConfig).at(-1)?.expression ?? ''; + variables && variables.length > 0 + ? substituteVariables(rawExpression, { + variables, + inputLanguage: 'promql', + }) + : rawExpression; // Shared helper for querying a Prometheus-compatible HTTP endpoint. async function queryPrometheusHttp( @@ -1491,6 +1489,15 @@ export const processAlert = async ( } }; + // Shared variables populated by whichever path runs (PromQL or SQL) + // and consumed by the shared tail below. + let sharedExpectedBuckets: Date[] = []; + let checkDataByBucket = new Map< + number, + { groupKey: string; value: number; attributes: Record }[] + >(); + let runAutoResolveForAll = false; + if (isPromQL) { const savedConfig = details.taskType === AlertTaskType.INLINE @@ -1630,198 +1637,26 @@ export const processAlert = async ( } } } - - for (const bucketStart of expectedBuckets) { - // Prometheus query_range aligns response timestamps to step boundaries, - // so we can do an exact ms lookup rather than a tolerance scan. - const seriesForBucket = bucketSeriesValues.get(bucketStart.getTime()); - - if (!seriesForBucket || seriesForBucket.size === 0) { - alertEvaluationsCounter.add(1, { outcome: 'empty_bucket' }); - logger.info( - { alertId: alert.id, bucketStart }, - 'No PromQL data for time bucket', - ); - - const zeroValueIsAlert = doesExceedThreshold(alert, 0); - - const hasAlertsInPreviousMap = previousMap - .values() - .some( - h => - h.state === AlertState.ALERT || h.state === AlertState.PENDING, - ); - - if (zeroValueIsAlert) { - const history = getOrCreateHistory(''); - history.lastValues.push({ count: 0, startTime: bucketStart }); - history.counts += 1; - if (shouldFireBasedOnConsecutiveWindows('')) { - history.state = AlertState.ALERT; - history.fired = true; - latestAlertContext.set('', { - value: 0, - attributes: {}, - startTime: bucketStart, - }); - } else { - history.state = AlertState.PENDING; - history.fired = - previousMap.get(computeHistoryMapKey(alert.id, ''))?.fired === - true; - } - } else if (!hasGroupBy || !hasAlertsInPreviousMap) { - const history = getOrCreateHistory(''); - history.lastValues.push({ count: 0, startTime: bucketStart }); - } - continue; - } - - const bucketEvaluations = new Map< - string, - { - value: number; - attributes: Record; - exceeds: boolean; - } - >(); - for (const [ - groupKey, - { value, attributes }, - ] of seriesForBucket.entries()) { - const exceeds = doesExceedThreshold(alert, value); - const existing = bucketEvaluations.get(groupKey); - if (!existing || (!existing.exceeds && exceeds)) { - bucketEvaluations.set(groupKey, { value, attributes, exceeds }); - } - } - - for (const [groupKey, evaluation] of bucketEvaluations.entries()) { - const history = getOrCreateHistory(groupKey); - - if (evaluation.exceeds) { - history.counts += 1; - if (shouldFireBasedOnConsecutiveWindows(groupKey)) { - history.state = AlertState.ALERT; - history.fired = true; - latestAlertContext.set(groupKey, { - value: evaluation.value, - attributes: evaluation.attributes, - startTime: bucketStart, - }); - } else { - history.state = AlertState.PENDING; - history.fired = - previousMap.get(computeHistoryMapKey(alert.id, groupKey)) - ?.fired === true; - } - } else { - history.state = AlertState.OK; - history.counts = 0; - } - history.lastValues.push({ - count: evaluation.value, - startTime: bucketStart, - }); - } - } - - // Auto-resolve: groups that were alerting/pending but absent from the newest bucket. - // PromQL always stores per-series history keyed by alertId||{labels}, - // regardless of hasGroupBy, so we always run this loop for PromQL alerts. - const lastExpectedBucket = expectedBuckets[expectedBuckets.length - 1]; - if (lastExpectedBucket) { - const groupsToCheck = new Set(); - if (previousMap) { - for (const previousKey of previousMap.keys()) { - groupsToCheck.add(extractGroupKeyFromMapKey(previousKey, alert.id)); - } - } - for (const groupKey of histories.keys()) { - groupsToCheck.add(groupKey); - } - - for (const groupKey of groupsToCheck) { - const history = histories.get(groupKey); - const previousHistory = previousMap?.get( - computeHistoryMapKey(alert.id, groupKey), - ); - - const lastEvaluated = - history?.lastValues[history.lastValues.length - 1]; - const isMissingFromLastBucket = - !lastEvaluated || - lastEvaluated.startTime.getTime() !== lastExpectedBucket.getTime(); - - const wasAlertingOrPending = - previousHistory?.state === AlertState.ALERT || - previousHistory?.state === AlertState.PENDING || - history?.state === AlertState.ALERT || - history?.state === AlertState.PENDING; - - if ( - isMissingFromLastBucket && - wasAlertingOrPending && - !doesExceedThreshold(alert, 0) - ) { - const h = getOrCreateHistory(groupKey); - h.lastValues.push({ - count: 0, - startTime: lastExpectedBucket, - }); - h.state = AlertState.OK; - h.counts = 0; - latestAlertContext.delete(groupKey); - } - } - } - - if (histories.size === 0) { - getOrCreateHistory(''); - } - - // Send notifications for state transitions - for (const [groupKey, history] of histories.entries()) { - const previousKey = computeHistoryMapKey(alert.id, groupKey); - let groupPrevious = previousMap.get(previousKey); - - const hitAlertThisRun = latestAlertContext.has(groupKey); - if (hitAlertThisRun) { - const context = latestAlertContext.get(groupKey); - if (context) { - await trySendNotification({ - state: AlertState.ALERT, - group: groupKey, - totalCount: context.value, - startTime: context.startTime, - attributes: context.attributes, - }); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - groupPrevious = { - ...(groupPrevious ?? {}), - state: AlertState.ALERT, - fired: true, - } as AggregatedAlertHistory; - } - } - - await sendNotificationIfResolved(groupPrevious, history, groupKey); - } - - flushNotificationTimings(); - const historyRecords = Array.from(histories.values()); - for (const record of historyRecords) { - record.analytics = evaluationAnalytics; - } - await alertProvider.updateAlertState( - alert.id, - historyRecords, - executionErrors, - dateRange, + // Normalise into the shared bucket-row format used by the state machine below. + // Each bucket maps to a list of { groupKey, value, attributes } rows. + checkDataByBucket = new Map( + Array.from(bucketSeriesValues.entries()).map(([bucketMs, seriesMap]) => [ + bucketMs, + Array.from( + seriesMap.entries(), + ).map(([groupKey, { value, attributes }]) => ({ + groupKey, + value, + attributes, + })), + ]), ); - return; + sharedExpectedBuckets = expectedBuckets; + // PromQL alerts always behave as grouped (every label-set is a distinct group). + runAutoResolveForAll = true; } + if (!isPromQL) { const chartConfig = getChartConfigFromAlert( details, connection.id, @@ -1863,6 +1698,7 @@ export const processAlert = async ( ); if (withClauses) { chartConfig.with = withClauses; + aliasWithClauses = withClauses; } } catch (e) { logger.warn( @@ -2033,7 +1869,7 @@ export const processAlert = async ( } // Standard time-series alert evaluation (Line/StackedBar charts). - const expectedBuckets = timeBucketByGranularity( + const sqlExpectedBuckets = timeBucketByGranularity( dateRange[0], dateRange[1], `${windowSizeInMins} minute`, @@ -2042,51 +1878,66 @@ export const processAlert = async ( // earlier evaluation ticks were missed (job delay, failed evaluations). evaluationAnalytics.backfilledBuckets = Math.max( 0, - expectedBuckets.length - 1, + sqlExpectedBuckets.length - 1, ); - // Group data by time bucket (grouped alerts may have multiple entries per time bucket) - const checkDataByBucket = new Map< + // Normalise SQL rows into the shared bucket-row format: + // Map + const sqlBucketRows = new Map< number, - Record[] + { groupKey: string; value: number; attributes: Record }[] >(); - for (const checkData of checksData.data) { const bucketStart = new Date(checkData[meta.timestampColumnName]); - if (!checkDataByBucket.has(bucketStart.getTime())) { - checkDataByBucket.set(bucketStart.getTime(), []); - } + const { value, extraFields } = parseAlertData(checkData, meta); + if (value == null) continue; // skip NULL (missing denominator, empty ratio, etc.) + const groupKey = hasGroupBy + ? extraFields.map(([k, v]) => `${k}:${v}`).join(', ') + : ''; + const attributes = hasGroupBy ? Object.fromEntries(extraFields) : {}; + const bucketMs = bucketStart.getTime(); + if (!sqlBucketRows.has(bucketMs)) sqlBucketRows.set(bucketMs, []); + sqlBucketRows.get(bucketMs)!.push({ groupKey, value, attributes }); + } - checkDataByBucket.get(bucketStart.getTime())!.push(checkData); + checkDataByBucket = sqlBucketRows; + sharedExpectedBuckets = sqlExpectedBuckets; + runAutoResolveForAll = false; // SQL grouped alerts already guard this with hasGroupBy } - for (const bucketStart of expectedBuckets) { - const dataForBucket = checkDataByBucket.get(bucketStart.getTime()); + // ─── Shared tail: bucket state machine + auto-resolve + notifications ─── + // + // Both paths (PromQL and SQL) produce: + // sharedExpectedBuckets: Date[] + // checkDataByBucket: Map + // runAutoResolveForAll: boolean + // + // Everything from here to updateAlertState runs exactly once regardless of + // which path populated the data above. - // Handle case where no data is available for this bucket - const bucketHasData = dataForBucket && dataForBucket.length > 0; - if (!bucketHasData) { + for (const bucketStart of sharedExpectedBuckets) { + const rowsForBucket = checkDataByBucket.get(bucketStart.getTime()) ?? []; + + if (rowsForBucket.length === 0) { alertEvaluationsCounter.add(1, { outcome: 'empty_bucket' }); logger.info( { alertId: alert.id, bucketStart }, - 'No data returned from ClickHouse for time bucket', + 'No data for time bucket', ); const zeroValueIsAlert = doesExceedThreshold(alert, 0); - const hasAlertsInPreviousMap = previousMap .values() .some( - history => - history.state === AlertState.ALERT || - history.state === AlertState.PENDING, + h => + h.state === AlertState.ALERT || h.state === AlertState.PENDING, ); if (zeroValueIsAlert) { const history = getOrCreateHistory(''); history.lastValues.push({ count: 0, startTime: bucketStart }); history.counts += 1; - if (shouldFireBasedOnConsecutiveWindows()) { + if (shouldFireBasedOnConsecutiveWindows('')) { history.state = AlertState.ALERT; history.fired = true; latestAlertContext.set('', { @@ -2096,48 +1947,27 @@ export const processAlert = async ( }); } else { history.state = AlertState.PENDING; - // Carry forward fired=true if a notification was previously sent and not yet resolved. history.fired = previousMap.get(computeHistoryMapKey(alert.id, ''))?.fired === true; } } else if (!hasGroupBy || !hasAlertsInPreviousMap) { - // For grouped alerts, if there are alerts in the previous map, - // we will handle creating a history as part of auto-resolve later const history = getOrCreateHistory(''); history.lastValues.push({ count: 0, startTime: bucketStart }); } - continue; } - // We have at least one data point for this bucket - - // Track the worst-case state for each group in this bucket to prevent - // a subsequent OK row in the SAME bucket from overwriting an ALERT row. + // Track worst-case state per group in this bucket: an OK row must not + // overwrite an ALERT row that appeared earlier in the same bucket. const bucketEvaluations = new Map< string, { value: number; attributes: Record; exceeds: boolean } >(); - for (const checkData of dataForBucket) { - const { value, extraFields } = parseAlertData(checkData, meta); - - // NULL means no data: a metric series with no row at this bucket, or - // a ratio with a missing/zero denominator. Skip the row instead of - // fabricating a state from a gap. - if (value == null) { - continue; - } - - const groupKey = hasGroupBy - ? extraFields.map(([k, v]) => `${k}:${v}`).join(', ') - : ''; - const attributes = hasGroupBy ? Object.fromEntries(extraFields) : {}; - + for (const { groupKey, value, attributes } of rowsForBucket) { const exceeds = doesExceedThreshold(alert, value); - const existing = bucketEvaluations.get(groupKey); - if (!existing || !existing.exceeds || exceeds) { + if (!existing || (!existing.exceeds && exceeds)) { bucketEvaluations.set(groupKey, { value, attributes, exceeds }); } } @@ -2157,15 +1987,11 @@ export const processAlert = async ( }); } else { history.state = AlertState.PENDING; - // Carry forward fired=true if a notification was previously sent and not yet resolved. history.fired = previousMap.get(computeHistoryMapKey(alert.id, groupKey)) ?.fired === true; } } else { - // If the threshold is not met, reset the state to OK. - // This ensures that if a previous window in this evaluation triggered an ALERT, - // a subsequent OK window correctly resolves it before the notification phase. history.state = AlertState.OK; history.counts = 0; } @@ -2176,16 +2002,16 @@ export const processAlert = async ( } } - // Handle missing groups: If current check found no data, check if any previously alerting/pending groups need to be resolved - // For group-by alerts, check if any previously alerting or pending groups are missing from current data - if (hasGroupBy) { - const lastExpectedBucket = expectedBuckets[expectedBuckets.length - 1]; - if (lastExpectedBucket) { + // Auto-resolve: groups that were alerting/pending but absent from the newest bucket. + // PromQL always runs this for every group (runAutoResolveForAll=true). + // SQL grouped alerts also run it when hasGroupBy is set. + if (runAutoResolveForAll || hasGroupBy) { + const lastBucket = + sharedExpectedBuckets[sharedExpectedBuckets.length - 1]; + if (lastBucket) { const groupsToCheck = new Set(); - if (previousMap) { - for (const previousKey of previousMap.keys()) { - groupsToCheck.add(extractGroupKeyFromMapKey(previousKey, alert.id)); - } + for (const previousKey of previousMap.keys()) { + groupsToCheck.add(extractGroupKeyFromMapKey(previousKey, alert.id)); } for (const groupKey of histories.keys()) { groupsToCheck.add(groupKey); @@ -2201,7 +2027,7 @@ export const processAlert = async ( history?.lastValues[history.lastValues.length - 1]; const isMissingFromLastBucket = !lastEvaluated || - lastEvaluated.startTime.getTime() !== lastExpectedBucket.getTime(); + lastEvaluated.startTime.getTime() !== lastBucket.getTime(); const wasAlertingOrPending = previousHistory?.state === AlertState.ALERT || @@ -2215,14 +2041,11 @@ export const processAlert = async ( !doesExceedThreshold(alert, 0) ) { logger.info( - { - alertId: alert.id, - group: groupKey, - }, - `Group "${groupKey}" is missing from current data but was previously ALERT/PENDING - creating OK history`, + { alertId: alert.id, group: groupKey }, + `Group "${groupKey}" absent from current data but was ALERT/PENDING — auto-resolving`, ); const h = getOrCreateHistory(groupKey); - h.lastValues.push({ count: 0, startTime: lastExpectedBucket }); + h.lastValues.push({ count: 0, startTime: lastBucket }); h.state = AlertState.OK; h.counts = 0; latestAlertContext.delete(groupKey); @@ -2231,19 +2054,16 @@ export const processAlert = async ( } } - // If no histories exist at all (no current data and no previous alerting groups), create a default OK history if (histories.size === 0) { getOrCreateHistory(''); } - // Check for state transitions and send notifications + // Send notifications for state transitions for (const [groupKey, history] of histories.entries()) { const previousKey = computeHistoryMapKey(alert.id, groupKey); let groupPrevious = previousMap.get(previousKey); const hitAlertThisRun = latestAlertContext.has(groupKey); - - // If it hit ALERT during this run, send the notification (re-notifying every tick if it continuously breaches) if (hitAlertThisRun) { const context = latestAlertContext.get(groupKey); if (context) { @@ -2254,12 +2074,11 @@ export const processAlert = async ( startTime: context.startTime, attributes: context.attributes, }); - - // Inject a mock previous history so the resolve check below catches it - // if the final state for this group is OK (i.e. it breached then resolved). + // Inject a synthetic previous so the resolve-check below fires if the + // group already returned to OK within the same evaluation batch. // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion groupPrevious = { - ...(groupPrevious || {}), + ...(groupPrevious ?? {}), state: AlertState.ALERT, fired: true, } as AggregatedAlertHistory; @@ -2269,7 +2088,6 @@ export const processAlert = async ( await sendNotificationIfResolved(groupPrevious, history, groupKey); } - // Save all history records and update alert state flushNotificationTimings(); const historyRecords = Array.from(histories.values()); for (const record of historyRecords) { diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx index 833bb2640d..bc0b8293bb 100644 --- a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useState } from 'react'; import { Control, useFieldArray, @@ -25,7 +25,7 @@ import { TileAlertEditor } from '@/components/DBEditTimeChartForm/TileAlertEdito import { SourceSelectControlled } from '@/components/SourceSelect'; import { IS_LOCAL_MODE } from '@/config'; import { usePromqlMetricNames } from '@/hooks/usePromqlMetadata'; -import { useSource, useSources } from '@/source'; +import { useSource } from '@/source'; import { DEFAULT_TILE_ALERT } from '@/utils/alerts'; import PromqlExpressionEditor from './PromqlExpressionEditor'; @@ -97,16 +97,6 @@ export default function PromqlChartEditor({ const displayType = useWatch({ control, name: 'displayType' }); const chartName = useWatch({ control, name: 'name' }); const { data: source } = useSource({ id: sourceId }); - const { data: sources } = useSources(); - - useEffect(() => { - if (!sourceId && sources) { - const firstPromqlSource = sources.find(s => s.kind === SourceKind.Promql); - if (firstPromqlSource) { - setValue('source', firstPromqlSource.id); - } - } - }, [sourceId, sources, setValue]); // The form can still hold a non-PromQL source right after switching a tile // into PromQL mode (the picker above only restricts future selections), and @@ -114,12 +104,10 @@ export default function PromqlChartEditor({ // against any other source's table. const promqlSource = source?.kind === SourceKind.Promql ? source : undefined; - const tableName = promqlSource?.from.tableName; - const { data: metricNames } = usePromqlMetricNames( promqlSource?.connection, promqlSource?.from.databaseName, - tableName, + promqlSource?.from.tableName, ); const displayTypeSupportsMultiExpression = @@ -222,7 +210,7 @@ export default function PromqlChartEditor({ ? additionalAlertWarnings.join(' ') : undefined } - tooltip="The threshold will be evaluated against the last value returned by the last PromQL expression" + tooltip="The threshold is checked against the last PromQL expression, evaluated at the end of each alert interval." /> )} diff --git a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx index d3cab44c4e..a9d1949642 100644 --- a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx @@ -309,6 +309,7 @@ export default function RawSqlChartEditor({ variant="subtle" data-testid="alert-button" size="sm" + color={'gray'} onClick={() => setValue('alert', { ...DEFAULT_TILE_ALERT, diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index dadf06dfd0..cef3f3b5a0 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1812,6 +1812,16 @@ export type PrometheusVectorResult = { value: [number, string]; }; +/** The top-level response envelope for a Prometheus range query (`/api/v1/query_range`). */ +export type PrometheusQueryRangeResponse = { + status: 'success' | 'error'; + data?: { + resultType: 'matrix'; + result: PrometheusMatrixResult[]; + }; + error?: string; +}; + /** How a range query's samples are client-side aggregated to a single value. */ export enum PromqlReducer { LastNotNull = 'lastNotNull', From 3bcde7e2154b5957d49f419d2657d81b0b99d075 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 2 Oct 2026 17:24:58 +0530 Subject: [PATCH 44/49] refactor(checkAlerts): consolidate duplicated ~250-line bucket evaluation tail --- .../__tests__/checkAlerts.int.test.ts | 4 +- packages/api/src/tasks/checkAlerts/index.ts | 67 +++++++------------ 2 files changed, 25 insertions(+), 46 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 37768206c0..98977d0fe9 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -9536,9 +9536,9 @@ describe('checkAlerts', () => { teamWebhooksById, ); - // Alert should be in OK state (both service-a and service-b resolved) + // Alert should be in ALERT state (service-b fired) const updatedAlert = await Alert.findById(details.alert.id); - expect(updatedAlert!.state).toBe('OK'); + expect(updatedAlert!.state).toBe('ALERT'); // Check alert histories after first run const firstRunHistories = await AlertHistory.find({ diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 8ce8d22e1e..e2215f807c 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1889,14 +1889,14 @@ export const processAlert = async ( >(); for (const checkData of checksData.data) { const bucketStart = new Date(checkData[meta.timestampColumnName]); + const bucketMs = bucketStart.getTime(); + if (!sqlBucketRows.has(bucketMs)) sqlBucketRows.set(bucketMs, []); const { value, extraFields } = parseAlertData(checkData, meta); if (value == null) continue; // skip NULL (missing denominator, empty ratio, etc.) const groupKey = hasGroupBy ? extraFields.map(([k, v]) => `${k}:${v}`).join(', ') : ''; const attributes = hasGroupBy ? Object.fromEntries(extraFields) : {}; - const bucketMs = bucketStart.getTime(); - if (!sqlBucketRows.has(bucketMs)) sqlBucketRows.set(bucketMs, []); sqlBucketRows.get(bucketMs)!.push({ groupKey, value, attributes }); } @@ -1905,7 +1905,7 @@ export const processAlert = async ( runAutoResolveForAll = false; // SQL grouped alerts already guard this with hasGroupBy } - // ─── Shared tail: bucket state machine + auto-resolve + notifications ─── + // Shared tail: bucket state machine + auto-resolve + notifications // // Both paths (PromQL and SQL) produce: // sharedExpectedBuckets: Date[] @@ -2002,53 +2002,32 @@ export const processAlert = async ( } } - // Auto-resolve: groups that were alerting/pending but absent from the newest bucket. + // Auto-resolve: groups that were alerting/pending but absent from all evaluated buckets. // PromQL always runs this for every group (runAutoResolveForAll=true). // SQL grouped alerts also run it when hasGroupBy is set. if (runAutoResolveForAll || hasGroupBy) { const lastBucket = sharedExpectedBuckets[sharedExpectedBuckets.length - 1]; - if (lastBucket) { - const groupsToCheck = new Set(); + if (lastBucket && previousMap) { for (const previousKey of previousMap.keys()) { - groupsToCheck.add(extractGroupKeyFromMapKey(previousKey, alert.id)); - } - for (const groupKey of histories.keys()) { - groupsToCheck.add(groupKey); - } - - for (const groupKey of groupsToCheck) { - const history = histories.get(groupKey); - const previousHistory = previousMap?.get( - computeHistoryMapKey(alert.id, groupKey), - ); - - const lastEvaluated = - history?.lastValues[history.lastValues.length - 1]; - const isMissingFromLastBucket = - !lastEvaluated || - lastEvaluated.startTime.getTime() !== lastBucket.getTime(); - - const wasAlertingOrPending = - previousHistory?.state === AlertState.ALERT || - previousHistory?.state === AlertState.PENDING || - history?.state === AlertState.ALERT || - history?.state === AlertState.PENDING; - - if ( - isMissingFromLastBucket && - wasAlertingOrPending && - !doesExceedThreshold(alert, 0) - ) { - logger.info( - { alertId: alert.id, group: groupKey }, - `Group "${groupKey}" absent from current data but was ALERT/PENDING — auto-resolving`, - ); - const h = getOrCreateHistory(groupKey); - h.lastValues.push({ count: 0, startTime: lastBucket }); - h.state = AlertState.OK; - h.counts = 0; - latestAlertContext.delete(groupKey); + const groupKey = extractGroupKeyFromMapKey(previousKey, alert.id); + // Only auto-resolve if the group did not appear in ANY bucket this run. + if (!histories.has(groupKey)) { + const previousHistory = previousMap.get(previousKey); + const wasAlertingOrPending = + previousHistory?.state === AlertState.ALERT || + previousHistory?.state === AlertState.PENDING; + + if (wasAlertingOrPending && !doesExceedThreshold(alert, 0)) { + logger.info( + { alertId: alert.id, group: groupKey }, + `Group "${groupKey}" absent from current data but was ALERT/PENDING — auto-resolving`, + ); + const h = getOrCreateHistory(groupKey); + h.lastValues.push({ count: 0, startTime: lastBucket }); + h.state = AlertState.OK; + h.counts = 0; + } } } } From cbbb3cff4545d5c56b045fbcc4bb7864cdb180a0 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 2 Oct 2026 17:32:31 +0530 Subject: [PATCH 45/49] lint --- packages/api/src/tasks/checkAlerts/index.ts | 479 ++++++++++---------- 1 file changed, 243 insertions(+), 236 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index e2215f807c..4b9d830374 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1640,16 +1640,18 @@ export const processAlert = async ( // Normalise into the shared bucket-row format used by the state machine below. // Each bucket maps to a list of { groupKey, value, attributes } rows. checkDataByBucket = new Map( - Array.from(bucketSeriesValues.entries()).map(([bucketMs, seriesMap]) => [ - bucketMs, - Array.from( - seriesMap.entries(), - ).map(([groupKey, { value, attributes }]) => ({ - groupKey, - value, - attributes, - })), - ]), + Array.from(bucketSeriesValues.entries()).map( + ([bucketMs, seriesMap]) => [ + bucketMs, + Array.from(seriesMap.entries()).map( + ([groupKey, { value, attributes }]) => ({ + groupKey, + value, + attributes, + }), + ), + ], + ), ); sharedExpectedBuckets = expectedBuckets; // PromQL alerts always behave as grouped (every label-set is a distinct group). @@ -1657,252 +1659,258 @@ export const processAlert = async ( } if (!isPromQL) { - const chartConfig = getChartConfigFromAlert( - details, - connection.id, - dateRange, - windowSizeInMins, - ); - - if (chartConfig == null) { - evalOutcome = 'error'; - logger.error( - { - chartConfig, - alertId: alert.id, - }, - 'Failed to build chart config', + const chartConfig = getChartConfigFromAlert( + details, + connection.id, + dateRange, + windowSizeInMins, ); - return; - } - // For saved search alerts, the WHERE clause may reference aliased columns - // from the saved search's select expression (e.g. `toString(Body) AS body`). - // The alert query itself uses count(*), not the saved search's select, - // so we render the saved search's select separately to discover aliases - // and inject them as WITH clauses into the alert query. - if (details.taskType === AlertTaskType.SAVED_SEARCH) { - if (!isBuilderChartConfig(chartConfig)) { - logger.error({ - chartConfig, - message: - 'Found non-builder chart config for saved search alert, cannot compute WITH clauses', - }); - throw new Error('Expected builder chart config for saved search alert'); + if (chartConfig == null) { + evalOutcome = 'error'; + logger.error( + { + chartConfig, + alertId: alert.id, + }, + 'Failed to build chart config', + ); + return; + } + + // For saved search alerts, the WHERE clause may reference aliased columns + // from the saved search's select expression (e.g. `toString(Body) AS body`). + // The alert query itself uses count(*), not the saved search's select, + // so we render the saved search's select separately to discover aliases + // and inject them as WITH clauses into the alert query. + if (details.taskType === AlertTaskType.SAVED_SEARCH) { + if (!isBuilderChartConfig(chartConfig)) { + logger.error({ + chartConfig, + message: + 'Found non-builder chart config for saved search alert, cannot compute WITH clauses', + }); + throw new Error( + 'Expected builder chart config for saved search alert', + ); + } + try { + const withClauses = await computeAliasWithClauses( + details.savedSearch, + details.source, + metadata, + ); + if (withClauses) { + chartConfig.with = withClauses; + aliasWithClauses = withClauses; + } + } catch (e) { + logger.warn( + { error: serializeError(e), alertId: alert.id }, + 'Failed to compute alias WITH clauses for alert check', + ); + } } + + // Optimize chart config with materialized views, if available. + // materializedViews exists on Log and Trace sources. + const mvSource = + source?.kind === SourceKind.Log || source?.kind === SourceKind.Trace + ? source + : undefined; + const optimizedChartConfig = + isBuilderChartConfig(chartConfig) && mvSource?.materializedViews?.length + ? await tryOptimizeConfigWithMaterializedView( + chartConfig, + metadata, + clickhouseClient, + undefined, + mvSource, + ) + : chartConfig; + + // Readonly = 2 means the query is readonly but can still specify query settings. + // This is done only for Raw SQL configs because it carries a minor risk of conflict with + // existing settings (which may have readonly = 1) and is not required for builder + // chart configs, which are always rendered as select statements. + const clickHouseSettings = isRawSqlChartConfig(optimizedChartConfig) + ? { readonly: '2' } + : {}; + + // Query for alert data. If the query fails, record the error and exit + // without touching alert state or creating an AlertHistory. + let checksData; + const queryStartedAt = performance.now(); try { - const withClauses = await computeAliasWithClauses( - details.savedSearch, - details.source, + checksData = await clickhouseClient.queryChartConfig({ + config: optimizedChartConfig, metadata, - ); - if (withClauses) { - chartConfig.with = withClauses; - aliasWithClauses = withClauses; - } + opts: { clickhouse_settings: clickHouseSettings }, + querySettings: source?.querySettings, + }); + // SLO signal for alert data fetching (distinct from the end-to-end + // evaluation SLI): did ClickHouse serve the alert query, and how fast. + const queryDurationMs = performance.now() - queryStartedAt; + evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); + recordOperationOutcome({ + operation: 'alerts.query', + outcome: 'success', + durationMs: queryDurationMs, + attributes: { alert_source: alert.source ?? 'unknown' }, + }); } catch (e) { - logger.warn( - { error: serializeError(e), alertId: alert.id }, - 'Failed to compute alias WITH clauses for alert check', + const queryDurationMs = performance.now() - queryStartedAt; + // Time-to-failure — for QUERY_TIMEOUT this is roughly the configured + // evaluation timeout. + evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); + recordOperationOutcome({ + operation: 'alerts.query', + outcome: 'error', + durationMs: queryDurationMs, + attributes: { alert_source: alert.source ?? 'unknown' }, + }); + evalOutcome = 'error'; + const alertError = makeQueryAlertError( + e, + clickhouseClient.requestTimeoutMs, ); + alertQueryFailuresCounter.add(1, { + error_type: + alertError.type === AlertErrorType.QUERY_TIMEOUT + ? 'timeout' + : 'error', + }); + logger.error( + { + alertId: alert.id, + errorType: alertError.type, + error: serializeError(e), + }, + 'Alert query failed, skipping state/history update', + ); + // Record the error on the alert and as an ERROR history row for this + // window. ERROR rows are excluded from the due-ness gate and date-range + // computation, so the failed window is still retried/backfilled. + await alertProvider.recordAlertErrors( + alert.id, + [alertError], + nowInMinsRoundDown, + evaluationAnalytics, + ); + return; } - } - // Optimize chart config with materialized views, if available. - // materializedViews exists on Log and Trace sources. - const mvSource = - source?.kind === SourceKind.Log || source?.kind === SourceKind.Trace - ? source - : undefined; - const optimizedChartConfig = - isBuilderChartConfig(chartConfig) && mvSource?.materializedViews?.length - ? await tryOptimizeConfigWithMaterializedView( - chartConfig, - metadata, - clickhouseClient, - undefined, - mvSource, - ) - : chartConfig; - - // Readonly = 2 means the query is readonly but can still specify query settings. - // This is done only for Raw SQL configs because it carries a minor risk of conflict with - // existing settings (which may have readonly = 1) and is not required for builder - // chart configs, which are always rendered as select statements. - const clickHouseSettings = isRawSqlChartConfig(optimizedChartConfig) - ? { readonly: '2' } - : {}; - - // Query for alert data. If the query fails, record the error and exit - // without touching alert state or creating an AlertHistory. - let checksData; - const queryStartedAt = performance.now(); - try { - checksData = await clickhouseClient.queryChartConfig({ - config: optimizedChartConfig, - metadata, - opts: { clickhouse_settings: clickHouseSettings }, - querySettings: source?.querySettings, - }); - // SLO signal for alert data fetching (distinct from the end-to-end - // evaluation SLI): did ClickHouse serve the alert query, and how fast. - const queryDurationMs = performance.now() - queryStartedAt; - evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); - recordOperationOutcome({ - operation: 'alerts.query', - outcome: 'success', - durationMs: queryDurationMs, - attributes: { alert_source: alert.source ?? 'unknown' }, - }); - } catch (e) { - const queryDurationMs = performance.now() - queryStartedAt; - // Time-to-failure — for QUERY_TIMEOUT this is roughly the configured - // evaluation timeout. - evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); - recordOperationOutcome({ - operation: 'alerts.query', - outcome: 'error', - durationMs: queryDurationMs, - attributes: { alert_source: alert.source ?? 'unknown' }, - }); - evalOutcome = 'error'; - const alertError = makeQueryAlertError( - e, - clickhouseClient.requestTimeoutMs, - ); - alertQueryFailuresCounter.add(1, { - error_type: - alertError.type === AlertErrorType.QUERY_TIMEOUT - ? 'timeout' - : 'error', - }); - logger.error( + logger.info( { alertId: alert.id, - errorType: alertError.type, - error: serializeError(e), + chartConfig, + optimizedChartConfig, + checksData, + checkStartTime: dateRange[0], + checkEndTime: dateRange[1], }, - 'Alert query failed, skipping state/history update', - ); - // Record the error on the alert and as an ERROR history row for this - // window. ERROR rows are excluded from the due-ness gate and date-range - // computation, so the failed window is still retried/backfilled. - await alertProvider.recordAlertErrors( - alert.id, - [alertError], - nowInMinsRoundDown, - evaluationAnalytics, + `Received alert metric [${alert.source} source]`, ); - return; - } - logger.info( - { - alertId: alert.id, - chartConfig, - optimizedChartConfig, - checksData, - checkStartTime: dateRange[0], - checkEndTime: dateRange[1], - }, - `Received alert metric [${alert.source} source]`, - ); - - const meta = getResponseMetadata(chartConfig, checksData); - if (!meta) { - evalOutcome = 'error'; - logger.error({ alertId: alert.id }, 'Failed to get response metadata'); - return; - } + const meta = getResponseMetadata(chartConfig, checksData); + if (!meta) { + evalOutcome = 'error'; + logger.error({ alertId: alert.id }, 'Failed to get response metadata'); + return; + } - // single_value type (Raw SQL Number charts) returns a single value with no - // timestamp column, and are assumed to not have groups. - if (meta.type === 'single_value') { - // Use the date range end as the alert timestamp. - const alertTimestamp = dateRange[1]; - const history = getOrCreateHistory(''); - - // The value is taken from the last numeric column of the first row. - // The value defaults to 0. - const value = - checksData.data.length > 0 - ? (parseAlertData(checksData.data[0], meta).value ?? 0) - : 0; - - history.lastValues.push({ count: value, startTime: alertTimestamp }); - const previous = previousMap.get(computeHistoryMapKey(alert.id, '')); - if (doesExceedThreshold(alert, value)) { - history.counts += 1; - if (shouldFireBasedOnConsecutiveWindows()) { - history.state = AlertState.ALERT; - history.fired = true; - await trySendNotification({ - state: AlertState.ALERT, - group: '', - totalCount: value, - startTime: alertTimestamp, - }); - } else { - history.state = AlertState.PENDING; - // Carry forward fired=true if a notification was previously sent and not yet resolved. - history.fired = previous?.fired === true; + // single_value type (Raw SQL Number charts) returns a single value with no + // timestamp column, and are assumed to not have groups. + if (meta.type === 'single_value') { + // Use the date range end as the alert timestamp. + const alertTimestamp = dateRange[1]; + const history = getOrCreateHistory(''); + + // The value is taken from the last numeric column of the first row. + // The value defaults to 0. + const value = + checksData.data.length > 0 + ? (parseAlertData(checksData.data[0], meta).value ?? 0) + : 0; + + history.lastValues.push({ count: value, startTime: alertTimestamp }); + const previous = previousMap.get(computeHistoryMapKey(alert.id, '')); + if (doesExceedThreshold(alert, value)) { + history.counts += 1; + if (shouldFireBasedOnConsecutiveWindows()) { + history.state = AlertState.ALERT; + history.fired = true; + await trySendNotification({ + state: AlertState.ALERT, + group: '', + totalCount: value, + startTime: alertTimestamp, + }); + } else { + history.state = AlertState.PENDING; + // Carry forward fired=true if a notification was previously sent and not yet resolved. + history.fired = previous?.fired === true; + } } - } - // Auto-resolve - await sendNotificationIfResolved(previous, history, ''); + // Auto-resolve + await sendNotificationIfResolved(previous, history, ''); - // Single-value evaluations always cover exactly the current window. - evaluationAnalytics.backfilledBuckets = 0; - flushNotificationTimings(); - const historyRecords = Array.from(histories.values()); - for (const record of historyRecords) { - record.analytics = evaluationAnalytics; + // Single-value evaluations always cover exactly the current window. + evaluationAnalytics.backfilledBuckets = 0; + flushNotificationTimings(); + const historyRecords = Array.from(histories.values()); + for (const record of historyRecords) { + record.analytics = evaluationAnalytics; + } + await alertProvider.updateAlertState( + alert.id, + historyRecords, + executionErrors, + dateRange, + ); + return; } - await alertProvider.updateAlertState( - alert.id, - historyRecords, - executionErrors, - dateRange, - ); - return; - } - // Standard time-series alert evaluation (Line/StackedBar charts). - const sqlExpectedBuckets = timeBucketByGranularity( - dateRange[0], - dateRange[1], - `${windowSizeInMins} minute`, - ); - // Buckets beyond the current window were backfilled in this run — - // earlier evaluation ticks were missed (job delay, failed evaluations). - evaluationAnalytics.backfilledBuckets = Math.max( - 0, - sqlExpectedBuckets.length - 1, - ); + // Standard time-series alert evaluation (Line/StackedBar charts). + const sqlExpectedBuckets = timeBucketByGranularity( + dateRange[0], + dateRange[1], + `${windowSizeInMins} minute`, + ); + // Buckets beyond the current window were backfilled in this run — + // earlier evaluation ticks were missed (job delay, failed evaluations). + evaluationAnalytics.backfilledBuckets = Math.max( + 0, + sqlExpectedBuckets.length - 1, + ); - // Normalise SQL rows into the shared bucket-row format: - // Map - const sqlBucketRows = new Map< - number, - { groupKey: string; value: number; attributes: Record }[] - >(); - for (const checkData of checksData.data) { - const bucketStart = new Date(checkData[meta.timestampColumnName]); - const bucketMs = bucketStart.getTime(); - if (!sqlBucketRows.has(bucketMs)) sqlBucketRows.set(bucketMs, []); - const { value, extraFields } = parseAlertData(checkData, meta); - if (value == null) continue; // skip NULL (missing denominator, empty ratio, etc.) - const groupKey = hasGroupBy - ? extraFields.map(([k, v]) => `${k}:${v}`).join(', ') - : ''; - const attributes = hasGroupBy ? Object.fromEntries(extraFields) : {}; - sqlBucketRows.get(bucketMs)!.push({ groupKey, value, attributes }); - } + // Normalise SQL rows into the shared bucket-row format: + // Map + const sqlBucketRows = new Map< + number, + { + groupKey: string; + value: number; + attributes: Record; + }[] + >(); + for (const checkData of checksData.data) { + const bucketStart = new Date(checkData[meta.timestampColumnName]); + const bucketMs = bucketStart.getTime(); + if (!sqlBucketRows.has(bucketMs)) sqlBucketRows.set(bucketMs, []); + const { value, extraFields } = parseAlertData(checkData, meta); + if (value == null) continue; // skip NULL (missing denominator, empty ratio, etc.) + const groupKey = hasGroupBy + ? extraFields.map(([k, v]) => `${k}:${v}`).join(', ') + : ''; + const attributes = hasGroupBy ? Object.fromEntries(extraFields) : {}; + sqlBucketRows.get(bucketMs)!.push({ groupKey, value, attributes }); + } - checkDataByBucket = sqlBucketRows; - sharedExpectedBuckets = sqlExpectedBuckets; - runAutoResolveForAll = false; // SQL grouped alerts already guard this with hasGroupBy + checkDataByBucket = sqlBucketRows; + sharedExpectedBuckets = sqlExpectedBuckets; + runAutoResolveForAll = false; // SQL grouped alerts already guard this with hasGroupBy } // Shared tail: bucket state machine + auto-resolve + notifications @@ -1929,8 +1937,7 @@ export const processAlert = async ( const hasAlertsInPreviousMap = previousMap .values() .some( - h => - h.state === AlertState.ALERT || h.state === AlertState.PENDING, + h => h.state === AlertState.ALERT || h.state === AlertState.PENDING, ); if (zeroValueIsAlert) { From 4654a9a361e99d60293fe0ea8276bb4903679fb9 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 2 Oct 2026 18:19:59 +0530 Subject: [PATCH 46/49] lint --- .../ChartEditor/RawSqlChartEditor.tsx | 1 - .../app/src/components/ChartEditor/utils.ts | 38 +------------------ packages/app/src/utils.ts | 19 +--------- 3 files changed, 3 insertions(+), 55 deletions(-) diff --git a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx index a9d1949642..d3cab44c4e 100644 --- a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx @@ -309,7 +309,6 @@ export default function RawSqlChartEditor({ variant="subtle" data-testid="alert-button" size="sm" - color={'gray'} onClick={() => setValue('alert', { ...DEFAULT_TILE_ALERT, diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 85a3476b92..9bbb675de4 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -12,7 +12,6 @@ import { validateRawSqlForAlert, } from '@hyperdx/common-utils/dist/core/utils'; import { - HEATMAP_ALLOWED_SOURCE_KINDS, isBuilderSavedChartConfig, isPromqlSavedChartConfig, isRawSqlSavedChartConfig, @@ -177,39 +176,6 @@ export const isPromqlDisplayType = ( displayType === DisplayType.Bar || displayType === DisplayType.Number; -const NON_PROMQL_SOURCE_KINDS = Object.values(SourceKind).filter( - kind => kind !== SourceKind.Promql, -); - -/** - * Search and event patterns list raw rows out of the source's `from` table. A - * metric source keeps its rows in `metricTables` and leaves `from.tableName` - * empty, so there is nothing for them to read. - */ -const ROW_LISTING_SOURCE_KINDS = NON_PROMQL_SOURCE_KINDS.filter( - kind => kind !== SourceKind.Metric, -); - -/** Source kinds the Data Source picker offers, given the editor mode and the display type. */ -export function getAllowedSourceKinds({ - configType, - displayType, -}: { - configType: ChartEditorFormState['configType']; - displayType: DisplayType | undefined; -}): SourceKind[] { - if (configType === 'promql' && isPromqlDisplayType(displayType)) { - return [SourceKind.Promql]; - } - if (displayType === DisplayType.Heatmap) { - return [...HEATMAP_ALLOWED_SOURCE_KINDS]; - } - if (isStringSelectDisplayType(displayType)) { - return ROW_LISTING_SOURCE_KINDS; - } - return NON_PROMQL_SOURCE_KINDS; -} - const isCustomOrderByDisplayType = ( displayType: DisplayType | undefined, ): displayType is DisplayType.Table | DisplayType.Bar | DisplayType.Pie => @@ -235,13 +201,12 @@ export function convertFormStateToSavedChartConfig( 'numberFormat', 'color', 'colorRules', - 'backgroundChart', 'granularity', 'compareToPreviousPeriod', 'fillNulls', 'alignDateRangeToGranularity', 'alternateRowBackground', - 'alert', + // 'alert', // TODO: Support alerts on PromQL (HDX-4636) ]), promqlExpression: formPromqlExpressions(form), connection: form.connection ?? '', @@ -334,7 +299,6 @@ export function convertFormStateToChartConfig( 'numberFormat', 'color', 'colorRules', - 'backgroundChart', 'granularity', 'compareToPreviousPeriod', 'fillNulls', diff --git a/packages/app/src/utils.ts b/packages/app/src/utils.ts index e9644d36e8..fb2783818d 100644 --- a/packages/app/src/utils.ts +++ b/packages/app/src/utils.ts @@ -91,24 +91,13 @@ export const useDebugMode = () => { return Boolean(query.debugMode) || Boolean(query.debug); }; -const returnFalse = () => false; // From: https://usehooks.com/useDebounce/ -export const useDebounce = ( - value: T, - delay: number, - immediate?: (value: T) => boolean, -) => { +export const useDebounce = (value: T, delay: number) => { // State and setters for debounced value const [debouncedValue, setDebouncedValue] = useState(value); - const shouldBeImmediate = (immediate ?? returnFalse)(value); useEffect( () => { - if (shouldBeImmediate) { - setDebouncedValue(value); - return () => {}; - } - // Update debounced value after delay const handler = setTimeout(() => { setDebouncedValue(value); @@ -120,12 +109,8 @@ export const useDebounce = ( clearTimeout(handler); }; }, - [value, delay, shouldBeImmediate], // Only re-call effect if value or delay changes + [value, delay], // Only re-call effect if value or delay changes ); - if (shouldBeImmediate) { - return value; - } - return debouncedValue; }; From 6522eaea67a4a526afd69e46cb1445258b17dfe8 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 2 Oct 2026 18:52:50 +0530 Subject: [PATCH 47/49] test: restore all jest mocks after each checkAlerts integration test --- .../src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 98977d0fe9..e7638d97fd 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -71,6 +71,10 @@ beforeAll(async () => { }); describe('checkAlerts', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + describe('doesExceedThreshold', () => { it('should return true when value exceeds ABOVE threshold', () => { expect( From 3a22cece92078f6056e137e68ea05e3e3d76ecdb Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 2 Oct 2026 19:25:50 +0530 Subject: [PATCH 48/49] test: restore all jest mocks after each checkAlerts integration test --- .../app/src/components/ChartEditor/utils.ts | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 9bbb675de4..85a3476b92 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -12,6 +12,7 @@ import { validateRawSqlForAlert, } from '@hyperdx/common-utils/dist/core/utils'; import { + HEATMAP_ALLOWED_SOURCE_KINDS, isBuilderSavedChartConfig, isPromqlSavedChartConfig, isRawSqlSavedChartConfig, @@ -176,6 +177,39 @@ export const isPromqlDisplayType = ( displayType === DisplayType.Bar || displayType === DisplayType.Number; +const NON_PROMQL_SOURCE_KINDS = Object.values(SourceKind).filter( + kind => kind !== SourceKind.Promql, +); + +/** + * Search and event patterns list raw rows out of the source's `from` table. A + * metric source keeps its rows in `metricTables` and leaves `from.tableName` + * empty, so there is nothing for them to read. + */ +const ROW_LISTING_SOURCE_KINDS = NON_PROMQL_SOURCE_KINDS.filter( + kind => kind !== SourceKind.Metric, +); + +/** Source kinds the Data Source picker offers, given the editor mode and the display type. */ +export function getAllowedSourceKinds({ + configType, + displayType, +}: { + configType: ChartEditorFormState['configType']; + displayType: DisplayType | undefined; +}): SourceKind[] { + if (configType === 'promql' && isPromqlDisplayType(displayType)) { + return [SourceKind.Promql]; + } + if (displayType === DisplayType.Heatmap) { + return [...HEATMAP_ALLOWED_SOURCE_KINDS]; + } + if (isStringSelectDisplayType(displayType)) { + return ROW_LISTING_SOURCE_KINDS; + } + return NON_PROMQL_SOURCE_KINDS; +} + const isCustomOrderByDisplayType = ( displayType: DisplayType | undefined, ): displayType is DisplayType.Table | DisplayType.Bar | DisplayType.Pie => @@ -201,12 +235,13 @@ export function convertFormStateToSavedChartConfig( 'numberFormat', 'color', 'colorRules', + 'backgroundChart', 'granularity', 'compareToPreviousPeriod', 'fillNulls', 'alignDateRangeToGranularity', 'alternateRowBackground', - // 'alert', // TODO: Support alerts on PromQL (HDX-4636) + 'alert', ]), promqlExpression: formPromqlExpressions(form), connection: form.connection ?? '', @@ -299,6 +334,7 @@ export function convertFormStateToChartConfig( 'numberFormat', 'color', 'colorRules', + 'backgroundChart', 'granularity', 'compareToPreviousPeriod', 'fillNulls', From faf2756c142b89e12cded12c7565119a5039f203 Mon Sep 17 00:00:00 2001 From: aryainguz Date: Fri, 2 Oct 2026 21:14:43 +0530 Subject: [PATCH 49/49] feat: add support for PromQL alerts in default alert check provider --- packages/api/src/tasks/checkAlerts/index.ts | 6 +- .../tasks/checkAlerts/providers/default.ts | 100 ++++++++++++++++-- 2 files changed, 97 insertions(+), 9 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 4b9d830374..2333c87fe8 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1924,9 +1924,11 @@ export const processAlert = async ( // which path populated the data above. for (const bucketStart of sharedExpectedBuckets) { - const rowsForBucket = checkDataByBucket.get(bucketStart.getTime()) ?? []; + const bucketMs = bucketStart.getTime(); + const hasBucket = checkDataByBucket.has(bucketMs); + const rowsForBucket = checkDataByBucket.get(bucketMs) ?? []; - if (rowsForBucket.length === 0) { + if (!hasBucket) { alertEvaluationsCounter.add(1, { outcome: 'empty_bucket' }); logger.info( { alertId: alert.id, bucketStart }, diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index 560637a0c8..9583a9fdb2 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -1,8 +1,15 @@ import PQueue from '@esm2cjs/p-queue'; -import { displayTypeSupportsRawSqlAlerts } from '@hyperdx/common-utils/dist/core/utils'; -import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { + displayTypeSupportsPromQLAlerts, + displayTypeSupportsRawSqlAlerts, +} from '@hyperdx/common-utils/dist/core/utils'; +import { + isPromqlSavedChartConfig, + isRawSqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { AlertChartConfig, + PromqlSavedChartConfig, RawSqlSavedChartConfig, Tile, } from '@hyperdx/common-utils/dist/types'; @@ -106,9 +113,9 @@ async function getSavedSearchDetails( * source macros keep evaluating correctly, and templates that do fail with a * visible query error recorded on the alert. */ -async function getRawSqlSourceMetadata( +async function getConnectionPinnedSourceMetadata( alert: IAlert, - chartConfig: RawSqlSavedChartConfig, + chartConfig: RawSqlSavedChartConfig | PromqlSavedChartConfig, ): Promise { if (!chartConfig.source) { return undefined; @@ -130,7 +137,7 @@ async function getRawSqlSourceMetadata( ) { logger.warn({ message: - 'raw sql alert source has moved to a different connection; ignoring its metadata', + 'alert source has moved to a different connection; ignoring its metadata', alertId: alert.id, sourceId: chartConfig.source, sourceConnectionId: String(sourceDoc.connection), @@ -201,7 +208,7 @@ async function getTileDetails( } // Optionally look up source for filter/macro metadata - const source = await getRawSqlSourceMetadata(alert, tile.config); + const source = await getConnectionPinnedSourceMetadata(alert, tile.config); return [ connection, @@ -215,6 +222,48 @@ async function getTileDetails( ]; } + if (isPromqlSavedChartConfig(tile.config)) { + if (!displayTypeSupportsPromQLAlerts(tile.config.displayType)) { + logger.warn({ + tileId, + dashboardId: dashboard._id, + alertId: alert.id, + message: + 'skipping alert with promql chart config, unsupported display type', + }); + return []; + } + + const connection = await Connection.findOne({ + _id: tile.config.connection, + team: alert.team, + }).select('+password'); + + if (!connection) { + logger.error({ + message: 'connection not found for promql tile', + connectionId: tile.config.connection, + tileId, + dashboardId: dashboard._id, + alertId: alert.id, + }); + return []; + } + + const source = await getConnectionPinnedSourceMetadata(alert, tile.config); + + return [ + connection, + { + alert, + source: source ? { ...source, connection: connection.id } : undefined, + taskType: AlertTaskType.TILE, + tile, + dashboard, + }, + ]; + } + const source = await Source.findOne({ _id: tile.config.source, team: alert.team, @@ -297,7 +346,7 @@ async function getInlineAlertDetails( } // Optionally look up source for filter/macro metadata - const source = await getRawSqlSourceMetadata(alert, chartConfig); + const source = await getConnectionPinnedSourceMetadata(alert, chartConfig); return [ connection, @@ -310,6 +359,43 @@ async function getInlineAlertDetails( ]; } + if (isPromqlSavedChartConfig(chartConfig)) { + if (!displayTypeSupportsPromQLAlerts(chartConfig.displayType)) { + logger.warn({ + alertId: alert.id, + message: + 'skipping inline alert with promql chart config, unsupported display type', + }); + return []; + } + + const connection = await Connection.findOne({ + _id: chartConfig.connection, + team: alert.team, + }).select('+password'); + + if (!connection) { + logger.error({ + message: 'connection not found for promql inline alert', + connectionId: chartConfig.connection, + alertId: alert.id, + }); + return []; + } + + const source = await getConnectionPinnedSourceMetadata(alert, chartConfig); + + return [ + connection, + { + alert, + source: source ? { ...source, connection: connection.id } : undefined, + taskType: AlertTaskType.INLINE, + chartConfig, + }, + ]; + } + const source = await Source.findOne({ _id: chartConfig.source, team: alert.team,