diff --git a/.changeset/external-api-promql-source-gate.md b/.changeset/external-api-promql-source-gate.md new file mode 100644 index 0000000000..c721f79d70 --- /dev/null +++ b/.changeset/external-api-promql-source-gate.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': patch +--- + +fix: Reject creating or updating PromQL sources through the external API when PromQL is not enabled + +`POST` and `PUT /api/v2/sources` now return a 400 for `kind: "promql"` unless `ENABLE_PROMQL` is set, since PromQL sources cannot be queried without it. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index de21409f23..f6ea5912f6 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -2446,15 +2446,24 @@ } }, "HeatmapChartConfig": { - "description": "Heatmap tile. Heatmap is builder-only (no Raw SQL variant). Omit heatmapMode (or set it to \"distribution\") for the distribution variant; set heatmapMode to \"series\" for the series variant.\n", + "description": "Heatmap tile. There is no Raw SQL variant. For the builder variant, omit heatmapMode (or set it to \"distribution\") for the distribution heatmap, or set it to \"series\" for the series heatmap. Set configType to \"promql\" for the PromQL variant.\n", "oneOf": [ { "$ref": "#/components/schemas/HeatmapDistributionChartConfig" }, { "$ref": "#/components/schemas/HeatmapSeriesChartConfig" + }, + { + "$ref": "#/components/schemas/HeatmapPromqlChartConfig" } - ] + ], + "discriminator": { + "propertyName": "configType", + "mapping": { + "promql": "#/components/schemas/HeatmapPromqlChartConfig" + } + } }, "SearchChartConfig": { "type": "object", @@ -2845,54 +2854,547 @@ } ] }, + "PromqlRangeExpression": { + "type": "object", + "required": [ + "expression" + ], + "description": "A PromQL expression evaluated over the dashboard time range.", + "properties": { + "expression": { + "type": "string", + "maxLength": 10000, + "description": "PromQL expression. Supports dashboard variables and the $__interval, $__rate_interval, and $__range macros.\n", + "example": "sum by (job) (rate(http_requests_total[$__rate_interval]))" + }, + "alias": { + "type": "string", + "maxLength": 10000, + "description": "Prefixed to every series name this expression produces.", + "example": "Requests" + } + } + }, + "PromqlInstantExpression": { + "description": "A PromQL expression on a table, number, pie, or bar tile, which can be evaluated at a single instant instead of over a range.\n", + "allOf": [ + { + "$ref": "#/components/schemas/PromqlRangeExpression" + }, + { + "type": "object", + "properties": { + "queryType": { + "type": "string", + "enum": [ + "range", + "instant" + ], + "description": "Whether to evaluate over the time range or at its end. Defaults to \"range\" when omitted.\n", + "example": "instant" + } + } + } + ] + }, + "PromqlReducibleExpression": { + "description": "A PromQL expression on a number, pie, or bar tile, which reduces each series of a range query to a single value.\n", + "allOf": [ + { + "$ref": "#/components/schemas/PromqlInstantExpression" + }, + { + "type": "object", + "properties": { + "reducer": { + "type": "string", + "enum": [ + "lastNotNull", + "min", + "max", + "mean", + "sum", + "count" + ], + "description": "How each series' samples are reduced to one value. Defaults to \"lastNotNull\" when omitted. It also applies to instant queries, where each series has a single sample, so e.g. \"count\" turns every value into 1.\n", + "example": "max" + } + } + } + ] + }, + "PromqlChartConfigBase": { + "type": "object", + "required": [ + "configType", + "sourceId" + ], + "description": "Shared fields for PromQL chart configs. Set configType to \"promql\" and provide sourceId + expressions. The connection is read from the source. PromQL is a preview feature and must be enabled on the deployment (ENABLE_PROMQL) before PromQL tiles can be queried.\n", + "properties": { + "configType": { + "type": "string", + "enum": [ + "promql" + ], + "description": "Must be \"promql\" to use the PromQL chart config variant.", + "example": "promql" + }, + "sourceId": { + "type": "string", + "description": "ID of the PromQL source to query. Must be a source of kind \"promql\".", + "example": "65f5e4a3b9e77c001a333333" + }, + "granularity": { + "type": "string", + "pattern": "^(\\d+ (second|minute|hour|day)|auto)$", + "description": "Resolution of range queries, used as the Prometheus step and the $__interval macro. Defaults to \"auto\", which picks one from the time range.\n", + "example": "5 minute" + }, + "numberFormat": { + "$ref": "#/components/schemas/NumberFormat", + "description": "Number formatting options for displayed values." + } + } + }, + "PromqlTimeChartConfigBase": { + "description": "Shared fields for PromQL time-series chart configs.", + "allOf": [ + { + "$ref": "#/components/schemas/PromqlChartConfigBase" + }, + { + "type": "object", + "required": [ + "expressions" + ], + "properties": { + "expressions": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "PromQL expressions to plot.", + "items": { + "$ref": "#/components/schemas/PromqlRangeExpression" + } + }, + "legendTemplate": { + "type": "string", + "maxLength": 1024, + "description": "Template for series names, using the series' Prometheus labels in double braces.\n", + "example": "{{job}} - {{instance}}" + }, + "fillNulls": { + "type": "boolean", + "description": "Fill missing time buckets with zero instead of leaving gaps.", + "default": true + }, + "alignDateRangeToGranularity": { + "type": "boolean", + "description": "Expand date range boundaries to the query granularity interval.", + "default": true + }, + "seriesLimit": { + "type": "integer", + "minimum": 0, + "description": "Maximum number of series rendered (top-N by value). Omit to use the default render cap, set 0 for unlimited, or a positive N to keep the top N series.\n", + "example": 5 + } + } + } + ] + }, + "LinePromqlChartConfig": { + "description": "PromQL configuration for a line time-series chart.", + "allOf": [ + { + "$ref": "#/components/schemas/PromqlTimeChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "line" + ], + "description": "Display as a line time-series chart.", + "example": "line" + }, + "compareToPreviousPeriod": { + "type": "boolean", + "description": "Overlay the equivalent previous time period for comparison.", + "default": false + }, + "lineInterpolation": { + "type": "string", + "enum": [ + "smooth", + "linear", + "step" + ], + "example": "step", + "description": "How points are connected. Defaults to \"smooth\" when omitted.\n" + } + } + } + ] + }, + "BarPromqlChartConfig": { + "description": "PromQL configuration for a stacked-bar time-series chart.", + "allOf": [ + { + "$ref": "#/components/schemas/PromqlTimeChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "stacked_bar" + ], + "description": "Display as a stacked-bar time-series chart.", + "example": "stacked_bar" + } + } + } + ] + }, + "StackedLinePromqlChartConfig": { + "description": "PromQL configuration for a stacked-line time-series chart.", + "allOf": [ + { + "$ref": "#/components/schemas/PromqlTimeChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "stacked_line" + ], + "description": "Display as a stacked-line time-series chart.", + "example": "stacked_line" + }, + "lineInterpolation": { + "type": "string", + "enum": [ + "smooth", + "linear", + "step" + ], + "example": "step", + "description": "How points are connected. Defaults to \"linear\" when omitted.\n" + } + } + } + ] + }, + "TablePromqlChartConfig": { + "description": "PromQL configuration for a table chart, with one row per sample: a column per label, a Value column, and a Timestamp column when the expression is a range query. An instant query gives one row per series.\n", + "allOf": [ + { + "$ref": "#/components/schemas/PromqlChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType", + "expressions" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "table" + ], + "description": "Display as a table chart.", + "example": "table" + }, + "expressions": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "PromQL expressions. Only the first is queried; the rest are kept but ignored.\n", + "items": { + "$ref": "#/components/schemas/PromqlInstantExpression" + } + } + } + } + ] + }, + "NumberPromqlChartConfig": { + "description": "PromQL configuration for a single big-number chart.", + "allOf": [ + { + "$ref": "#/components/schemas/PromqlChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType", + "expressions" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "number" + ], + "description": "Display as a single big-number chart.", + "example": "number" + }, + "expressions": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "PromQL expressions. Only the first is queried; the rest are kept but ignored. The tile displays the first value.\n", + "items": { + "$ref": "#/components/schemas/PromqlReducibleExpression" + } + }, + "color": { + "$ref": "#/components/schemas/ChartPaletteToken", + "description": "Optional static color applied to the displayed number." + }, + "colorRules": { + "type": "array", + "maxItems": 10, + "description": "Ordered conditional color rules evaluated against the displayed value (last match wins). Falls back to color, then the default text color when no rule matches.\n", + "items": { + "$ref": "#/components/schemas/NumberTileColorCondition" + } + }, + "backgroundChart": { + "$ref": "#/components/schemas/BackgroundChart", + "description": "Optional trend sparkline drawn behind the number. Only rendered when the expression uses a range query.\n" + } + } + } + ] + }, + "PiePromqlChartConfig": { + "description": "PromQL configuration for a pie chart, with one slice per series.", + "allOf": [ + { + "$ref": "#/components/schemas/PromqlChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType", + "expressions" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "pie" + ], + "description": "Display as a pie chart.", + "example": "pie" + }, + "expressions": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "PromQL expressions. Only the first is queried; the rest are kept but ignored.\n", + "items": { + "$ref": "#/components/schemas/PromqlReducibleExpression" + } + }, + "legendTemplate": { + "type": "string", + "maxLength": 1024, + "description": "Template for slice names, using the series' Prometheus labels in double braces.\n", + "example": "{{job}}" + }, + "limit": { + "type": "integer", + "minimum": 0, + "description": "Maximum number of slices (top-N by value), at most 500. Omit or set 0 to show up to 500.\n", + "example": 10 + } + } + } + ] + }, + "CategoricalBarPromqlChartConfig": { + "description": "PromQL configuration for a categorical bar chart, with one bar per series.", + "allOf": [ + { + "$ref": "#/components/schemas/PromqlChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType", + "expressions" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "bar" + ], + "description": "Display as a categorical bar chart.", + "example": "bar" + }, + "expressions": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "PromQL expressions. Only the first is queried; the rest are kept but ignored.\n", + "items": { + "$ref": "#/components/schemas/PromqlReducibleExpression" + } + }, + "legendTemplate": { + "type": "string", + "maxLength": 1024, + "description": "Template for bar names, using the series' Prometheus labels in double braces.\n", + "example": "{{job}}" + }, + "limit": { + "type": "integer", + "minimum": 0, + "description": "Maximum number of bars (top-N by value), at most 500. Omit or set 0 to show up to 500.\n", + "example": 10 + } + } + } + ] + }, + "HeatmapPromqlChartConfig": { + "description": "PromQL configuration for a heatmap.", + "allOf": [ + { + "$ref": "#/components/schemas/PromqlChartConfigBase" + }, + { + "type": "object", + "required": [ + "displayType", + "expressions" + ], + "properties": { + "displayType": { + "type": "string", + "enum": [ + "heatmap" + ], + "description": "Display as a heatmap.", + "example": "heatmap" + }, + "expressions": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "PromQL expressions. Only the first is queried; the rest are kept but ignored.\n", + "items": { + "$ref": "#/components/schemas/PromqlRangeExpression" + } + }, + "heatmapMode": { + "type": "string", + "enum": [ + "distribution", + "series", + "histogram" + ], + "description": "\"distribution\" buckets the values of all series over time, \"series\" draws one row per series, and \"histogram\" draws one row per Prometheus histogram bucket (series need an \"le\" label). Defaults to \"distribution\" when omitted.\n", + "example": "histogram" + }, + "heatmapScaleType": { + "type": "string", + "enum": [ + "log", + "linear" + ], + "description": "Y-axis scale of a distribution heatmap.", + "example": "log" + }, + "legendTemplate": { + "type": "string", + "maxLength": 1024, + "description": "Template for row names of a series heatmap, using the series' Prometheus labels in double braces.\n", + "example": "{{job}}" + } + } + } + ] + }, "LineChartConfig": { - "description": "Line chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "description": "Line chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate), or to \"promql\" for the PromQL variant (requires sourceId and expressions).\n", "oneOf": [ { "$ref": "#/components/schemas/LineBuilderChartConfig" }, { "$ref": "#/components/schemas/LineRawSqlChartConfig" + }, + { + "$ref": "#/components/schemas/LinePromqlChartConfig" } ], "discriminator": { "propertyName": "configType", "mapping": { - "sql": "#/components/schemas/LineRawSqlChartConfig" + "sql": "#/components/schemas/LineRawSqlChartConfig", + "promql": "#/components/schemas/LinePromqlChartConfig" } } }, "BarChartConfig": { - "description": "Stacked-bar chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "description": "Stacked-bar chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate), or to \"promql\" for the PromQL variant (requires sourceId and expressions).\n", "oneOf": [ { "$ref": "#/components/schemas/BarBuilderChartConfig" }, { "$ref": "#/components/schemas/BarRawSqlChartConfig" + }, + { + "$ref": "#/components/schemas/BarPromqlChartConfig" } ], "discriminator": { "propertyName": "configType", "mapping": { - "sql": "#/components/schemas/BarRawSqlChartConfig" + "sql": "#/components/schemas/BarRawSqlChartConfig", + "promql": "#/components/schemas/BarPromqlChartConfig" } } }, "StackedLineChartConfig": { - "description": "Stacked-line chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "description": "Stacked-line chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate), or to \"promql\" for the PromQL variant (requires sourceId and expressions).\n", "oneOf": [ { "$ref": "#/components/schemas/StackedLineBuilderChartConfig" }, { "$ref": "#/components/schemas/StackedLineRawSqlChartConfig" + }, + { + "$ref": "#/components/schemas/StackedLinePromqlChartConfig" } ], "discriminator": { "propertyName": "configType", "mapping": { - "sql": "#/components/schemas/StackedLineRawSqlChartConfig" + "sql": "#/components/schemas/StackedLineRawSqlChartConfig", + "promql": "#/components/schemas/StackedLinePromqlChartConfig" } } }, @@ -3103,75 +3605,91 @@ } }, "TableChartConfig": { - "description": "Table chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "description": "Table chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate), or to \"promql\" for the PromQL variant (requires sourceId and expressions).\n", "oneOf": [ { "$ref": "#/components/schemas/TableBuilderChartConfig" }, { "$ref": "#/components/schemas/TableRawSqlChartConfig" + }, + { + "$ref": "#/components/schemas/TablePromqlChartConfig" } ], "discriminator": { "propertyName": "configType", "mapping": { - "sql": "#/components/schemas/TableRawSqlChartConfig" + "sql": "#/components/schemas/TableRawSqlChartConfig", + "promql": "#/components/schemas/TablePromqlChartConfig" } } }, "NumberChartConfig": { - "description": "Single big-number chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "description": "Single big-number chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate), or to \"promql\" for the PromQL variant (requires sourceId and expressions).\n", "oneOf": [ { "$ref": "#/components/schemas/NumberBuilderChartConfig" }, { "$ref": "#/components/schemas/NumberRawSqlChartConfig" + }, + { + "$ref": "#/components/schemas/NumberPromqlChartConfig" } ], "discriminator": { "propertyName": "configType", "mapping": { - "sql": "#/components/schemas/NumberRawSqlChartConfig" + "sql": "#/components/schemas/NumberRawSqlChartConfig", + "promql": "#/components/schemas/NumberPromqlChartConfig" } } }, "PieChartConfig": { - "description": "Pie chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "description": "Pie chart. Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate), or to \"promql\" for the PromQL variant (requires sourceId and expressions).\n", "oneOf": [ { "$ref": "#/components/schemas/PieBuilderChartConfig" }, { "$ref": "#/components/schemas/PieRawSqlChartConfig" + }, + { + "$ref": "#/components/schemas/PiePromqlChartConfig" } ], "discriminator": { "propertyName": "configType", "mapping": { - "sql": "#/components/schemas/PieRawSqlChartConfig" + "sql": "#/components/schemas/PieRawSqlChartConfig", + "promql": "#/components/schemas/PiePromqlChartConfig" } } }, "CategoricalBarChartConfig": { - "description": "Categorical bar chart (one bar per group value; not a time series). Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate).\n", + "description": "Categorical bar chart (one bar per group value; not a time series). Omit configType for the builder variant (requires sourceId and select). Set configType to \"sql\" for the Raw SQL variant (requires connectionId and sqlTemplate), or to \"promql\" for the PromQL variant (requires sourceId and expressions).\n", "oneOf": [ { "$ref": "#/components/schemas/CategoricalBarBuilderChartConfig" }, { "$ref": "#/components/schemas/CategoricalBarRawSqlChartConfig" + }, + { + "$ref": "#/components/schemas/CategoricalBarPromqlChartConfig" } ], "discriminator": { "propertyName": "configType", "mapping": { - "sql": "#/components/schemas/CategoricalBarRawSqlChartConfig" + "sql": "#/components/schemas/CategoricalBarRawSqlChartConfig", + "promql": "#/components/schemas/CategoricalBarPromqlChartConfig" } } }, "TileConfig": { - "description": "Tile chart configuration. displayType is the primary discriminant and determines which variant group applies. For displayTypes that support both builder and Raw SQL modes (line, stacked_bar, stacked_line, table, number, pie, bar), configType is the secondary discriminant: omit it for the builder variant or set it to \"sql\" for the Raw SQL variant. The heatmap, search, event_patterns, and markdown displayTypes only have a builder variant.\n", + "description": "Tile chart configuration. displayType is the primary discriminant and determines which variant group applies. configType is the secondary discriminant: omit it for the builder variant, set it to \"sql\" for the Raw SQL variant, or set it to \"promql\" for the PromQL variant. Line, stacked_bar, stacked_line, table, number, pie, and bar support all three. Heatmap supports builder and PromQL. Search, event_patterns, and markdown only have a builder variant.\n", "oneOf": [ { "$ref": "#/components/schemas/LineChartConfig" @@ -3653,7 +4171,7 @@ }, "PrometheusLabelFilterInput": { "type": "object", - "description": "A filter whose dropdown lists the values for a Prometheus label,\nread from a PromQL source over the dashboard's time range.\nSupports variable mode only.\n", + "description": "A filter whose dropdown lists the values for a Prometheus label,\nread from a PromQL source over the dashboard's time range.\nSupports variable mode only. PromQL is a preview feature and must be\nenabled on the deployment (ENABLE_PROMQL) before these filters can\nlist values.\n", "required": [ "type", "name", @@ -5157,7 +5675,7 @@ }, "PromqlSource": { "type": "object", - "description": "A source backed by a Prometheus-compatible endpoint, queried with PromQL. The referenced connection should be a Prometheus connection (isPrometheusEndpoint set to true).", + "description": "A source queried with PromQL. It reads either a ClickHouse TimeSeries table (through a ClickHouse connection) or a Prometheus-compatible endpoint (through a connection with isPrometheusEndpoint set to true). PromQL is a preview feature, so creating or updating a PromQL source is rejected unless it is enabled on the deployment (ENABLE_PROMQL).", "required": [ "name", "kind", @@ -5199,12 +5717,12 @@ }, "connection": { "type": "string", - "description": "ID of the connection used by this source. Should reference a Prometheus-compatible connection.", + "description": "ID of the connection used by this source. Either a ClickHouse connection holding the TimeSeries table, or a Prometheus-compatible connection.", "example": "507f1f77bcf86cd799439012" }, "from": { "$ref": "#/components/schemas/SourceFrom", - "description": "Required by the API for all source kinds; not used when querying a Prometheus endpoint (empty strings are not accepted — use placeholder values such as \"default\")." + "description": "The database and TimeSeries table to query on a ClickHouse connection. Not used for a Prometheus-compatible connection, but still required (empty strings are not accepted — use placeholder values such as \"default\")." }, "querySettings": { "type": "array", @@ -5216,7 +5734,7 @@ }, "timestampValueExpression": { "type": "string", - "description": "Required by the API for all source kinds; not used when querying a Prometheus endpoint.", + "description": "Required by the API for all source kinds; not used by PromQL queries.", "example": "timestamp" } } diff --git a/packages/api/src/routers/external-api/__tests__/sources-crud.int.test.ts b/packages/api/src/routers/external-api/__tests__/sources-crud.int.test.ts index 360eedb232..f7286a13c2 100644 --- a/packages/api/src/routers/external-api/__tests__/sources-crud.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/sources-crud.int.test.ts @@ -285,6 +285,55 @@ describe('External API v2 Sources CRUD', () => { }); }); + describe('PromQL sources with PromQL disabled', () => { + const promqlSourceBody = () => ({ + kind: SourceKind.Promql, + name: 'Prometheus Metrics', + from: { databaseName: 'default', tableName: 'default' }, + timestampValueExpression: 'timestamp', + connection: connection._id.toString(), + }); + + beforeEach(() => { + jest.replaceProperty(config, 'IS_PROMQL_ENABLED', false); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('rejects creating a PromQL source', async () => { + const response = await authRequest('post', BASE_URL) + .send(promqlSourceBody()) + .expect(400); + + expect(response.body.message).toBe( + 'PromQL is not enabled on this deployment', + ); + expect(await Source.findOne({ team: team._id })).toBeNull(); + }); + + it('rejects updating a source to PromQL', async () => { + const logSource = await LogSource.create({ + ...logSourceBody(), + team: team._id, + }); + + const response = await authRequest('put', `${BASE_URL}/${logSource._id}`) + .send(promqlSourceBody()) + .expect(400); + + expect(response.body.message).toBe( + 'PromQL is not enabled on this deployment', + ); + expect((await Source.findById(logSource._id))!.kind).toBe(SourceKind.Log); + }); + + it('still creates other source kinds', async () => { + await authRequest('post', BASE_URL).send(logSourceBody()).expect(200); + }); + }); + describe('PUT /api/v2/sources/:id', () => { it('should return 401 when user is not authenticated', async () => { await request(server.getHttpServer()) diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 276c28c59e..77d3b31361 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -1235,12 +1235,18 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * * HeatmapChartConfig: * description: > - * Heatmap tile. Heatmap is builder-only (no Raw SQL variant). Omit - * heatmapMode (or set it to "distribution") for the distribution - * variant; set heatmapMode to "series" for the series variant. + * Heatmap tile. There is no Raw SQL variant. For the builder variant, + * omit heatmapMode (or set it to "distribution") for the distribution + * heatmap, or set it to "series" for the series heatmap. Set + * configType to "promql" for the PromQL variant. * oneOf: * - $ref: '#/components/schemas/HeatmapDistributionChartConfig' * - $ref: '#/components/schemas/HeatmapSeriesChartConfig' + * - $ref: '#/components/schemas/HeatmapPromqlChartConfig' + * discriminator: + * propertyName: configType + * mapping: + * promql: '#/components/schemas/HeatmapPromqlChartConfig' * * SearchChartConfig: * type: object @@ -1514,44 +1520,425 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * description: Display as a categorical bar chart. * example: "bar" * + * PromqlRangeExpression: + * type: object + * required: + * - expression + * description: A PromQL expression evaluated over the dashboard time range. + * properties: + * expression: + * type: string + * maxLength: 10000 + * description: > + * PromQL expression. Supports dashboard variables and the + * $__interval, $__rate_interval, and $__range macros. + * example: "sum by (job) (rate(http_requests_total[$__rate_interval]))" + * alias: + * type: string + * maxLength: 10000 + * description: Prefixed to every series name this expression produces. + * example: "Requests" + * + * PromqlInstantExpression: + * description: > + * A PromQL expression on a table, number, pie, or bar tile, which can be + * evaluated at a single instant instead of over a range. + * allOf: + * - $ref: '#/components/schemas/PromqlRangeExpression' + * - type: object + * properties: + * queryType: + * type: string + * enum: [range, instant] + * description: > + * Whether to evaluate over the time range or at its end. + * Defaults to "range" when omitted. + * example: "instant" + * + * PromqlReducibleExpression: + * description: > + * A PromQL expression on a number, pie, or bar tile, which reduces each + * series of a range query to a single value. + * allOf: + * - $ref: '#/components/schemas/PromqlInstantExpression' + * - type: object + * properties: + * reducer: + * type: string + * enum: [lastNotNull, min, max, mean, sum, count] + * description: > + * How each series' samples are reduced to one value. Defaults + * to "lastNotNull" when omitted. It also applies to instant + * queries, where each series has a single sample, so e.g. + * "count" turns every value into 1. + * example: "max" + * + * PromqlChartConfigBase: + * type: object + * required: + * - configType + * - sourceId + * description: > + * Shared fields for PromQL chart configs. Set configType to "promql" and + * provide sourceId + expressions. The connection is read from the source. + * PromQL is a preview feature and must be enabled on the deployment + * (ENABLE_PROMQL) before PromQL tiles can be queried. + * properties: + * configType: + * type: string + * enum: [promql] + * description: Must be "promql" to use the PromQL chart config variant. + * example: "promql" + * sourceId: + * type: string + * description: ID of the PromQL source to query. Must be a source of kind "promql". + * example: "65f5e4a3b9e77c001a333333" + * granularity: + * type: string + * pattern: '^(\d+ (second|minute|hour|day)|auto)$' + * description: > + * Resolution of range queries, used as the Prometheus step and the + * $__interval macro. Defaults to "auto", which picks one from the + * time range. + * example: "5 minute" + * numberFormat: + * $ref: '#/components/schemas/NumberFormat' + * description: Number formatting options for displayed values. + * + * PromqlTimeChartConfigBase: + * description: Shared fields for PromQL time-series chart configs. + * allOf: + * - $ref: '#/components/schemas/PromqlChartConfigBase' + * - type: object + * required: + * - expressions + * properties: + * expressions: + * type: array + * minItems: 1 + * maxItems: 10 + * description: PromQL expressions to plot. + * items: + * $ref: '#/components/schemas/PromqlRangeExpression' + * legendTemplate: + * type: string + * maxLength: 1024 + * description: > + * Template for series names, using the series' Prometheus labels + * in double braces. + * example: "{{job}} - {{instance}}" + * fillNulls: + * type: boolean + * description: Fill missing time buckets with zero instead of leaving gaps. + * default: true + * alignDateRangeToGranularity: + * type: boolean + * description: Expand date range boundaries to the query granularity interval. + * default: true + * seriesLimit: + * type: integer + * minimum: 0 + * description: > + * Maximum number of series rendered (top-N by value). Omit to use + * the default render cap, set 0 for unlimited, or a positive N to + * keep the top N series. + * example: 5 + * + * LinePromqlChartConfig: + * description: PromQL configuration for a line time-series chart. + * allOf: + * - $ref: '#/components/schemas/PromqlTimeChartConfigBase' + * - type: object + * required: + * - displayType + * properties: + * displayType: + * type: string + * enum: [line] + * description: Display as a line time-series chart. + * example: "line" + * compareToPreviousPeriod: + * type: boolean + * description: Overlay the equivalent previous time period for comparison. + * default: false + * lineInterpolation: + * type: string + * enum: [smooth, linear, step] + * example: "step" + * description: > + * How points are connected. Defaults to "smooth" when omitted. + * + * BarPromqlChartConfig: + * description: PromQL configuration for a stacked-bar time-series chart. + * allOf: + * - $ref: '#/components/schemas/PromqlTimeChartConfigBase' + * - type: object + * required: + * - displayType + * properties: + * displayType: + * type: string + * enum: [stacked_bar] + * description: Display as a stacked-bar time-series chart. + * example: "stacked_bar" + * + * StackedLinePromqlChartConfig: + * description: PromQL configuration for a stacked-line time-series chart. + * allOf: + * - $ref: '#/components/schemas/PromqlTimeChartConfigBase' + * - type: object + * required: + * - displayType + * properties: + * displayType: + * type: string + * enum: [stacked_line] + * description: Display as a stacked-line time-series chart. + * example: "stacked_line" + * lineInterpolation: + * type: string + * enum: [smooth, linear, step] + * example: "step" + * description: > + * How points are connected. Defaults to "linear" when omitted. + * + * TablePromqlChartConfig: + * description: > + * PromQL configuration for a table chart, with one row per sample: a + * column per label, a Value column, and a Timestamp column when the + * expression is a range query. An instant query gives one row per + * series. + * allOf: + * - $ref: '#/components/schemas/PromqlChartConfigBase' + * - type: object + * required: + * - displayType + * - expressions + * properties: + * displayType: + * type: string + * enum: [table] + * description: Display as a table chart. + * example: "table" + * expressions: + * type: array + * minItems: 1 + * maxItems: 10 + * description: > + * PromQL expressions. Only the first is queried; the rest are + * kept but ignored. + * items: + * $ref: '#/components/schemas/PromqlInstantExpression' + * + * NumberPromqlChartConfig: + * description: PromQL configuration for a single big-number chart. + * allOf: + * - $ref: '#/components/schemas/PromqlChartConfigBase' + * - type: object + * required: + * - displayType + * - expressions + * properties: + * displayType: + * type: string + * enum: [number] + * description: Display as a single big-number chart. + * example: "number" + * expressions: + * type: array + * minItems: 1 + * maxItems: 10 + * description: > + * PromQL expressions. Only the first is queried; the rest are + * kept but ignored. The tile displays the first value. + * items: + * $ref: '#/components/schemas/PromqlReducibleExpression' + * color: + * $ref: '#/components/schemas/ChartPaletteToken' + * description: Optional static color applied to the displayed number. + * colorRules: + * type: array + * maxItems: 10 + * description: > + * Ordered conditional color rules evaluated against the displayed + * value (last match wins). Falls back to color, then the default + * text color when no rule matches. + * items: + * $ref: '#/components/schemas/NumberTileColorCondition' + * backgroundChart: + * $ref: '#/components/schemas/BackgroundChart' + * description: > + * Optional trend sparkline drawn behind the number. Only rendered + * when the expression uses a range query. + * + * PiePromqlChartConfig: + * description: PromQL configuration for a pie chart, with one slice per series. + * allOf: + * - $ref: '#/components/schemas/PromqlChartConfigBase' + * - type: object + * required: + * - displayType + * - expressions + * properties: + * displayType: + * type: string + * enum: [pie] + * description: Display as a pie chart. + * example: "pie" + * expressions: + * type: array + * minItems: 1 + * maxItems: 10 + * description: > + * PromQL expressions. Only the first is queried; the rest are + * kept but ignored. + * items: + * $ref: '#/components/schemas/PromqlReducibleExpression' + * legendTemplate: + * type: string + * maxLength: 1024 + * description: > + * Template for slice names, using the series' Prometheus labels + * in double braces. + * example: "{{job}}" + * limit: + * type: integer + * minimum: 0 + * description: > + * Maximum number of slices (top-N by value), at most 500. Omit + * or set 0 to show up to 500. + * example: 10 + * + * CategoricalBarPromqlChartConfig: + * description: PromQL configuration for a categorical bar chart, with one bar per series. + * allOf: + * - $ref: '#/components/schemas/PromqlChartConfigBase' + * - type: object + * required: + * - displayType + * - expressions + * properties: + * displayType: + * type: string + * enum: [bar] + * description: Display as a categorical bar chart. + * example: "bar" + * expressions: + * type: array + * minItems: 1 + * maxItems: 10 + * description: > + * PromQL expressions. Only the first is queried; the rest are + * kept but ignored. + * items: + * $ref: '#/components/schemas/PromqlReducibleExpression' + * legendTemplate: + * type: string + * maxLength: 1024 + * description: > + * Template for bar names, using the series' Prometheus labels in + * double braces. + * example: "{{job}}" + * limit: + * type: integer + * minimum: 0 + * description: > + * Maximum number of bars (top-N by value), at most 500. Omit or + * set 0 to show up to 500. + * example: 10 + * + * HeatmapPromqlChartConfig: + * description: PromQL configuration for a heatmap. + * allOf: + * - $ref: '#/components/schemas/PromqlChartConfigBase' + * - type: object + * required: + * - displayType + * - expressions + * properties: + * displayType: + * type: string + * enum: [heatmap] + * description: Display as a heatmap. + * example: "heatmap" + * expressions: + * type: array + * minItems: 1 + * maxItems: 10 + * description: > + * PromQL expressions. Only the first is queried; the rest are + * kept but ignored. + * items: + * $ref: '#/components/schemas/PromqlRangeExpression' + * heatmapMode: + * type: string + * enum: [distribution, series, histogram] + * description: > + * "distribution" buckets the values of all series over time, + * "series" draws one row per series, and "histogram" draws one + * row per Prometheus histogram bucket (series need an "le" + * label). Defaults to "distribution" when omitted. + * example: "histogram" + * heatmapScaleType: + * type: string + * enum: [log, linear] + * description: Y-axis scale of a distribution heatmap. + * example: "log" + * legendTemplate: + * type: string + * maxLength: 1024 + * description: > + * Template for row names of a series heatmap, using the series' + * Prometheus labels in double braces. + * example: "{{job}}" + * * LineChartConfig: * description: > * Line chart. Omit configType for the builder variant (requires sourceId * and select). Set configType to "sql" for the Raw SQL variant (requires - * connectionId and sqlTemplate). + * connectionId and sqlTemplate), or to "promql" for the PromQL variant + * (requires sourceId and expressions). * oneOf: * - $ref: '#/components/schemas/LineBuilderChartConfig' * - $ref: '#/components/schemas/LineRawSqlChartConfig' + * - $ref: '#/components/schemas/LinePromqlChartConfig' * discriminator: * propertyName: configType * mapping: * sql: '#/components/schemas/LineRawSqlChartConfig' + * promql: '#/components/schemas/LinePromqlChartConfig' * * BarChartConfig: * description: > * Stacked-bar chart. Omit configType for the builder variant (requires * sourceId and select). Set configType to "sql" for the Raw SQL variant - * (requires connectionId and sqlTemplate). + * (requires connectionId and sqlTemplate), or to "promql" for the PromQL + * variant (requires sourceId and expressions). * oneOf: * - $ref: '#/components/schemas/BarBuilderChartConfig' * - $ref: '#/components/schemas/BarRawSqlChartConfig' + * - $ref: '#/components/schemas/BarPromqlChartConfig' * discriminator: * propertyName: configType * mapping: * sql: '#/components/schemas/BarRawSqlChartConfig' + * promql: '#/components/schemas/BarPromqlChartConfig' * * StackedLineChartConfig: * description: > * Stacked-line chart. Omit configType for the builder variant (requires * sourceId and select). Set configType to "sql" for the Raw SQL variant - * (requires connectionId and sqlTemplate). + * (requires connectionId and sqlTemplate), or to "promql" for the PromQL + * variant (requires sourceId and expressions). * oneOf: * - $ref: '#/components/schemas/StackedLineBuilderChartConfig' * - $ref: '#/components/schemas/StackedLineRawSqlChartConfig' + * - $ref: '#/components/schemas/StackedLinePromqlChartConfig' * discriminator: * propertyName: configType * mapping: * sql: '#/components/schemas/StackedLineRawSqlChartConfig' + * promql: '#/components/schemas/StackedLinePromqlChartConfig' * * OnClickFilterTemplate: * type: object @@ -1710,63 +2097,76 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * description: > * Table chart. Omit configType for the builder variant (requires sourceId * and select). Set configType to "sql" for the Raw SQL variant (requires - * connectionId and sqlTemplate). + * connectionId and sqlTemplate), or to "promql" for the PromQL variant + * (requires sourceId and expressions). * oneOf: * - $ref: '#/components/schemas/TableBuilderChartConfig' * - $ref: '#/components/schemas/TableRawSqlChartConfig' + * - $ref: '#/components/schemas/TablePromqlChartConfig' * discriminator: * propertyName: configType * mapping: * sql: '#/components/schemas/TableRawSqlChartConfig' + * promql: '#/components/schemas/TablePromqlChartConfig' * * NumberChartConfig: * description: > * Single big-number chart. Omit configType for the builder variant * (requires sourceId and select). Set configType to "sql" for the Raw - * SQL variant (requires connectionId and sqlTemplate). + * SQL variant (requires connectionId and sqlTemplate), or to "promql" + * for the PromQL variant (requires sourceId and expressions). * oneOf: * - $ref: '#/components/schemas/NumberBuilderChartConfig' * - $ref: '#/components/schemas/NumberRawSqlChartConfig' + * - $ref: '#/components/schemas/NumberPromqlChartConfig' * discriminator: * propertyName: configType * mapping: * sql: '#/components/schemas/NumberRawSqlChartConfig' + * promql: '#/components/schemas/NumberPromqlChartConfig' * * PieChartConfig: * description: > * Pie chart. Omit configType for the builder variant (requires sourceId * and select). Set configType to "sql" for the Raw SQL variant (requires - * connectionId and sqlTemplate). + * connectionId and sqlTemplate), or to "promql" for the PromQL variant + * (requires sourceId and expressions). * oneOf: * - $ref: '#/components/schemas/PieBuilderChartConfig' * - $ref: '#/components/schemas/PieRawSqlChartConfig' + * - $ref: '#/components/schemas/PiePromqlChartConfig' * discriminator: * propertyName: configType * mapping: * sql: '#/components/schemas/PieRawSqlChartConfig' + * promql: '#/components/schemas/PiePromqlChartConfig' * * CategoricalBarChartConfig: * description: > * Categorical bar chart (one bar per group value; not a time series). * Omit configType for the builder variant (requires sourceId and * select). Set configType to "sql" for the Raw SQL variant (requires - * connectionId and sqlTemplate). + * connectionId and sqlTemplate), or to "promql" for the PromQL variant + * (requires sourceId and expressions). * oneOf: * - $ref: '#/components/schemas/CategoricalBarBuilderChartConfig' * - $ref: '#/components/schemas/CategoricalBarRawSqlChartConfig' + * - $ref: '#/components/schemas/CategoricalBarPromqlChartConfig' * discriminator: * propertyName: configType * mapping: * sql: '#/components/schemas/CategoricalBarRawSqlChartConfig' + * promql: '#/components/schemas/CategoricalBarPromqlChartConfig' * * TileConfig: * description: > * Tile chart configuration. displayType is the primary discriminant and - * determines which variant group applies. For displayTypes that support - * both builder and Raw SQL modes (line, stacked_bar, stacked_line, table, - * number, pie, bar), configType is the secondary discriminant: omit it for - * the builder variant or set it to "sql" for the Raw SQL variant. The heatmap, - * search, event_patterns, and markdown displayTypes only have a builder variant. + * determines which variant group applies. configType is the secondary + * discriminant: omit it for the builder variant, set it to "sql" for the + * Raw SQL variant, or set it to "promql" for the PromQL variant. Line, + * stacked_bar, stacked_line, table, number, pie, and bar support all + * three. Heatmap supports builder and PromQL. Search, event_patterns, + * and markdown only have a builder variant. * oneOf: * - $ref: '#/components/schemas/LineChartConfig' * - $ref: '#/components/schemas/BarChartConfig' @@ -2192,7 +2592,9 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * description: | * A filter whose dropdown lists the values for a Prometheus label, * read from a PromQL source over the dashboard's time range. - * Supports variable mode only. + * Supports variable mode only. PromQL is a preview feature and must be + * enabled on the deployment (ENABLE_PROMQL) before these filters can + * list values. * required: * - type * - name diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 38c9d2b2f5..fb66b16915 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -7,6 +7,7 @@ import { import express from 'express'; import { z } from 'zod'; +import * as config from '@/config'; import { validateConnectionId } from '@/controllers/connection'; import { createSource, @@ -111,6 +112,21 @@ async function requireValidConnectionId( } } +// PromQL is a preview feature: without it enabled, a PromQL source cannot be +// queried. Runs after body validation so req.body is the parsed shape. +function rejectPromqlSourceWhenDisabled( + req: express.Request, + res: express.Response, + next: express.NextFunction, +) { + if (req.body?.kind === SourceKind.Promql && !config.IS_PROMQL_ENABLED) { + return res + .status(400) + .json({ message: 'PromQL is not enabled on this deployment' }); + } + next(); +} + function mapSourceToExternalSource(source: TSource): TSource { if (!('materializedViews' in source)) return source; @@ -847,7 +863,7 @@ function formatExternalSource(source: SourceDocument) { * example: ResourceAttributes * PromqlSource: * type: object - * description: A source backed by a Prometheus-compatible endpoint, queried with PromQL. The referenced connection should be a Prometheus connection (isPrometheusEndpoint set to true). + * description: A source queried with PromQL. It reads either a ClickHouse TimeSeries table (through a ClickHouse connection) or a Prometheus-compatible endpoint (through a connection with isPrometheusEndpoint set to true). PromQL is a preview feature, so creating or updating a PromQL source is rejected unless it is enabled on the deployment (ENABLE_PROMQL). * required: * - name * - kind @@ -881,11 +897,11 @@ function formatExternalSource(source: SourceDocument) { * example: promql * connection: * type: string - * description: ID of the connection used by this source. Should reference a Prometheus-compatible connection. + * description: ID of the connection used by this source. Either a ClickHouse connection holding the TimeSeries table, or a Prometheus-compatible connection. * example: 507f1f77bcf86cd799439012 * from: * $ref: '#/components/schemas/SourceFrom' - * description: Required by the API for all source kinds; not used when querying a Prometheus endpoint (empty strings are not accepted — use placeholder values such as "default"). + * description: The database and TimeSeries table to query on a ClickHouse connection. Not used for a Prometheus-compatible connection, but still required (empty strings are not accepted — use placeholder values such as "default"). * querySettings: * type: array * description: Optional ClickHouse query settings applied when querying this source. @@ -894,7 +910,7 @@ function formatExternalSource(source: SourceDocument) { * nullable: true * timestampValueExpression: * type: string - * description: Required by the API for all source kinds; not used when querying a Prometheus endpoint. + * description: Required by the API for all source kinds; not used by PromQL queries. * example: timestamp * Source: * oneOf: @@ -1115,6 +1131,7 @@ router.post( validateRequest({ body: SourceSchemaNoId, }), + rejectPromqlSourceWhenDisabled, requireValidConnectionId, async (req, res, next) => { try { @@ -1220,6 +1237,7 @@ router.put( }), body: SourceSchemaNoId, }), + rejectPromqlSourceWhenDisabled, requireValidConnectionId, async (req, res, next) => { try {