diff --git a/.changeset/promql-source-id.md b/.changeset/promql-source-id.md new file mode 100644 index 0000000000..aa8ded27e6 --- /dev/null +++ b/.changeset/promql-source-id.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +fix: Resolve PromQL queries through the source instead of a caller-supplied connection and table + +The `/api/v1/prometheus/*` routes now take `sourceId` instead of `connectionId`, and read the connection and TimeSeries table from that PromQL source. Caller-supplied `database`/`table` params are ignored for ClickHouse and no longer forwarded to external Prometheus endpoints. Saved PromQL chart configs now require `source` and no longer persist `connection`, so provisioned dashboards with PromQL tiles must set `source`. diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index fd1b734d40..028164edbe 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -10,7 +10,10 @@ import { isTimeSeriesDisplayType, splitAndTrimWithBracket, } from '@hyperdx/common-utils/dist/core/utils'; -import { isBuilderSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { + isBuilderSavedChartConfig, + isPromqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import { UnknownVariableError } from '@hyperdx/common-utils/dist/macroErrors'; import type { ChartConfigWithDateRange, @@ -618,6 +621,11 @@ export async function runConfigTile( } } + // PromQL is evaluated by the Prometheus API, not the SQL renderer. + if (isPromqlSavedChartConfig(savedConfig)) { + return mcpUserError('PromQL tiles cannot be queried through MCP yet.'); + } + // Raw SQL tile — hydrate source fields for macro support ($__sourceTable, $__filters) let sourceFields: { from?: { databaseName: string; tableName: string }; diff --git a/packages/api/src/middleware/__tests__/queryAttribution.int.test.ts b/packages/api/src/middleware/__tests__/queryAttribution.int.test.ts index 67d32a9327..6632c046a8 100644 --- a/packages/api/src/middleware/__tests__/queryAttribution.int.test.ts +++ b/packages/api/src/middleware/__tests__/queryAttribution.int.test.ts @@ -1,5 +1,6 @@ import { buildLogComment } from '@hyperdx/common-utils/dist/clickhouse'; import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; import * as config from '@/config'; import { @@ -11,6 +12,7 @@ import { seedTimeSeriesTagsTable, } from '@/fixtures'; import Connection from '@/models/connection'; +import { Source } from '@/models/source'; declare global { // Set by jest.setup.ts before it stubs `fetch`. @@ -118,14 +120,18 @@ describe('request query attribution', () => { username: config.CLICKHOUSE_USER, password: config.CLICKHOUSE_PASSWORD, }); + const source = await Source.create({ + kind: SourceKind.Promql, + team: team._id, + connection: conn._id, + name: 'PromQL', + from: { databaseName: DEFAULT_DATABASE, tableName: TABLE }, + timestampValueExpression: 'timestamp', + }); await agent .get('/v1/prometheus/labels') - .query({ - connectionId: conn._id.toString(), - database: DEFAULT_DATABASE, - table: TABLE, - }) + .query({ sourceId: source._id.toString() }) .expect(200); const logged = await findRouteQueries(); @@ -153,6 +159,14 @@ describe('request query attribution', () => { username: config.CLICKHOUSE_USER, password: config.CLICKHOUSE_PASSWORD, }); + const source = await Source.create({ + kind: SourceKind.Promql, + team: team._id, + connection: conn._id, + name: 'PromQL', + from: { databaseName: DEFAULT_DATABASE, tableName: TABLE }, + timestampValueExpression: 'timestamp', + }); const tile = `tile-${Date.now()}`; await agent @@ -170,9 +184,7 @@ describe('request query attribution', () => { .query({ query: 'attribution_metric', time: '1700003600', - connectionId: conn._id.toString(), - database: DEFAULT_DATABASE, - table: TABLE, + sourceId: source._id.toString(), }) .expect(200); @@ -202,6 +214,14 @@ describe('request query attribution', () => { username: config.CLICKHOUSE_USER, password: config.CLICKHOUSE_PASSWORD, }); + const source = await Source.create({ + kind: SourceKind.Promql, + team: team._id, + connection: conn._id, + name: 'PromQL', + from: { databaseName: DEFAULT_DATABASE, tableName: TABLE }, + timestampValueExpression: 'timestamp', + }); const tile = `tile-${Date.now()}`; await agent @@ -213,9 +233,7 @@ describe('request query attribution', () => { .query({ query: 'attribution_metric', time: '1700003600', - connectionId: conn._id.toString(), - database: DEFAULT_DATABASE, - table: TABLE, + sourceId: source._id.toString(), }) .expect(200); diff --git a/packages/api/src/routers/api/__tests__/iac.int.test.ts b/packages/api/src/routers/api/__tests__/iac.int.test.ts index 131e1391e5..f2de03b6b2 100644 --- a/packages/api/src/routers/api/__tests__/iac.int.test.ts +++ b/packages/api/src/routers/api/__tests__/iac.int.test.ts @@ -97,7 +97,7 @@ describe('iac router', () => { config: { configType: 'promql', promqlExpression: 'up', - connection: randomMongoId(), + source: randomMongoId(), displayType: 'line', }, }, diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts index 1033e270d8..f0e9bb826f 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts @@ -3,6 +3,7 @@ import { QUERY_ATTRIBUTION_HEADER, } from '@hyperdx/common-utils/dist/clickhouse'; import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { Types } from 'mongoose'; import * as config from '@/config'; @@ -15,6 +16,7 @@ import { seedTimeSeriesTagsTable, } from '@/fixtures'; import Connection from '@/models/connection'; +import { Source } from '@/models/source'; import { CLICKHOUSE_PROMETHEUS_API_PREFIX, clickhousePrometheusUpstream, @@ -80,11 +82,28 @@ describe('prometheus router', () => { await server.stop(); }); - const seedPrometheusConnection = async ( + const seedPromqlSource = ( + connection: { _id: Types.ObjectId | string; team: Types.ObjectId }, + { + name = 'PromQL', + databaseName = DEFAULT_DATABASE, + tableName = 'metrics_ts', + }: { name?: string; databaseName?: string; tableName?: string } = {}, + ) => + Source.create({ + kind: SourceKind.Promql, + team: connection.team, + connection: connection._id, + name, + from: { databaseName, tableName }, + timestampValueExpression: 'timestamp', + }); + + const seedPrometheusSource = async ( teamId: Types.ObjectId, host = 'http://prom.example.com', ) => { - return Connection.create({ + const connection = await Connection.create({ team: teamId, name: 'Prom', host, @@ -92,16 +111,21 @@ describe('prometheus router', () => { password: '', isPrometheusEndpoint: true, }); + return seedPromqlSource(connection); }; - const seedClickHouseConnection = async (teamId: Types.ObjectId) => { - return Connection.create({ + const seedClickHouseSource = async ( + teamId: Types.ObjectId, + from?: { databaseName?: string; tableName?: string }, + ) => { + const connection = await Connection.create({ team: teamId, name: 'CH', host: config.CLICKHOUSE_HOST, username: config.CLICKHOUSE_USER, password: config.CLICKHOUSE_PASSWORD, }); + return seedPromqlSource(connection, from); }; describe('GET /v1/prometheus/query_range', () => { @@ -114,7 +138,7 @@ describe('prometheus router', () => { const { agent } = await getLoggedInAgent(server); const res = await agent .get('/v1/prometheus/query_range') - .query({ connectionId: new Types.ObjectId().toString() }) + .query({ sourceId: new Types.ObjectId().toString() }) .expect(400); expect(res.body).toMatchObject({ status: 'error', @@ -123,32 +147,50 @@ describe('prometheus router', () => { }); }); - it('returns 400 when connectionId parameter is missing', async () => { + it('returns 400 when sourceId parameter is missing', async () => { const { agent } = await getLoggedInAgent(server); const res = await agent .get('/v1/prometheus/query_range') .query({ query: 'up' }) .expect(400); - expect(res.body).toMatchObject({ + expect(res.body).toEqual({ status: 'error', errorType: 'bad_data', - error: expect.stringContaining('connectionId'), + error: 'missing required parameter: sourceId', }); }); - it('returns 404 when connection does not exist', async () => { + it('returns 404 when source does not exist', async () => { const { agent } = await getLoggedInAgent(server); const res = await agent .get('/v1/prometheus/query_range') .query({ query: 'up', - connectionId: new Types.ObjectId().toString(), + sourceId: new Types.ObjectId().toString(), }) .expect(404); + expect(res.body).toMatchObject({ + status: 'error', + error: 'Source not found', + }); + }); + + it("returns 404 when the source's connection no longer exists", async () => { + const { agent, team } = await getLoggedInAgent(server); + const source = await seedPromqlSource({ + _id: new Types.ObjectId(), + team: team._id, + }); + + const res = await agent + .get('/v1/prometheus/query_range') + .query({ query: 'up', sourceId: source._id.toString() }) + .expect(404); expect(res.body).toMatchObject({ status: 'error', error: 'Connection not found', }); + expect(mockFetch).not.toHaveBeenCalled(); }); // The connection host is member-configured, so its response body is untrusted @@ -157,7 +199,7 @@ describe('prometheus router', () => { // render as script here. it('never forwards a non-JSON upstream content-type, and always sends nosniff', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); mockFetch.mockResolvedValueOnce( fakeUpstreamResponse( @@ -173,7 +215,7 @@ describe('prometheus router', () => { query: 'up', start: '1700000000', end: '1700000060', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -189,7 +231,7 @@ describe('prometheus router', () => { // separate Content-Type headers into exactly this shape. it('does not forward a JSON content-type that carries a second media type', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); mockFetch.mockResolvedValueOnce( fakeUpstreamResponse( @@ -205,7 +247,7 @@ describe('prometheus router', () => { query: 'up', start: '1700000000', end: '1700000060', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -220,7 +262,7 @@ describe('prometheus router', () => { // fallback: the upstream value must not survive even when it is valid JSON. it('relabels even a legitimate non-standard JSON content-type', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); mockFetch.mockResolvedValueOnce( fakeUpstreamResponse( @@ -236,7 +278,7 @@ describe('prometheus router', () => { query: 'up', start: '1700000000', end: '1700000060', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -251,7 +293,7 @@ describe('prometheus router', () => { // connection host into a body the browser shows. it('sends nosniff on a 502 and redacts credentials from the message', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://user:s3cr3t@prom.example.com', ); @@ -268,7 +310,7 @@ describe('prometheus router', () => { query: 'up', start: '1700000000', end: '1700000060', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(502); @@ -284,7 +326,7 @@ describe('prometheus router', () => { // secret-shaped parts of it, or this leaks into the 502 body. it('does not leak a secret pinned in the host query string into the 502 message', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://prom.example.com?authKey=super-secret-token', ); @@ -301,7 +343,7 @@ describe('prometheus router', () => { query: 'up', start: '1700000000', end: '1700000060', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(502); @@ -316,7 +358,7 @@ describe('prometheus router', () => { // `target` here specifically) would still be caught. it('does not leak credentials or a host query-string secret into the 504 message', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://user:s3cr3t@prom.example.com?authKey=super-secret-token', ); @@ -333,7 +375,7 @@ describe('prometheus router', () => { query: 'up', start: '1700000000', end: '1700000060', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(504); @@ -346,7 +388,7 @@ describe('prometheus router', () => { // which echo the caller-supplied host — carry it too. it('sends nosniff on its own error responses', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id, 'not-a-valid-url'); + const source = await seedPrometheusSource(team._id, 'not-a-valid-url'); const res = await agent .get('/v1/prometheus/query_exemplars') @@ -354,7 +396,7 @@ describe('prometheus router', () => { query: 'up', start: '1700000000', end: '1700000060', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(400); @@ -363,7 +405,7 @@ describe('prometheus router', () => { it('returns 400 for a scheme-less connection host instead of 502', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id, 'prometheus:9090'); + const source = await seedPrometheusSource(team._id, 'prometheus:9090'); const res = await agent .get('/v1/prometheus/query_range') @@ -372,7 +414,7 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '15s', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(400); @@ -389,7 +431,7 @@ describe('prometheus router', () => { // interpolate the raw host back in would fail the suite. it('never echoes the connection host (or any credentials in it) in the 400 body', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'user:secret-password@prometheus:9090', ); @@ -401,7 +443,7 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '15s', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(400); @@ -413,7 +455,7 @@ describe('prometheus router', () => { it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); const promResponse = { status: 'success', @@ -428,7 +470,7 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '15s', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -438,14 +480,14 @@ describe('prometheus router', () => { expect(calledUrl).toContain('http://prom.example.com'); expect(calledUrl).toContain('/api/v1/query_range'); expect(calledUrl).toContain('query=up'); - expect(calledUrl).not.toContain('connectionId'); + expect(calledUrl).not.toContain('sourceId'); }); // Pins the call site, not just the helper: reverting // `url = new URL(path, upstreamHost)` would drop `/select/0/prometheus`. it('keeps a VictoriaMetrics cluster path prefix on the upstream URL', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://vmselect:8481/select/0/prometheus', ); @@ -464,7 +506,7 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '15s', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -480,7 +522,7 @@ describe('prometheus router', () => { // silently stop authenticating. it('still sends real credentials to the actual upstream fetch call', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://user:pw@prom.example.com', ); @@ -499,7 +541,7 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '15s', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -508,7 +550,7 @@ describe('prometheus router', () => { it('leaves a host-pinned param untouched when the request never references it', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://vmselect:8481/select/0/prometheus?extra_label=namespace%3Dprod', ); @@ -527,7 +569,7 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '15s', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -543,7 +585,7 @@ describe('prometheus router', () => { // connection return the same series regardless of what was requested). it('lets a request query param override a same-named one pinned on the connection host', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://vmselect:8481/select/0/prometheus?query=host_pinned_query', ); @@ -562,7 +604,7 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '15s', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -577,7 +619,7 @@ describe('prometheus router', () => { // guarantee for every param this list claims to cover. it('forwards a request timeout, overriding one pinned on the connection host', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://prom.example.com?timeout=5s', ); @@ -597,7 +639,7 @@ describe('prometheus router', () => { end: '1700000060', step: '15s', timeout: '30s', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -610,7 +652,7 @@ describe('prometheus router', () => { // be caught. it('forwards a request stats value, overriding one pinned on the connection host', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://prom.example.com?stats=all', ); @@ -630,7 +672,7 @@ describe('prometheus router', () => { end: '1700000060', step: '15s', stats: 'none', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -647,7 +689,7 @@ describe('prometheus router', () => { // param just by naming it. it('does not let a request override a non-Prometheus param pinned on the connection host', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://vmselect:8481/select/0/prometheus?extra_label=namespace%3Dprod', ); @@ -667,7 +709,7 @@ describe('prometheus router', () => { end: '1700000060', step: '15s', extra_label: 'namespace=other', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -678,7 +720,7 @@ describe('prometheus router', () => { it('lets the request step override a step pinned on the connection host', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://prom.example.com?step=5m', ); @@ -697,7 +739,7 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '15s', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -705,9 +747,12 @@ describe('prometheus router', () => { expect(requested.searchParams.get('step')).toBe('15s'); }); - it('proxies a ClickHouse connection to its prometheus_api_v1 handler with db/table and credentials', async () => { + it("proxies a ClickHouse connection to its prometheus_api_v1 handler with the source's db/table and credentials", async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + databaseName: 'otel', + tableName: 'metrics_ts', + }); const promResponse = { status: 'success', data: { resultType: 'matrix', result: [] }, @@ -723,9 +768,7 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '15s', - database: 'otel', - table: 'metrics_ts', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -746,16 +789,48 @@ describe('prometheus router', () => { expect(calledUrl).toContain('table=metrics_ts'); expect(calledUrl).toContain('max_execution_time=30'); expect(calledUrl).toContain('max_result_rows=100000'); - expect(calledUrl).not.toContain('connectionId'); + expect(calledUrl).not.toContain('sourceId'); expect(init?.headers).toMatchObject({ 'X-ClickHouse-User': config.CLICKHOUSE_USER, 'X-ClickHouse-Key': config.CLICKHOUSE_PASSWORD, }); }); + it.each(['query_range', 'query'])( + "ignores a caller-supplied database/table on /%s and queries the source's", + async endpoint => { + const { agent, team } = await getLoggedInAgent(server); + const source = await seedClickHouseSource(team._id, { + databaseName: 'otel', + tableName: 'metrics_ts', + }); + mockFetch.mockResolvedValueOnce(handlerProbeResponse()); + + await agent + .get(`/v1/prometheus/${endpoint}`) + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + step: '15s', + database: 'other_db', + table: 'other_table', + sourceId: source._id.toString(), + }) + .expect(200); + + const calledUrl = new URL(String(mockFetch.mock.calls[1][0])); + expect(calledUrl.pathname).toBe( + `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/${endpoint}`, + ); + expect(calledUrl.searchParams.getAll('database')).toEqual(['otel']); + expect(calledUrl.searchParams.getAll('table')).toEqual(['metrics_ts']); + }, + ); + it('tags a request proxied to prometheus_api_v1 with log_comment and query_id', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id); mockFetch.mockResolvedValueOnce(handlerProbeResponse()); await agent @@ -769,10 +844,9 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '15s', - table: 'metrics_ts', // Not caller-settable, so this never reaches the upstream. log_comment: 'from-caller', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -794,7 +868,7 @@ describe('prometheus router', () => { it('does not let the request loosen the pinned ClickHouse limits', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id); mockFetch.mockResolvedValueOnce(handlerProbeResponse()); await agent @@ -804,10 +878,9 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '15s', - table: 'metrics_ts', max_execution_time: '9999', max_result_rows: '0', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -822,7 +895,9 @@ describe('prometheus router', () => { it('falls back to prometheusQueryRange() when the server has no prometheus_api_v1 handler', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: 'no_such_table_for_fallback', + }); // A stock ClickHouse answers the probe with a plain-text 404. mockFetch.mockResolvedValueOnce({ status: 404, @@ -840,8 +915,7 @@ describe('prometheus router', () => { start: '1700000000', end: '1700000060', step: '60s', - table: 'no_such_table_for_fallback', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(400); @@ -889,9 +963,11 @@ describe('prometheus router', () => { mockFetch.mockImplementation(realFetch); }); - it('answers a range query with the series from the requested table', async () => { + it("answers a range query with the series from the source's table", async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: TABLE, + }); const res = await agent .get('/v1/prometheus/query_range') @@ -900,9 +976,7 @@ describe('prometheus router', () => { start: String(START), end: String(START + 60), step: '60s', - database: DEFAULT_DATABASE, - table: TABLE, - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -923,16 +997,16 @@ describe('prometheus router', () => { it('answers an instant query', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: TABLE, + }); const res = await agent .get('/v1/prometheus/query') .query({ query: 'count(live_up)', time: String(START + 60), - database: DEFAULT_DATABASE, - table: TABLE, - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -1007,14 +1081,15 @@ describe('prometheus router', () => { it('relays a PromQL parse error as a Prometheus 400', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: TABLE, + }); const res = await agent .get('/v1/prometheus/query') .query({ query: 'live_up(', - table: TABLE, - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(400); @@ -1061,7 +1136,9 @@ describe('prometheus router', () => { it('evaluates through prometheusQueryRange() instead of proxying', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: TABLE, + }); const res = await agent .get('/v1/prometheus/query_range') @@ -1070,9 +1147,7 @@ describe('prometheus router', () => { start: String(START), end: String(START + 60), step: '60s', - database: DEFAULT_DATABASE, - table: TABLE, - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -1089,16 +1164,16 @@ describe('prometheus router', () => { it('evaluates an instant query through prometheusQuery()', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: TABLE, + }); const res = await agent .get('/v1/prometheus/query') .query({ query: 'fallback_up', time: String(START + 60), - database: DEFAULT_DATABASE, - table: TABLE, - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -1111,29 +1186,40 @@ describe('prometheus router', () => { }, ]); }); - }); - - it('returns 400 before proxying when a ClickHouse connection has no table', async () => { - const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); - - const res = await agent - .get('/v1/prometheus/query_range') - .query({ - query: 'up', - start: '1700000000', - end: '1700000060', - connectionId: conn._id.toString(), - }) - .expect(400); - expect(res.body.error).toContain('table parameter required'); - expect(mockFetch).not.toHaveBeenCalled(); + // A caller-named table that does not exist would fail the table function, + // so a 200 with the seeded series shows the source's table was queried. + it.each([ + ['query_range', { start: String(START), end: String(START + 60) }], + ['query', { time: String(START + 60) }], + ])( + "ignores a caller-supplied database/table on /%s and queries the source's", + async (endpoint, window) => { + const { agent, team } = await getLoggedInAgent(server); + const source = await seedClickHouseSource(team._id, { + tableName: TABLE, + }); + + const res = await agent + .get(`/v1/prometheus/${endpoint}`) + .query({ + query: 'fallback_up', + ...window, + database: 'no_such_database', + table: 'no_such_table', + sourceId: source._id.toString(), + }) + .expect(200); + + expect(res.body.status).toBe('success'); + expect(res.body.data.result).toHaveLength(1); + }, + ); }); it('returns 400 with Prometheus-compatible error when resolution exceeds 11,000 points', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id); const res = await agent .get('/v1/prometheus/query_range') @@ -1142,8 +1228,7 @@ describe('prometheus router', () => { start: '0', end: '1700000000', step: '1s', - connectionId: conn._id.toString(), - table: 'metrics_ts', + sourceId: source._id.toString(), }) .expect(400); expect(res.body).toMatchObject({ @@ -1160,11 +1245,11 @@ describe('prometheus router', () => { const { agent } = await getLoggedInAgent(server); await agent .get('/v1/prometheus/query') - .query({ connectionId: new Types.ObjectId().toString() }) + .query({ sourceId: new Types.ObjectId().toString() }) .expect(400); }); - it('returns 400 when connectionId parameter is missing', async () => { + it('returns 400 when sourceId parameter is missing', async () => { const { agent } = await getLoggedInAgent(server); await agent .get('/v1/prometheus/query') @@ -1172,20 +1257,20 @@ describe('prometheus router', () => { .expect(400); }); - it('returns 404 when connection does not exist', async () => { + it('returns 404 when source does not exist', async () => { const { agent } = await getLoggedInAgent(server); await agent .get('/v1/prometheus/query') .query({ query: 'up', - connectionId: new Types.ObjectId().toString(), + sourceId: new Types.ObjectId().toString(), }) .expect(404); }); it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); const promResponse = { status: 'success', @@ -1195,7 +1280,7 @@ describe('prometheus router', () => { const res = await agent .get('/v1/prometheus/query') - .query({ query: 'up', connectionId: conn._id.toString() }) + .query({ query: 'up', sourceId: source._id.toString() }) .expect(200); expect(res.body).toEqual(promResponse); @@ -1207,7 +1292,9 @@ describe('prometheus router', () => { it('proxies a ClickHouse connection to its prometheus_api_v1 handler', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: 'metrics_ts', + }); mockFetch .mockResolvedValueOnce(handlerProbeResponse()) .mockResolvedValueOnce( @@ -1221,8 +1308,7 @@ describe('prometheus router', () => { .get('/v1/prometheus/query') .query({ query: 'up', - table: 'metrics_ts', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -1240,7 +1326,7 @@ describe('prometheus router', () => { }); describe('GET /v1/prometheus/label/:name/values', () => { - it('returns 400 when connectionId parameter is missing', async () => { + it('returns 400 when sourceId parameter is missing', async () => { const { agent } = await getLoggedInAgent(server); await agent.get('/v1/prometheus/label/__name__/values').expect(400); }); @@ -1249,40 +1335,40 @@ describe('prometheus router', () => { // instead turns a client error into a cast failure that the handler's // catch-all reports as a backend fault. it.each(['', 'not-an-object-id'])( - 'returns 400 for connectionId %p', - async connectionId => { + 'returns 400 for sourceId %p', + async sourceId => { const { agent } = await getLoggedInAgent(server); const res = await agent .get('/v1/prometheus/label/__name__/values') - .query({ connectionId }) + .query({ sourceId }) .expect(400); expect(res.body).toMatchObject({ status: 'error', errorType: 'bad_data', - error: expect.stringContaining('connectionId'), + error: expect.stringContaining('sourceId'), }); }, ); - it('returns 404 when connection does not exist', async () => { + it('returns 404 when source does not exist', async () => { const { agent } = await getLoggedInAgent(server); await agent .get('/v1/prometheus/label/__name__/values') - .query({ connectionId: new Types.ObjectId().toString() }) + .query({ sourceId: new Types.ObjectId().toString() }) .expect(404); }); it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); const promResponse = { status: 'success', data: ['up', 'requests'] }; mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse)); const res = await agent .get('/v1/prometheus/label/__name__/values') - .query({ connectionId: conn._id.toString() }) + .query({ sourceId: source._id.toString() }) .expect(200); expect(res.body).toEqual(promResponse); @@ -1292,12 +1378,12 @@ describe('prometheus router', () => { it('forwards normalized bounds upstream', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/label/job/values') .query({ - connectionId: conn._id.toString(), + sourceId: source._id.toString(), start: '2023-11-14T22:13:20Z', end: '1700000060', }) @@ -1310,11 +1396,11 @@ describe('prometheus router', () => { it('sends no bounds upstream when the caller gives none', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/label/job/values') - .query({ connectionId: conn._id.toString() }) + .query({ sourceId: source._id.toString() }) .expect(200); const requested = new URL(String(mockFetch.mock.calls[0][0])); @@ -1326,11 +1412,11 @@ describe('prometheus router', () => { // epoch, which is what parseTimestamp('') -> Number('') would make it. it('treats an empty bound as absent rather than as the epoch', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/label/job/values') - .query({ connectionId: conn._id.toString(), start: '', end: '' }) + .query({ sourceId: source._id.toString(), start: '', end: '' }) .expect(200); const requested = new URL(String(mockFetch.mock.calls[0][0])); @@ -1342,11 +1428,11 @@ describe('prometheus router', () => { // would 400 every request carrying a limit. it('forwards limit upstream', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/label/job/values') - .query({ connectionId: conn._id.toString(), limit: '25' }) + .query({ sourceId: source._id.toString(), limit: '25' }) .expect(200); const requested = new URL(String(mockFetch.mock.calls[0][0])); @@ -1355,11 +1441,11 @@ describe('prometheus router', () => { it('treats an empty limit as absent', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/label/job/values') - .query({ connectionId: conn._id.toString(), limit: '' }) + .query({ sourceId: source._id.toString(), limit: '' }) .expect(200); const requested = new URL(String(mockFetch.mock.calls[0][0])); @@ -1370,11 +1456,11 @@ describe('prometheus router', () => { // limit means. Rejecting it would break a caller Prometheus itself accepts. it('drops a zero limit and still proxies', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/label/job/values') - .query({ connectionId: conn._id.toString(), limit: '0' }) + .query({ sourceId: source._id.toString(), limit: '0' }) .expect(200); const requested = new URL(String(mockFetch.mock.calls[0][0])); @@ -1389,14 +1475,14 @@ describe('prometheus router', () => { // reaching upstream, which the previous test guards separately). it('lets a request limit of 0 clear a non-zero limit pinned on the connection host', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://prom.example.com?limit=100', ); await agent .get('/v1/prometheus/label/job/values') - .query({ connectionId: conn._id.toString(), limit: '0' }) + .query({ sourceId: source._id.toString(), limit: '0' }) .expect(200); const requested = new URL(String(mockFetch.mock.calls[0][0])); @@ -1405,11 +1491,11 @@ describe('prometheus router', () => { it('rejects a negative limit', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/label/job/values') - .query({ connectionId: conn._id.toString(), limit: '-1' }) + .query({ sourceId: source._id.toString(), limit: '-1' }) .expect(400); expect(mockFetch).not.toHaveBeenCalled(); @@ -1417,11 +1503,11 @@ describe('prometheus router', () => { it('rejects a non-numeric limit', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/label/job/values') - .query({ connectionId: conn._id.toString(), limit: 'lots' }) + .query({ sourceId: source._id.toString(), limit: 'lots' }) .expect(400); expect(mockFetch).not.toHaveBeenCalled(); @@ -1429,11 +1515,11 @@ describe('prometheus router', () => { it('rejects a malformed bound before reaching upstream', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); const res = await agent .get('/v1/prometheus/label/job/values') - .query({ connectionId: conn._id.toString(), start: 'not-a-time' }) + .query({ sourceId: source._id.toString(), start: 'not-a-time' }) .expect(400); expect(res.body).toMatchObject({ @@ -1448,12 +1534,12 @@ describe('prometheus router', () => { // backends answering the same request the same way. it('rejects an inverted range before reaching upstream', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); const res = await agent .get('/v1/prometheus/label/job/values') .query({ - connectionId: conn._id.toString(), + sourceId: source._id.toString(), start: '1700000000', end: '1600000000', }) @@ -1472,12 +1558,12 @@ describe('prometheus router', () => { // forwarded verbatim — a selector is PromQL, which only upstream parses. it('forwards every match[] selector verbatim', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/label/job/values') .query( - `connectionId=${conn._id.toString()}&start=2023-11-14T22:13:20Z` + + `sourceId=${source._id.toString()}&start=2023-11-14T22:13:20Z` + `&match[]=${encodeURIComponent('up{env="prod"}')}` + `&match[]=${encodeURIComponent('down')}`, ) @@ -1496,11 +1582,11 @@ describe('prometheus router', () => { // Prometheus reads. it('restores the bracketed name for an unbracketed match', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/label/job/values') - .query({ connectionId: conn._id.toString(), match: 'up' }) + .query({ sourceId: source._id.toString(), match: 'up' }) .expect(200); const requested = new URL(String(mockFetch.mock.calls[0][0])); @@ -1510,11 +1596,11 @@ describe('prometheus router', () => { it('treats an empty match as absent', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/label/job/values') - .query({ connectionId: conn._id.toString(), match: '' }) + .query({ sourceId: source._id.toString(), match: '' }) .expect(200); const requested = new URL(String(mockFetch.mock.calls[0][0])); @@ -1528,7 +1614,7 @@ describe('prometheus router', () => { // return the wrong series list). it('replaces a host-pinned match[] with the request values, not appends alongside them', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, 'http://prom.example.com?match%5B%5D=host-pinned-series', ); @@ -1536,7 +1622,7 @@ describe('prometheus router', () => { await agent .get('/v1/prometheus/label/job/values') .query({ - connectionId: conn._id.toString(), + sourceId: source._id.toString(), match: ['requested-series-1', 'requested-series-2'], }) .expect(200); @@ -1567,15 +1653,12 @@ describe('prometheus router', () => { { labelName = '__name__', match = [] as string[] } = {}, ) => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: TABLE, + }); const res = await agent .get(`/v1/prometheus/label/${labelName}/values`) - .query({ - connectionId: conn._id.toString(), - database: DEFAULT_DATABASE, - table: TABLE, - ...query, - }) + .query({ sourceId: source._id.toString(), ...query }) .query(matchQueryString(match)) .expect(200); return res.body; @@ -1628,14 +1711,14 @@ describe('prometheus router', () => { // inverted gap, while the proxy path 400s the identical request. it('rejects an inverted range instead of answering', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: TABLE, + }); const res = await agent .get('/v1/prometheus/label/__name__/values') .query({ - connectionId: conn._id.toString(), - database: DEFAULT_DATABASE, - table: TABLE, + sourceId: source._id.toString(), start: String(RECENT_START), end: String(OLD_START), }) @@ -1717,14 +1800,14 @@ describe('prometheus router', () => { it('answers 400 when a selector is not valid PromQL', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: TABLE, + }); const res = await agent .get('/v1/prometheus/label/__name__/values') .query({ - connectionId: conn._id.toString(), - database: DEFAULT_DATABASE, - table: TABLE, + sourceId: source._id.toString(), 'match[]': 'up{', }) .expect(400); @@ -1739,22 +1822,22 @@ describe('prometheus router', () => { }); describe('GET /v1/prometheus/labels', () => { - it('returns 400 when connectionId parameter is missing', async () => { + it('returns 400 when sourceId parameter is missing', async () => { const { agent } = await getLoggedInAgent(server); await agent.get('/v1/prometheus/labels').expect(400); }); - it('returns 404 when connection does not exist', async () => { + it('returns 404 when source does not exist', async () => { const { agent } = await getLoggedInAgent(server); await agent .get('/v1/prometheus/labels') - .query({ connectionId: new Types.ObjectId().toString() }) + .query({ sourceId: new Types.ObjectId().toString() }) .expect(404); }); it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); const promResponse = { status: 'success', @@ -1764,7 +1847,7 @@ describe('prometheus router', () => { const res = await agent .get('/v1/prometheus/labels') - .query({ connectionId: conn._id.toString() }) + .query({ sourceId: source._id.toString() }) .expect(200); expect(res.body).toEqual(promResponse); @@ -1774,12 +1857,12 @@ describe('prometheus router', () => { it('forwards normalized bounds, limit, and match[] upstream', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); await agent .get('/v1/prometheus/labels') .query( - `connectionId=${conn._id.toString()}&start=2023-11-14T22:13:20Z` + + `sourceId=${source._id.toString()}&start=2023-11-14T22:13:20Z` + `&end=1700000060&limit=25&match[]=${encodeURIComponent('up')}`, ) .expect(200); @@ -1793,11 +1876,11 @@ describe('prometheus router', () => { it('rejects a malformed bound before reaching upstream', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); const res = await agent .get('/v1/prometheus/labels') - .query({ connectionId: conn._id.toString(), start: 'not-a-time' }) + .query({ sourceId: source._id.toString(), start: 'not-a-time' }) .expect(400); expect(res.body).toMatchObject({ @@ -1821,15 +1904,12 @@ describe('prometheus router', () => { { match = [] as string[] } = {}, ) => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: TABLE, + }); const res = await agent .get('/v1/prometheus/labels') - .query({ - connectionId: conn._id.toString(), - database: DEFAULT_DATABASE, - table: TABLE, - ...query, - }) + .query({ sourceId: source._id.toString(), ...query }) .query(matchQueryString(match)) .expect(200); return res.body; @@ -1883,18 +1963,15 @@ describe('prometheus router', () => { }); }); - it('returns 400 when table is missing', async () => { - const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); - - const res = await agent - .get('/v1/prometheus/labels') - .query({ connectionId: conn._id.toString() }) - .expect(400); - - expect(res.body).toMatchObject({ - status: 'error', - error: expect.stringContaining('table'), + it("ignores a caller-supplied database/table and reads the source's", async () => { + expect( + await labels({ + database: 'no_such_database', + table: 'no_such_table', + }), + ).toEqual({ + status: 'success', + data: ['__name__', 'job', 'region'], }); }); @@ -1925,14 +2002,14 @@ describe('prometheus router', () => { it('answers 400 when a selector is not valid PromQL', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id, { + tableName: TABLE, + }); const res = await agent .get('/v1/prometheus/labels') .query({ - connectionId: conn._id.toString(), - database: DEFAULT_DATABASE, - table: TABLE, + sourceId: source._id.toString(), 'match[]': 'up{', }) .expect(400); @@ -1951,7 +2028,7 @@ describe('prometheus router', () => { const { agent } = await getLoggedInAgent(server); const res = await agent .get('/v1/prometheus/query_exemplars') - .query({ connectionId: new Types.ObjectId().toString() }) + .query({ sourceId: new Types.ObjectId().toString() }) .expect(400); expect(res.body).toMatchObject({ status: 'error', @@ -1960,7 +2037,7 @@ describe('prometheus router', () => { }); }); - it('returns 400 when connectionId is missing', async () => { + it('returns 400 when sourceId is missing', async () => { const { agent } = await getLoggedInAgent(server); const res = await agent .get('/v1/prometheus/query_exemplars') @@ -1969,33 +2046,31 @@ describe('prometheus router', () => { expect(res.body).toMatchObject({ status: 'error', errorType: 'bad_data', - error: expect.stringContaining('connectionId'), + error: expect.stringContaining('sourceId'), }); expect(mockFetch).not.toHaveBeenCalled(); }); - it('returns 404 for a connection owned by another team', async () => { + it('returns 404 for a source owned by another team', async () => { const { agent } = await getLoggedInAgent(server); - // A real, resolvable connection — just not this team's. The 404 must come + // A real, resolvable source — just not this team's. The 404 must come // from the team scoping, not from the id simply not existing. - const otherTeamConn = await seedPrometheusConnection( - new Types.ObjectId(), - ); + const otherTeamSource = await seedPrometheusSource(new Types.ObjectId()); const res = await agent .get('/v1/prometheus/query_exemplars') - .query({ query: 'up', connectionId: otherTeamConn._id.toString() }) + .query({ query: 'up', sourceId: otherTeamSource._id.toString() }) .expect(404); expect(res.body).toMatchObject({ status: 'error', - error: 'Connection not found', + error: 'Source not found', }); expect(mockFetch).not.toHaveBeenCalled(); }); it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); const promResponse = { status: 'success', data: [] }; mockFetch.mockResolvedValueOnce(fakeUpstreamResponse(promResponse)); @@ -2006,7 +2081,7 @@ describe('prometheus router', () => { query: 'up', start: '1700000000', end: '1700000060', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -2022,7 +2097,7 @@ describe('prometheus router', () => { // the clamp to bite and inspects what was actually requested. it('narrows an over-wide window on the outgoing request and keeps end', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedPrometheusConnection(team._id); + const source = await seedPrometheusSource(team._id); mockFetch.mockResolvedValueOnce( fakeUpstreamResponse({ status: 'success', data: [] }), @@ -2037,7 +2112,7 @@ describe('prometheus router', () => { query: 'up', start: String(end - thirtyDays), end: String(end), - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -2056,7 +2131,7 @@ describe('prometheus router', () => { // below can only pass if the request's value actually won -- if the // host's pinned value were used instead, `end` wouldn't match. const hostEnd = end - 3600; - const conn = await seedPrometheusConnection( + const source = await seedPrometheusSource( team._id, `http://prom.example.com?start=${hostEnd - thirtyDays}&end=${hostEnd}`, ); @@ -2071,7 +2146,7 @@ describe('prometheus router', () => { query: 'up', start: String(end - thirtyDays), end: String(end), - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -2083,7 +2158,7 @@ describe('prometheus router', () => { it('returns an empty result for ClickHouse-backed connections (no native exemplar table function)', async () => { const { agent, team } = await getLoggedInAgent(server); - const conn = await seedClickHouseConnection(team._id); + const source = await seedClickHouseSource(team._id); const res = await agent .get('/v1/prometheus/query_exemplars') @@ -2091,7 +2166,7 @@ describe('prometheus router', () => { query: 'up', start: '1700000000', end: '1700000060', - connectionId: conn._id.toString(), + sourceId: source._id.toString(), }) .expect(200); @@ -2106,15 +2181,15 @@ describe('prometheus router', () => { // enough to pin rather than leave to a comment. it('validates start/end only on the branch that reaches Prometheus', async () => { const { agent, team } = await getLoggedInAgent(server); - const prom = await seedPrometheusConnection(team._id); - const ch = await seedClickHouseConnection(team._id); + const prom = await seedPrometheusSource(team._id); + const ch = await seedClickHouseSource(team._id); await agent .get('/v1/prometheus/query_exemplars') .query({ query: 'up', start: 'not-a-time', - connectionId: prom._id.toString(), + sourceId: prom._id.toString(), }) .expect(400); @@ -2123,11 +2198,126 @@ describe('prometheus router', () => { .query({ query: 'up', start: 'not-a-time', - connectionId: ch._id.toString(), + sourceId: ch._id.toString(), }) .expect(200); expect(res.body).toEqual({ status: 'success', data: [] }); }); }); + + describe('source access', () => { + const ROUTES = [ + ['get', '/v1/prometheus/query_range'], + ['post', '/v1/prometheus/query_range'], + ['get', '/v1/prometheus/query'], + ['post', '/v1/prometheus/query'], + ['get', '/v1/prometheus/query_exemplars'], + ['post', '/v1/prometheus/query_exemplars'], + ['get', '/v1/prometheus/labels'], + ['get', '/v1/prometheus/label/job/values'], + ] as const; + + // A superset of what any one route reads; the label routes drop the + // query/step keys they don't take. + const send = ( + agent: ReturnType, + method: 'get' | 'post', + path: string, + sourceId: string, + extra: Record = {}, + ) => { + const params = { + query: 'up', + start: '1700000000', + end: '1700000060', + step: '15s', + sourceId, + ...extra, + }; + return method === 'get' + ? agent.get(path).query(params) + : agent.post(path).type('form').send(params); + }; + + it.each(ROUTES)( + '%s %s returns 404 for a source that is not a PromQL source', + async (method, path) => { + const { agent, team } = await getLoggedInAgent(server); + const promql = await seedPrometheusSource(team._id); + const logSource = await Source.create({ + kind: SourceKind.Log, + team: team._id, + connection: promql.connection, + name: 'Logs', + from: { databaseName: DEFAULT_DATABASE, tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + }); + + const res = await send( + agent, + method, + path, + logSource._id.toString(), + ).expect(404); + + expect(res.body).toMatchObject({ error: 'Source not found' }); + expect(mockFetch).not.toHaveBeenCalled(); + }, + ); + + describe('on an external Prometheus endpoint', () => { + const FORWARDING_ROUTES = [ + '/v1/prometheus/query_range', + '/v1/prometheus/query', + '/v1/prometheus/query_exemplars', + ]; + + // They select a ClickHouse TimeSeries table and mean nothing to a real + // Prometheus. + it.each(FORWARDING_ROUTES)( + 'does not forward database/table from %s', + async path => { + const { agent, team } = await getLoggedInAgent(server); + const source = await seedPrometheusSource(team._id); + + await send(agent, 'get', path, source._id.toString(), { + database: 'caller_db', + table: 'caller_table', + }).expect(200); + + const requested = new URL(String(mockFetch.mock.calls[0][0])); + expect(requested.searchParams.has('database')).toBe(false); + expect(requested.searchParams.has('table')).toBe(false); + expect(requested.searchParams.get('query')).toBe('up'); + }, + ); + + // A host can be a ClickHouse prometheus_api_v1 endpoint that pins its + // own table; the caller must not be able to repoint it. + it.each(FORWARDING_ROUTES)( + 'keeps database/table pinned on the connection host for %s', + async path => { + const { agent, team } = await getLoggedInAgent(server); + const source = await seedPrometheusSource( + team._id, + 'http://prom.example.com?database=pinned_db&table=pinned_table', + ); + + await send(agent, 'get', path, source._id.toString(), { + database: 'caller_db', + table: 'caller_table', + }).expect(200); + + const requested = new URL(String(mockFetch.mock.calls[0][0])); + expect(requested.searchParams.getAll('database')).toEqual([ + 'pinned_db', + ]); + expect(requested.searchParams.getAll('table')).toEqual([ + 'pinned_table', + ]); + }, + ); + }); + }); }); diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index 60238868e5..95cdf6e5bd 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -8,7 +8,9 @@ import { getCurrentQueryAttribution, withQueryAttribution, } from '@hyperdx/common-utils/dist/clickhouse/node'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; import express from 'express'; +import { omit } from 'lodash'; import { performance } from 'perf_hooks'; import { Readable } from 'stream'; import { pipeline } from 'stream/promises'; @@ -16,6 +18,7 @@ import z from 'zod'; import { ClickhouseClient } from '@/clickhouse'; import { getConnectionById } from '@/controllers/connection'; +import { getSource } from '@/controllers/sources'; import { connectionSupportsPrometheusHttpApi, PROMETHEUS_MAX_EXECUTION_SEC, @@ -127,6 +130,63 @@ function getParams(req: express.Request): Record { }; } +type PromqlTarget = { + connection: NonNullable>>; + databaseName: string; + tableName: string; +}; + +/** + * Resolve the PromQL source a request names into the connection and table to + * query. The table always comes from the source, never from the request, so a + * caller can only reach data through a source. + */ +async function resolvePromqlTarget( + req: express.Request, + sourceId: string | undefined, + { selectPassword }: { selectPassword: boolean }, +): Promise { + if (!sourceId) { + return { status: 400, error: 'missing required parameter: sourceId' }; + } + + const { teamId } = getNonNullUserWithTeam(req); + + const source = await getSource(teamId.toString(), sourceId); + + if (!source || source.kind !== SourceKind.Promql) { + return { status: 404, error: 'Source not found' }; + } + + const connection = await getConnectionById( + teamId.toString(), + source.connection.toString(), + selectPassword, + ); + + if (!connection) { + return { status: 404, error: 'Connection not found' }; + } + + return { + connection, + databaseName: source.from.databaseName, + tableName: source.from.tableName, + }; +} + +/** + * Params for an external Prometheus endpoint. `database` and `table` select a + * ClickHouse TimeSeries table and mean nothing to a real Prometheus, so they + * are dropped rather than forwarded: a host that pins them (a ClickHouse + * prometheus_api_v1 endpoint) keeps its own values. + */ +function prometheusEndpointParams>( + params: T, +): Omit { + return omit(params, ['database', 'table']); +} + // -------------------------- // Prometheus proxy (for real Prometheus backends) // -------------------------- @@ -517,7 +577,6 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { const startedAt = performance.now(); let backend: PrometheusBackend = 'unknown'; try { - const { teamId } = getNonNullUserWithTeam(req); const params = getParams(req); const query = params.query; @@ -529,29 +588,20 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { }); } - const connectionId = params.connectionId; - if (!connectionId) { - return res.status(400).json({ - status: 'error', - errorType: 'bad_data', - error: 'missing required parameter: connectionId', - }); - } + const target = await resolvePromqlTarget(req, params.sourceId, { + selectPassword: true, + }); - // Resolve connection to determine backend (Prometheus or ClickHouse) - const connection = await getConnectionById( - teamId.toString(), - connectionId, - true, - ); - if (!connection) { - return res.status(404).json({ + if ('error' in target) { + return res.status(target.status).json({ status: 'error', errorType: 'bad_data', - error: 'Connection not found', + error: target.error, }); } + const { connection, databaseName, tableName } = target; + // If the connection points at a Prometheus-compatible endpoint, proxy // directly to connection.host instead of running a ClickHouse query. if (connection.isPrometheusEndpoint) { @@ -559,7 +609,7 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { const status = await proxyToPrometheus( connection.host, '/api/v1/query_range', - params, + prometheusEndpointParams(params), res, ); recordProxyOutcome(status, 'query_range', backend); @@ -567,14 +617,6 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { } backend = 'clickhouse'; - const table = params.table; - if (!table) { - return res.status(400).json({ - status: 'error', - errorType: 'bad_data', - error: `table parameter required for querying clickhouse via promql`, - }); - } const start = parseTimestamp(params.start); const end = parseTimestamp(params.end); const step = parseDuration(params.step ?? '60s'); @@ -596,7 +638,7 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { const status = await proxyToPrometheus( upstream, `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query_range`, - params, + { ...params, database: databaseName, table: tableName }, res, headers, ); @@ -607,9 +649,9 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => { // fallback for clickhouse version < 26.6 const result = await queryRangeViaTableFunction({ client, - connectionId, - databaseName: params.database ?? 'default', - tableName: table, + connectionId: connection.id, + databaseName, + tableName, expr: query, startMs: Math.floor(start * 1000), endMs: Math.floor(end * 1000), @@ -645,7 +687,6 @@ const queryHandler: express.RequestHandler = async (req, res) => { const startedAt = performance.now(); let backend: PrometheusBackend = 'unknown'; try { - const { teamId } = getNonNullUserWithTeam(req); const params = getParams(req); const query = params.query; @@ -657,34 +698,24 @@ const queryHandler: express.RequestHandler = async (req, res) => { }); } - const connectionId = params.connectionId; - if (!connectionId) { - return res.status(400).json({ - status: 'error', - errorType: 'bad_data', - error: 'missing required parameter: connectionId', - }); - } - - const connection = await getConnectionById( - teamId.toString(), - connectionId, - true, - ); - if (!connection) { - return res.status(404).json({ + const target = await resolvePromqlTarget(req, params.sourceId, { + selectPassword: true, + }); + if ('error' in target) { + return res.status(target.status).json({ status: 'error', errorType: 'bad_data', - error: 'Connection not found', + error: target.error, }); } + const { connection, databaseName, tableName } = target; if (connection.isPrometheusEndpoint) { backend = 'prometheus'; const status = await proxyToPrometheus( connection.host, '/api/v1/query', - params, + prometheusEndpointParams(params), res, ); recordProxyOutcome(status, 'query', backend); @@ -692,15 +723,6 @@ const queryHandler: express.RequestHandler = async (req, res) => { } backend = 'clickhouse'; - const table = params.table; - if (!table) { - return res.status(400).json({ - status: 'error', - errorType: 'bad_data', - error: `table parameter required for querying clickhouse via promql`, - }); - } - const client = newClickhouseClient(connection); if (await clickhouseServesPrometheusHttpApi(client, connection)) { @@ -708,7 +730,7 @@ const queryHandler: express.RequestHandler = async (req, res) => { const status = await proxyToPrometheus( upstream, `${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query`, - params, + { ...params, database: databaseName, table: tableName }, res, headers, ); @@ -719,8 +741,8 @@ const queryHandler: express.RequestHandler = async (req, res) => { // fallback for clickhouse version < 26.6 const result = await queryInstantViaTableFunction({ client, - databaseName: params.database ?? 'default', - tableName: table, + databaseName, + tableName, expr: query, evalMs: params.time ? Math.floor(parseTimestamp(params.time) * 1000) @@ -791,7 +813,6 @@ const queryExemplarsHandler: express.RequestHandler = async (req, res) => { const startedAt = performance.now(); let backend: PrometheusBackend = 'unknown'; try { - const { teamId } = getNonNullUserWithTeam(req); const params = getParams(req); const query = params.query; @@ -803,25 +824,19 @@ const queryExemplarsHandler: express.RequestHandler = async (req, res) => { }); } - const connectionId = params.connectionId; - if (!connectionId) { - return res.status(400).json({ - status: 'error', - errorType: 'bad_data', - error: 'missing required parameter: connectionId', - }); - } - // No `selectPassword` — neither branch builds a ClickhouseClient, so there // is no reason to pull the connection secret into request scope. - const connection = await getConnectionById(teamId.toString(), connectionId); - if (!connection) { - return res.status(404).json({ + const target = await resolvePromqlTarget(req, params.sourceId, { + selectPassword: false, + }); + if ('error' in target) { + return res.status(target.status).json({ status: 'error', errorType: 'bad_data', - error: 'Connection not found', + error: target.error, }); } + const { connection } = target; if (connection.isPrometheusEndpoint) { backend = 'prometheus'; @@ -856,7 +871,7 @@ const queryExemplarsHandler: express.RequestHandler = async (req, res) => { connection.host, '/api/v1/query_exemplars', { - ...params, + ...prometheusEndpointParams(params), start: String(window.start), end: String(window.end), }, @@ -923,7 +938,7 @@ const prometheusTimestampSchema = z const labelLookupRequestQuerySchema = z .object({ - connectionId: objectIdSchema, + sourceId: objectIdSchema, start: prometheusTimestampSchema.optional(), end: prometheusTimestampSchema.optional(), /** @@ -933,8 +948,6 @@ const labelLookupRequestQuerySchema = z * than forwarded for upstream to reject. */ match: stringListQueryParam.optional(), - database: z.string().optional(), - table: z.string().optional(), limit: z.preprocess( v => (v === '' ? undefined : v), z.coerce.number().int().nonnegative().optional(), @@ -992,8 +1005,6 @@ async function handleLabelLookup( const startedAt = performance.now(); let backend: PrometheusBackend = 'unknown'; try { - const { teamId } = getNonNullUserWithTeam(req); - const parseResult = labelLookupRequestQuerySchema.safeParse(req.query); if (!parseResult.success) { return res.status(400).json({ @@ -1003,20 +1014,18 @@ async function handleLabelLookup( }); } - const params = parseResult.data; - const { connectionId, start, end, limit, match } = params; - const connection = await getConnectionById( - teamId.toString(), - connectionId, - true, - ); - if (!connection) { - return res.status(404).json({ + const { sourceId, start, end, limit, match } = parseResult.data; + const target = await resolvePromqlTarget(req, sourceId, { + selectPassword: true, + }); + if ('error' in target) { + return res.status(target.status).json({ status: 'error', errorType: 'bad_data', - error: 'Connection not found', + error: target.error, }); } + const { connection, databaseName, tableName } = target; // Proxy to Prometheus if endpoint is set if (connection.isPrometheusEndpoint) { @@ -1043,17 +1052,6 @@ async function handleLabelLookup( } backend = 'clickhouse'; - - const database = params.database ?? 'default'; - const table = params.table; - if (!table) { - return res.status(400).json({ - status: 'error', - errorType: 'bad_data', - error: `table parameter required for querying clickhouse via promql`, - }); - } - const client = newClickhouseClient(connection); const startMs = start != null ? Math.floor(start * 1000) : undefined; @@ -1061,8 +1059,8 @@ async function handleLabelLookup( const values = await queryClickHouse({ client, connectionId: connection.id, - databaseName: database, - tableName: table, + databaseName, + tableName, startMs, endMs, limit, diff --git a/packages/app/src/__tests__/ChartUtils.test.ts b/packages/app/src/__tests__/ChartUtils.test.ts index 429fc4c39c..80f90cc490 100644 --- a/packages/app/src/__tests__/ChartUtils.test.ts +++ b/packages/app/src/__tests__/ChartUtils.test.ts @@ -1291,6 +1291,7 @@ describe('ChartUtils', () => { configType: 'promql' as const, displayType: DisplayType.Table, connection: 'conn', + source: 'source-promql', promqlExpression: [{ expression: 'up' }, { expression: 'down' }], dateRange: [ new Date('2025-11-26T00:00:14.076Z'), @@ -1347,6 +1348,7 @@ describe('ChartUtils', () => { configType: 'promql' as const, displayType: DisplayType.Number, connection: 'conn', + source: 'source-promql', promqlExpression: [ { expression: 'up', queryType: 'range' as const }, { expression: 'down', queryType: 'range' as const }, diff --git a/packages/app/src/__tests__/prometheusApi.test.ts b/packages/app/src/__tests__/prometheusApi.test.ts index 8371e70f65..a26f005c0c 100644 --- a/packages/app/src/__tests__/prometheusApi.test.ts +++ b/packages/app/src/__tests__/prometheusApi.test.ts @@ -12,6 +12,8 @@ jest.mock('ky-universal', () => { import { prometheusApi } from '@/api'; +const SOURCE_ID = 'source-promql'; + /** The search params of the single request the call issued. */ const requestedParams = () => new URLSearchParams(get.mock.calls[0][1].searchParams); @@ -20,6 +22,17 @@ const requestedParams = () => const postedParams = () => new URLSearchParams(post.mock.calls[0][1].searchParams); +/** + * The server resolves the connection and table from the source, so naming any + * of them would only invite it to be trusted over the source. + */ +const expectOnlySourceNamed = (params: URLSearchParams) => { + expect(params.get('sourceId')).toBe(SOURCE_ID); + expect(params.has('connectionId')).toBe(false); + expect(params.has('database')).toBe(false); + expect(params.has('table')).toBe(false); +}; + describe('prometheusApi.labelValues', () => { beforeEach(() => { get.mockReset(); @@ -31,9 +44,7 @@ describe('prometheusApi.labelValues', () => { it('sends the selector under the repeatable name Prometheus expects', async () => { await prometheusApi.labelValues({ label: 'pod', - connectionId: 'conn', - database: 'db', - table: 'tbl', + sourceId: SOURCE_ID, start: 100, end: 200, match: 'up{job="api"}', @@ -47,12 +58,13 @@ describe('prometheusApi.labelValues', () => { expect(params.getAll('match[]')).toEqual(['up{job="api"}']); expect(params.get('start')).toBe('100'); expect(params.get('end')).toBe('200'); + expectOnlySourceNamed(params); }); it('sends the attribution as a header', async () => { await prometheusApi.labelValues({ label: 'pod', - connectionId: 'conn', + sourceId: SOURCE_ID, attribution: { surface: 'metadata', dashboard: 'd1' }, }); @@ -65,7 +77,7 @@ describe('prometheusApi.labelValues', () => { }); it('omits the selector when there is none', async () => { - await prometheusApi.labelValues({ label: 'pod', connectionId: 'conn' }); + await prometheusApi.labelValues({ label: 'pod', sourceId: SOURCE_ID }); expect(requestedParams().has('match[]')).toBe(false); }); @@ -78,7 +90,7 @@ describe('prometheusApi.labelValues', () => { const resp = await prometheusApi.labelValues({ label: 'pod', - connectionId: 'conn', + sourceId: SOURCE_ID, }); // Mantine throws on duplicate options, taking the page down with it. @@ -91,6 +103,20 @@ describe('prometheusApi.labels', () => { get.mockReset(); }); + it('names the source the label names are read from', async () => { + get.mockReturnValue({ + json: () => Promise.resolve({ status: 'success', data: ['instance'] }), + }); + + await prometheusApi.labels({ sourceId: SOURCE_ID, start: 100, end: 200 }); + + expect(get).toHaveBeenCalledWith('v1/prometheus/labels', expect.anything()); + const params = requestedParams(); + expect(params.get('start')).toBe('100'); + expect(params.get('end')).toBe('200'); + expectOnlySourceNamed(params); + }); + it('reports a repeated label name once', async () => { get.mockReturnValue({ json: () => @@ -100,7 +126,7 @@ describe('prometheusApi.labels', () => { }), }); - const resp = await prometheusApi.labels({ connectionId: 'conn' }); + const resp = await prometheusApi.labels({ sourceId: SOURCE_ID }); expect(resp.data).toEqual(['byoc', 'instance']); }); @@ -111,7 +137,7 @@ describe('prometheusApi.labels', () => { }); await prometheusApi.labels({ - connectionId: 'conn', + sourceId: SOURCE_ID, attribution: { surface: 'metadata' }, }); @@ -127,12 +153,46 @@ describe('prometheusApi.labels', () => { json: () => Promise.resolve({ status: 'error', error: 'nope' }), }); - const resp = await prometheusApi.labels({ connectionId: 'conn' }); + const resp = await prometheusApi.labels({ sourceId: SOURCE_ID }); expect(resp).toEqual({ status: 'error', error: 'nope' }); }); }); +describe('prometheusApi.queryRange', () => { + beforeEach(() => { + post.mockReset(); + post.mockReturnValue({ + json: () => + Promise.resolve({ + status: 'success', + data: { resultType: 'matrix', result: [] }, + }), + }); + }); + + it('names only the source on a range query', async () => { + await prometheusApi.queryRange({ + query: 'sum(up)', + start: 100, + end: 200, + step: '15s', + sourceId: SOURCE_ID, + }); + + expect(post).toHaveBeenCalledWith( + 'v1/prometheus/query_range', + expect.anything(), + ); + const params = postedParams(); + expect(params.get('query')).toBe('sum(up)'); + expect(params.get('start')).toBe('100'); + expect(params.get('end')).toBe('200'); + expect(params.get('step')).toBe('15s'); + expectOnlySourceNamed(params); + }); +}); + describe('prometheusApi.query', () => { beforeEach(() => { post.mockReset(); @@ -149,18 +209,14 @@ describe('prometheusApi.query', () => { await prometheusApi.query({ query: 'sum(up)', time: 1700000000, - connectionId: 'conn', - database: 'db', - table: 'tbl', + sourceId: SOURCE_ID, }); expect(post).toHaveBeenCalledWith('v1/prometheus/query', expect.anything()); const params = postedParams(); expect(params.get('query')).toBe('sum(up)'); expect(params.get('time')).toBe('1700000000'); - expect(params.get('connectionId')).toBe('conn'); - expect(params.get('database')).toBe('db'); - expect(params.get('table')).toBe('tbl'); + expectOnlySourceNamed(params); // The instant endpoint takes no range, and sending one would be forwarded // upstream as an unrecognized param. expect(params.has('start')).toBe(false); @@ -172,33 +228,21 @@ describe('prometheusApi.query', () => { await prometheusApi.query({ query: 'up', time: 1, - connectionId: 'conn', + sourceId: SOURCE_ID, limit: 25, }); expect(postedParams().get('limit')).toBe('25'); post.mockClear(); - await prometheusApi.query({ query: 'up', time: 1, connectionId: 'conn' }); + await prometheusApi.query({ query: 'up', time: 1, sourceId: SOURCE_ID }); expect(postedParams().has('limit')).toBe(false); }); - it('omits the table params a Prometheus-backed connection has no use for', async () => { - await prometheusApi.query({ - query: 'up', - time: 1, - connectionId: 'conn', - }); - - const params = postedParams(); - expect(params.has('database')).toBe(false); - expect(params.has('table')).toBe(false); - }); - it('sends the attribution as a log_comment payload in a header', async () => { await prometheusApi.query({ query: 'up', time: 1, - connectionId: 'conn', + sourceId: SOURCE_ID, attribution: { surface: 'dashboard', dashboard: 'd1', tile: 't1' }, }); @@ -215,7 +259,7 @@ describe('prometheusApi.query', () => { await prometheusApi.query({ query: 'up', time: 1, - connectionId: 'conn', + sourceId: SOURCE_ID, attribution: {}, }); expect(post.mock.calls[0][1].headers).toBeUndefined(); @@ -234,7 +278,7 @@ describe('prometheusApi.query', () => { const resp = await prometheusApi.query({ query: 'sum(', time: 1, - connectionId: 'conn', + sourceId: SOURCE_ID, }); expect(resp).toEqual({ diff --git a/packages/app/src/__tests__/source.test.ts b/packages/app/src/__tests__/source.test.ts index 7f6fd08cb2..731669c3db 100644 --- a/packages/app/src/__tests__/source.test.ts +++ b/packages/app/src/__tests__/source.test.ts @@ -18,6 +18,7 @@ import { useChartNumberFormats, useSingleSeriesNumberFormat, useSources, + useUpdateSource, } from '@/source'; jest.mock('../api', () => ({ hdxServer: jest.fn() })); @@ -31,7 +32,7 @@ jest.mock('@tanstack/react-query', () => ({ useQueryClient: jest.fn(), })); -import { useQuery } from '@tanstack/react-query'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; import { hdxServer } from '@/api'; @@ -231,6 +232,30 @@ describe('getTraceDurationNumberFormat', () => { }); }); +describe('useUpdateSource', () => { + it("drops the updated source's cached PromQL lookups", () => { + const invalidateQueries = jest.fn(); + // @ts-expect-error return value type + jest.mocked(useQueryClient).mockReturnValue({ invalidateQueries }); + let onSuccess: ((data: unknown, vars: unknown) => void) | undefined; + // @ts-expect-error param type + jest.mocked(useMutation).mockImplementation(options => { + // @ts-expect-error return value type + onSuccess = options.onSuccess; + }); + + useUpdateSource(); + onSuccess?.(undefined, { source: { id: 'source-promql' } }); + + expect(invalidateQueries.mock.calls.map(([arg]) => arg.queryKey)).toEqual([ + ['sources'], + ['promql-metric-names', 'source-promql'], + ['promql-label-names', 'source-promql'], + ['dashboard-filter-promql-label-values', 'source-promql'], + ]); + }); +}); + describe('useSources validation notifications', () => { const mockUseQuery = useQuery as jest.Mock; const mockHdxServer = hdxServer as jest.Mock; diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts index 2f60022f92..34a49630c2 100644 --- a/packages/app/src/api.ts +++ b/packages/app/src/api.ts @@ -896,9 +896,7 @@ export const prometheusApi = { start: number; end: number; step: string; - connectionId: string; - database?: string; - table?: string; + sourceId: string; signal?: AbortSignal; attribution?: QueryAttribution; }): Promise => @@ -909,9 +907,7 @@ export const prometheusApi = { start: String(params.start), end: String(params.end), step: params.step, - connectionId: params.connectionId, - ...(params.database ? { database: params.database } : {}), - ...(params.table ? { table: params.table } : {}), + sourceId: params.sourceId, }, params.signal, params.attribution, @@ -920,9 +916,7 @@ export const prometheusApi = { query: (params: { query: string; time: number; - connectionId: string; - database?: string; - table?: string; + sourceId: string; /** Maximum number of series to return. */ limit?: number; signal?: AbortSignal; @@ -933,19 +927,15 @@ export const prometheusApi = { { query: params.query, time: String(params.time), - connectionId: params.connectionId, + sourceId: params.sourceId, ...(params.limit ? { limit: String(params.limit) } : {}), - ...(params.database ? { database: params.database } : {}), - ...(params.table ? { table: params.table } : {}), }, params.signal, params.attribution, ), labels: (params: { - connectionId: string; - database?: string; - table?: string; + sourceId: string; start?: number; end?: number; attribution?: QueryAttribution; @@ -960,9 +950,7 @@ export const prometheusApi = { labelValues: (params: { label: string; - connectionId: string; - database?: string; - table?: string; + sourceId: string; start?: number; end?: number; match?: string; @@ -980,17 +968,13 @@ export const prometheusApi = { }; function labelLookupSearchParams(params: { - connectionId: string; - database?: string; - table?: string; + sourceId: string; start?: number; end?: number; match?: string; }): Record { return { - connectionId: params.connectionId, - ...(params.database ? { database: params.database } : {}), - ...(params.table ? { table: params.table } : {}), + sourceId: params.sourceId, ...(params.start != null ? { start: String(params.start) } : {}), ...(params.end != null ? { end: String(params.end) } : {}), ...(params.match ? { 'match[]': params.match } : {}), diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx index 5dac49f90a..838e3ec737 100644 --- a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -85,14 +85,9 @@ export default function PromqlChartEditor({ const { data: source } = useSource({ id: sourceId }); // The form can still hold a non-PromQL source right after switching a tile // into PromQL mode (the picker above only restricts future selections), and - // the metric-name lookup reads a TimeSeries engine table, so it would fail - // against any other source's table. + // the metric-name lookup only answers for PromQL sources. const promqlSource = source?.kind === SourceKind.Promql ? source : undefined; - const { data: metricNames } = usePromqlMetricNames( - promqlSource?.connection, - promqlSource?.from.databaseName, - promqlSource?.from.tableName, - ); + const { data: metricNames } = usePromqlMetricNames(promqlSource?.id); const displayTypeSupportsMultiExpression = isTimeSeriesDisplayType(displayType); diff --git a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts index a45eb39922..aa6c554dfa 100644 --- a/packages/app/src/components/ChartEditor/__tests__/utils.test.ts +++ b/packages/app/src/components/ChartEditor/__tests__/utils.test.ts @@ -733,6 +733,16 @@ describe('PromQL expressions', () => { ).not.toHaveProperty('heatmap', expect.anything()); }); + it('carries the PromQL source on the saved and queried configs', () => { + const form = promqlForm([{ expression: 'up' }]); + expect( + convertFormStateToSavedChartConfig(form, promqlSource), + ).toMatchObject({ source: 'source-promql' }); + expect( + convertFormStateToChartConfig(form, dateRange, promqlSource), + ).toMatchObject({ source: 'source-promql' }); + }); + it('saves the form rows as the expression list', () => { expect( convertFormStateToSavedChartConfig( @@ -901,7 +911,7 @@ describe('PromQL expressions', () => { configType: 'promql', displayType: DisplayType.Line, promqlExpression: 'up', - connection: 'conn-1', + source: 'source-promql', legendTemplate: '{{pod}}', }); expect(result.configType).toBe('promql'); @@ -917,7 +927,7 @@ describe('PromQL expressions', () => { convertSavedChartConfigToFormState({ configType: 'promql', displayType: DisplayType.Line, - connection: 'conn-1', + source: 'source-promql', promqlExpression: [], }).promqlExpressions, ).toEqual([{ expression: '', alias: '' }]); @@ -1321,6 +1331,55 @@ describe('validateChartForm', () => { expect(errors.filter(e => e.path === 'source')).toHaveLength(0); }); + describe('PromQL charts', () => { + const promqlChartForm = (overrides: Partial) => + makeForm({ + configType: 'promql', + promqlExpressions: [{ expression: 'up' }], + series: [], + ...overrides, + }); + + it('returns no errors with a PromQL source', () => { + const errors = validateChartForm( + promqlChartForm({ source: 'source-promql' }), + promqlSource, + jest.fn(), + ); + expect(errors).toHaveLength(0); + }); + + it('errors without a source', () => { + const errors = validateChartForm( + promqlChartForm({ source: undefined }), + undefined, + jest.fn(), + ); + expect(errors).toContainEqual( + expect.objectContaining({ + path: 'source', + message: 'Source is required', + }), + ); + }); + + // The editor resolves the selection against the kinds PromQL mode allows, + // so a leftover builder source arrives here as no source at all. + it('errors when the selected source is not a PromQL source', () => { + const errors = validateChartForm( + promqlChartForm({ source: 'source-log' }), + undefined, + jest.fn(), + ); + expect(errors).toContainEqual( + expect.objectContaining({ + path: 'source', + message: 'Source is required', + }), + ); + }); + }); + // ── Series valueExpression validation ──────────────────────────────── it('errors when a non-count aggFn series is missing valueExpression', () => { diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 4885b61306..d028efd563 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -278,8 +278,7 @@ export function convertFormStateToSavedChartConfig( // 'alert', // TODO: Support alerts on PromQL (HDX-4636) ]), promqlExpression: formPromqlExpressions(form), - connection: form.connection ?? '', - source: form.source || undefined, + source: source?.id ?? form.source ?? '', legendTemplate: form.legendTemplate?.trim() || undefined, heatmap: formPromqlHeatmap(form), }; @@ -381,7 +380,7 @@ export function convertFormStateToChartConfig( ]), promqlExpression: formPromqlExpressions(form), connection: source?.connection ?? form.connection ?? '', - source: form.source || undefined, + source: source?.id ?? form.source ?? '', from: source?.from, legendTemplate: form.legendTemplate?.trim() || undefined, heatmap: formPromqlHeatmap(form), diff --git a/packages/app/src/components/DBEditTimeChartForm/__tests__/ChartPreviewPanel.test.tsx b/packages/app/src/components/DBEditTimeChartForm/__tests__/ChartPreviewPanel.test.tsx index 609fff93df..9ac69def6d 100644 --- a/packages/app/src/components/DBEditTimeChartForm/__tests__/ChartPreviewPanel.test.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/__tests__/ChartPreviewPanel.test.tsx @@ -309,6 +309,7 @@ describe('ChartPreviewPanel', () => { displayType: DisplayType.Line, promqlExpression: EXPRESSION, connection: 'default', + source: 'source-promql', from: { databaseName: 'default', tableName: 'metrics' }, dateRange, ...overrides, diff --git a/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts b/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts index 55c9a14a9b..3db717adf2 100644 --- a/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts +++ b/packages/app/src/components/DBEditTimeChartForm/__tests__/utils.test.ts @@ -167,6 +167,7 @@ describe('isQueryReady', () => { configType: 'promql', promqlExpression: [{ expression: 'up' }], connection: 'local', + source: 'source-promql', dateRange, }), ).toBeTruthy(); @@ -176,6 +177,7 @@ describe('isQueryReady', () => { configType: 'promql', promqlExpression: 'up', connection: 'local', + source: 'source-promql', dateRange, }), ).toBeTruthy(); @@ -187,6 +189,7 @@ describe('isQueryReady', () => { configType: 'promql', promqlExpression: [{ expression: '' }], connection: 'local', + source: 'source-promql', dateRange, }), ).toBeFalsy(); @@ -201,6 +204,7 @@ describe('isQueryReady', () => { displayType: DisplayType.Number, promqlExpression: [{ expression: '' }, { expression: 'up' }], connection: 'local', + source: 'source-promql', dateRange, }), ).toBeFalsy(); @@ -210,6 +214,7 @@ describe('isQueryReady', () => { displayType: DisplayType.Line, promqlExpression: [{ expression: '' }, { expression: 'up' }], connection: 'local', + source: 'source-promql', dateRange, }), ).toBeTruthy(); @@ -227,6 +232,7 @@ describe('buildRenderedPromqlExpression', () => { ): ChartConfigWithDateRange => ({ configType: 'promql', connection: 'local', + source: 'source-promql', displayType: DisplayType.Line, promqlExpression: [{ expression: 'up' }], dateRange, @@ -596,6 +602,7 @@ describe('resolvePreviewVariables', () => { configType: 'promql', promqlExpression: 'up{service=~"$service"}', connection: 'local', + source: 'source-promql', dateRange, }; @@ -644,6 +651,7 @@ describe('resolveTilePreviewFilters', () => { configType: 'promql', promqlExpression: 'up{service=~"$service"}', connection: 'local', + source: 'source-promql', dateRange, }; diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts index a2e676221f..83ed9150ce 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts +++ b/packages/app/src/components/DBHeatmapChart/__tests__/heatmapQueries.test.ts @@ -71,6 +71,7 @@ describe('PromQL heatmaps', () => { displayType: DisplayType.Heatmap, promqlExpression: [{ expression: 'up' }, { expression: 'down' }], connection: 'test-connection', + source: 'source-promql', dateRange: [new Date(T0 + 90_000), new Date(T0 + HOUR + 90_000)], granularity: '5 minute', legendTemplate: '{{job}}', diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/useCalculatedHeatmapData.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/useCalculatedHeatmapData.test.ts index d5147ddbff..9f8c8de288 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/useCalculatedHeatmapData.test.ts +++ b/packages/app/src/components/DBHeatmapChart/__tests__/useCalculatedHeatmapData.test.ts @@ -22,6 +22,7 @@ const config: PromqlConfigWithDateRange = { displayType: DisplayType.Heatmap, promqlExpression: [{ expression: 'up' }], connection: 'test-connection', + source: 'promql-source', dateRange, granularity: 'auto', }; diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/useHistogramHeatmapData.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/useHistogramHeatmapData.test.ts index 51994a60a1..89f9159695 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/useHistogramHeatmapData.test.ts +++ b/packages/app/src/components/DBHeatmapChart/__tests__/useHistogramHeatmapData.test.ts @@ -20,6 +20,7 @@ const config: PromqlConfigWithDateRange = { displayType: DisplayType.Heatmap, promqlExpression: [{ expression: 'sum by (le) (rate(x_bucket[5m]))' }], connection: 'test-connection', + source: 'promql-source', dateRange: [new Date(T0), new Date(T0 + 2 * HOUR)], granularity: '1 hour', heatmap: { mode: 'histogram' }, diff --git a/packages/app/src/components/DBHeatmapChart/__tests__/useSeriesHeatmapData.test.ts b/packages/app/src/components/DBHeatmapChart/__tests__/useSeriesHeatmapData.test.ts index 9a810da34f..d6472f085d 100644 --- a/packages/app/src/components/DBHeatmapChart/__tests__/useSeriesHeatmapData.test.ts +++ b/packages/app/src/components/DBHeatmapChart/__tests__/useSeriesHeatmapData.test.ts @@ -218,6 +218,7 @@ describe('useSeriesHeatmapData', () => { displayType: DisplayType.Heatmap, promqlExpression: [{ expression: 'up' }], connection: 'test-connection', + source: 'source-promql', dateRange, granularity: 'auto', }; diff --git a/packages/app/src/components/DashboardFiltersModal/PromqlLabelFilterEditForm.tsx b/packages/app/src/components/DashboardFiltersModal/PromqlLabelFilterEditForm.tsx index ab5e6cb644..bb3d90a8f0 100644 --- a/packages/app/src/components/DashboardFiltersModal/PromqlLabelFilterEditForm.tsx +++ b/packages/app/src/components/DashboardFiltersModal/PromqlLabelFilterEditForm.tsx @@ -45,16 +45,8 @@ export const PromqlLabelFilterEditForm = ({ const sourceId = useWatch({ control, name: 'source' }); const source = sources?.find(s => s.id === sourceId); const promqlSource = source && isPromqlSource(source) ? source : undefined; - const { data: labelNames } = usePromqlLabelNames( - promqlSource?.connection, - promqlSource?.from.databaseName, - promqlSource?.from.tableName, - ); - const { data: metricNames } = usePromqlMetricNames( - promqlSource?.connection, - promqlSource?.from.databaseName, - promqlSource?.from.tableName, - ); + const { data: labelNames } = usePromqlLabelNames(promqlSource?.id); + const { data: metricNames } = usePromqlMetricNames(promqlSource?.id); return ( <> diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/PromqlLabelFilterEditForm.test.tsx b/packages/app/src/components/DashboardFiltersModal/__tests__/PromqlLabelFilterEditForm.test.tsx index ef16cc44eb..b63fb204a0 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/PromqlLabelFilterEditForm.test.tsx +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/PromqlLabelFilterEditForm.test.tsx @@ -88,9 +88,7 @@ describe('PromqlLabelFilterEditForm', () => { expect(await screen.findByText('instance')).toBeInTheDocument(); expect(screen.getByText('job')).toBeInTheDocument(); expect(mockLabels).toHaveBeenCalledWith({ - connectionId: 'clickhouse-conn', - database: 'telemetry', - table: 'metrics', + sourceId: 'promql-source', attribution: expect.objectContaining({ surface: 'metadata' }), }); }); diff --git a/packages/app/src/components/__tests__/DBNumberChart.test.tsx b/packages/app/src/components/__tests__/DBNumberChart.test.tsx index a53c75d8ce..556e78610b 100644 --- a/packages/app/src/components/__tests__/DBNumberChart.test.tsx +++ b/packages/app/src/components/__tests__/DBNumberChart.test.tsx @@ -779,6 +779,7 @@ describe('DBNumberChart', () => { displayType: DisplayType.Number, promqlExpression: 'e2e_service_up', connection: 'test-connection', + source: 'source-promql', dateRange: [new Date(), new Date()] as [Date, Date], }; diff --git a/packages/app/src/components/__tests__/DBPieChart.test.tsx b/packages/app/src/components/__tests__/DBPieChart.test.tsx index 06bf826c15..06a23f9bfe 100644 --- a/packages/app/src/components/__tests__/DBPieChart.test.tsx +++ b/packages/app/src/components/__tests__/DBPieChart.test.tsx @@ -305,6 +305,7 @@ describe('DBPieChart', () => { configType: 'promql' as const, displayType: DisplayType.Pie, connection: 'test-connection', + source: 'source-promql', promqlExpression: [{ expression: 'up' }, { expression: 'down' }], dateRange: [ new Date('2025-11-26T00:00:14.076Z'), diff --git a/packages/app/src/components/__tests__/DBTableChart.test.tsx b/packages/app/src/components/__tests__/DBTableChart.test.tsx index b755c3de10..23d3669b85 100644 --- a/packages/app/src/components/__tests__/DBTableChart.test.tsx +++ b/packages/app/src/components/__tests__/DBTableChart.test.tsx @@ -501,6 +501,7 @@ describe('DBTableChart', () => { configType: 'promql' as const, dateRange: [new Date(), new Date()] as [Date, Date], connection: 'test-connection', + source: 'source-promql', promqlExpression: [{ expression: 'up' }], numberFormat: { output: 'number' as const }, }; diff --git a/packages/app/src/components/__tests__/NumberTileBackgroundChart.test.tsx b/packages/app/src/components/__tests__/NumberTileBackgroundChart.test.tsx index ad68501ef8..835cae7857 100644 --- a/packages/app/src/components/__tests__/NumberTileBackgroundChart.test.tsx +++ b/packages/app/src/components/__tests__/NumberTileBackgroundChart.test.tsx @@ -125,6 +125,7 @@ describe('buildSparklineQueryConfig', () => { configType: 'promql' as const, displayType: DisplayType.Number, connection: 'test-connection', + source: 'source-promql', dateRange: baseConfig.dateRange, promqlExpression: [ { @@ -245,6 +246,7 @@ describe('NumberTileBackgroundChart', () => { { expression: 'e2e_service_up', queryType: 'range' as const }, ], connection: 'test-connection', + source: 'source-promql', granularity: '1 minute' as const, dateRange: [ new Date('2024-01-01T00:00:00Z'), diff --git a/packages/app/src/components/alerts/__tests__/AlertDetailChart.test.tsx b/packages/app/src/components/alerts/__tests__/AlertDetailChart.test.tsx index 6ec4d6d9d1..f64b776e3c 100644 --- a/packages/app/src/components/alerts/__tests__/AlertDetailChart.test.tsx +++ b/packages/app/src/components/alerts/__tests__/AlertDetailChart.test.tsx @@ -89,7 +89,7 @@ describe('buildAlertChartConfig', () => { build({ configType: 'promql', promqlExpression: 'up', - connection: 'conn-1', + source: 'source-promql', displayType: DisplayType.Line, }), ).toBeUndefined(); diff --git a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx index c5b0be47ec..a20788a176 100644 --- a/packages/app/src/hooks/__tests__/useChartConfig.test.tsx +++ b/packages/app/src/hooks/__tests__/useChartConfig.test.tsx @@ -405,6 +405,7 @@ describe('useChartConfig', () => { configType: 'promql', promqlExpression: 'e2e_service_up', connection: 'foo', + source: 'source-promql', dateRange: [ new Date('2023-01-10 00:00:00'), new Date('2023-01-10 01:00:00'), @@ -412,6 +413,14 @@ describe('useChartConfig', () => { ...overrides, }) as ChartConfigWithOptDateRange; + /** The server resolves the connection and table from the source alone. */ + const expectOnlySourceNamed = (params: object) => { + expect(params).toHaveProperty('sourceId', 'source-promql'); + expect(params).not.toHaveProperty('connectionId'); + expect(params).not.toHaveProperty('database'); + expect(params).not.toHaveProperty('table'); + }; + const matrixResponse = { status: 'success' as const, data: { @@ -769,6 +778,9 @@ describe('useChartConfig', () => { await waitFor(() => expect(result.current.isSuccess).toBe(true)); expect(prometheusApi.queryRange).not.toHaveBeenCalled(); + expectOnlySourceNamed( + jest.mocked(prometheusApi.query).mock.calls[0][0], + ); // Evaluated at the end of the window, and timeless: a Date column // here would be read back as a time series. expect(jest.mocked(prometheusApi.query).mock.calls[0][0].time).toBe( @@ -802,6 +814,9 @@ describe('useChartConfig', () => { await waitFor(() => expect(result.current.isSuccess).toBe(true)); expect(prometheusApi.query).not.toHaveBeenCalled(); expect(prometheusApi.queryRange).toHaveBeenCalledTimes(1); + expectOnlySourceNamed( + jest.mocked(prometheusApi.queryRange).mock.calls[0][0], + ); }); it('reports an expression that evaluates to a string', async () => { diff --git a/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx b/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx index de5ba12414..2f2e4ace05 100644 --- a/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx +++ b/packages/app/src/hooks/__tests__/useOffsetPaginatedQuery.test.tsx @@ -1196,6 +1196,7 @@ describe('useOffsetPaginatedQuery', () => { const promqlConfig: PromqlConfigWithDateRange = { configType: 'promql', connection: 'foo', + source: 'source-promql', promqlExpression: [{ expression: 'up' }], dateRange: [ new Date('2024-01-01T00:00:00Z'), diff --git a/packages/app/src/hooks/__tests__/usePromqlLabelFilterValues.test.tsx b/packages/app/src/hooks/__tests__/usePromqlLabelFilterValues.test.tsx index e036e2a61c..a0719b5cc7 100644 --- a/packages/app/src/hooks/__tests__/usePromqlLabelFilterValues.test.tsx +++ b/packages/app/src/hooks/__tests__/usePromqlLabelFilterValues.test.tsx @@ -97,9 +97,7 @@ describe('usePromqlLabelFilterValues', () => { expect(mockLabelValues).toHaveBeenCalledTimes(1); expect(mockLabelValues).toHaveBeenCalledWith({ label: 'job', - connectionId: 'clickhouse-conn', - database: 'telemetry', - table: 'metrics', + sourceId: 'promql-source', start: 1704067200, end: 1704153600, attribution: expect.objectContaining({ surface: 'metadata' }), @@ -126,6 +124,29 @@ describe('usePromqlLabelFilterValues', () => { }); }); + // Access is checked per source, so two sources over one table are not interchangeable. + it('queries separately for two sources reading the same table', async () => { + mockSourcesData.push({ + id: 'promql-source-2', + kind: SourceKind.Promql, + name: 'Prometheus copy', + connection: 'clickhouse-conn', + from: { databaseName: 'telemetry', tableName: 'metrics' }, + }); + + const { result } = renderFilters([ + filter(), + filter({ id: 'promql2', source: 'promql-source-2' }), + ]); + + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + expect(mockLabelValues).toHaveBeenCalledTimes(2); + expect(mockLabelValues).toHaveBeenCalledWith( + expect.objectContaining({ sourceId: 'promql-source-2' }), + ); + }); + it('queries separately for a different label on the same source', async () => { const { result } = renderFilters([ filter(), diff --git a/packages/app/src/hooks/usePromqlLabelFilterValues.tsx b/packages/app/src/hooks/usePromqlLabelFilterValues.tsx index 03b2a9c278..ed66bd5384 100644 --- a/packages/app/src/hooks/usePromqlLabelFilterValues.tsx +++ b/packages/app/src/hooks/usePromqlLabelFilterValues.tsx @@ -21,9 +21,7 @@ import { mapKeyBy } from '@/utils'; type LabelValuesCall = { filterId: string; - connectionId: string; - database?: string; - table?: string; + sourceId: string; label: string; match?: string; }; @@ -76,9 +74,7 @@ export function usePromqlLabelFilterValues({ if (resolvedMatch?.error) continue; resolved.push({ filterId: filter.id, - connectionId: source.connection, - database: source.from.databaseName, - table: source.from.tableName, + sourceId: source.id, label: filter.label, match: resolvedMatch?.match, }); @@ -96,9 +92,7 @@ export function usePromqlLabelFilterValues({ // values as placeholders instead of blanking the dropdown. const queryKeyPrefix = [ 'dashboard-filter-promql-label-values', - call.connectionId, - call.database, - call.table, + call.sourceId, call.label, call.match, ]; @@ -122,9 +116,7 @@ export function usePromqlLabelFilterValues({ queryFn: async (): Promise => { const resp = await prometheusApi.labelValues({ label: call.label, - connectionId: call.connectionId, - database: call.database, - table: call.table, + sourceId: call.sourceId, start: startSec, end: endSec, match: call.match, diff --git a/packages/app/src/hooks/usePromqlMetadata.ts b/packages/app/src/hooks/usePromqlMetadata.ts index 351cc7632c..b8102ac745 100644 --- a/packages/app/src/hooks/usePromqlMetadata.ts +++ b/packages/app/src/hooks/usePromqlMetadata.ts @@ -3,50 +3,35 @@ import { useQuery } from '@tanstack/react-query'; import { prometheusApi } from '@/api'; import { useMetadataQueryAttribution } from '@/queryAttribution'; -export function usePromqlMetricNames( - connectionId: string | undefined, - database?: string, - table?: string, -) { +export function usePromqlMetricNames(sourceId: string | undefined) { const attribution = useMetadataQueryAttribution(); return useQuery({ - queryKey: ['promql-metric-names', connectionId, database, table], + queryKey: ['promql-metric-names', sourceId], queryFn: async () => { - if (!connectionId) return []; + if (!sourceId) return []; const resp = await prometheusApi.labelValues({ label: '__name__', - connectionId, - database, - table, + sourceId, attribution, }); return resp.data ?? []; }, - enabled: !!connectionId, + enabled: !!sourceId, staleTime: 60_000, }); } -/** The label names carried by any series the PromQL connection can see. */ -export function usePromqlLabelNames( - connectionId: string | undefined, - database?: string, - table?: string, -) { +/** The label names carried by any series the PromQL source can see. */ +export function usePromqlLabelNames(sourceId: string | undefined) { const attribution = useMetadataQueryAttribution(); return useQuery({ - queryKey: ['promql-label-names', connectionId, database, table], + queryKey: ['promql-label-names', sourceId], queryFn: async () => { - if (!connectionId) return []; - const resp = await prometheusApi.labels({ - connectionId, - database, - table, - attribution, - }); + if (!sourceId) return []; + const resp = await prometheusApi.labels({ sourceId, attribution }); return resp.data ?? []; }, - enabled: !!connectionId, + enabled: !!sourceId, staleTime: 60_000, }); } diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts index 6b956a27b2..b98ee52b2b 100644 --- a/packages/app/src/source.ts +++ b/packages/app/src/source.ts @@ -205,6 +205,14 @@ export function useSource({ }); } +// PromQL lookups are cached by source ID alone, so a source pointed at another +// table would keep serving the old table's values until they go stale. +const PROMQL_SOURCE_QUERY_KEYS = [ + 'promql-metric-names', + 'promql-label-names', + 'dashboard-filter-promql-label-values', +]; + export function useUpdateSource() { const queryClient = useQueryClient(); @@ -219,8 +227,11 @@ export function useUpdateSource() { json: source, }); }, - onSuccess: () => { + onSuccess: (_data, { source }) => { queryClient.invalidateQueries({ queryKey: ['sources'] }); + for (const key of PROMQL_SOURCE_QUERY_KEYS) { + queryClient.invalidateQueries({ queryKey: [key, source.id] }); + } }, }); } diff --git a/packages/app/src/utils/__tests__/alerts.test.ts b/packages/app/src/utils/__tests__/alerts.test.ts index 92e6bd968a..efee2c398c 100644 --- a/packages/app/src/utils/__tests__/alerts.test.ts +++ b/packages/app/src/utils/__tests__/alerts.test.ts @@ -339,7 +339,7 @@ describe('buildInlineAlertPayload', () => { buildInlineAlertPayload({ configType: 'promql', promqlExpression: 'up', - connection: 'conn-1', + source: 'source-promql', displayType: DisplayType.Line, alert, }), diff --git a/packages/app/src/utils/__tests__/chartConfig.test.ts b/packages/app/src/utils/__tests__/chartConfig.test.ts index 694d49f94c..8767ff2851 100644 --- a/packages/app/src/utils/__tests__/chartConfig.test.ts +++ b/packages/app/src/utils/__tests__/chartConfig.test.ts @@ -6,6 +6,7 @@ describe('stripClientSideConfigFields', () => { const promqlConfig = { configType: 'promql' as const, connection: 'conn', + source: 'source-promql', promqlExpression: [ { expression: 'up', diff --git a/packages/app/src/utils/__tests__/promqlChartQuery.test.ts b/packages/app/src/utils/__tests__/promqlChartQuery.test.ts index 3689762d53..5c497f0879 100644 --- a/packages/app/src/utils/__tests__/promqlChartQuery.test.ts +++ b/packages/app/src/utils/__tests__/promqlChartQuery.test.ts @@ -310,14 +310,52 @@ describe('queryPromqlChartConfig', () => { status: 'success', data: { resultType: 'vector', result: [] }, }); + jest + .mocked(prometheusApi.queryRange) + .mockReset() + .mockResolvedValue({ + status: 'success', + data: { resultType: 'matrix', result: [] }, + }); }); + // The server resolves the table from the source, so the table a config still + // carries must not be sent. + it.each([ + ['instant', prometheusApi.query], + ['range', prometheusApi.queryRange], + ] as const)( + 'names only the source on a %s query', + async (queryType, endpoint) => { + await queryPromqlChartConfig( + { + configType: 'promql', + displayType: DisplayType.Number, + connection: 'conn', + source: 'source-promql', + from: { databaseName: 'db', tableName: 'tbl' }, + promqlExpression: [{ expression: 'up', queryType }], + }, + dateRange, + new AbortController().signal, + {}, + ); + + const params = jest.mocked(endpoint).mock.calls[0][0]; + expect(params).toHaveProperty('sourceId', 'source-promql'); + expect(params).not.toHaveProperty('connectionId'); + expect(params).not.toHaveProperty('database'); + expect(params).not.toHaveProperty('table'); + }, + ); + it("caps an instant table query's series at the table's row cap", async () => { await queryPromqlChartConfig( { configType: 'promql', displayType: DisplayType.Table, connection: 'conn', + source: 'source-promql', promqlExpression: [{ expression: 'up', queryType: 'instant' }], }, dateRange, @@ -337,6 +375,7 @@ describe('queryPromqlChartConfig', () => { configType: 'promql', displayType: DisplayType.Table, connection: 'conn', + source: 'source-promql', promqlExpression: [{ expression: 'up', queryType: 'instant' }], }, dateRange, @@ -357,6 +396,7 @@ describe('queryPromqlChartConfig', () => { configType: 'promql', displayType: DisplayType.Number, connection: 'conn', + source: 'source-promql', promqlExpression: [{ expression: 'up', queryType: 'instant' }], }, dateRange, @@ -391,6 +431,7 @@ describe('queryPromqlChartConfig', () => { configType: 'promql', displayType: DisplayType.Heatmap, connection: 'conn', + source: 'source-promql', granularity: '5 minute', legendTemplate: '{{job}}', promqlExpression: [ @@ -446,6 +487,7 @@ describe('queryPromqlChartConfig', () => { configType: 'promql', displayType: DisplayType.Heatmap, connection: 'conn', + source: 'source-promql', legendTemplate: '{{job}}', promqlExpression: [{ expression: 'sum by (le) (x_bucket)' }], heatmap: { mode: 'histogram' }, diff --git a/packages/app/src/utils/promqlChartQuery.ts b/packages/app/src/utils/promqlChartQuery.ts index 8d77dde0ce..e7cb4fa2e6 100644 --- a/packages/app/src/utils/promqlChartQuery.ts +++ b/packages/app/src/utils/promqlChartQuery.ts @@ -91,14 +91,12 @@ async function fetchInstantExpression( const response = await prometheusApi.query({ query: series.expression, time: evalTime.getTime() / 1000, - connectionId: config.connection, + sourceId: config.source, // A vector is one sample per series, so capping series caps a table's // rows exactly. Range queries go uncapped: series can be sparse, so no // series count bounds their samples without dropping rows that would fit. limit: config.displayType === DisplayType.Table ? MAX_TABLE_ROWS : undefined, - database: config.from?.databaseName, - table: config.from?.tableName, signal, attribution, }); @@ -127,9 +125,7 @@ async function fetchRangeExpression( dateRange, config.minGranularitySeconds, ), - connectionId: config.connection, - database: config.from?.databaseName, - table: config.from?.tableName, + sourceId: config.source, signal, attribution, }); diff --git a/packages/app/tests/e2e/features/dashboard.spec.ts b/packages/app/tests/e2e/features/dashboard.spec.ts index 467879f9d8..924006f1a3 100644 --- a/packages/app/tests/e2e/features/dashboard.spec.ts +++ b/packages/app/tests/e2e/features/dashboard.spec.ts @@ -3414,7 +3414,7 @@ test.describe('Dashboard', { tag: ['@dashboard'] }, () => { ` { _id: ObjectId(${JSON.stringify(dashboardId)}) },`, " { $set: { tiles: [{ id: 'promql-1', x: 0, y: 0, w: 4, h: 2,", " config: { configType: 'promql', promqlExpression: 'up',", - " connection: 'c1', displayType: 'line' } }] } }", + " connection: 'c1', source: 's1', displayType: 'line' } }] } }", ')));', ].join('\n'), ); diff --git a/packages/cli/src/__tests__/tileConfig.test.ts b/packages/cli/src/__tests__/tileConfig.test.ts index df164606eb..ce2bd7b93a 100644 --- a/packages/cli/src/__tests__/tileConfig.test.ts +++ b/packages/cli/src/__tests__/tileConfig.test.ts @@ -197,7 +197,7 @@ describe('resolveTileConfig', () => { name: 'PromQL', configType: 'promql', promqlExpression: 'up', - connection: 'conn-1', + source: 'source-1', displayType: DisplayType.Line, } as SavedChartConfig, source: undefined, diff --git a/packages/common-utils/src/__tests__/builderToRawSql.test.ts b/packages/common-utils/src/__tests__/builderToRawSql.test.ts index 8cc8138f2e..aa4d24d7d2 100644 --- a/packages/common-utils/src/__tests__/builderToRawSql.test.ts +++ b/packages/common-utils/src/__tests__/builderToRawSql.test.ts @@ -566,6 +566,7 @@ describe('renderBuilderConfigAsSqlTemplate', () => { configType: 'promql', promqlExpression: 'up', connection: 'test-connection', + source: 'promql-source', }, mockMetadata, ), diff --git a/packages/common-utils/src/__tests__/dashboardFilterValues.test.ts b/packages/common-utils/src/__tests__/dashboardFilterValues.test.ts index f90a424e11..69652d0bea 100644 --- a/packages/common-utils/src/__tests__/dashboardFilterValues.test.ts +++ b/packages/common-utils/src/__tests__/dashboardFilterValues.test.ts @@ -811,6 +811,7 @@ describe('dashboardFilterValues', () => { configType: 'promql', promqlExpression: 'up', connection: 'local', + source: 'promql-source', }; it('applies to a builder tile', () => { @@ -864,6 +865,7 @@ describe('dashboardFilterValues', () => { configType: 'promql', promqlExpression: 'up', connection: 'local', + source: 'promql-source', }; const unsatisfied = [ diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 20b8566135..8e54b666cf 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -3846,6 +3846,7 @@ describe('renderChartConfig', () => { configType: 'promql' as const, promqlExpression: 'rate(http_requests_total[5m])', connection: 'test-connection', + source: 'promql-source', displayType: DisplayType.Line, dateRange: [ new Date('2025-01-01T00:00:00Z'), diff --git a/packages/common-utils/src/__tests__/types.test.ts b/packages/common-utils/src/__tests__/types.test.ts index 4f072291b7..3c0786dda5 100644 --- a/packages/common-utils/src/__tests__/types.test.ts +++ b/packages/common-utils/src/__tests__/types.test.ts @@ -1003,6 +1003,7 @@ describe('PromQL legend templates on saved chart configs', () => { configType: 'promql' as const, promqlExpression: 'up', connection: 'conn-1', + source: 'source-1', legendTemplate, }); @@ -1021,3 +1022,33 @@ describe('PromQL legend templates on saved chart configs', () => { ).toBe(false); }); }); + +describe('PromQL saved chart config source', () => { + const base = { + configType: 'promql' as const, + promqlExpression: 'up', + }; + + // The PromQL API resolves the table from the source, so a tile without one + // could never be queried. + it('requires a source', () => { + expect(SavedChartConfigSchema.safeParse(base).success).toBe(false); + expect( + SavedChartConfigSchema.safeParse({ ...base, source: '' }).success, + ).toBe(false); + expect( + SavedChartConfigSchema.safeParse({ ...base, source: 'source-1' }).success, + ).toBe(true); + }); + + // The connection comes from the source at query time, so a stored copy is + // dropped rather than kept around to go stale. + it('drops a persisted connection', () => { + const parsed = SavedChartConfigSchema.parse({ + ...base, + source: 'source-1', + connection: 'conn-1', + }); + expect(parsed).not.toHaveProperty('connection'); + }); +}); diff --git a/packages/common-utils/src/__tests__/variables.test.ts b/packages/common-utils/src/__tests__/variables.test.ts index d8701b5acc..81d60adf4c 100644 --- a/packages/common-utils/src/__tests__/variables.test.ts +++ b/packages/common-utils/src/__tests__/variables.test.ts @@ -1473,6 +1473,7 @@ describe('filterReferencedVariables', () => { configType: 'promql', promqlExpression: 'up{service=~"$service"}', connection: 'local', + source: 'promql-source', }, variables, ), @@ -1489,6 +1490,7 @@ describe('filterReferencedVariables', () => { { expression: 'up{service=~"$service"}' }, ], connection: 'local', + source: 'promql-source', }, variables, ), @@ -1576,6 +1578,7 @@ describe('getAlertVariableWarning', () => { configType: 'promql', promqlExpression: 'up{service=~"$service"}', connection: 'local', + source: 'promql-source', }, variables, ), diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 26935e84a8..013869fa45 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1914,8 +1914,7 @@ export type PromqlExpressionList = z.infer; const PromqlBaseChartConfigSchema = SharedChartSettingsSchema.extend({ configType: z.literal('promql'), promqlExpression: PromqlExpressionListSchema, - connection: z.string(), - source: z.string().optional(), + source: z.string().min(1), step: z.string().optional(), legendTemplate: z.string().max(MAX_LEGEND_TEMPLATE_LENGTH).optional(), heatmap: HeatmapSettingsSchema.extend({ @@ -1926,6 +1925,9 @@ const PromqlBaseChartConfigSchema = SharedChartSettingsSchema.extend({ /** Schema describing PromQL chart configs with runtime-only fields */ const PromqlChartConfigSchema = PromqlBaseChartConfigSchema.extend({ + // Copied from the source at query time and never persisted, so it cannot go + // stale if the source moves to another connection. + connection: z.string(), filters: z.array(FilterSchema).optional(), from: z .object({ databaseName: z.string(), tableName: z.string() })