Skip to content

Commit c0a220d

Browse files
Merge branch 'main' into feat/instance-label
2 parents 01e667f + d8b1cdf commit c0a220d

19 files changed

Lines changed: 517 additions & 26 deletions

File tree

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
'@hyperdx/common-utils': minor
3+
'@hyperdx/api': minor
4+
'@hyperdx/app': minor
5+
---
6+
7+
feat: Attribute PromQL queries to the dashboard, tile or search that issued them, through both the `prometheusQuery`/`prometheusQueryRange` table functions and ClickHouse's Prometheus HTTP API

‎agent_docs/observability.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,22 @@ everything below it, including the materialized-view `EXPLAIN` probes.
294294
Order of precedence: the client's default, then the scope around the current
295295
request or job, then anything passed with an individual query.
296296

297+
**Browser to API route** — for requests that reach ClickHouse through an API
298+
route instead of a ClickHouse client. PromQL is the case today: the browser
299+
calls `/api/v1/prometheus/query{,_range}`, and `/labels` and
300+
`/label/:name/values` for autocomplete and dashboard filters (tagged with
301+
`useMetadataQueryAttribution()`), over fetch. The caller sends
302+
`buildLogComment(attribution)` in the `x-hyperdx-query-attribution` header
303+
(`QUERY_ATTRIBUTION_HEADER`), and the router parses it with `parseLogComment`
304+
into a `withQueryAttribution` scope layered over the auth middleware's `api`
305+
scope. The browser's surface and ids win; `trace` and `label` stay the
306+
server's. Queries the route runs through a ClickHouse client (label lookups,
307+
and the `prometheusQuery()`/`prometheusQueryRange()` fallback) pick the scope
308+
up as usual. Requests proxied to ClickHouse's Prometheus HTTP API on 26.6+ are tagged
309+
by `clickhousePrometheusRequest`: `log_comment` as a URL setting, `query_id` as
310+
the `X-ClickHouse-Query-Id` header, because the handler rejects `query_id` as an
311+
unknown setting in the URL.
312+
297313
### Rules
298314

299315
- **Values from the browser are hints, not proof.** The client controls them.

‎packages/api/src/middleware/__tests__/queryAttribution.int.test.ts‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
import { buildLogComment } from '@hyperdx/common-utils/dist/clickhouse';
2+
import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
3+
14
import * as config from '@/config';
25
import {
36
DEFAULT_DATABASE,
@@ -9,6 +12,11 @@ import {
912
} from '@/fixtures';
1013
import Connection from '@/models/connection';
1114

15+
declare global {
16+
// Set by jest.setup.ts before it stubs `fetch`.
17+
var realFetch: typeof fetch;
18+
}
19+
1220
// Covers the wiring the unit tests can't: auth middleware opens the
1321
// AsyncLocalStorage scope, and a query issued deep inside the route handler
1422
// still reads it. If that scope is lost, queries fall back to hdx-unknown-*.
@@ -29,11 +37,23 @@ describe('request query attribution', () => {
2937
endSec: 1700003600,
3038
},
3139
],
40+
withSamples: true,
3241
});
3342
});
3443

44+
// jest.setup.ts installs `fetch` as a plain jest.fn(), not a spy, so
45+
// restoreAllMocks can't put it back; and restoreAllMocks would also undo
46+
// the setup's console spies. Each mock is reset by hand instead.
47+
const fetchStub = jest.mocked(global.fetch).getMockImplementation();
48+
let serverVersionSpy: jest.SpiedFunction<
49+
Metadata['getServerVersion']
50+
> | null = null;
51+
3552
afterEach(async () => {
3653
await server.clearDBs();
54+
jest.mocked(global.fetch).mockImplementation(fetchStub);
55+
serverVersionSpy?.mockRestore();
56+
serverVersionSpy = null;
3757
});
3858

3959
afterAll(async () => {
@@ -66,6 +86,29 @@ describe('request query attribution', () => {
6686
return [];
6787
}
6888

89+
// As findRouteQueries, for the PromQL evaluation a tile's request ran.
90+
async function findTileQueries(queryText: string, tile: string) {
91+
const client = await getTestFixtureClickHouseClient();
92+
for (let attempt = 0; attempt < 10; attempt++) {
93+
await client.command({ query: 'SYSTEM FLUSH LOGS' });
94+
const result = await client.query({
95+
query: `
96+
SELECT query_id, log_comment
97+
FROM system.query_log
98+
WHERE type = 'QueryFinish'
99+
AND position(query, {queryText:String}) > 0
100+
AND JSONExtractString(log_comment, 'tile') = {tile:String}`,
101+
query_params: { queryText, tile },
102+
format: 'JSON',
103+
});
104+
const { data }: { data: { query_id: string; log_comment: string }[] } =
105+
await result.json();
106+
if (data.length > 0) return data;
107+
await new Promise(resolve => setTimeout(resolve, 250));
108+
}
109+
return [];
110+
}
111+
69112
it('tags queries from an authenticated route with the api surface', async () => {
70113
const { agent, team } = await getLoggedInAgent(server);
71114
const conn = await Connection.create({
@@ -95,4 +138,95 @@ describe('request query attribution', () => {
95138
});
96139
}
97140
});
141+
142+
it('tags PromQL table-function queries with the attribution the browser sent', async () => {
143+
// Below 26.6 the router evaluates through prometheusQuery() rather than
144+
// proxying to ClickHouse's Prometheus HTTP API.
145+
serverVersionSpy = jest
146+
.spyOn(Metadata.prototype, 'getServerVersion')
147+
.mockResolvedValue([26, 5, 0, 0]);
148+
const { agent, team } = await getLoggedInAgent(server);
149+
const conn = await Connection.create({
150+
team: team._id,
151+
name: 'CH',
152+
host: config.CLICKHOUSE_HOST,
153+
username: config.CLICKHOUSE_USER,
154+
password: config.CLICKHOUSE_PASSWORD,
155+
});
156+
const tile = `tile-${Date.now()}`;
157+
158+
await agent
159+
.get('/v1/prometheus/query')
160+
.set(
161+
'x-hyperdx-query-attribution',
162+
buildLogComment({
163+
surface: 'dashboard',
164+
dashboard: 'dash-1',
165+
tile,
166+
label: 'from-browser',
167+
trace: 'from-browser',
168+
})!,
169+
)
170+
.query({
171+
query: 'attribution_metric',
172+
time: '1700003600',
173+
connectionId: conn._id.toString(),
174+
database: DEFAULT_DATABASE,
175+
table: TABLE,
176+
})
177+
.expect(200);
178+
179+
const logged = await findTileQueries('prometheusQuery(', tile);
180+
181+
expect(logged).toHaveLength(1);
182+
expect(logged[0].query_id).toMatch(/^hdx-dashboard-/);
183+
expect(JSON.parse(logged[0].log_comment)).toMatchObject({
184+
surface: 'dashboard',
185+
dashboard: 'dash-1',
186+
tile,
187+
// The server's, not the browser's.
188+
label: '/v1/prometheus',
189+
});
190+
expect(JSON.parse(logged[0].log_comment).trace).not.toBe('from-browser');
191+
});
192+
193+
// The CI ClickHouse serves prometheus_api_v1, which evaluates the PromQL
194+
// itself: this pins that it honours the log_comment and query_id we send.
195+
it('tags PromQL proxied to the prometheus_api_v1 handler', async () => {
196+
jest.mocked(global.fetch).mockImplementation(globalThis.realFetch);
197+
const { agent, team } = await getLoggedInAgent(server);
198+
const conn = await Connection.create({
199+
team: team._id,
200+
name: 'CH',
201+
host: config.CLICKHOUSE_HOST,
202+
username: config.CLICKHOUSE_USER,
203+
password: config.CLICKHOUSE_PASSWORD,
204+
});
205+
const tile = `tile-${Date.now()}`;
206+
207+
await agent
208+
.get('/v1/prometheus/query')
209+
.set(
210+
'x-hyperdx-query-attribution',
211+
buildLogComment({ surface: 'dashboard', dashboard: 'dash-1', tile })!,
212+
)
213+
.query({
214+
query: 'attribution_metric',
215+
time: '1700003600',
216+
connectionId: conn._id.toString(),
217+
database: DEFAULT_DATABASE,
218+
table: TABLE,
219+
})
220+
.expect(200);
221+
222+
const logged = await findTileQueries('prometheus_query_step', tile);
223+
expect(logged).toHaveLength(1);
224+
expect(logged[0].query_id).toMatch(/^hdx-dashboard-/);
225+
expect(JSON.parse(logged[0].log_comment)).toMatchObject({
226+
surface: 'dashboard',
227+
dashboard: 'dash-1',
228+
tile,
229+
label: '/v1/prometheus',
230+
});
231+
});
98232
});

‎packages/api/src/routers/api/__tests__/prometheus.int.test.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
import {
2+
buildLogComment,
3+
QUERY_ATTRIBUTION_HEADER,
4+
} from '@hyperdx/common-utils/dist/clickhouse';
15
import { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
26
import { Types } from 'mongoose';
37

@@ -749,6 +753,45 @@ describe('prometheus router', () => {
749753
});
750754
});
751755

756+
it('tags a request proxied to prometheus_api_v1 with log_comment and query_id', async () => {
757+
const { agent, team } = await getLoggedInAgent(server);
758+
const conn = await seedClickHouseConnection(team._id);
759+
mockFetch.mockResolvedValueOnce(handlerProbeResponse());
760+
761+
await agent
762+
.get('/v1/prometheus/query_range')
763+
.set(
764+
QUERY_ATTRIBUTION_HEADER,
765+
buildLogComment({ surface: 'dashboard', tile: 'tile-1' })!,
766+
)
767+
.query({
768+
query: 'up',
769+
start: '1700000000',
770+
end: '1700000060',
771+
step: '15s',
772+
table: 'metrics_ts',
773+
// Not caller-settable, so this never reaches the upstream.
774+
log_comment: 'from-caller',
775+
connectionId: conn._id.toString(),
776+
})
777+
.expect(200);
778+
779+
const [calledUrl, init] = mockFetch.mock.calls[1];
780+
const url = new URL(String(calledUrl));
781+
expect(url.searchParams.getAll('log_comment')).toHaveLength(1);
782+
expect(JSON.parse(url.searchParams.get('log_comment')!)).toMatchObject({
783+
surface: 'dashboard',
784+
tile: 'tile-1',
785+
label: '/v1/prometheus',
786+
});
787+
// In the URL, ClickHouse fails the request on query_id as an unknown
788+
// setting.
789+
expect(url.searchParams.has('query_id')).toBe(false);
790+
expect(init?.headers).toMatchObject({
791+
'X-ClickHouse-Query-Id': expect.stringMatching(/^hdx-dashboard-/),
792+
});
793+
});
794+
752795
it('does not let the request loosen the pinned ClickHouse limits', async () => {
753796
const { agent, team } = await getLoggedInAgent(server);
754797
const conn = await seedClickHouseConnection(team._id);

‎packages/api/src/routers/api/prometheus.ts‎

Lines changed: 61 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,13 @@
1+
import {
2+
buildLogComment,
3+
buildQueryId,
4+
parseLogComment,
5+
QUERY_ATTRIBUTION_HEADER,
6+
} from '@hyperdx/common-utils/dist/clickhouse';
7+
import {
8+
getCurrentQueryAttribution,
9+
withQueryAttribution,
10+
} from '@hyperdx/common-utils/dist/clickhouse/node';
111
import express from 'express';
212
import { performance } from 'perf_hooks';
313
import { Readable } from 'stream';
@@ -53,6 +63,20 @@ router.use((_req, res, next) => {
5363
next();
5464
});
5565

66+
// The browser reaches this router over fetch, not through a ClickHouse client,
67+
// so it sends the tile, dashboard or search a query belongs to in a header.
68+
// Its surface and ids win over the `api` scope the auth middleware opened, but
69+
// the trace id and route label are the server's to state: a client-sent trace
70+
// would link the query_log row to a request that never ran it.
71+
router.use((req, _res, next) => {
72+
const attribution = parseLogComment(req.get(QUERY_ATTRIBUTION_HEADER));
73+
if (!attribution) return next();
74+
// Dropped rather than overwritten: with no active span there is no server
75+
// trace to overwrite with, and the browser's would survive the merge.
76+
const { label: _label, trace: _trace, ...fromBrowser } = attribution;
77+
withQueryAttribution(fromBrowser, () => next());
78+
});
79+
5680
// --------------------------
5781
// Param parsing helpers
5882
// --------------------------
@@ -210,14 +234,45 @@ export function clickhousePrometheusUpstream(
210234
{
211235
maxExecutionSec = PROMETHEUS_MAX_EXECUTION_SEC,
212236
maxResultRows = PROMETHEUS_MAX_RESULT_ROWS,
237+
logComment,
238+
}: {
239+
maxExecutionSec?: number;
240+
maxResultRows?: number;
241+
logComment?: string;
213242
} = {},
214243
): string {
215244
const url = new URL(host);
216245
url.searchParams.set('max_execution_time', String(maxExecutionSec));
217246
url.searchParams.set('max_result_rows', String(maxResultRows));
247+
if (logComment) url.searchParams.set('log_comment', logComment);
218248
return url.toString();
219249
}
220250

251+
/**
252+
* Where and how to proxy a PromQL request to the connection's prometheus_api_v1
253+
* handler, tagged with the query attribution in scope. The handler runs the
254+
* query itself rather than through our ClickHouse client, so the tags are set
255+
* here. `log_comment` goes in the URL like the limits. `query_id` goes in a
256+
* header: the handler reads every URL param as a setting, and `query_id` in
257+
* the URL fails the request as an unknown one.
258+
*/
259+
function clickhousePrometheusRequest(connection: {
260+
host: string;
261+
username: string;
262+
password?: string;
263+
}): { upstream: string; headers: Record<string, string> } {
264+
const attribution = getCurrentQueryAttribution();
265+
return {
266+
upstream: clickhousePrometheusUpstream(connection.host, {
267+
logComment: buildLogComment(attribution),
268+
}),
269+
headers: {
270+
...clickhouseAuthHeaders(connection),
271+
'X-ClickHouse-Query-Id': buildQueryId(attribution),
272+
},
273+
};
274+
}
275+
221276
function newClickhouseClient(connection: {
222277
host: string;
223278
username: string;
@@ -537,12 +592,13 @@ const queryRangeHandler: express.RequestHandler = async (req, res) => {
537592
// forward-compatible while TimeSeries is in preview, so it is preferred
538593
// wherever the server has it (26.6+); the table function is the fallback.
539594
if (await clickhouseServesPrometheusHttpApi(client, connection)) {
595+
const { upstream, headers } = clickhousePrometheusRequest(connection);
540596
const status = await proxyToPrometheus(
541-
clickhousePrometheusUpstream(connection.host),
597+
upstream,
542598
`${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query_range`,
543599
params,
544600
res,
545-
clickhouseAuthHeaders(connection),
601+
headers,
546602
);
547603
recordProxyOutcome(status, 'query_range', backend);
548604
return;
@@ -648,12 +704,13 @@ const queryHandler: express.RequestHandler = async (req, res) => {
648704
const client = newClickhouseClient(connection);
649705

650706
if (await clickhouseServesPrometheusHttpApi(client, connection)) {
707+
const { upstream, headers } = clickhousePrometheusRequest(connection);
651708
const status = await proxyToPrometheus(
652-
clickhousePrometheusUpstream(connection.host),
709+
upstream,
653710
`${CLICKHOUSE_PROMETHEUS_API_PREFIX}/query`,
654711
params,
655712
res,
656-
clickhouseAuthHeaders(connection),
713+
headers,
657714
);
658715
recordProxyOutcome(status, 'query', backend);
659716
return;

0 commit comments

Comments
 (0)