+ {/* The corner above the label column is otherwise empty, so the
+ zoom hint lives here instead of in its own row above the chart.
+ It reuses the tick label class and inherits the axis font so it
+ reads as part of the axis header. */}
+ >
+
+ ⌘/Ctrl + scroll to zoom
+
+
{rowVirtualizer.getVirtualItems().map(virtualRow => {
const row = rows[virtualRow.index];
diff --git a/packages/app/src/components/TimelineChart/__tests__/TimelineChart.test.tsx b/packages/app/src/components/TimelineChart/__tests__/TimelineChart.test.tsx
new file mode 100644
index 000000000..10ca8856f
--- /dev/null
+++ b/packages/app/src/components/TimelineChart/__tests__/TimelineChart.test.tsx
@@ -0,0 +1,39 @@
+import { screen } from '@testing-library/react';
+
+import { TimelineChart } from '@/components/TimelineChart/TimelineChart';
+import type { TTimelineEvent } from '@/components/TimelineChart/TimelineChartRowEvents';
+
+function makeRow(id: string, start: number, end: number) {
+ const event: TTimelineEvent = {
+ id,
+ start,
+ end,
+ tooltip: id,
+ color: 'white',
+ backgroundColor: 'gray',
+ body:
{id},
+ minWidthPx: 1,
+ };
+ return { id, label:
{id}, events: [event] };
+}
+
+describe('TimelineChart', () => {
+ it('shows the zoom hint in the corner above the labels, inside the chart', () => {
+ renderWithMantine(
+
,
+ );
+
+ const hint = screen.getByTestId('timeline-zoom-hint');
+ expect(hint).toHaveTextContent('⌘/Ctrl + scroll to zoom');
+ // Lives in the corner cell above the span labels rather than in a row of
+ // its own above the chart.
+ expect(hint.closest('[data-testid="timeline-corner"]')).toBeInTheDocument();
+ });
+});
diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.compactHeader.test.ts b/packages/app/src/components/__tests__/DBRowSidePanel.compactHeader.test.ts
new file mode 100644
index 000000000..6f7b43f22
--- /dev/null
+++ b/packages/app/src/components/__tests__/DBRowSidePanel.compactHeader.test.ts
@@ -0,0 +1,72 @@
+import {
+ getHeaderHighlightExpressions,
+ isSpanNameShownInTitle,
+} from '@/components/DBRowSidePanel';
+import { makeLogSource, makeTraceSource } from '@/llm/__fixtures__/sources';
+
+describe('isSpanNameShownInTitle', () => {
+ const trace = makeTraceSource();
+
+ it('is true when the lone title crumb is a short span name', () => {
+ expect(
+ isSpanNameShownInTitle(trace, 'GET /checkout', ['GET /checkout']),
+ ).toBe(true);
+ });
+
+ it('allows names up to 40 characters', () => {
+ const name = 'x'.repeat(40);
+ expect(isSpanNameShownInTitle(trace, name, [name])).toBe(true);
+ });
+
+ it('keeps the body for longer names the title can cut off', () => {
+ const name = 'x'.repeat(41);
+ expect(isSpanNameShownInTitle(trace, name, [name])).toBe(false);
+ });
+
+ it('keeps the body in a breadcrumb trail, where crumbs share the row', () => {
+ // A hop between two spans with the same name is still a trail.
+ expect(
+ isSpanNameShownInTitle(trace, 'GET /checkout', [
+ 'GET /checkout',
+ 'GET /checkout',
+ ]),
+ ).toBe(false);
+ });
+
+ it('keeps the body when the title is not the span name', () => {
+ expect(isSpanNameShownInTitle(trace, 'GET /checkout', ['Trace'])).toBe(
+ false,
+ );
+ });
+
+ it('keeps the body when the span has no body value', () => {
+ expect(isSpanNameShownInTitle(trace, undefined, ['Trace'])).toBe(false);
+ });
+
+ it('keeps the body on log panels', () => {
+ expect(
+ isSpanNameShownInTitle(makeLogSource(), 'a log line', ['a log line']),
+ ).toBe(false);
+ });
+});
+
+describe('getHeaderHighlightExpressions', () => {
+ it('does not repeat the service as a chip on trace panels', () => {
+ expect(getHeaderHighlightExpressions(makeTraceSource())).toEqual([]);
+ });
+
+ it('still highlights the service on log panels', () => {
+ expect(getHeaderHighlightExpressions(makeLogSource())).toEqual([
+ { sqlExpression: 'ServiceName' },
+ ]);
+ });
+
+ it('keeps highlights configured on the source', () => {
+ const highlight = { sqlExpression: "SpanAttributes['http.method']" };
+ expect(
+ getHeaderHighlightExpressions(
+ makeTraceSource({ highlightedRowAttributeExpressions: [highlight] }),
+ ),
+ ).toEqual([highlight]);
+ });
+});
diff --git a/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx
index 0bd32b7f3..9b49e56c3 100644
--- a/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx
+++ b/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx
@@ -1,7 +1,9 @@
import React from 'react';
import { screen } from '@testing-library/react';
+import type { HighlightedAttribute } from '@/components/DBHighlightedAttributesList';
import DBRowSidePanelHeader from '@/components/DBRowSidePanelHeader';
+import { makeTraceSource } from '@/llm/__fixtures__/sources';
// Stub modules that touch user preferences, AI APIs, or DOM observers; the
// behavior under test is the body-paper visibility logic, not those deps.
@@ -22,7 +24,9 @@ jest.mock('../AISummarizeButton', () => ({
}));
jest.mock('../DBHighlightedAttributesList', () => ({
- DBHighlightedAttributesList: () => null,
+ DBHighlightedAttributesList: ({ attributes }: { attributes: unknown[] }) => (
+
{attributes.length}
+ ),
}));
jest.mock('../DBRowSidePanel', () => ({
@@ -84,4 +88,47 @@ describe('DBRowSidePanelHeader: body section (HDX-4373)', () => {
renderWithMantine(
);
expect(screen.queryByText('No body for this event.')).toBeInTheDocument();
});
+
+ it('omits the body paper when the panel title already shows the body', () => {
+ renderWithMantine(
+
,
+ );
+ expect(screen.queryByText('GET /checkout')).not.toBeInTheDocument();
+ expect(screen.queryByText('SpanName')).not.toBeInTheDocument();
+ expect(
+ screen.queryByText('No body for this event.'),
+ ).not.toBeInTheDocument();
+ });
+});
+
+describe('DBRowSidePanelHeader: highlighted attributes', () => {
+ const attribute: HighlightedAttribute = {
+ source: makeTraceSource(),
+ displayedKey: 'http.method',
+ value: 'GET',
+ sql: "SpanAttributes['http.method']",
+ };
+
+ it('renders the attribute list when there are attributes', () => {
+ renderWithMantine(
+
,
+ );
+ expect(screen.getByTestId('highlighted-attributes')).toHaveTextContent('1');
+ });
+
+ it('renders nothing for the attribute list when there are none', () => {
+ renderWithMantine(
+
,
+ );
+ expect(
+ screen.queryByTestId('highlighted-attributes'),
+ ).not.toBeInTheDocument();
+ });
});
From 253a690d49776c390d2c5b7e4824edeb011f027d Mon Sep 17 00:00:00 2001
From: Baris Can Vural
Date: Wed, 4 Nov 2026 21:37:01 +0300
Subject: [PATCH 027/103] docs(mcp): recommend quoted Lucene values for exact
matches and bare terms for body text (#3548)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Refs #3538
## What
Only descriptions and the query guide change; Lucene processing is untouched.
- **Query tool `where` / `whereLanguage` descriptions** (`packages/api/src/mcp/tools/query/schemas.ts`):
- for an exact match, quote the value: `SpanKind:"Server"` compiles to `=`, so a sort-key column such as `ServiceName` can use the primary key (SQL `=` is still listed);
- for text in the log body, write the term or phrase without a field name. That uses the source's token/text index; `Body:term`, `Body LIKE` and `Body ILIKE` read the whole column, and `Body:"…"` is exact equality on the whole body.
- **Query guide prompt** (`packages/api/src/mcp/prompts/dashboards/content.ts`): the same two points in the SUBSTRING MATCHING section and in common mistake 15; the "Contains" example now uses `StatusMessage` instead of `Body LIKE '%…%'`.
- `prompts.test.ts` asserts both points are in the guide.
## Why
Agents read "for exact match, use SQL" and mostly kept writing `ServiceName:x`, which matches `x-jobs` too and skips the primary key (one hour of logs: 108.9M vs 1.65M rows read). For body text, the descriptions pointed to `LIKE`, which can't use the index (27.7 GiB vs 121 MiB for the same matches). Details in #3538.
What each recommended form compiles to on `main`:
| Lucene | SQL |
|---|---|
| `ServiceName:api` | `ServiceName ILIKE '%api%'` |
| `ServiceName:"api"` | `ServiceName = 'api'` |
| `timeout` | `hasAllTokens(lower(Body), lower('timeout'))` (with a text index) |
| `"connection refused"` | `hasAllTokens(lower(Body), lower('connection refused')) AND lower(Body) LIKE lower('%connection refused%')` |
| `Body:"connection refused"` | `Body = 'connection refused'` |
The optional hint ("`ServiceName:api` matched 2 values") is not part of this PR; happy to follow up if you want it.
## Tests
- `npx jest src/mcp --testPathIgnorePatterns '\.int\.test\.ts$'` in `packages/api`: 25 suites, 484 tests pass.
- `tsc --noEmit`, `eslint` (no new warnings) and `prettier` clean.
---
.changeset/mcp-lucene-exact-match.md | 7 ++++++
.../mcp/__tests__/dashboards/prompts.test.ts | 7 +++++-
packages/api/src/mcp/__tests__/query.test.ts | 16 ++++++++++++++
.../api/src/mcp/prompts/dashboards/content.ts | 22 ++++++++++++++-----
packages/api/src/mcp/tools/query/schemas.ts | 12 ++++++++--
5 files changed, 55 insertions(+), 9 deletions(-)
create mode 100644 .changeset/mcp-lucene-exact-match.md
diff --git a/.changeset/mcp-lucene-exact-match.md b/.changeset/mcp-lucene-exact-match.md
new file mode 100644
index 000000000..853bb32a8
--- /dev/null
+++ b/.changeset/mcp-lucene-exact-match.md
@@ -0,0 +1,7 @@
+---
+'@hyperdx/api': patch
+---
+
+docs: MCP tool descriptions and the query guide now recommend the quoted Lucene
+form `field:"value"` for exact matches, and bare terms or phrases for log body
+text, so agents keep the primary key and the text index
diff --git a/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts b/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts
index 124e020cd..00daf1c76 100644
--- a/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts
+++ b/packages/api/src/mcp/__tests__/dashboards/prompts.test.ts
@@ -175,8 +175,13 @@ describe('MCP Dashboard Prompts', () => {
const luceneBody = prompt.slice(luceneIdx);
expect(luceneBody).toMatch(/SUBSTRING MATCHING/);
expect(luceneBody).toMatch(/ilike\(field, '%value%'\)/);
- // The canonical fix is SQL with =.
+ // The canonical fix is SQL with =, or a quoted Lucene value.
expect(luceneBody).toMatch(/SpanKind = 'Server'/);
+ expect(luceneBody).toMatch(
+ /SpanKind:"Server" compiles to SpanKind = 'Server'/,
+ );
+ // Body text goes through the token/text index as a bare term, not Body:term.
+ expect(luceneBody).toMatch(/bare Lucene term or phrase/);
// Wildcard limitation is documented alongside substring matching.
expect(luceneBody).toMatch(/prefix-within-substring/);
expect(luceneBody).toMatch(/LIKE 'api%'/);
diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts
index 0855c40bc..8d4ea7a42 100644
--- a/packages/api/src/mcp/__tests__/query.test.ts
+++ b/packages/api/src/mcp/__tests__/query.test.ts
@@ -28,6 +28,8 @@ import {
getMetricSelectIssues,
mcpQuantileLevelSchema,
validateMetricSelectItems,
+ whereLanguageSchema,
+ whereSchema,
} from '@/mcp/tools/query/schemas';
import { resolveOrderBy } from '@/mcp/tools/query/table';
import { getErrorCategory } from '@/mcp/utils/errors';
@@ -1414,3 +1416,17 @@ describe('mcpQuantileLevelSchema', () => {
expect(optional.safeParse('0.97').success).toBe(false);
});
});
+
+describe('where / whereLanguage descriptions', () => {
+ it('steer exact matches to the quoted Lucene form and body text to bare terms', () => {
+ expect(whereSchema.description).toContain(
+ 'quote the value: SpanKind:"Server"',
+ );
+ expect(whereSchema.description).toContain(
+ 'BODY TEXT: write the term or phrase without a field name',
+ );
+ expect(whereLanguageSchema.description).toContain(
+ 'quote the value (field:"value", case-sensitive)',
+ );
+ });
+});
diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts
index a45d9fcf8..159e391db 100644
--- a/packages/api/src/mcp/prompts/dashboards/content.ts
+++ b/packages/api/src/mcp/prompts/dashboards/content.ts
@@ -862,7 +862,7 @@ Used in the "where" field of select items and search tiles.
OR: level:error OR level:fatal
NOT: NOT level:debug
Wildcard: ServiceName:front*
- Phrase: Body:"connection refused"
+ Phrase: "connection refused" (default search column, e.g. the log body)
Exists: _exists_:http.route
Range (numeric): Duration:>1000000000
Range (inclusive): http.status_code:[400 TO 499]
@@ -883,13 +883,22 @@ Lucene field:value does NOT mean exact equality. It translates to ilike(field, '
ServiceName:api* does NOT mean "starts with api". It means "contains a substring starting with api".
In practice this often works for prefix matching, but it is NOT equivalent to SQL LIKE 'api%'.
- For exact matching or true prefix/suffix/contains patterns, use SQL instead:
+ For exact matching in Lucene, quote the value: SpanKind:"Server" compiles to SpanKind = 'Server'
+ (case-sensitive, unlike the unquoted form).
+
+ For true prefix/suffix/contains patterns, use SQL instead:
Exact: whereLanguage: "sql", where: "SpanKind = 'Server'"
Prefix: whereLanguage: "sql", where: "ServiceName LIKE 'api%'"
- Contains: whereLanguage: "sql", where: "Body LIKE '%timeout%'"
+ Contains: whereLanguage: "sql", where: "StatusMessage LIKE '%timeout%'"
Regex: whereLanguage: "sql", where: "match(ServiceName, '^api-v[0-9]+')"
- When Lucene substring matching is FINE: free-text columns like Body, StatusMessage where substring is the intent.
+ For text in the log body, write a bare Lucene term or phrase (timeout, "connection refused"). It searches the
+ source's default search column (the body for logs) and uses its text index when the source has one. With an
+ index it matches whole tokens (timeout does not match ConnectionTimeoutException); for a substring inside a word, use
+ SQL ILIKE. Body:timeout, Body LIKE and Body ILIKE read the whole column, and Body:"..." is exact equality on
+ the whole body.
+
+ When Lucene substring matching is FINE: free-text columns like StatusMessage where substring is the intent.
When Lucene substring matching is WRONG: enum-like columns (SpanKind, SeverityText, StatusCode, ServiceName) where you want exact or true prefix match.
*** MAP-ATTRIBUTE GOTCHA ***
@@ -923,7 +932,7 @@ Alternative to Lucene. Set whereLanguage: "sql" when using SQL syntax.
Basic: SeverityText = 'error'
AND/OR: ServiceName = 'api' AND StatusCode = 'STATUS_CODE_ERROR'
IN: ServiceName IN ('api', 'web', 'worker')
- LIKE: Body LIKE '%timeout%'
+ LIKE: StatusMessage LIKE '%timeout%'
Comparison: Duration > 1000000000
Map access: SpanAttributes['http.status_code'] = '500'
@@ -1562,8 +1571,9 @@ Example: find top patterns for production services over the last 4 hours:
Wrong: where: "SpanKind:Server" (whereLanguage: "lucene"; translates to ilike(SpanKind, '%Server%') and matches "ServerStreaming", "InternalServer", etc.)
Wrong: where: "ServiceName:api*" (whereLanguage: "lucene"; NOT a true prefix match -- it is prefix-within-substring)
Correct: where: "SpanKind = 'Server'" (whereLanguage: "sql"; exact match)
+ Correct: where: 'SpanKind:"Server"' (whereLanguage: "lucene"; a quoted value is an exact match)
Correct: where: "ServiceName LIKE 'api%'" (whereLanguage: "sql"; true prefix match)
- Lucene field:value is a substring match (ilike with surrounding wildcards), not exact equality. Lucene field:value* is prefix-within-substring, not SQL LIKE 'value%'. Fine for free-text columns (Body, StatusMessage); wrong for enum-like columns where exact match matters (SpanKind, SeverityText, StatusCode) and wrong when true prefix matching is needed. Use SQL for exact match (=), prefix (LIKE 'x%'), contains (LIKE '%x%'), or regex (match(col, 'pattern')).
+ Lucene field:value is a substring match (ilike with surrounding wildcards), not exact equality. Lucene field:value* is prefix-within-substring, not SQL LIKE 'value%'. Fine for free-text columns (StatusMessage; for Body use a bare term or phrase, which uses the index); wrong for enum-like columns where exact match matters (SpanKind, SeverityText, StatusCode) and wrong when true prefix matching is needed. Use SQL for exact match (=), prefix (LIKE 'x%'), contains (LIKE '%x%'), or regex (match(col, 'pattern')).
16. Forgetting that map-attribute values are often empty strings
Wrong: groupBy on SpanAttributes['db.collection.name'] without filtering empty values; the result has an empty-string bucket alongside the real ones, which renders as a blank row in tables and an unlabelled slice in pies.
diff --git a/packages/api/src/mcp/tools/query/schemas.ts b/packages/api/src/mcp/tools/query/schemas.ts
index e3e70664b..b48dbe460 100644
--- a/packages/api/src/mcp/tools/query/schemas.ts
+++ b/packages/api/src/mcp/tools/query/schemas.ts
@@ -20,7 +20,14 @@ const WHERE_DESCRIPTION =
'WRONG: level = "error" (SQL syntax with whereLanguage:"lucene")\n\n' +
'SUBSTRING TRAP: Lucene field:value matches ANY row containing "value" as a substring, not exact equality.\n' +
' SpanKind:Server matches "Server", "ServerStreaming", "InternalServer", etc.\n' +
- " For exact match, use SQL: SpanKind = 'Server'";
+ ' For exact match, quote the value: SpanKind:"Server" (compiles to a case-sensitive =, so a sort-key\n' +
+ " column such as ServiceName can use the primary key), or use SQL: SpanKind = 'Server'\n\n" +
+ 'BODY TEXT: write the term or phrase without a field name, e.g. timeout or "connection refused".\n' +
+ " That searches the source's default search column (the body for logs) and uses its text index\n" +
+ ' when the source has one. With an index it matches whole tokens: timeout does not match ConnectionTimeoutException.\n' +
+ " For a substring inside a word, use SQL ILIKE '%timeout%' (reads the whole column).\n" +
+ ' Body:timeout, Body LIKE and Body ILIKE read the whole column, and Body:"..." is exact equality on the\n' +
+ ' whole body.';
const WHERE_LANGUAGE_DESCRIPTION =
'Language of the "where" filter. The syntax in "where" MUST match this setting.\n' +
@@ -30,7 +37,8 @@ const WHERE_LANGUAGE_DESCRIPTION =
'and map attributes via dot notation. Use "sql" for IN(...) lists, complex expressions, or function calls.\n' +
'IMPORTANT: Lucene field:value is a SUBSTRING match (ilike), not exact equality. ' +
'field:val* is prefix-within-substring, not a true prefix match. ' +
- "For exact matching or reliable wildcards, use SQL: WHERE field = 'value' or WHERE field LIKE 'val%'.\n\n" +
+ 'For exact matching, quote the value (field:"value", case-sensitive) or use SQL: WHERE field = \'value\'. ' +
+ "For reliable wildcards, use SQL: WHERE field LIKE 'val%'.\n\n" +
'Common mistake: writing Column:value (Lucene) but setting whereLanguage to "sql". ' +
'If your filter uses colon syntax, use "lucene".';
From af75b2b59d0891a3528782891a572da3d190ea74 Mon Sep 17 00:00:00 2001
From: Spencer Torres
Date: Wed, 4 Nov 2026 11:53:45 -0800
Subject: [PATCH 028/103] feat: browser events in trace waterfall (#3443)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Closes HDX-5774 Stacks on #3440
## Summary
Newer OTel browser SDKs no longer put network timing on fetch and XHR spans. They send one `browser.resource_timing` log per request instead, and the click that started the request is a separate log with no trace id. This draws the timing logs as dots on their spans, like the old SDK's span events, and adds a "Triggered by" row for the click. Old-SDK and backend traces are unchanged.
### Changes
- **Gate** (`hooks/useBrowserTraceEvents.ts`, `utils/browserTraceContext.ts`): both queries run only for traces with a `browser.document.url.full` span, when the correlated log source has Event Name, Event Attributes, Trace Id and Span Id expressions. They start after the waterfall loads and fail quietly.
- **Timing dots** (`utils/resourceTimingMarkers.ts`): `browser.resource_timing` logs are read by TraceId and drawn on their span through the existing span event markers, named like sdk-trace-web's events. The last dot's tooltip shows duration, sizes and protocol, since new-SDK spans don't carry the response size. Dots follow the **Span events** chip.
- **Timing rows**: once the dots load, the log rows they replace leave the waterfall and the log count, so they show for about one round trip first. They stay when the dots aren't visible (**Spans** or **Span events** off, a logs filter) or when the row is selected.
- **Triggered by** (`utils/traceTriggerClick.ts`): the latest `browser.user_action.click` with the same `session.id` and service in the 250 ms before the first browser span, shown first as "Triggered by: Clicked … · 3ms before". Clicking it opens the click log. It doesn't match on page URL: the SDK records the click's URL before the handler changes the route or query string.
- **Shared code**: E2's session log query parts move to `getSessionLogEventQueryParts` (`utils/sessionLogEvents.ts`) for the trigger query. E2's SQL is unchanged.
### Screenshots
Example of a browser event triggering this trace, along with timing events on the span:
---
.changeset/trace-waterfall-browser-events.md | 5 +
.../src/components/DBTraceWaterfallChart.tsx | 86 ++++-
.../TimelineChart/TimelineSpanEventMarker.tsx | 1 +
.../__tests__/DBTraceWaterfallChart.test.tsx | 329 ++++++++++++++++++
.../hooks/__fixtures__/browserTraceEvents.ts | 90 +++++
.../__tests__/useBrowserTraceEvents.test.tsx | 275 +++++++++++++++
.../app/src/hooks/useBrowserTraceEvents.ts | 220 ++++++++++++
packages/app/src/hooks/useRowHighlightHint.ts | 22 +-
packages/app/src/hooks/useSessionLogEvents.ts | 2 +-
.../__tests__/browserTraceContext.test.ts | 144 ++++++++
.../__tests__/resourceTimingMarkers.test.ts | 326 +++++++++++++++++
.../utils/__tests__/traceTriggerClick.test.ts | 152 ++++++++
packages/app/src/utils/browserEvents.ts | 1 +
packages/app/src/utils/browserTraceContext.ts | 140 ++++++++
packages/app/src/utils/httpSpanAttributes.ts | 2 +-
.../app/src/utils/resourceTimingMarkers.ts | 250 +++++++++++++
packages/app/src/utils/sessionLogEvents.ts | 158 ++++++---
packages/app/src/utils/traceTriggerClick.ts | 103 ++++++
.../app/tests/e2e/features/sessions.spec.ts | 26 +-
packages/app/tests/e2e/seed-otel-sessions.ts | 29 +-
20 files changed, 2272 insertions(+), 89 deletions(-)
create mode 100644 .changeset/trace-waterfall-browser-events.md
create mode 100644 packages/app/src/hooks/__fixtures__/browserTraceEvents.ts
create mode 100644 packages/app/src/hooks/__tests__/useBrowserTraceEvents.test.tsx
create mode 100644 packages/app/src/hooks/useBrowserTraceEvents.ts
create mode 100644 packages/app/src/utils/__tests__/browserTraceContext.test.ts
create mode 100644 packages/app/src/utils/__tests__/resourceTimingMarkers.test.ts
create mode 100644 packages/app/src/utils/__tests__/traceTriggerClick.test.ts
create mode 100644 packages/app/src/utils/browserTraceContext.ts
create mode 100644 packages/app/src/utils/resourceTimingMarkers.ts
create mode 100644 packages/app/src/utils/traceTriggerClick.ts
diff --git a/.changeset/trace-waterfall-browser-events.md b/.changeset/trace-waterfall-browser-events.md
new file mode 100644
index 000000000..26be802d3
--- /dev/null
+++ b/.changeset/trace-waterfall-browser-events.md
@@ -0,0 +1,5 @@
+---
+'@hyperdx/app': patch
+---
+
+feat: Trace waterfall shows OpenTelemetry browser SDK request timings as dots on fetch and XHR spans, and the click that triggered a browser trace
diff --git a/packages/app/src/components/DBTraceWaterfallChart.tsx b/packages/app/src/components/DBTraceWaterfallChart.tsx
index 15c7206f7..9b22ee2f0 100644
--- a/packages/app/src/components/DBTraceWaterfallChart.tsx
+++ b/packages/app/src/components/DBTraceWaterfallChart.tsx
@@ -35,6 +35,7 @@ import {
IconChevronsDown,
IconChevronsRight,
IconLogs,
+ IconPointer,
} from '@tabler/icons-react';
import { ContactSupportText } from '@/components/ContactSupportText';
@@ -51,6 +52,7 @@ import {
TRACE_WATERFALL_FETCH_LIMIT,
TRACE_WATERFALL_ROW_LIMIT,
} from '@/components/traceWaterfallLimits';
+import { useBrowserTraceEvents } from '@/hooks/useBrowserTraceEvents';
import { useColumns } from '@/hooks/useMetadata';
import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery';
import {
@@ -107,6 +109,7 @@ export type SpanRow = {
Attributes: Record;
}>;
__hdx_hidden?: boolean | 1 | 0;
+ triggeredBy?: string;
};
type TimestampedRow = {
@@ -785,11 +788,12 @@ export function DBTraceWaterfallChartContainer({
logColumns?.some(column => column.name === 'SeverityNumber') ?? false;
const {
- rows: logRowsData,
+ rows: fetchedLogRows,
isFetching: logIsFetching,
meta: logRowsMeta,
filterError: logFilterError,
fatalError: logFatalError,
+ isTruncated: logIsTruncated,
} = useFilteredEventsAroundFocus({
// search data if logTableModel exist
// search invalid date range if no logTableModel(react hook need execute no matter what)
@@ -816,8 +820,33 @@ export function DBTraceWaterfallChartContainer({
// shown), otherwise the fatal error (no logs could be loaded at all).
const logError = logFilterError ?? logFatalError;
+ const [showSpanEvents, setShowSpanEvents] = useState(true);
+ const [showSpans, setShowSpans] = useState(true);
+ const [showLogs, setShowLogs] = useState(true);
+
+ const {
+ logRows: logRowsData,
+ resourceTimingBySpanId,
+ triggerRow,
+ } = useBrowserTraceEvents({
+ traceRows: traceRowsData,
+ logRows: fetchedLogRows,
+ logSource: logTableSource,
+ traceId,
+ highlightedRowWhere,
+ initialRowHighlightHint,
+ // Only while the dots show. A log filter asks for the rows it matches. A
+ // capped list can keep a log that shares a timing log's key and drop the
+ // timing log, and the fold would then hide that other log.
+ foldTimingRows: showSpans && showSpanEvents && !logWhere && !logIsTruncated,
+ enabled: !traceIsFetching,
+ });
+
const rows: any[] = useMemo(() => {
- const nextRows: Array<(typeof traceRowsData)[number] & TimestampedRow> = [
+ // The trigger row goes first so a click with the same timestamp as the
+ // root span sorts before it (the sort is stable).
+ const nextRows: TimestampedRow[] = [
+ ...(triggerRow ? [triggerRow] : []),
...traceRowsData,
...logRowsData,
];
@@ -833,7 +862,7 @@ export function DBTraceWaterfallChartContainer({
});
return nextRows;
- }, [traceRowsData, logRowsData]);
+ }, [triggerRow, traceRowsData, logRowsData]);
const validSpanIDs = useMemo(() => {
return new Set(
@@ -983,9 +1012,6 @@ export function DBTraceWaterfallChartContainer({
};
const [collapsedIds, setCollapsedIds] = useState>(new Set());
- const [showSpanEvents, setShowSpanEvents] = useState(true);
- const [showSpans, setShowSpans] = useState(true);
- const [showLogs, setShowLogs] = useState(true);
const { nodesMap, flattenedNodes, parentIdsByLevel } = useMemo(() => {
const rootNodes: Node[] = [];
@@ -1128,6 +1154,8 @@ export function DBTraceWaterfallChartContainer({
const visibleNodes = useMemo(() => {
if (showSpans && showLogs) return flattenedNodes;
return flattenedNodes.filter(node => {
+ // The click belongs with the spans it started, not with the logs.
+ if (node.triggeredBy) return showSpans;
if (node.type === SourceKind.Log) return showLogs;
return showSpans;
});
@@ -1137,7 +1165,7 @@ export function DBTraceWaterfallChartContainer({
node => node.type !== SourceKind.Log,
).length;
const logCount = visibleNodes.filter(
- node => node.type === SourceKind.Log,
+ node => node.type === SourceKind.Log && !node.triggeredBy,
).length;
const errorCount = visibleNodes.filter(
node => getRowSeverity(node) === 'error',
@@ -1178,6 +1206,7 @@ export function DBTraceWaterfallChartContainer({
id,
type,
aliasWith,
+ triggeredBy,
} = result;
let body = `${_body}`;
try {
@@ -1193,12 +1222,16 @@ export function DBTraceWaterfallChartContainer({
const httpUrl = isRpcSpan(spanAttributes)
? undefined
: getHttpUrl(spanAttributes);
- const displayText = httpUrl ? `${body} ${httpUrl}` : body;
+ const displayText =
+ triggeredBy ?? (httpUrl ? `${body} ${httpUrl}` : body);
+ const startedAt = !isNaN(startOffset)
+ ? ` | Started at ${formatTime(new Date(startOffset), { format: 'withMs' })}`
+ : '';
// Process span events into markers (only if showSpanEvents is enabled)
- const markers =
- showSpanEvents && result.SpanEvents
- ? result.SpanEvents.map(spanEvent => {
+ const markers = showSpanEvents
+ ? [
+ ...(result.SpanEvents ?? []).map(spanEvent => {
const epochMs = parseTimestampToMs(spanEvent.Timestamp);
return {
timestamp: epochMs - minOffset,
@@ -1206,8 +1239,17 @@ export function DBTraceWaterfallChartContainer({
name: spanEvent.Name,
attributes: spanEvent.Attributes || {},
};
- })
- : [];
+ }),
+ // Log rows carry their parent span's SpanId.
+ ...(type !== SourceKind.Log
+ ? (resourceTimingBySpanId.get(result.SpanId) ?? [])
+ : []
+ ).map(marker => ({
+ ...marker,
+ timestamp: marker.epochMs - minOffset,
+ })),
+ ]
+ : [];
const severity = getRowSeverity(result);
const isError = severity === 'error';
@@ -1316,7 +1358,13 @@ export function DBTraceWaterfallChartContainer({
)}
- {type === SourceKind.Log ? (
+ {triggeredBy ? (
+
+ ) : type === SourceKind.Log ? (
{serviceName && <>{serviceName} >}
@@ -1347,7 +1398,9 @@ export function DBTraceWaterfallChartContainer({
aliasWith,
start,
end,
- tooltip: `${displayText} ${tookMs >= 0 ? `took ${tookMs.toFixed(4)}ms` : ''} ${status ? `| Status: ${status}` : ''}${!isNaN(startOffset) ? ` | Started at ${formatTime(new Date(startOffset), { format: 'withMs' })}` : ''}`,
+ tooltip: triggeredBy
+ ? `${triggeredBy}${startedAt}`
+ : `${displayText} ${tookMs >= 0 ? `took ${tookMs.toFixed(4)}ms` : ''} ${status ? `| Status: ${status}` : ''}${startedAt}`,
color: 'var(--color-text-inverted)',
backgroundColor: barColor({
isHighlighted,
@@ -1373,6 +1426,7 @@ export function DBTraceWaterfallChartContainer({
highlightedRowWhere,
minOffset,
onClick,
+ resourceTimingBySpanId,
serviceColorMap,
showSpanEvents,
toggleCollapse,
diff --git a/packages/app/src/components/TimelineChart/TimelineSpanEventMarker.tsx b/packages/app/src/components/TimelineChart/TimelineSpanEventMarker.tsx
index ad142ad5b..219e88ecd 100644
--- a/packages/app/src/components/TimelineChart/TimelineSpanEventMarker.tsx
+++ b/packages/app/src/components/TimelineChart/TimelineSpanEventMarker.tsx
@@ -81,6 +81,7 @@ export const TimelineSpanEventMarker = memo(function ({
}}
>
({
jest.mock('../DBRowDataPanel', () => ({
getJSONColumnNames: jest.fn().mockReturnValue([]),
}));
+jest.mock('@/hooks/useBrowserTraceEvents', () => ({
+ useBrowserTraceEvents: jest.fn(),
+}));
+const mockUseBrowserTraceEvents = jest.mocked(useBrowserTraceEvents);
+const passBrowserTraceEventsThrough: typeof useBrowserTraceEvents = ({
+ logRows,
+}) => ({ logRows, resourceTimingBySpanId: new Map(), triggerRow: undefined });
// Lightweight stub: the real SearchWhereInput renders SearchInputV2, which
// pulls in useMe()/metadata hooks that require a QueryClientProvider not present
@@ -215,6 +226,7 @@ describe('DBTraceWaterfallChartContainer', () => {
// Reset mocks before each test
beforeEach(() => {
jest.clearAllMocks();
+ mockUseBrowserTraceEvents.mockImplementation(passBrowserTraceEventsThrough);
mockUseRowWhere.mockReturnValue(() => ({ where: 'row-id', aliasWith: [] }));
MockTimelineChart.latestProps = {};
mockUseWaterfallSearchState.mockReturnValue(makeWaterfallSearchState());
@@ -225,6 +237,9 @@ describe('DBTraceWaterfallChartContainer', () => {
const renderComponent = (
logTableSource: typeof mockLogTableSource | null = mockLogTableSource,
traceId: string = mockTraceId,
+ props: Partial<
+ React.ComponentProps
+ > = {},
) => {
return renderWithMantine(
@@ -234,6 +249,7 @@ describe('DBTraceWaterfallChartContainer', () => {
traceId={traceId}
dateRange={mockDateRange}
focusDate={mockFocusDate}
+ {...props}
/>
,
);
@@ -789,6 +805,50 @@ describe('DBTraceWaterfallChartContainer', () => {
expect(screen.getByText(label)).toBeInTheDocument();
});
+ const findRow = (type: string) =>
+ MockTimelineChart.latestProps.rows.find(
+ (row: { type?: string }) => row.type === type,
+ );
+
+ it('builds the tooltip and title of a span with a URL and of a log', async () => {
+ setupQueryMocks({
+ traceData: {
+ data: [
+ { ...mockTraceData.data[0], SpanAttributes: { 'url.full': apiUrl } },
+ ],
+ meta: [{ totalCount: 1 }],
+ },
+ logData: {
+ data: [{ ...mockLogData.data[0], ServiceName: 'test-service' }],
+ meta: [{ totalCount: 1 }],
+ },
+ });
+ renderComponent();
+ await waitForLoading();
+
+ const spanRow = findRow(SourceKind.Trace);
+ expect(spanRow.events[0].tooltip).toBe(
+ `test span ${apiUrl} took 100.0000ms | Started at Jan 1 1:00:00.000 AM`,
+ );
+ const spanLabel = renderWithMantine(spanRow.label);
+ expect(
+ spanLabel.getByTitle(`test-service test span ${apiUrl}`),
+ ).toBeInTheDocument();
+ expect(
+ spanLabel.queryByLabelText('Correlated Log Line'),
+ ).not.toBeInTheDocument();
+
+ const logRow = findRow(SourceKind.Log);
+ expect(logRow.events[0].tooltip).toBe(
+ 'test log took 0.0000ms | Status: warn | Started at Jan 1 1:00:00.000 AM',
+ );
+ const logLabel = renderWithMantine(logRow.label);
+ expect(logLabel.getByTitle('test-service')).toHaveTextContent(
+ 'test-service test log',
+ );
+ expect(logLabel.getByLabelText('Correlated Log Line')).toBeInTheDocument();
+ });
+
// Main drew icons only for exact lowercase 'error'/'warn' and counted only
// SeverityText 'error'.
it.each([
@@ -1051,6 +1111,275 @@ describe('DBTraceWaterfallChartContainer', () => {
expect(screen.getByTestId('trace-search-input')).toBeInTheDocument();
expect(screen.queryByTestId('log-search-input')).not.toBeInTheDocument();
});
+
+ describe('browser trace events', () => {
+ const spanStartMs = Date.parse('2024-01-01T06:00:00.000Z');
+ const responseEnd = {
+ epochMs: spanStartMs + 20,
+ name: 'responseEnd',
+ attributes: { duration: '19.2ms' },
+ };
+ const mockTimingMarkers = () =>
+ mockUseBrowserTraceEvents.mockImplementation(({ logRows }) => ({
+ logRows,
+ resourceTimingBySpanId: new Map([['span-1', [responseEnd]]]),
+ triggerRow: undefined,
+ }));
+
+ it('draws span events, then timing dots, on the span and not on its log, and none with span events off', async () => {
+ mockTimingMarkers();
+ setupQueryMocks({
+ traceData: {
+ data: [
+ {
+ ...mockTraceData.data[0],
+ SpanEvents: [
+ {
+ Timestamp: '2024-01-01T06:00:00.050000000Z',
+ Name: 'exception',
+ Attributes: { 'exception.message': 'boom' },
+ },
+ {
+ Timestamp: '2024-01-01T06:00:00.075000000Z',
+ Name: 'cache miss',
+ Attributes: {},
+ },
+ ],
+ },
+ ],
+ meta: [{ totalCount: 1 }],
+ },
+ logData: mockLogData,
+ });
+ renderComponent();
+ await waitForLoading();
+
+ expect(findRow(SourceKind.Trace).events[0].markers).toEqual([
+ {
+ timestamp: 50,
+ epochMs: spanStartMs + 50,
+ name: 'exception',
+ attributes: { 'exception.message': 'boom' },
+ },
+ {
+ timestamp: 75,
+ epochMs: spanStartMs + 75,
+ name: 'cache miss',
+ attributes: {},
+ },
+ { timestamp: 20, ...responseEnd },
+ ]);
+ expect(findRow(SourceKind.Log).events[0].markers).toEqual([]);
+
+ await userEvent.click(screen.getByTestId('show-span-events-chip'));
+ await waitFor(() => {
+ expect(findRow(SourceKind.Trace).events[0].markers).toEqual([]);
+ });
+ });
+
+ it('draws timing dots on a span without span events', async () => {
+ mockTimingMarkers();
+ setupQueryMocks({ traceData: mockTraceData });
+ renderComponent(null);
+ await waitForLoading();
+
+ expect(findRow(SourceKind.Trace).events[0].markers).toEqual([
+ { timestamp: 20, ...responseEnd },
+ ]);
+ });
+
+ it('renders the log rows the hook keeps', async () => {
+ mockUseBrowserTraceEvents.mockImplementation(() => ({
+ logRows: [],
+ resourceTimingBySpanId: new Map(),
+ triggerRow: undefined,
+ }));
+ setupQueryMocks({ traceData: mockTraceData, logData: mockLogData });
+ renderComponent();
+ await waitForLoading();
+
+ expect(MockTimelineChart.latestProps.rows).toHaveLength(1);
+ expect(screen.getByText('1 span,')).toBeInTheDocument();
+ });
+
+ it('holds the hook while the trace fetches', () => {
+ setupQueryMocks({ traceData: mockTraceData, isFetching: true });
+ renderComponent();
+
+ expect(mockUseBrowserTraceEvents).toHaveBeenLastCalledWith(
+ expect.objectContaining({ enabled: false }),
+ );
+ });
+
+ it('passes the trace, its log rows, the selection and the hint', async () => {
+ const hint = {
+ timestamp: mockLogData.data[0].Timestamp,
+ spanId: 'span-1',
+ body: 'test log',
+ };
+ setupQueryMocks({ traceData: mockTraceData, logData: mockLogData });
+ renderComponent(mockLogTableSource, mockTraceId, {
+ highlightedRowWhere: 'selected-row',
+ initialRowHighlightHint: hint,
+ });
+ await waitForLoading();
+
+ expect(mockUseBrowserTraceEvents).toHaveBeenLastCalledWith({
+ traceRows: [expect.objectContaining({ SpanId: 'span-1' })],
+ logRows: [expect.objectContaining({ Body: 'test log' })],
+ logSource: mockLogTableSource,
+ traceId: mockTraceId,
+ highlightedRowWhere: 'selected-row',
+ initialRowHighlightHint: hint,
+ foldTimingRows: true,
+ enabled: true,
+ });
+ });
+
+ const lastFold = () =>
+ mockUseBrowserTraceEvents.mock.lastCall?.[0].foldTimingRows;
+
+ it.each(['show-span-events-chip', 'show-spans-chip'])(
+ 'keeps the timing rows while %s is off',
+ async chip => {
+ setupQueryMocks({ traceData: mockTraceData, logData: mockLogData });
+ renderComponent();
+ await waitForLoading();
+
+ await userEvent.click(screen.getByTestId(chip));
+ await waitFor(() => expect(lastFold()).toBe(false));
+ },
+ );
+
+ it('keeps the timing rows under a log filter', async () => {
+ mockUseWaterfallSearchState.mockReturnValue({
+ ...makeWaterfallSearchState(),
+ logWhere: 'EventName:"browser.resource_timing"',
+ });
+ setupQueryMocks({ traceData: mockTraceData, logData: mockLogData });
+ renderComponent();
+ await waitForLoading();
+
+ expect(lastFold()).toBe(false);
+ });
+
+ it('keeps the timing rows when the log list is capped', async () => {
+ setupQueryMocks({
+ traceData: mockTraceData,
+ logData: {
+ data: Array.from(
+ { length: TRACE_WATERFALL_ROW_LIMIT + 1 },
+ () => mockLogData.data[0],
+ ),
+ meta: [{ totalCount: TRACE_WATERFALL_ROW_LIMIT + 1 }],
+ },
+ });
+ renderComponent();
+ await waitForLoading();
+
+ expect(lastFold()).toBe(false);
+ });
+
+ it('sends the same log queries whether or not the source has browser event fields', () => {
+ const logQueries = (logSource: TLogSource) => {
+ mockUseOffsetPaginatedQuery.mockClear();
+ renderComponent(logSource).unmount();
+ return mockUseOffsetPaginatedQuery.mock.calls
+ .filter(([query]) => query?.from?.tableName === 'log_table')
+ .map(([query, opts]) => [query, opts?.enabled]);
+ };
+ setupQueryMocks({ traceData: mockTraceData, logData: mockLogData });
+ const correlatedSource = {
+ ...mockLogTableSource,
+ traceIdExpression: 'TraceId',
+ spanIdExpression: 'SpanId',
+ };
+
+ const withEventFields = logQueries({
+ ...correlatedSource,
+ eventNameExpression: 'EventName',
+ eventAttributesExpression: 'LogAttributes',
+ });
+
+ expect(withEventFields.some(([, enabled]) => enabled)).toBe(true);
+ expect(withEventFields).toEqual(logQueries(correlatedSource));
+ expect(JSON.stringify(withEventFields)).not.toContain('EventName');
+ });
+
+ describe('triggered-by row', () => {
+ const triggeredBy = 'Triggered by: Clicked testid=checkout · 3ms before';
+ const triggerRow: TraceTriggerRow = {
+ Timestamp: mockTraceData.data[0].Timestamp,
+ Body: '',
+ SpanId: '',
+ type: SourceKind.Log,
+ id: 'click-row',
+ aliasWith: [],
+ __hdx_hidden: false,
+ triggeredBy,
+ };
+ const mockTriggerRow = () =>
+ mockUseBrowserTraceEvents.mockImplementation(({ logRows }) => ({
+ logRows,
+ resourceTimingBySpanId: new Map(),
+ triggerRow,
+ }));
+ const rowTypes = () =>
+ MockTimelineChart.latestProps.rows.map(
+ (row: { id: string; type?: string }) => `${row.type}:${row.id}`,
+ );
+
+ it('comes first, labeled as the triggering click, and opens the click log', async () => {
+ mockTriggerRow();
+ const onClick = jest.fn();
+ setupQueryMocks({ traceData: mockTraceData, logData: mockLogData });
+ renderComponent(mockLogTableSource, mockTraceId, { onClick });
+ await waitForLoading();
+
+ expect(rowTypes()).toEqual([
+ 'log:click-row',
+ 'trace:row-id',
+ 'log:row-id',
+ ]);
+ const [row] = MockTimelineChart.latestProps.rows;
+ expect(row.events[0].tooltip).toBe(
+ `${triggeredBy} | Started at Jan 1 1:00:00.000 AM`,
+ );
+ const label = renderWithMantine(row.label);
+ expect(label.getByTitle(triggeredBy)).toHaveTextContent(triggeredBy);
+ expect(label.getByLabelText('Triggering click')).toBeInTheDocument();
+ expect(
+ label.queryByLabelText('Correlated Log Line'),
+ ).not.toBeInTheDocument();
+ expect(screen.getByText('1 span, 1 log,')).toBeInTheDocument();
+
+ await userEvent.click(label.getByTitle(triggeredBy));
+ expect(onClick).toHaveBeenCalledWith({
+ id: 'click-row',
+ type: SourceKind.Log,
+ aliasWith: [],
+ });
+ });
+
+ it('follows the Spans chip, not the Logs chip', async () => {
+ mockTriggerRow();
+ setupQueryMocks({ traceData: mockTraceData, logData: mockLogData });
+ renderComponent();
+ await waitForLoading();
+
+ await userEvent.click(screen.getByTestId('show-logs-chip'));
+ await waitFor(() => {
+ expect(rowTypes()).toEqual(['log:click-row', 'trace:row-id']);
+ });
+
+ await userEvent.click(screen.getByTestId('show-logs-chip'));
+ await userEvent.click(screen.getByTestId('show-spans-chip'));
+ await waitFor(() => {
+ expect(rowTypes()).toEqual(['log:row-id']);
+ });
+ });
+ });
+ });
});
describe('useEventsAroundFocus', () => {
diff --git a/packages/app/src/hooks/__fixtures__/browserTraceEvents.ts b/packages/app/src/hooks/__fixtures__/browserTraceEvents.ts
new file mode 100644
index 000000000..72d88ccd6
--- /dev/null
+++ b/packages/app/src/hooks/__fixtures__/browserTraceEvents.ts
@@ -0,0 +1,90 @@
+import { SourceKind } from '@hyperdx/common-utils/dist/types';
+
+import type { BrowserEventLogSource } from '@/utils/browserTraceContext';
+
+export const logSource: BrowserEventLogSource = {
+ id: 'logs',
+ kind: SourceKind.Log,
+ name: 'Logs',
+ connection: 'conn',
+ from: { databaseName: 'default', tableName: 'otel_logs' },
+ timestampValueExpression: 'Timestamp',
+ defaultTableSelectExpression: 'Timestamp, Body',
+ implicitColumnExpression: 'Body',
+ serviceNameExpression: 'ServiceName',
+ eventNameExpression: 'EventName',
+ eventAttributesExpression: 'LogAttributes',
+ traceIdExpression: 'TraceId',
+ spanIdExpression: 'SpanId',
+};
+
+export const PAGE_URL = 'http://localhost:5173/';
+export const SPAN_START = '2026-10-22T04:52:37.818000000Z';
+export const SPAN_START_MS = Date.parse('2026-10-22T04:52:37.818Z');
+export const TIMING_TIMESTAMP = '2026-10-22T04:52:37.838000000Z';
+export const CLICK_TIMESTAMP = '2026-10-22T04:52:37.815000000Z';
+
+export const browserSpan = {
+ Timestamp: SPAN_START,
+ Duration: 0.012,
+ SpanId: 'span-1',
+ ServiceName: 'web',
+ SpanAttributes: {
+ 'browser.document.url.full': PAGE_URL,
+ 'session.id': 's1',
+ },
+};
+export const legacySpan = {
+ ...browserSpan,
+ SpanAttributes: { 'http.url': 'http://localhost:5173/api' },
+};
+
+const timingLogRow = {
+ id: 'timing-row',
+ Timestamp: TIMING_TIMESTAMP,
+ SpanId: 'span-1',
+ Body: '',
+};
+export const consoleLogRow = {
+ id: 'console-row',
+ Timestamp: '2026-10-22T04:52:37.825000000Z',
+ SpanId: 'span-1',
+ Body: 'loading cart',
+};
+export const logRows = [timingLogRow, consoleLogRow];
+
+export const timingResult = {
+ data: [
+ {
+ span_id: 'span-1',
+ timestamp: TIMING_TIMESTAMP,
+ attributes: {
+ 'browser.resource_timing.fetch_start': '1000.5',
+ 'browser.resource_timing.response_end': '1011.5',
+ },
+ },
+ ],
+ meta: [{ name: 'timestamp', type: 'DateTime64(9)' }],
+};
+
+export const clickResult = {
+ data: [
+ {
+ timestamp: CLICK_TIMESTAMP,
+ event_name: 'browser.user_action.click',
+ session_id: 's1',
+ body: '',
+ attributes: {
+ 'browser.tag_name': 'BUTTON',
+ 'browser.element.attributes.testid': 'checkout',
+ },
+ },
+ ],
+ meta: [
+ { name: 'timestamp', type: 'DateTime64(9)' },
+ { name: 'event_name', type: 'String' },
+ { name: 'session_id', type: 'String' },
+ { name: 'body', type: 'String' },
+ { name: 'attributes', type: 'Map(LowCardinality(String), String)' },
+ ],
+};
diff --git a/packages/app/src/hooks/__tests__/useBrowserTraceEvents.test.tsx b/packages/app/src/hooks/__tests__/useBrowserTraceEvents.test.tsx
new file mode 100644
index 000000000..2a4874a62
--- /dev/null
+++ b/packages/app/src/hooks/__tests__/useBrowserTraceEvents.test.tsx
@@ -0,0 +1,275 @@
+import { SourceKind, TLogSource } from '@hyperdx/common-utils/dist/types';
+import { renderHook } from '@testing-library/react';
+
+import {
+ browserSpan,
+ CLICK_TIMESTAMP,
+ clickResult,
+ consoleLogRow,
+ legacySpan,
+ logRows,
+ logSource,
+ TIMING_TIMESTAMP,
+ timingResult,
+} from '@/hooks/__fixtures__/browserTraceEvents';
+import { useBrowserTraceEvents } from '@/hooks/useBrowserTraceEvents';
+
+const mockUseQueriedChartConfig = jest.fn();
+const mockUseFieldExpressionGenerator = jest.fn();
+
+jest.mock('@/hooks/useChartConfig', () => ({
+ useQueriedChartConfig: (...args: unknown[]) =>
+ mockUseQueriedChartConfig(...args),
+}));
+jest.mock('@/hooks/useFieldExpressionGenerator', () => ({
+ __esModule: true,
+ default: (source?: TLogSource) => {
+ mockUseFieldExpressionGenerator(source);
+ return {
+ isLoading: false,
+ getFieldExpression: source
+ ? (column: string, key: string) => `${column}['${key}']`
+ : undefined,
+ };
+ },
+}));
+
+const TIMING = 'TraceResourceTiming';
+const TRIGGER = 'TraceTriggerClick';
+
+type Props = Parameters[0];
+type QueryState = { data?: unknown; isError?: boolean };
+
+function setup(
+ props: Partial = {},
+ queries: { [TIMING]?: QueryState; [TRIGGER]?: QueryState } = {},
+) {
+ const states: Record = {
+ [TIMING]: { data: timingResult, ...queries[TIMING] },
+ [TRIGGER]: { data: clickResult, ...queries[TRIGGER] },
+ };
+ mockUseQueriedChartConfig.mockImplementation(
+ (_config: unknown, { queryKeyPrefix }: { queryKeyPrefix: string }) => ({
+ isError: false,
+ ...states[queryKeyPrefix],
+ }),
+ );
+ const initialProps: Props = {
+ traceRows: [browserSpan],
+ logRows,
+ logSource,
+ traceId: 't1',
+ foldTimingRows: true,
+ enabled: true,
+ ...props,
+ };
+ const hook = renderHook((p: Props) => useBrowserTraceEvents(p), {
+ initialProps,
+ });
+ return { ...hook, initialProps };
+}
+
+const lastQuery = (prefix: string) =>
+ mockUseQueriedChartConfig.mock.calls.findLast(
+ ([, options]) => options.queryKeyPrefix === prefix,
+ );
+const queriesEnabled = () => ({
+ timing: lastQuery(TIMING)?.[1].enabled,
+ trigger: lastQuery(TRIGGER)?.[1].enabled,
+});
+
+describe('useBrowserTraceEvents', () => {
+ beforeEach(() => {
+ mockUseQueriedChartConfig.mockReset();
+ mockUseFieldExpressionGenerator.mockReset();
+ });
+
+ it.each([
+ ['legacy SDK spans', { traceRows: [legacySpan] }],
+ [
+ 'a log source without an event name',
+ { logSource: { ...logSource, eventNameExpression: undefined } },
+ ],
+ ['no log source', { logSource: null }],
+ ])('sends nothing and returns the rows as given for %s', (_name, props) => {
+ const { result, rerender, initialProps } = setup(props);
+
+ expect(queriesEnabled()).toEqual({ timing: false, trigger: false });
+ expect(mockUseFieldExpressionGenerator).toHaveBeenLastCalledWith(undefined);
+ expect(result.current.triggerRow).toBeUndefined();
+ expect(result.current.logRows).toBe(logRows);
+ const markers = result.current.resourceTimingBySpanId;
+ expect(markers.size).toBe(0);
+
+ rerender({ ...initialProps, traceRows: [...initialProps.traceRows] });
+ expect(result.current.resourceTimingBySpanId).toBe(markers);
+ });
+
+ it('holds both queries while the trace fetches', () => {
+ setup({ enabled: false });
+
+ expect(queriesEnabled()).toEqual({ timing: false, trigger: false });
+ });
+
+ it('queries the trace timing logs from the browser spans', () => {
+ setup();
+
+ expect(lastQuery(TIMING)).toEqual([
+ expect.objectContaining({
+ source: 'logs',
+ where: "TraceId = 't1'",
+ dateRange: [
+ new Date('2026-10-22T04:52:37.000Z'),
+ new Date('2026-10-22T04:57:37.830Z'),
+ ],
+ }),
+ { enabled: true, queryKeyPrefix: TIMING, staleTime: Infinity },
+ ]);
+ });
+
+ it("queries the clicks before the first browser span in the span's session", () => {
+ setup();
+
+ expect(mockUseFieldExpressionGenerator).toHaveBeenLastCalledWith(logSource);
+ expect(lastQuery(TRIGGER)).toEqual([
+ expect.objectContaining({
+ source: 'logs',
+ dateRange: [
+ new Date('2026-10-22T04:52:37.000Z'),
+ new Date('2026-10-22T04:52:37.818Z'),
+ ],
+ filters: expect.arrayContaining([
+ { type: 'lucene', condition: 'LogAttributes.session.id:"s1"' },
+ ]),
+ }),
+ { enabled: true, queryKeyPrefix: TRIGGER, staleTime: Infinity },
+ ]);
+ });
+
+ it('shows the click as a row with its log row id', () => {
+ const { result } = setup();
+
+ expect(result.current.triggerRow).toEqual({
+ Timestamp: CLICK_TIMESTAMP,
+ Body: '',
+ SpanId: '',
+ type: SourceKind.Log,
+ id: `Timestamp=parseDateTime64BestEffort('${CLICK_TIMESTAMP}', 9) AND toString(EventName)='browser.user_action.click' AND LogAttributes['session.id']='s1' AND Body=''`,
+ aliasWith: [],
+ __hdx_hidden: false,
+ triggeredBy: 'Triggered by: Clicked testid=checkout · 3ms before',
+ });
+ });
+
+ it('hides the click row with the span it anchors on', () => {
+ const { result } = setup({
+ traceRows: [{ ...browserSpan, __hdx_hidden: true }],
+ });
+
+ expect(result.current.triggerRow?.__hdx_hidden).toBe(true);
+ });
+
+ it('skips the click lookup, metadata included, when the span has no session id', () => {
+ const { result } = setup({
+ traceRows: [
+ {
+ ...browserSpan,
+ SpanAttributes: {
+ 'browser.document.url.full': 'http://localhost:5173/',
+ },
+ },
+ ],
+ });
+
+ expect(mockUseFieldExpressionGenerator).toHaveBeenLastCalledWith(undefined);
+ expect(queriesEnabled()).toEqual({ timing: true, trigger: false });
+ expect(result.current.triggerRow).toBeUndefined();
+ });
+
+ it('shows no click row when the row id cannot be built', () => {
+ const consoleError = jest
+ .spyOn(console, 'error')
+ .mockImplementation(() => {});
+ const { result } = setup(
+ {},
+ {
+ [TRIGGER]: {
+ data: { ...clickResult, meta: clickResult.meta.slice(0, 1) },
+ },
+ },
+ );
+
+ expect(result.current.triggerRow).toBeUndefined();
+ expect(consoleError).toHaveBeenCalled();
+ consoleError.mockRestore();
+ });
+
+ it('draws the timing log as dots and folds its row, keeping another log on the span', () => {
+ const { result } = setup();
+
+ expect(
+ result.current.resourceTimingBySpanId.get('span-1')?.map(m => m.name),
+ ).toEqual(['fetchStart', 'responseEnd']);
+ expect(result.current.logRows).toEqual([consoleLogRow]);
+ });
+
+ it("keeps a log in the timing log's millisecond on its span, and the timing row", () => {
+ const errorRow = {
+ ...consoleLogRow,
+ id: 'error-row',
+ Timestamp: TIMING_TIMESTAMP,
+ Body: '',
+ };
+ const { result } = setup({ logRows: [...logRows, errorRow] });
+
+ expect(result.current.resourceTimingBySpanId.has('span-1')).toBe(true);
+ expect(result.current.logRows).toEqual([...logRows, errorRow]);
+ });
+
+ it('keeps the timing row while its dots are off screen', () => {
+ const { result, rerender, initialProps } = setup({
+ foldTimingRows: false,
+ });
+
+ expect(result.current.resourceTimingBySpanId.has('span-1')).toBe(true);
+ expect(result.current.logRows).toBe(logRows);
+
+ rerender({ ...initialProps, foldTimingRows: true });
+ expect(result.current.logRows).toEqual([consoleLogRow]);
+ });
+
+ it('changes nothing when the queries fail', () => {
+ const { result } = setup(
+ {},
+ { [TIMING]: { isError: true }, [TRIGGER]: { isError: true } },
+ );
+
+ expect(result.current.logRows).toBe(logRows);
+ expect(result.current.resourceTimingBySpanId.size).toBe(0);
+ expect(result.current.triggerRow).toBeUndefined();
+ });
+
+ it('keeps the selected timing row until another row is selected', () => {
+ const { result, rerender, initialProps } = setup({
+ highlightedRowWhere: 'timing-row',
+ });
+
+ expect(result.current.logRows).toEqual(logRows);
+ expect(result.current.resourceTimingBySpanId.has('span-1')).toBe(true);
+
+ rerender({ ...initialProps, highlightedRowWhere: 'console-row' });
+ expect(result.current.logRows).toEqual([consoleLogRow]);
+ });
+
+ it('keeps the timing row the panel was opened from', () => {
+ const { result } = setup({
+ initialRowHighlightHint: {
+ timestamp: TIMING_TIMESTAMP,
+ spanId: 'span-1',
+ body: '',
+ },
+ });
+
+ expect(result.current.logRows).toEqual(logRows);
+ });
+});
diff --git a/packages/app/src/hooks/useBrowserTraceEvents.ts b/packages/app/src/hooks/useBrowserTraceEvents.ts
new file mode 100644
index 000000000..0f72b6c29
--- /dev/null
+++ b/packages/app/src/hooks/useBrowserTraceEvents.ts
@@ -0,0 +1,220 @@
+import { useMemo } from 'react';
+import { SourceKind, TLogSource } from '@hyperdx/common-utils/dist/types';
+
+import { useQueriedChartConfig } from '@/hooks/useChartConfig';
+import useFieldExpressionGenerator from '@/hooks/useFieldExpressionGenerator';
+import {
+ matchesRowHighlightHint,
+ RowHighlightHint,
+} from '@/hooks/useRowHighlightHint';
+import useRowWhere, { WithClause } from '@/hooks/useRowWhere';
+import { PLACEHOLDER_CONFIG } from '@/hooks/useSessionLogEvents';
+import { formatDurationMs } from '@/utils';
+import { getBrowserEventLabel } from '@/utils/browserEvents';
+import {
+ BrowserTraceRow,
+ getBrowserTraceContext,
+ isBrowserEventLogSource,
+} from '@/utils/browserTraceContext';
+import {
+ buildResourceTimingQuery,
+ getFoldedRowKeys,
+ getResourceTimingMarkers,
+ ResourceTimingMarker,
+ resourceTimingRowKey,
+} from '@/utils/resourceTimingMarkers';
+import { SESSION_LOG_EVENT_KEY_COLUMNS } from '@/utils/sessionLogEvents';
+import {
+ buildTriggerClickQuery,
+ pickTriggerClick,
+} from '@/utils/traceTriggerClick';
+
+type TraceRow = BrowserTraceRow & { SpanId?: unknown };
+
+type LogRow = {
+ id: string;
+ Timestamp: string;
+ SpanId?: unknown;
+ Body?: unknown;
+};
+
+/** The click that triggered the trace, as a waterfall log row. */
+export type TraceTriggerRow = {
+ Timestamp: string;
+ Body: string;
+ SpanId: string;
+ type: SourceKind.Log;
+ id: string;
+ aliasWith: WithClause[];
+ __hdx_hidden: boolean;
+ triggeredBy: string;
+};
+
+// Stable, so the waterfall's row memos don't rerun while there is nothing.
+const NO_RESOURCE_TIMING: ReturnType = {
+ markersBySpanId: new Map(),
+ drawnRowCounts: new Map(),
+};
+
+/**
+ * Traces without an OTel browser span send no queries. The queries never hold
+ * the waterfall, and a failed one changes nothing.
+ */
+export function useBrowserTraceEvents({
+ traceRows,
+ logRows,
+ logSource,
+ traceId,
+ highlightedRowWhere,
+ initialRowHighlightHint,
+ foldTimingRows,
+ enabled,
+}: {
+ traceRows: readonly TraceRow[];
+ logRows: T[];
+ logSource: TLogSource | null;
+ traceId: string;
+ highlightedRowWhere?: string | null;
+ initialRowHighlightHint?: RowHighlightHint;
+ /** Whether the dots are on screen to replace the rows. */
+ foldTimingRows: boolean;
+ /** False while the trace rows fetch, so the anchor can't move mid-flight. */
+ enabled: boolean;
+}): {
+ logRows: T[];
+ resourceTimingBySpanId: ReadonlyMap;
+ triggerRow: TraceTriggerRow | undefined;
+} {
+ const context = useMemo(() => getBrowserTraceContext(traceRows), [traceRows]);
+ const source =
+ context != null && isBrowserEventLogSource(logSource)
+ ? logSource
+ : undefined;
+
+ const timingQuery = useMemo(
+ () =>
+ source != null && context != null
+ ? buildResourceTimingQuery({ logSource: source, context, traceId })
+ : null,
+ [source, context, traceId],
+ );
+ // Never stale, like the waterfall's own queries: another window is another
+ // key. Otherwise each trace refetch would rerun both.
+ const timing = useQueriedChartConfig(timingQuery ?? PLACEHOLDER_CONFIG, {
+ enabled: enabled && timingQuery != null,
+ queryKeyPrefix: 'TraceResourceTiming',
+ staleTime: Infinity,
+ });
+ const timingResult =
+ timingQuery != null && !timing.isError ? timing.data : undefined;
+ const { markersBySpanId, drawnRowCounts } = useMemo(
+ () =>
+ timingResult?.data.length
+ ? getResourceTimingMarkers(timingResult, traceRows)
+ : NO_RESOURCE_TIMING,
+ [timingResult, traceRows],
+ );
+
+ // The selected row and the row the panel opened from stay: folding them
+ // would leave the detail panel showing a row the waterfall lacks, and the
+ // hint would find nothing to select.
+ const keptLogRows = useMemo(() => {
+ if (!foldTimingRows || drawnRowCounts.size === 0) {
+ return logRows;
+ }
+ const foldedRowKeys = getFoldedRowKeys(logRows, drawnRowCounts);
+ if (foldedRowKeys.size === 0) {
+ return logRows;
+ }
+ return logRows.filter(
+ row =>
+ !foldedRowKeys.has(resourceTimingRowKey(row.SpanId, row.Timestamp)) ||
+ row.id === highlightedRowWhere ||
+ matchesRowHighlightHint(row, initialRowHighlightHint),
+ );
+ }, [
+ logRows,
+ foldTimingRows,
+ drawnRowCounts,
+ highlightedRowWhere,
+ initialRowHighlightHint,
+ ]);
+
+ // Only the trigger query reads a single attribute, so only it needs the
+ // JSON column lookup.
+ const triggerSource = context?.sessionId != null ? source : undefined;
+ const { getFieldExpression } = useFieldExpressionGenerator(triggerSource);
+ // A string, not the generator: the generator is a new function each render.
+ const sessionIdExpression =
+ triggerSource &&
+ getFieldExpression?.(triggerSource.eventAttributesExpression, 'session.id');
+ const triggerQuery = useMemo(
+ () =>
+ triggerSource != null && context?.sessionId != null && sessionIdExpression
+ ? buildTriggerClickQuery({
+ logSource: triggerSource,
+ context,
+ sessionId: context.sessionId,
+ sessionIdExpression,
+ })
+ : null,
+ [triggerSource, context, sessionIdExpression],
+ );
+ const trigger = useQueriedChartConfig(
+ triggerQuery?.config ?? PLACEHOLDER_CONFIG,
+ {
+ enabled: enabled && triggerQuery != null,
+ queryKeyPrefix: 'TraceTriggerClick',
+ staleTime: Infinity,
+ },
+ );
+ const triggerResult =
+ triggerQuery != null && !trigger.isError ? trigger.data : undefined;
+ const getRowWhere = useRowWhere({
+ meta: triggerResult?.meta,
+ aliasMap: triggerQuery?.aliasMap,
+ primaryKeyColumns: SESSION_LOG_EVENT_KEY_COLUMNS,
+ });
+ const anchorMs = context?.anchorMs;
+ const anchorHidden = context?.anchorHidden ?? false;
+ const triggerRow = useMemo((): TraceTriggerRow | undefined => {
+ const picked =
+ anchorMs != null && triggerResult?.data.length
+ ? pickTriggerClick(triggerResult, anchorMs)
+ : undefined;
+ if (picked == null) {
+ return undefined;
+ }
+ // useRowWhere throws on a column without meta, which must not take down
+ // the waterfall.
+ let id: string;
+ try {
+ id = getRowWhere(picked.row).where;
+ } catch (e) {
+ console.error(e);
+ return undefined;
+ }
+ const { timelineText } = getBrowserEventLabel({
+ eventName: String(picked.row.event_name ?? ''),
+ body: picked.row.body,
+ attributes: picked.row.attributes,
+ });
+ return {
+ Timestamp: String(picked.row.timestamp),
+ Body: '',
+ SpanId: '',
+ type: SourceKind.Log,
+ id,
+ // The row id is made of raw expressions, so it needs no WITH clauses.
+ aliasWith: [],
+ __hdx_hidden: anchorHidden,
+ triggeredBy: `Triggered by: ${timelineText} · ${formatDurationMs(picked.gapMs)} before`,
+ };
+ }, [triggerResult, anchorMs, anchorHidden, getRowWhere]);
+
+ return {
+ logRows: keptLogRows,
+ resourceTimingBySpanId: markersBySpanId,
+ triggerRow,
+ };
+}
diff --git a/packages/app/src/hooks/useRowHighlightHint.ts b/packages/app/src/hooks/useRowHighlightHint.ts
index 3a1031163..b54e7bcc1 100644
--- a/packages/app/src/hooks/useRowHighlightHint.ts
+++ b/packages/app/src/hooks/useRowHighlightHint.ts
@@ -8,6 +8,18 @@ export type RowHighlightHint = {
body: string;
};
+export function matchesRowHighlightHint(
+ row: { Timestamp?: unknown; SpanId?: unknown; Body?: unknown },
+ hint: RowHighlightHint | undefined,
+): boolean {
+ return (
+ hint != null &&
+ row.Timestamp === hint.timestamp &&
+ row.SpanId === hint.spanId &&
+ row.Body === hint.body
+ );
+}
+
type HighlightRow = {
id: string;
type?: string;
@@ -63,13 +75,9 @@ export function useRowHighlightHint({
return;
}
- const initialRowHighlightIndex = rows.findIndex(row => {
- return (
- row.Timestamp === initialRowHighlightHint.timestamp &&
- row.SpanId === initialRowHighlightHint.spanId &&
- row.Body === initialRowHighlightHint.body
- );
- });
+ const initialRowHighlightIndex = rows.findIndex(row =>
+ matchesRowHighlightHint(row, initialRowHighlightHint),
+ );
if (initialRowHighlightIndex !== -1) {
appliedHighlightHintRef.current = hintKey;
diff --git a/packages/app/src/hooks/useSessionLogEvents.ts b/packages/app/src/hooks/useSessionLogEvents.ts
index 7247e2f61..6869fc787 100644
--- a/packages/app/src/hooks/useSessionLogEvents.ts
+++ b/packages/app/src/hooks/useSessionLogEvents.ts
@@ -29,7 +29,7 @@ const NO_LOG_EVENTS: SessionLogEvent[] = [];
// Twin of linkedSpans' usePlaceholderConfig, without a source because the log
// source may not resolve. Never fetched: the query is disabled while it is used.
-const PLACEHOLDER_CONFIG: BuilderChartConfigWithDateRange = {
+export const PLACEHOLDER_CONFIG: BuilderChartConfigWithDateRange = {
connection: '',
from: { databaseName: '', tableName: '' },
select: [{ valueExpression: '1' }],
diff --git a/packages/app/src/utils/__tests__/browserTraceContext.test.ts b/packages/app/src/utils/__tests__/browserTraceContext.test.ts
new file mode 100644
index 000000000..4a4608ab0
--- /dev/null
+++ b/packages/app/src/utils/__tests__/browserTraceContext.test.ts
@@ -0,0 +1,144 @@
+import {
+ browserSpan,
+ logSource,
+ PAGE_URL,
+ SPAN_START_MS,
+} from '@/hooks/__fixtures__/browserTraceEvents';
+import {
+ getBrowserTraceContext,
+ hasSubSecondTimestamps,
+ isBrowserEventLogSource,
+} from '@/utils/browserTraceContext';
+
+const backendSpan = {
+ Timestamp: '2026-10-22T04:52:37.000000000Z',
+ Duration: 5,
+ ServiceName: 'api',
+ SpanAttributes: { 'http.route': '/api/checkout' },
+};
+
+describe('getBrowserTraceContext', () => {
+ it.each([
+ ['no rows', []],
+ ['backend spans', [backendSpan]],
+ [
+ 'an AI span with a session id and no page URL',
+ [{ ...backendSpan, SpanAttributes: { 'session.id': 's1' } }],
+ ],
+ [
+ 'an empty page URL',
+ [
+ {
+ ...browserSpan,
+ SpanAttributes: { 'browser.document.url.full': '' },
+ },
+ ],
+ ],
+ [
+ 'an unparsable timestamp',
+ [{ ...browserSpan, Timestamp: 'not-a-timestamp' }],
+ ],
+ ])('is undefined for %s', (_name, rows) => {
+ expect(getBrowserTraceContext(rows)).toBeUndefined();
+ });
+
+ it('anchors on the earliest browser span, ignoring earlier backend spans', () => {
+ const laterSpan = {
+ ...browserSpan,
+ Timestamp: '2026-10-22T04:52:37.900000000Z',
+ Duration: 0.2,
+ ServiceName: 'web-worker',
+ SpanAttributes: {
+ 'browser.document.url.full': PAGE_URL,
+ 'session.id': 's2',
+ },
+ };
+ expect(
+ getBrowserTraceContext([laterSpan, backendSpan, browserSpan]),
+ ).toEqual({
+ serviceNames: ['web', 'web-worker'],
+ anchorMs: SPAN_START_MS,
+ endMs: Date.parse('2026-10-22T04:52:38.100Z'),
+ anchorHidden: false,
+ sessionId: 's1',
+ serviceName: 'web',
+ });
+ });
+
+ it('ends at most an hour after the anchor', () => {
+ expect(
+ getBrowserTraceContext([{ ...browserSpan, Duration: 86400 }])?.endMs,
+ ).toBe(SPAN_START_MS + 60 * 60 * 1000);
+ });
+
+ it('reads nested JSON attributes', () => {
+ const context = getBrowserTraceContext([
+ {
+ ...browserSpan,
+ SpanAttributes: {
+ browser: { document: { url: { full: PAGE_URL } } },
+ session: { id: 's1' },
+ },
+ },
+ ]);
+ expect(context).toMatchObject({
+ anchorMs: SPAN_START_MS,
+ sessionId: 's1',
+ });
+ });
+
+ it('dedupes service names and keeps an unset session id unset', () => {
+ const context = getBrowserTraceContext([
+ {
+ ...browserSpan,
+ SpanAttributes: { 'browser.document.url.full': PAGE_URL },
+ },
+ { ...browserSpan, Timestamp: '2026-10-22T04:52:37.830000000Z' },
+ ]);
+ expect(context).toMatchObject({
+ serviceNames: ['web'],
+ sessionId: undefined,
+ serviceName: 'web',
+ });
+ });
+
+ it.each([
+ [1, true],
+ [0, false],
+ ])('reports a hidden anchor for __hdx_hidden %p', (hidden, expected) => {
+ expect(
+ getBrowserTraceContext([{ ...browserSpan, __hdx_hidden: hidden }])
+ ?.anchorHidden,
+ ).toBe(expected);
+ });
+});
+
+describe('hasSubSecondTimestamps', () => {
+ it.each([
+ ["Nullable(DateTime64(9, 'UTC'))", true],
+ ['DateTime64(0)', false],
+ ['DateTime', false],
+ ])('reads a %s timestamp column as %p', (type, expected) => {
+ expect(hasSubSecondTimestamps([{ name: 'timestamp', type }])).toBe(
+ expected,
+ );
+ });
+
+ it('is false without a timestamp column', () => {
+ expect(hasSubSecondTimestamps(undefined)).toBe(false);
+ expect(
+ hasSubSecondTimestamps([{ name: 'ts', type: 'DateTime64(9)' }]),
+ ).toBe(false);
+ });
+});
+
+describe('isBrowserEventLogSource', () => {
+ it.each([
+ 'eventNameExpression',
+ 'eventAttributesExpression',
+ 'traceIdExpression',
+ 'spanIdExpression',
+ ] as const)('rejects a source without %s', field => {
+ expect(isBrowserEventLogSource({ ...logSource, [field]: '' })).toBe(false);
+ });
+});
diff --git a/packages/app/src/utils/__tests__/resourceTimingMarkers.test.ts b/packages/app/src/utils/__tests__/resourceTimingMarkers.test.ts
new file mode 100644
index 000000000..28d9e2890
--- /dev/null
+++ b/packages/app/src/utils/__tests__/resourceTimingMarkers.test.ts
@@ -0,0 +1,326 @@
+import {
+ logSource,
+ SPAN_START,
+ SPAN_START_MS,
+ TIMING_TIMESTAMP,
+} from '@/hooks/__fixtures__/browserTraceEvents';
+import {
+ buildResourceTimingQuery,
+ getFoldedRowKeys,
+ getResourceTimingMarkers,
+ resourceTimingRowKey,
+} from '@/utils/resourceTimingMarkers';
+
+const traceRows = [{ SpanId: 'span-1', Timestamp: SPAN_START }];
+const TIMESTAMP_META = [{ name: 'timestamp', type: 'DateTime64(9)' }];
+
+const prefixed = (values: Record) =>
+ Object.fromEntries(
+ Object.entries(values).map(([key, value]) => [
+ `browser.resource_timing.${key}`,
+ value,
+ ]),
+ );
+
+const sameOrigin = prefixed({
+ fetch_start: '1000.5',
+ domain_lookup_start: '1001',
+ domain_lookup_end: '1001.25',
+ connect_start: '1001.5',
+ secure_connection_start: '0',
+ connect_end: '1002',
+ request_start: '1002',
+ response_start: '1009',
+ response_end: '1011.5',
+ redirect_start: '0',
+ duration: '11',
+ transfer_size: '558',
+ encoded_body_size: '1536',
+ decoded_body_size: '20480',
+ next_hop_protocol: 'http/1.1',
+ initiator_type: 'fetch',
+});
+
+const timingRow = (attributes: unknown, overrides: object = {}) => ({
+ span_id: 'span-1',
+ timestamp: TIMING_TIMESTAMP,
+ attributes,
+ ...overrides,
+});
+
+const draw = (
+ row: Record,
+ spans: Parameters[1] = traceRows,
+ meta = TIMESTAMP_META,
+) => getResourceTimingMarkers({ data: [row], meta }, spans);
+
+describe('getResourceTimingMarkers', () => {
+ it('places dots in sdk-trace-web order relative to the span start', () => {
+ const { markersBySpanId, drawnRowCounts } = draw(timingRow(sameOrigin));
+
+ expect(markersBySpanId.get('span-1')).toEqual([
+ { name: 'fetchStart', epochMs: SPAN_START_MS, attributes: {} },
+ {
+ name: 'domainLookupStart',
+ epochMs: SPAN_START_MS + 0.5,
+ attributes: {},
+ },
+ {
+ name: 'domainLookupEnd',
+ epochMs: SPAN_START_MS + 0.75,
+ attributes: {},
+ },
+ { name: 'connectStart', epochMs: SPAN_START_MS + 1, attributes: {} },
+ { name: 'connectEnd', epochMs: SPAN_START_MS + 1.5, attributes: {} },
+ { name: 'requestStart', epochMs: SPAN_START_MS + 1.5, attributes: {} },
+ { name: 'responseStart', epochMs: SPAN_START_MS + 8.5, attributes: {} },
+ {
+ name: 'responseEnd',
+ epochMs: SPAN_START_MS + 11,
+ attributes: {
+ duration: '11ms',
+ transfer_size: '558 B',
+ encoded_body_size: '1.5 KB',
+ decoded_body_size: '20.0 KB',
+ next_hop_protocol: 'http/1.1',
+ },
+ },
+ ]);
+ expect([...drawnRowCounts]).toEqual([
+ [resourceTimingRowKey('span-1', TIMING_TIMESTAMP), 1],
+ ]);
+ });
+
+ it('skips the phases and sizes a cross-origin request leaves at zero, and empty tooltip values', () => {
+ const crossOrigin = prefixed({
+ fetch_start: '1000.5',
+ domain_lookup_start: '0',
+ connect_start: '0',
+ request_start: '0',
+ response_start: '0',
+ response_end: '1011.5',
+ duration: '11',
+ transfer_size: '0',
+ encoded_body_size: '0',
+ decoded_body_size: '0',
+ next_hop_protocol: '',
+ });
+
+ expect(draw(timingRow(crossOrigin)).markersBySpanId.get('span-1')).toEqual([
+ { name: 'fetchStart', epochMs: SPAN_START_MS, attributes: {} },
+ {
+ name: 'responseEnd',
+ epochMs: SPAN_START_MS + 11,
+ attributes: { duration: '11ms' },
+ },
+ ]);
+ });
+
+ it('keeps a zero transfer size next to a body size, as for a cache hit', () => {
+ const cached = prefixed({
+ fetch_start: '1000.5',
+ response_end: '1011.5',
+ transfer_size: '0',
+ encoded_body_size: '221',
+ decoded_body_size: '221',
+ });
+
+ expect(
+ draw(timingRow(cached)).markersBySpanId.get('span-1')?.at(-1)?.attributes,
+ ).toEqual({
+ transfer_size: '0 B',
+ encoded_body_size: '221 B',
+ decoded_body_size: '221 B',
+ });
+ });
+
+ it('reads nested JSON numbers and quoted Int64 values', () => {
+ const json = {
+ browser: {
+ resource_timing: {
+ fetch_start: 1000.5,
+ response_end: 1011.5,
+ transfer_size: '558',
+ },
+ },
+ };
+
+ expect(draw(timingRow(json)).markersBySpanId.get('span-1')).toEqual([
+ { name: 'fetchStart', epochMs: SPAN_START_MS, attributes: {} },
+ {
+ name: 'responseEnd',
+ epochMs: SPAN_START_MS + 11,
+ attributes: { transfer_size: '558 B' },
+ },
+ ]);
+ });
+
+ it.each([
+ ['without fetch_start', timingRow(prefixed({ response_end: '1011.5' }))],
+ ['with a zero fetch_start', timingRow(prefixed({ fetch_start: '0' }))],
+ [
+ 'on a span outside the trace',
+ timingRow(sameOrigin, { span_id: 'other' }),
+ ],
+ ['without a span id', timingRow(sameOrigin, { span_id: '' })],
+ ['with attributes that are not a map', timingRow('not a map')],
+ ])('draws nothing and keeps the row for a log %s', (_name, row) => {
+ const { markersBySpanId, drawnRowCounts } = draw(row);
+ expect(markersBySpanId.size).toBe(0);
+ expect(drawnRowCounts.size).toBe(0);
+ });
+
+ it.each([
+ [
+ 'its span is hidden',
+ [{ ...traceRows[0], __hdx_hidden: true }],
+ TIMESTAMP_META,
+ ],
+ [
+ 'the timestamp column is whole seconds',
+ traceRows,
+ [{ name: 'timestamp', type: 'DateTime' }],
+ ],
+ ])('draws a log but keeps its row when %s', (_name, spans, meta) => {
+ const { markersBySpanId, drawnRowCounts } = draw(
+ timingRow(sameOrigin),
+ spans,
+ meta,
+ );
+ expect(markersBySpanId.get('span-1')).toHaveLength(8);
+ expect(drawnRowCounts.size).toBe(0);
+ });
+
+ it('counts a timing log ingested twice once per copy', () => {
+ const row = timingRow(sameOrigin);
+ const { drawnRowCounts } = getResourceTimingMarkers(
+ { data: [row, row], meta: TIMESTAMP_META },
+ traceRows,
+ );
+ expect([...drawnRowCounts]).toEqual([
+ [resourceTimingRowKey('span-1', TIMING_TIMESTAMP), 2],
+ ]);
+ });
+});
+
+describe('getFoldedRowKeys', () => {
+ const key = resourceTimingRowKey('span-1', TIMING_TIMESTAMP);
+ const timingLogRow = { SpanId: 'span-1', Timestamp: TIMING_TIMESTAMP };
+ const otherLogRow = {
+ SpanId: 'span-1',
+ Timestamp: '2026-10-22T04:52:37.825000000Z',
+ };
+
+ it.each([
+ ['folds the timing row alone', [timingLogRow, otherLogRow], 1, [key]],
+ [
+ 'keeps an exception in the same millisecond and the timing row',
+ [timingLogRow, { ...timingLogRow, Body: '', SeverityNumber: 17 }],
+ 1,
+ [],
+ ],
+ [
+ 'keeps a console error in the same millisecond and the timing row',
+ [timingLogRow, { ...timingLogRow, Body: 'boom', SeverityText: 'error' }],
+ 1,
+ [],
+ ],
+ [
+ 'folds two copies of a timing log drawn twice',
+ [timingLogRow, timingLogRow],
+ 2,
+ [key],
+ ],
+ ['ignores rows on other keys', [otherLogRow], 1, []],
+ ])('%s', (_name, logRows, drawn, expected) => {
+ expect([...getFoldedRowKeys(logRows, new Map([[key, drawn]]))]).toEqual(
+ expected,
+ );
+ });
+});
+
+describe('buildResourceTimingQuery', () => {
+ const source = {
+ ...logSource,
+ timestampValueExpression: 'TimestampTime',
+ displayedTimestampValueExpression: 'Timestamp',
+ };
+ const context = {
+ serviceNames: ['web', 'web-worker'],
+ anchorMs: SPAN_START_MS,
+ endMs: Date.parse('2026-10-22T04:52:38.100Z'),
+ anchorHidden: false,
+ sessionId: 's1',
+ serviceName: 'web',
+ };
+
+ it('selects the span id, displayed timestamp and attributes of the trace timing logs', () => {
+ expect(
+ buildResourceTimingQuery({ logSource: source, context, traceId: "t'1" }),
+ ).toEqual({
+ source: 'logs',
+ connection: 'conn',
+ from: { databaseName: 'default', tableName: 'otel_logs' },
+ timestampValueExpression: 'TimestampTime',
+ select: [
+ { valueExpression: 'SpanId', alias: 'span_id' },
+ { valueExpression: 'Timestamp', alias: 'timestamp' },
+ { valueExpression: 'LogAttributes', alias: 'attributes' },
+ ],
+ where: "TraceId = 't\\'1'",
+ whereLanguage: 'sql',
+ filters: [
+ {
+ type: 'sql',
+ condition: "toString(EventName) = 'browser.resource_timing'",
+ },
+ {
+ type: 'sql',
+ condition: "toString(ServiceName) IN ('web', 'web-worker')",
+ },
+ ],
+ dateRange: [
+ new Date('2026-10-22T04:52:37.000Z'),
+ new Date('2026-10-22T04:57:38.100Z'),
+ ],
+ limit: { limit: 1000 },
+ });
+ });
+
+ it('places a JSON path service name inside toString', () => {
+ const query = buildResourceTimingQuery({
+ logSource: {
+ ...source,
+ serviceNameExpression: 'ResourceAttributes.service.name',
+ },
+ context,
+ traceId: 't1',
+ });
+ expect(query.filters?.[1]).toEqual({
+ type: 'sql',
+ condition:
+ "toString(ResourceAttributes.service.name) IN ('web', 'web-worker')",
+ });
+ });
+
+ it.each([
+ [
+ 'the source has no service name',
+ { ...source, serviceNameExpression: undefined },
+ context,
+ ],
+ [
+ 'the spans have no service name',
+ source,
+ { ...context, serviceNames: [] },
+ ],
+ ])('leaves out the service filter when %s', (_name, querySource, ctx) => {
+ expect(
+ buildResourceTimingQuery({
+ logSource: querySource,
+ context: ctx,
+ traceId: 't1',
+ }).filters,
+ ).toHaveLength(1);
+ });
+});
diff --git a/packages/app/src/utils/__tests__/traceTriggerClick.test.ts b/packages/app/src/utils/__tests__/traceTriggerClick.test.ts
new file mode 100644
index 000000000..b05eb0347
--- /dev/null
+++ b/packages/app/src/utils/__tests__/traceTriggerClick.test.ts
@@ -0,0 +1,152 @@
+import {
+ logSource,
+ SPAN_START,
+ SPAN_START_MS,
+} from '@/hooks/__fixtures__/browserTraceEvents';
+import type { BrowserTraceContext } from '@/utils/browserTraceContext';
+import {
+ buildTriggerClickQuery,
+ pickTriggerClick,
+} from '@/utils/traceTriggerClick';
+
+const context: BrowserTraceContext = {
+ serviceNames: ['web'],
+ anchorMs: SPAN_START_MS,
+ endMs: SPAN_START_MS + 12,
+ anchorHidden: false,
+ sessionId: 's1',
+ serviceName: 'web',
+};
+
+const args = {
+ logSource: {
+ ...logSource,
+ timestampValueExpression: 'TimestampTime, Timestamp',
+ displayedTimestampValueExpression: 'Timestamp',
+ },
+ context,
+ sessionId: 's1',
+ sessionIdExpression: "LogAttributes['session.id']",
+};
+
+const click = (timestamp: string, attributes: object = {}) => ({
+ timestamp,
+ event_name: 'browser.user_action.click',
+ body: '',
+ attributes: { 'browser.tag_name': 'BUTTON', ...attributes },
+});
+
+const clicks = (...data: ReturnType[]) => ({
+ data,
+ meta: [{ name: 'timestamp', type: 'DateTime64(9)' }],
+});
+
+describe('buildTriggerClickQuery', () => {
+ it("looks up the latest clicks of the anchor span's session and service", () => {
+ const { config, aliasMap } = buildTriggerClickQuery(args);
+
+ expect(aliasMap).toEqual({
+ timestamp: 'Timestamp',
+ timestamp_key: 'TimestampTime',
+ event_name: 'toString(EventName)',
+ session_id: "LogAttributes['session.id']",
+ body: 'Body',
+ attributes: 'LogAttributes',
+ });
+ expect(config).toMatchObject({
+ source: 'logs',
+ from: logSource.from,
+ connection: 'conn',
+ timestampValueExpression: 'TimestampTime, Timestamp',
+ filters: [
+ { type: 'lucene', condition: 'LogAttributes.session.id:"s1"' },
+ {
+ type: 'sql',
+ condition: "toString(EventName) = 'browser.user_action.click'",
+ },
+ { type: 'sql', condition: "ServiceName = 'web'" },
+ ],
+ orderBy: 'Timestamp DESC',
+ limit: { limit: 10 },
+ });
+ });
+
+ it.each([
+ [
+ 'the lookback across a second boundary',
+ Date.parse('2026-10-22T04:52:38.100Z'),
+ ['2026-10-22T04:52:37.000Z', '2026-10-22T04:52:38.100Z'],
+ ],
+ [
+ 'an anchor on a whole second',
+ Date.parse('2026-10-22T04:52:38.000Z'),
+ ['2026-10-22T04:52:37.000Z', '2026-10-22T04:52:38.000Z'],
+ ],
+ ])(
+ 'floors the window start to the second and ends at the anchor for %s',
+ (_case, anchorMs, range) => {
+ expect(
+ buildTriggerClickQuery({ ...args, context: { ...context, anchorMs } })
+ .config.dateRange,
+ ).toEqual(range.map(iso => new Date(iso)));
+ },
+ );
+});
+
+describe('pickTriggerClick', () => {
+ it('picks the latest click at or before the anchor', () => {
+ const latest = click('2026-10-22T04:52:37.815000000Z');
+ const rows = clicks(
+ click('2026-10-22T04:52:37.819000000Z'),
+ latest,
+ click('2026-10-22T04:52:37.700000000Z'),
+ );
+
+ expect(pickTriggerClick(rows, SPAN_START_MS)).toEqual({
+ row: latest,
+ gapMs: 3,
+ });
+ });
+
+ it.each([
+ ['in the same millisecond as the anchor', SPAN_START, 0],
+ ['at the edge of the lookback', '2026-10-22T04:52:37.568000000Z', 250],
+ ])('takes a click %s', (_case, timestamp, gapMs) => {
+ const row = click(timestamp);
+
+ expect(pickTriggerClick(clicks(row), SPAN_START_MS)).toEqual({
+ row,
+ gapMs,
+ });
+ });
+
+ it.each([
+ ['after the anchor in the same second', '2026-10-22T04:52:37.900000000Z'],
+ ['older than the lookback', '2026-10-22T04:52:37.567000000Z'],
+ ['with an unparsable timestamp', 'not-a-timestamp'],
+ ])('ignores a click %s', (_case, timestamp) => {
+ expect(
+ pickTriggerClick(clicks(click(timestamp)), SPAN_START_MS),
+ ).toBeUndefined();
+ });
+
+ it('ignores clicks in a whole-second timestamp column', () => {
+ expect(
+ pickTriggerClick(
+ {
+ data: [click('2026-10-22T04:52:37Z')],
+ meta: [{ name: 'timestamp', type: 'DateTime' }],
+ },
+ Date.parse('2026-10-22T04:52:37.100Z'),
+ ),
+ ).toBeUndefined();
+ });
+
+ it('matches a click recorded on another page URL', () => {
+ const row = click('2026-10-22T04:52:37.816000000Z', {
+ 'browser.document.url.full': 'http://localhost:5173/cart?tab=items',
+ });
+
+ expect(pickTriggerClick(clicks(row), SPAN_START_MS)?.row).toBe(row);
+ });
+});
diff --git a/packages/app/src/utils/browserEvents.ts b/packages/app/src/utils/browserEvents.ts
index f60e6a6d2..6591d35d1 100644
--- a/packages/app/src/utils/browserEvents.ts
+++ b/packages/app/src/utils/browserEvents.ts
@@ -16,6 +16,7 @@ export const BROWSER_EVENT_NAMES = {
NAVIGATION: 'browser.navigation',
CONSOLE: 'browser.console',
EXCEPTION: 'exception',
+ RESOURCE_TIMING: 'browser.resource_timing',
} as const;
export const CONSOLE_METHOD_ATTRIBUTE = 'browser.console.method';
diff --git a/packages/app/src/utils/browserTraceContext.ts b/packages/app/src/utils/browserTraceContext.ts
new file mode 100644
index 000000000..40c8cb2ab
--- /dev/null
+++ b/packages/app/src/utils/browserTraceContext.ts
@@ -0,0 +1,140 @@
+import { get } from 'lodash';
+import type { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse';
+import { classifyTimestampType } from '@hyperdx/common-utils/dist/core/utils';
+import type { TLogSource } from '@hyperdx/common-utils/dist/types';
+
+import { parseTimestampToMs } from '@/utils';
+import {
+ isSessionLogEventSource,
+ SessionLogEventSource,
+} from '@/utils/sessionLogEvents';
+import { isSet } from '@/utils/spanAttributeReaders';
+
+// The waterfall reads an hour either side of its focus. One long or broken
+// span would otherwise stretch the timing query over its whole duration.
+const MAX_TRACE_MS = 60 * 60 * 1000;
+
+export type BrowserTraceContext = {
+ serviceNames: string[];
+ /** Earliest browser span start. */
+ anchorMs: number;
+ /** Latest browser span end, at most an hour after the anchor. */
+ endMs: number;
+ /** Whether the trace filter hid the anchor span. */
+ anchorHidden: boolean;
+ /** Of the anchor span. */
+ sessionId: string | undefined;
+ /** Of the anchor span. */
+ serviceName: string | undefined;
+};
+
+export type BrowserTraceRow = {
+ Timestamp?: unknown;
+ Duration?: unknown;
+ ServiceName?: unknown;
+ SpanAttributes?: unknown;
+ __hdx_hidden?: unknown;
+};
+
+export type BrowserEventLogSource = SessionLogEventSource &
+ Required>;
+
+export function isBrowserEventLogSource(
+ source: TLogSource | null | undefined,
+): source is BrowserEventLogSource {
+ return (
+ isSessionLogEventSource(source) &&
+ !!source.traceIdExpression &&
+ !!source.spanIdExpression
+ );
+}
+
+/**
+ * Twin of the guard in the waterfall's traceExtent: timestamp-nano reads a
+ * non-date as a finite instant.
+ */
+export function parseRowTimeMs(timestamp: unknown): number | undefined {
+ if (typeof timestamp !== 'string' || Number.isNaN(Date.parse(timestamp))) {
+ return undefined;
+ }
+ try {
+ const ms = parseTimestampToMs(timestamp);
+ return Number.isFinite(ms) ? ms : undefined;
+ } catch {
+ return undefined;
+ }
+}
+
+export type BrowserEventResult = Pick<
+ ResponseJSON>,
+ 'data' | 'meta'
+>;
+
+export function hasSubSecondTimestamps(
+ meta: BrowserEventResult['meta'],
+): boolean {
+ const type = meta?.find(column => column.name === 'timestamp')?.type;
+ return (classifyTimestampType(type)?.precision ?? 0) > 0;
+}
+
+/**
+ * The start is floored to the second: a DateTime time column compares its
+ * truncated value and would otherwise lose the start's second. The end is
+ * inclusive, so a truncated value is never past it.
+ */
+export function getBrowserEventDateRange(
+ startMs: number,
+ endMs: number,
+): [Date, Date] {
+ return [new Date(Math.floor(startMs / 1000) * 1000), new Date(endMs)];
+}
+
+function optionalString(value: unknown): string | undefined {
+ return isSet(value) ? String(value) : undefined;
+}
+
+/**
+ * Undefined unless the trace has a span from an OTel browser SDK, which sets
+ * the page URL on every span. AI and backend spans can carry `session.id`
+ * too, and legacy SDK spans have neither. Never anchored on the trace's
+ * earliest span overall: backend spans run on another clock.
+ */
+export function getBrowserTraceContext(
+ traceRows: readonly BrowserTraceRow[],
+): BrowserTraceContext | undefined {
+ let anchor: { row: BrowserTraceRow; startMs: number } | undefined;
+ let endMs = Number.NEGATIVE_INFINITY;
+ const serviceNames = new Set();
+ for (const row of traceRows) {
+ if (!isSet(get(row.SpanAttributes, 'browser.document.url.full'))) {
+ continue;
+ }
+ const startMs = parseRowTimeMs(row.Timestamp);
+ if (startMs == null) {
+ continue;
+ }
+ const durationSec = Number(row.Duration);
+ endMs = Math.max(
+ endMs,
+ startMs + (Number.isFinite(durationSec) ? durationSec : 0) * 1000,
+ );
+ const serviceName = optionalString(row.ServiceName);
+ if (serviceName != null) {
+ serviceNames.add(serviceName);
+ }
+ if (anchor == null || startMs < anchor.startMs) {
+ anchor = { row, startMs };
+ }
+ }
+ if (anchor == null) {
+ return undefined;
+ }
+ return {
+ serviceNames: [...serviceNames].sort(),
+ anchorMs: anchor.startMs,
+ endMs: Math.min(endMs, anchor.startMs + MAX_TRACE_MS),
+ anchorHidden: !!anchor.row.__hdx_hidden,
+ sessionId: optionalString(get(anchor.row.SpanAttributes, 'session.id')),
+ serviceName: optionalString(anchor.row.ServiceName),
+ };
+}
diff --git a/packages/app/src/utils/httpSpanAttributes.ts b/packages/app/src/utils/httpSpanAttributes.ts
index ca1acc1b4..f183b03b3 100644
--- a/packages/app/src/utils/httpSpanAttributes.ts
+++ b/packages/app/src/utils/httpSpanAttributes.ts
@@ -133,7 +133,7 @@ export function getUserAgent(attributes: Attributes): string | undefined {
// One decimal above 1 KB, so a body and total of 1536 and 1900 bytes don't
// both round to "2 KB".
-function formatBytes(bytes: number): string {
+export function formatBytes(bytes: number): string {
return formatNumber(bytes, {
output: 'byte',
mantissa: Math.abs(bytes) >= 1024 ? 1 : 0,
diff --git a/packages/app/src/utils/resourceTimingMarkers.ts b/packages/app/src/utils/resourceTimingMarkers.ts
new file mode 100644
index 000000000..89c695538
--- /dev/null
+++ b/packages/app/src/utils/resourceTimingMarkers.ts
@@ -0,0 +1,250 @@
+import { countBy, get } from 'lodash';
+import SqlString from 'sqlstring';
+import type {
+ BuilderChartConfigWithDateRange,
+ Filter,
+} from '@hyperdx/common-utils/dist/types';
+
+import type { TTimelineSpanEventMarker } from '@/components/TimelineChart/TimelineSpanEventMarker';
+import { getDisplayedTimestampValueExpression } from '@/source';
+import { formatDurationMs } from '@/utils';
+import { BROWSER_EVENT_NAMES } from '@/utils/browserEvents';
+import {
+ BrowserEventLogSource,
+ BrowserEventResult,
+ BrowserTraceContext,
+ getBrowserEventDateRange,
+ hasSubSecondTimestamps,
+ parseRowTimeMs,
+} from '@/utils/browserTraceContext';
+import { formatBytes } from '@/utils/httpSpanAttributes';
+import { isSet, toFiniteNumber } from '@/utils/spanAttributeReaders';
+
+const ATTRIBUTE_PREFIX = 'browser.resource_timing.';
+
+// Same names and order as the network span events @opentelemetry/sdk-trace-web
+// adds to fetch and XHR spans, which the legacy SDK drew as dots.
+const PHASES = [
+ ['fetch_start', 'fetchStart'],
+ ['domain_lookup_start', 'domainLookupStart'],
+ ['domain_lookup_end', 'domainLookupEnd'],
+ ['connect_start', 'connectStart'],
+ ['secure_connection_start', 'secureConnectionStart'],
+ ['connect_end', 'connectEnd'],
+ ['request_start', 'requestStart'],
+ ['response_start', 'responseStart'],
+ ['response_end', 'responseEnd'],
+] as const;
+
+// Five keys with duration and next_hop_protocol: the marker tooltip's limit.
+// New SDK spans carry no response size, and the dots replace the timing row.
+const SIZE_ATTRIBUTES = [
+ 'transfer_size',
+ 'encoded_body_size',
+ 'decoded_body_size',
+] as const;
+const PROTOCOL_ATTRIBUTE = 'next_hop_protocol';
+
+// A timing log's timestamp is its emit time, after the request ends. The SDK
+// flushes within 1 s; the margin covers slow exports.
+const EMIT_LAG_MS = 5 * 60 * 1000;
+const RESOURCE_TIMING_LIMIT = 1000;
+
+export type ResourceTimingMarker = Omit<
+ TTimelineSpanEventMarker,
+ 'timestamp'
+> & {
+ attributes: Record;
+};
+
+/**
+ * Matches a timing log to its waterfall log row: both queries select the same
+ * timestamp expression. Another log on the span can share the key, so the
+ * fold goes through getFoldedRowKeys.
+ */
+export function resourceTimingRowKey(spanId: unknown, timestamp: unknown) {
+ return `${String(spanId)}|${String(timestamp)}`;
+}
+
+/**
+ * Log rows carry no event name, and an exception or console error on the span
+ * in the same millisecond has the timing log's key. A key folds only when the
+ * dots query drew a timing log for each log row with it. All rows with a key
+ * share one timestamp, so an uncapped log list holds all of them or none.
+ */
+export function getFoldedRowKeys(
+ logRows: readonly { SpanId?: unknown; Timestamp?: unknown }[],
+ drawnRowCounts: ReadonlyMap,
+): ReadonlySet {
+ const rowCounts = countBy(logRows, row =>
+ resourceTimingRowKey(row.SpanId, row.Timestamp),
+ );
+ return new Set(
+ [...drawnRowCounts]
+ .filter(([key, drawn]) => rowCounts[key] <= drawn)
+ .map(([key]) => key),
+ );
+}
+
+export function buildResourceTimingQuery({
+ logSource,
+ context,
+ traceId,
+}: {
+ logSource: BrowserEventLogSource;
+ context: BrowserTraceContext;
+ traceId: string;
+}): BuilderChartConfigWithDateRange {
+ // toString: a JSON path is Dynamic, which IN rejects.
+ const filters: Filter[] = [
+ {
+ type: 'sql',
+ condition: SqlString.format('? = ?', [
+ SqlString.raw(`toString(${logSource.eventNameExpression})`),
+ BROWSER_EVENT_NAMES.RESOURCE_TIMING,
+ ]),
+ },
+ ];
+ if (logSource.serviceNameExpression && context.serviceNames.length > 0) {
+ filters.push({
+ type: 'sql',
+ condition: SqlString.format('? IN (?)', [
+ SqlString.raw(`toString(${logSource.serviceNameExpression})`),
+ context.serviceNames,
+ ]),
+ });
+ }
+ return {
+ source: logSource.id,
+ connection: logSource.connection,
+ from: logSource.from,
+ timestampValueExpression: logSource.timestampValueExpression,
+ select: [
+ { valueExpression: logSource.spanIdExpression, alias: 'span_id' },
+ {
+ valueExpression: getDisplayedTimestampValueExpression(logSource),
+ alias: 'timestamp',
+ },
+ // The whole map: a single key would need a JSON metadata lookup first.
+ {
+ valueExpression: logSource.eventAttributesExpression,
+ alias: 'attributes',
+ },
+ ],
+ where: `${logSource.traceIdExpression} = ${SqlString.escape(traceId)}`,
+ whereLanguage: 'sql',
+ filters,
+ dateRange: getBrowserEventDateRange(
+ context.anchorMs,
+ context.endMs + EMIT_LAG_MS,
+ ),
+ limit: { limit: RESOURCE_TIMING_LIMIT },
+ };
+}
+
+function readNumber(attributes: unknown, key: string): number | undefined {
+ // lodash get reads both a flat Map key and a nested JSON path. Map values
+ // are strings, and JSON Int64 values come quoted.
+ return toFiniteNumber(get(attributes, `${ATTRIBUTE_PREFIX}${key}`));
+}
+
+function getTooltipAttributes(attributes: unknown): Record {
+ const result: Record = {};
+ const duration = readNumber(attributes, 'duration');
+ if (duration != null) {
+ result.duration = formatDurationMs(duration);
+ }
+ // A cross-origin response without Timing-Allow-Origin reports every size as
+ // 0, which isn't its size. A lone 0 transfer size is a cache hit and stays.
+ const sizes = SIZE_ATTRIBUTES.map(
+ key => [key, readNumber(attributes, key)] as const,
+ );
+ if (sizes.some(([, size]) => size)) {
+ for (const [key, size] of sizes) {
+ if (size != null) {
+ result[key] = formatBytes(size);
+ }
+ }
+ }
+ const protocol: unknown = get(
+ attributes,
+ `${ATTRIBUTE_PREFIX}${PROTOCOL_ATTRIBUTE}`,
+ );
+ if (isSet(protocol)) {
+ result[PROTOCOL_ATTRIBUTE] = String(protocol);
+ }
+ return result;
+}
+
+/**
+ * Phases are milliseconds from the page's time origin, and the log timestamp
+ * is its emit time, so dots are placed relative to the span start. Zero
+ * phases weren't measured (a cross-origin request without
+ * Timing-Allow-Origin) and are skipped, as sdk-trace-web does.
+ */
+function toMarkers(
+ attributes: unknown,
+ spanStartMs: number,
+ fetchStart: number,
+): ResourceTimingMarker[] {
+ const markers = PHASES.flatMap(([key, name]) => {
+ const phase = readNumber(attributes, key);
+ return phase != null && phase !== 0
+ ? [{ epochMs: spanStartMs + (phase - fetchStart), name, attributes: {} }]
+ : [];
+ });
+ markers[markers.length - 1].attributes = getTooltipAttributes(attributes);
+ return markers;
+}
+
+/**
+ * A log on a span outside the trace isn't drawn, so its row stays. A drawn
+ * log's row stays too when the trace filter hides its span, since the dots
+ * don't show, or when the timestamp column is whole seconds, since the key
+ * would match every log on the span in that second.
+ */
+export function getResourceTimingMarkers(
+ { data, meta }: BrowserEventResult,
+ traceRows: readonly {
+ SpanId?: unknown;
+ Timestamp?: unknown;
+ __hdx_hidden?: unknown;
+ }[],
+): {
+ markersBySpanId: ReadonlyMap;
+ drawnRowCounts: ReadonlyMap;
+} {
+ const spanStarts = new Map();
+ const shownSpanIds = new Set();
+ for (const row of traceRows) {
+ const startMs = parseRowTimeMs(row.Timestamp);
+ if (isSet(row.SpanId) && startMs != null) {
+ const spanId = String(row.SpanId);
+ spanStarts.set(spanId, startMs);
+ if (!row.__hdx_hidden) {
+ shownSpanIds.add(spanId);
+ }
+ }
+ }
+
+ const foldable = hasSubSecondTimestamps(meta);
+ const markersBySpanId = new Map();
+ const drawnRowCounts = new Map();
+ for (const row of data) {
+ const spanId = isSet(row.span_id) ? String(row.span_id) : undefined;
+ const spanStartMs = spanId != null ? spanStarts.get(spanId) : undefined;
+ const fetchStart = readNumber(row.attributes, 'fetch_start');
+ if (spanId == null || spanStartMs == null || !fetchStart) {
+ continue;
+ }
+ markersBySpanId.set(spanId, [
+ ...(markersBySpanId.get(spanId) ?? []),
+ ...toMarkers(row.attributes, spanStartMs, fetchStart),
+ ]);
+ if (foldable && shownSpanIds.has(spanId)) {
+ const key = resourceTimingRowKey(spanId, row.timestamp);
+ drawnRowCounts.set(key, (drawnRowCounts.get(key) ?? 0) + 1);
+ }
+ }
+ return { markersBySpanId, drawnRowCounts };
+}
diff --git a/packages/app/src/utils/sessionLogEvents.ts b/packages/app/src/utils/sessionLogEvents.ts
index 0d557c1a0..3050b686e 100644
--- a/packages/app/src/utils/sessionLogEvents.ts
+++ b/packages/app/src/utils/sessionLogEvents.ts
@@ -4,6 +4,7 @@ import SqlString from 'sqlstring';
import type {
BuilderChartConfigWithDateRange,
Filter,
+ SelectList,
TLogSource,
} from '@hyperdx/common-utils/dist/types';
@@ -91,6 +92,86 @@ function eventNameCondition(
]);
}
+/** A log source with the fields every session log event query reads. */
+export type SessionLogEventSource = TLogSource &
+ Required<
+ Pick
+ >;
+
+export function isSessionLogEventSource(
+ source: TLogSource | null | undefined,
+): source is SessionLogEventSource {
+ return !!source?.eventNameExpression && !!source.eventAttributesExpression;
+}
+
+/** The select and filters every session log event query shares. */
+export function getSessionLogEventQueryParts({
+ logSource,
+ sessionId,
+ serviceName,
+ sessionIdExpression,
+}: {
+ logSource: SessionLogEventSource;
+ sessionId: string;
+ serviceName: string | undefined;
+ sessionIdExpression: string;
+}): {
+ aliasMap: Record;
+ select: SelectList;
+ sessionFilter: Filter;
+ serviceFilters: Filter[];
+} {
+ const attrs = logSource.eventAttributesExpression;
+ // toString: a JSON path is Dynamic, which IN rejects.
+ const eventName = `toString(${logSource.eventNameExpression})`;
+ const timestamp = getDisplayedTimestampValueExpression(logSource);
+ const timestampKey = getFirstTimestampValueExpression(
+ logSource.timestampValueExpression,
+ );
+ const body = getEventBody(logSource);
+ const aliasMap: Record = {
+ timestamp,
+ ...(timestampKey !== timestamp ? { timestamp_key: timestampKey } : {}),
+ event_name: eventName,
+ session_id: sessionIdExpression,
+ ...(body ? { body } : {}),
+ attributes: attrs,
+ };
+ return {
+ aliasMap,
+ select: Object.entries(aliasMap).map(([alias, valueExpression]) => ({
+ valueExpression,
+ alias,
+ })),
+ sessionFilter: {
+ type: 'lucene',
+ condition: sessionIdFilter(
+ {
+ resource:
+ logSource.resourceAttributesExpression ?? 'ResourceAttributes',
+ attrs,
+ },
+ sessionId,
+ 'session.id',
+ ),
+ },
+ // Matches the replay query, and keeps backend exceptions that carry a
+ // propagated session.id off the browser timeline.
+ serviceFilters:
+ logSource.serviceNameExpression && serviceName
+ ? [
+ {
+ type: 'sql',
+ condition: SqlString.format('? = ?', [
+ SqlString.raw(logSource.serviceNameExpression),
+ serviceName,
+ ]),
+ },
+ ]
+ : [],
+ };
+}
+
/** Null when the session has no browser log events to query. */
export function buildSessionLogEventsQuery({
logSource,
@@ -116,83 +197,48 @@ export function buildSessionLogEventsQuery({
config: BuilderChartConfigWithDateRange;
aliasMap: Record;
} | null {
- const attrs = logSource?.eventAttributesExpression;
// Only OTel browser SDKs write these events, and the search box is written
// against trace fields.
if (
sessionIdKey !== 'session.id' ||
searchedQuery !== '' ||
- !logSource?.eventNameExpression ||
- !attrs ||
+ !isSessionLogEventSource(logSource) ||
!sessionIdExpression ||
!consoleMethodExpression
) {
return null;
}
- // toString: a JSON path is Dynamic, which IN rejects.
- const eventName = `toString(${logSource.eventNameExpression})`;
- const timestamp = getDisplayedTimestampValueExpression(logSource);
- const timestampKey = getFirstTimestampValueExpression(
- logSource.timestampValueExpression,
- );
- const body = getEventBody(logSource);
- const aliasMap: Record = {
- timestamp,
- ...(timestampKey !== timestamp ? { timestamp_key: timestampKey } : {}),
- event_name: eventName,
- session_id: sessionIdExpression,
- ...(body ? { body } : {}),
- attributes: attrs,
- };
- const filters: Filter[] = [
- {
- type: 'lucene',
- condition: sessionIdFilter(
- {
- resource:
- logSource.resourceAttributesExpression ?? 'ResourceAttributes',
- attrs,
- },
- sessionId,
- 'session.id',
- ),
- },
- {
- type: 'sql',
- condition: eventNameCondition(
- eventName,
- consoleMethodExpression,
- tab === 'highlighted',
- ),
- },
- ];
- // Matches the replay query, and keeps backend exceptions that carry a
- // propagated session.id off the browser timeline.
- if (logSource.serviceNameExpression && serviceName) {
- filters.push({
- type: 'sql',
- condition: SqlString.format('? = ?', [
- SqlString.raw(logSource.serviceNameExpression),
- serviceName,
- ]),
+ const { aliasMap, select, sessionFilter, serviceFilters } =
+ getSessionLogEventQueryParts({
+ logSource,
+ sessionId,
+ serviceName,
+ sessionIdExpression,
});
- }
return {
aliasMap,
config: {
source: logSource.id,
- select: Object.entries(aliasMap).map(([alias, valueExpression]) => ({
- valueExpression,
- alias,
- })),
+ select,
from: logSource.from,
connection: logSource.connection,
timestampValueExpression: logSource.timestampValueExpression,
dateRange,
where: '',
whereLanguage: 'sql',
- filters,
- orderBy: `${timestamp} ASC`,
+ filters: [
+ sessionFilter,
+ {
+ type: 'sql',
+ condition: eventNameCondition(
+ aliasMap.event_name,
+ consoleMethodExpression,
+ tab === 'highlighted',
+ ),
+ },
+ ...serviceFilters,
+ ],
+ orderBy: `${aliasMap.timestamp} ASC`,
limit: { limit: 4000, offset: 0 },
},
};
diff --git a/packages/app/src/utils/traceTriggerClick.ts b/packages/app/src/utils/traceTriggerClick.ts
new file mode 100644
index 000000000..c7bfe717c
--- /dev/null
+++ b/packages/app/src/utils/traceTriggerClick.ts
@@ -0,0 +1,103 @@
+import SqlString from 'sqlstring';
+import type { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
+
+import { BROWSER_EVENT_NAMES } from '@/utils/browserEvents';
+import {
+ BrowserEventLogSource,
+ BrowserEventResult,
+ BrowserTraceContext,
+ getBrowserEventDateRange,
+ hasSubSecondTimestamps,
+ parseRowTimeMs,
+} from '@/utils/browserTraceContext';
+import { getSessionLogEventQueryParts } from '@/utils/sessionLogEvents';
+
+// Clicks land 0-5 ms before the request they start; the click before that is
+// usually several hundred ms earlier.
+const TRIGGER_LOOKBACK_MS = 250;
+// The floored window holds at most about 2 s of one session's clicks.
+const TRIGGER_CLICK_LIMIT = 10;
+
+export function buildTriggerClickQuery({
+ logSource,
+ context,
+ sessionId,
+ sessionIdExpression,
+}: {
+ logSource: BrowserEventLogSource;
+ context: BrowserTraceContext;
+ sessionId: string;
+ sessionIdExpression: string;
+}): {
+ config: BuilderChartConfigWithDateRange;
+ aliasMap: Record;
+} {
+ const { aliasMap, select, sessionFilter, serviceFilters } =
+ getSessionLogEventQueryParts({
+ logSource,
+ sessionId,
+ serviceName: context.serviceName,
+ sessionIdExpression,
+ });
+ return {
+ aliasMap,
+ config: {
+ source: logSource.id,
+ select,
+ from: logSource.from,
+ connection: logSource.connection,
+ timestampValueExpression: logSource.timestampValueExpression,
+ // pickTriggerClick clips to the exact window.
+ dateRange: getBrowserEventDateRange(
+ context.anchorMs - TRIGGER_LOOKBACK_MS,
+ context.anchorMs,
+ ),
+ where: '',
+ whereLanguage: 'sql',
+ filters: [
+ sessionFilter,
+ {
+ type: 'sql',
+ condition: SqlString.format('? = ?', [
+ SqlString.raw(aliasMap.event_name),
+ BROWSER_EVENT_NAMES.CLICK,
+ ]),
+ },
+ ...serviceFilters,
+ ],
+ orderBy: `${aliasMap.timestamp} DESC`,
+ limit: { limit: TRIGGER_CLICK_LIMIT },
+ },
+ };
+}
+
+/**
+ * The latest click in the lookback before the anchor. The page URL isn't
+ * compared: the SDK records a click's URL before the app's handler runs, and
+ * the request's after, so a click that changes the route or query string
+ * would never match. Clicks in a whole-second timestamp column can't be
+ * placed within the lookback, so none match.
+ */
+export function pickTriggerClick(
+ { data, meta }: BrowserEventResult,
+ anchorMs: number,
+): { row: Record; gapMs: number } | undefined {
+ if (!hasSubSecondTimestamps(meta)) {
+ return undefined;
+ }
+ let latest: { row: Record; ms: number } | undefined;
+ for (const row of data) {
+ const ms = parseRowTimeMs(row.timestamp);
+ if (
+ ms != null &&
+ ms >= anchorMs - TRIGGER_LOOKBACK_MS &&
+ ms <= anchorMs &&
+ (latest == null || ms > latest.ms)
+ ) {
+ latest = { row, ms };
+ }
+ }
+ return latest != null
+ ? { row: latest.row, gapMs: anchorMs - latest.ms }
+ : undefined;
+}
diff --git a/packages/app/tests/e2e/features/sessions.spec.ts b/packages/app/tests/e2e/features/sessions.spec.ts
index 54f89ac6c..6e84754f1 100644
--- a/packages/app/tests/e2e/features/sessions.spec.ts
+++ b/packages/app/tests/e2e/features/sessions.spec.ts
@@ -63,9 +63,29 @@ test.describe('Client Sessions Functionality', { tag: ['@sessions'] }, () => {
await card.click();
await expect(sessionsPage.sessionSidePanel).toBeVisible();
- await expect(
- sessionsPage.getSessionEventRows().filter({ hasText: '500 GET' }),
- ).toBeVisible({ timeout: 15000 });
+ const span = sessionsPage
+ .getSessionEventRows()
+ .filter({ hasText: '500 GET' });
+ await expect(span).toBeVisible({ timeout: 15000 });
+
+ await test.step('Its trace shows the triggering click and the request timings', async () => {
+ await span.click();
+ const trace = page.getByTestId('side-panel-tab-trace');
+ const trigger = trace.getByText(
+ 'Triggered by: Clicked testid=checkout · 3ms before',
+ );
+ await expect(trigger).toBeVisible({ timeout: 15000 });
+ await expect(trace.getByTestId('span-event-marker')).toHaveCount(4);
+ // The timing log is drawn as the dots, so it isn't counted as a log.
+ await expect(trace.getByText(/^1 span, 1 error/)).toBeVisible();
+
+ await trigger.click();
+ await expect(
+ trace
+ .getByText('#root > main > button.checkout', { exact: true })
+ .first(),
+ ).toBeVisible();
+ });
});
test('shows browser log events from the OpenTelemetry SDK and opens them', async ({
diff --git a/packages/app/tests/e2e/seed-otel-sessions.ts b/packages/app/tests/e2e/seed-otel-sessions.ts
index c1c6a0da1..74ede603f 100644
--- a/packages/app/tests/e2e/seed-otel-sessions.ts
+++ b/packages/app/tests/e2e/seed-otel-sessions.ts
@@ -61,16 +61,35 @@ export function generateOtelSessionSpans(seedRef: number): string {
].join(',\n');
}
-/** Browser events the OTel SDK records as log events, after the +500 ms span. */
+// Four dots inside the 12 ms span.
+const RESOURCE_TIMING = {
+ fetch_start: '1000.5',
+ request_start: '1002',
+ response_start: '1009',
+ response_end: '1011.5',
+ duration: '11',
+ transfer_size: '558',
+ next_hop_protocol: 'http/1.1',
+};
+
+/**
+ * OTel SDK log events around the +500 ms span: the click 3 ms before it that
+ * triggers its trace, its request timing, and a console error.
+ */
export function generateOtelSessionLogEvents(seedRef: number): string {
const ns = (offsetMs: number) =>
(seedRef - 5 * 60 * 1000 + offsetMs) * 1_000_000;
const resource = `{'service.name':'${SERVICE}'}`;
+ const { newSdk, bothKeys } = OTEL_SESSIONS;
const click = (sessionId: string) =>
- `('${ns(600)}', '', '', 0, '', 9, '${SERVICE}', '', '', ${resource}, '', '', '', {}, {'session.id':'${sessionId}','browser.tag_name':'BUTTON','browser.css_selector':'#root > main > button.checkout','browser.document.url.full':'${PAGE_URL}','browser.element.attributes.testid':'checkout'}, 'browser.user_action.click')`;
+ `('${ns(497)}', '', '', 0, '', 9, '${SERVICE}', '', '', ${resource}, '', '', '', {}, {'session.id':'${sessionId}','browser.tag_name':'BUTTON','browser.css_selector':'#root > main > button.checkout','browser.document.url.full':'${PAGE_URL}','browser.element.attributes.testid':'checkout'}, 'browser.user_action.click')`;
+ const timing = Object.entries(RESOURCE_TIMING)
+ .map(([key, value]) => `'browser.resource_timing.${key}':'${value}'`)
+ .join(',');
return [
- click(OTEL_SESSIONS.newSdk),
- `('${ns(800)}', '', '', 0, 'error', 17, '${SERVICE}', 'Error: checkout failed', '', ${resource}, '', '', '', {}, {'session.id':'${OTEL_SESSIONS.newSdk}','browser.console.method':'error','browser.document.url.full':'${PAGE_URL}'}, 'browser.console')`,
- click(OTEL_SESSIONS.bothKeys),
+ click(newSdk),
+ `('${ns(520)}', '${newSdk}-trace', '${newSdk}-span', 0, '', 9, '${SERVICE}', '', '', ${resource}, '', '@opentelemetry/browser-instrumentation/resource-timing', '', {}, {'session.id':'${newSdk}','browser.document.url.full':'${PAGE_URL}','url.full':'${PAGE_URL}api/error',${timing}}, 'browser.resource_timing')`,
+ `('${ns(800)}', '', '', 0, 'error', 17, '${SERVICE}', 'Error: checkout failed', '', ${resource}, '', '', '', {}, {'session.id':'${newSdk}','browser.console.method':'error','browser.document.url.full':'${PAGE_URL}'}, 'browser.console')`,
+ click(bothKeys),
].join(',\n');
}
From 04f55d12cbf57bd4817977636271796db986b7ca Mon Sep 17 00:00:00 2001
From: Spencer Torres
Date: Wed, 4 Nov 2026 12:04:12 -0800
Subject: [PATCH 029/103] feat: parse JavaScript stack traces in the exception
panel (HDX-5888) (#3534)
Closes HDX-5888
## Summary
Our new OTel JS SDKs send `exception.stacktrace` as raw text, so the exception panel showed a text block. This parses it into frames on `webjs`/`nodejs` rows and shows each `Caused by:` error as a collapsible section. Old `parsed_stacktrace` rows and other languages are unchanged. This is just a visual change to the exceptions panel.
### Changes
- **Parser**: Sentry's Chrome/Gecko line parsers, ported with the MIT notice. No new dependency.
- **Causes**: collapsed one-line rows that expand to their own frames. Full message with line breaks when open. These get stacked (the SDK limits it to 5 by default)
- **Frames**: library frames labeled "Library or framework code". No bare "in" on unnamed frames for when the parser/sourcemap can't find the parent func.
### Screenshots
#### Before
Old SDK error for example (as seen in traces browser). As you can see this is undisturbed and the old SDK works fine.
#### After
New Exceptions view panel (as seen on logs browser, where the SDKs send browser events)
Raw stack trace (note the source maps used this data in the previous screenshot, and that caused by was parsed and separated)
When source maps are unavailable:
When there's no a stack trace, only a message:
---
.../javascript-exception-stack-frames.md | 5 +
.../app/src/components/DBRowOverviewPanel.tsx | 38 +-
.../app/src/components/ExceptionSubpanel.tsx | 439 ++++++++++-------
.../DBRowOverviewPanel.exception.test.tsx | 131 +++++-
.../__tests__/ExceptionSubpanel.test.tsx | 371 ++++++++++++++-
packages/app/src/types.ts | 4 +-
.../utils/__tests__/javascriptStack.test.ts | 441 ++++++++++++++++++
packages/app/src/utils/javascriptStack.ts | 124 +++++
packages/app/src/utils/sentryStackParser.ts | 330 +++++++++++++
packages/app/styles/LogSidePanel.module.scss | 4 +
10 files changed, 1710 insertions(+), 177 deletions(-)
create mode 100644 .changeset/javascript-exception-stack-frames.md
create mode 100644 packages/app/src/utils/__tests__/javascriptStack.test.ts
create mode 100644 packages/app/src/utils/javascriptStack.ts
create mode 100644 packages/app/src/utils/sentryStackParser.ts
diff --git a/.changeset/javascript-exception-stack-frames.md b/.changeset/javascript-exception-stack-frames.md
new file mode 100644
index 000000000..48d845b14
--- /dev/null
+++ b/.changeset/javascript-exception-stack-frames.md
@@ -0,0 +1,5 @@
+---
+'@hyperdx/app': patch
+---
+
+feat: Exception panel parses JavaScript stack traces from OpenTelemetry SDKs into frames, with a section per `Caused by:` error
diff --git a/packages/app/src/components/DBRowOverviewPanel.tsx b/packages/app/src/components/DBRowOverviewPanel.tsx
index 6f7602a2f..ebad17c2e 100644
--- a/packages/app/src/components/DBRowOverviewPanel.tsx
+++ b/packages/app/src/components/DBRowOverviewPanel.tsx
@@ -7,6 +7,10 @@ import { Accordion, Box, Flex, Text } from '@mantine/core';
import { WithClause } from '@/hooks/useRowWhere';
import { getEventBody } from '@/source';
import { getHighlightedAttributesFromData } from '@/utils/highlightedAttributes';
+import {
+ isJavaScriptRuntime,
+ parseJavaScriptException,
+} from '@/utils/javascriptStack';
import { resolveRowTimestampAnchor } from '@/utils/rowTimestamps';
import { getRequestPanelTitle } from '@/utils/rpcSpanAttributes';
import { isSet } from '@/utils/spanAttributeReaders';
@@ -175,18 +179,36 @@ export function RowOverviewPanel({
// do nothing
}
+ const exception = {
+ stacktrace: parsedStacktrace,
+ type: parsedEvents?.['exception.type'],
+ value:
+ typeof parsedEvents?.['exception.message'] !== 'string'
+ ? JSON.stringify(parsedEvents?.['exception.message'])
+ : parsedEvents?.['exception.message'],
+ mechanism: parsedEvents?.['exception.mechanism'],
+ };
+
+ const parsed =
+ typeof parsedStacktrace === 'string' &&
+ isJavaScriptRuntime(resourceAttributes)
+ ? parseJavaScriptException(parsedStacktrace)
+ : undefined;
+ if (parsed == null) {
+ return [exception];
+ }
return [
{
- stacktrace: parsedStacktrace,
- type: parsedEvents?.['exception.type'],
- value:
- typeof parsedEvents?.['exception.message'] !== 'string'
- ? JSON.stringify(parsedEvents?.['exception.message'])
- : parsedEvents?.['exception.message'],
- mechanism: parsedEvents?.['exception.mechanism'],
+ ...exception,
+ stacktrace: { frames: parsed.frames },
+ rawStacktrace: parsedStacktrace,
},
+ ...parsed.causes.map(({ frames, ...cause }) => ({
+ ...cause,
+ stacktrace: { frames },
+ })),
];
- }, [exceptionAttributes]);
+ }, [exceptionAttributes, resourceAttributes]);
const hasException = exceptionAttributes != null;
diff --git a/packages/app/src/components/ExceptionSubpanel.tsx b/packages/app/src/components/ExceptionSubpanel.tsx
index 54bb41f13..add6b41e2 100644
--- a/packages/app/src/components/ExceptionSubpanel.tsx
+++ b/packages/app/src/components/ExceptionSubpanel.tsx
@@ -3,7 +3,7 @@ import React, { useMemo } from 'react';
import cx from 'classnames';
import { ErrorBoundary } from 'react-error-boundary';
import { Badge } from '@clickhouse/click-ui';
-import { Button, Text, Tooltip } from '@mantine/core';
+import { Button, Text, Tooltip, UnstyledButton } from '@mantine/core';
import { Group, Loader } from '@mantine/core';
import {
IconChevronDown,
@@ -13,12 +13,14 @@ import {
} from '@tabler/icons-react';
import { ColumnDef, Row, Table as TanstackTable } from '@tanstack/react-table';
+import { CopySnippet } from '@/components/ClickStackOnboarding/CopySnippet';
import type {
StacktraceBreadcrumb,
StacktraceFrame as TStacktraceFrame,
} from '@/types';
import { FormatTime } from '@/useFormatTime';
import { useSourceMappedFrame } from '@/useSourceMappedFrame';
+import { UNKNOWN_FUNCTION } from '@/utils/sentryStackParser';
import { Table, TableCellButton } from './Table';
@@ -66,8 +68,8 @@ const StacktraceFrame = ({
}: {
filename: string;
function?: string;
- lineno: number;
- colno: number;
+ lineno?: number;
+ colno?: number;
isLoading?: boolean;
}) => {
return (
@@ -80,21 +82,26 @@ const StacktraceFrame = ({
}}
>
{filename}
-
- :{lineno}:{colno}
-
- {' in '}
- {functionName && (
-
- {functionName}
+ {lineno != null && (
+
+ :{lineno}
+ {colno != null && `:${colno}`}
)}
+ {functionName && functionName !== UNKNOWN_FUNCTION && (
+ <>
+ {' in '}
+
+ {functionName}
+
+ >
+ )}
{isLoading && }
@@ -180,6 +187,8 @@ const StacktraceRowExpandButton = ({
);
};
+const NOT_IN_APP_LABEL = 'Library or framework code';
+
const StacktraceRow = ({
row,
table,
@@ -218,12 +227,16 @@ const StacktraceRow = ({
{!augmentedFrame.in_app && (
-
+
)}
{augmentedFrame && (
@@ -267,7 +280,7 @@ const StacktraceRow = ({
{augmentedFrame.post_context?.map((line, i) => (
- {augmentedFrame.lineno + i + 1}
+ {(augmentedFrame.lineno ?? 0) + i + 1}
{line}
@@ -420,9 +433,9 @@ export const useShowMoreRows =
({
return { visibleRows, hiddenRowsCount, handleToggleMoreRows, isExpanded };
};
-type ExceptionValues = {
+type ExceptionValue = {
type: string;
- value: string;
+ value?: string;
mechanism?: {
type: string;
handled: boolean;
@@ -433,10 +446,226 @@ type ExceptionValues = {
target?: string;
};
};
- stacktrace?: {
- frames: TStacktraceFrame[];
- };
-}[];
+ /** A string when the stack text could not be parsed into frames. */
+ stacktrace?: { frames: TStacktraceFrame[] } | string;
+ /** The unparsed stack text, set when `stacktrace` was parsed from it. */
+ rawStacktrace?: string;
+};
+
+const getFrames = ({ stacktrace }: ExceptionValue) =>
+ typeof stacktrace === 'string' ? undefined : stacktrace?.frames;
+
+const getExceptionTitle = ({ type, value }: ExceptionValue) =>
+ [type, value?.trim()].filter(Boolean).join(': ');
+
+const ExceptionTitle = ({
+ exception,
+ isCause,
+ singleLine,
+}: {
+ exception: ExceptionValue;
+ isCause: boolean;
+ singleLine: boolean;
+}) => {
+ const message = exception.value?.trim() ?? '';
+
+ return (
+
+ {isCause && Caused by: }
+ {exception.type && (
+
+ {exception.type}
+
+ )}
+
+ {exception.type && message && ': '}
+ {singleLine ? message.replace(/\s+/g, ' ') : message}
+
+
+ );
+};
+
+const ExceptionStacktrace = ({ exception }: { exception: ExceptionValue }) => {
+ const stacktraceFrames = useMemo(
+ () => getFrames(exception)?.slice().reverse() ?? [],
+ [exception],
+ );
+
+ const firstFrameIndex = useMemo(() => {
+ const firstFrame = getFirstFrame(stacktraceFrames);
+ return firstFrame ? stacktraceFrames.indexOf(firstFrame) : -1;
+ }, [stacktraceFrames]);
+
+ const {
+ handleToggleMoreRows: handleStacktraceToggleMoreRows,
+ hiddenRowsCount: stacktraceHiddenRowsCount,
+ visibleRows: stacktraceVisibleRows,
+ isExpanded: stacktraceExpanded,
+ } = useShowMoreRows({
+ rows: stacktraceFrames,
+ maxRows: Math.max(5, firstFrameIndex + 1),
+ });
+
+ return (
+ <>
+ {
+ console.error(err);
+ }}
+ fallbackRender={() => (
+
+ An error occurred while rendering stacktrace
+
+ )}
+ >
+
+
+
+ {stacktraceHiddenRowsCount ? (
+
+ ) : null}
+ >
+ );
+};
+
+const ExceptionValueSection = ({
+ exception,
+ isCause,
+}: {
+ exception: ExceptionValue;
+ isCause: boolean;
+}) => {
+ const hasFrames = !!getFrames(exception)?.length;
+
+ const unparsedStack =
+ !isCause && typeof exception.stacktrace === 'string'
+ ? exception.stacktrace
+ : '';
+
+ const collapsible = isCause && hasFrames;
+ const [expanded, setExpanded] = React.useState(!isCause);
+ const open = expanded || !collapsible;
+ const Chevron = expanded ? IconChevronDown : IconChevronRight;
+
+ return (
+
+ {collapsible ? (
+ setExpanded(!expanded)}
+ >
+
+
+
+ ) : (
+
+
+
+ )}
+
+ {open && exception.mechanism && (
+
+
+ true
+ ) : (
+ false
+ )
+ }
+ />
+
+ {exception.mechanism?.data?.function ? (
+
+ ) : null}
+ {exception.mechanism?.data?.handler ? (
+
+ ) : null}
+ {exception.mechanism?.data?.target ? (
+
+ ) : null}
+
+ )}
+ >
+ }
+ >
+ {open && hasFrames && }
+ {unparsedStack ? (
+
+ {unparsedStack}
+
+ ) : null}
+
+ );
+};
export const ExceptionSubpanel = ({
logData,
@@ -445,29 +674,21 @@ export const ExceptionSubpanel = ({
}: {
logData?: any;
breadcrumbs?: StacktraceBreadcrumb[];
- exceptionValues: ExceptionValues;
+ /** The thrown error first, then each error in its `cause` chain. */
+ exceptionValues: ExceptionValue[];
}) => {
const firstException = exceptionValues[0];
const shouldShowSourceMapFtux = useMemo(() => {
- return firstException?.stacktrace?.frames?.some(
- f =>
- f.filename.startsWith('http://') || f.filename.startsWith('https://'),
+ return (
+ firstException &&
+ getFrames(firstException)?.some(
+ f =>
+ f.filename.startsWith('http://') || f.filename.startsWith('https://'),
+ )
);
- }, [firstException?.stacktrace?.frames]);
-
- const stacktraceFrames = useMemo(() => {
- if (!firstException?.stacktrace?.frames) {
- return [];
- }
- return firstException?.stacktrace.frames.slice().reverse();
}, [firstException]);
- const firstFrameIndex = useMemo(() => {
- const firstFrame = getFirstFrame(stacktraceFrames);
- return firstFrame ? stacktraceFrames.indexOf(firstFrame) : -1;
- }, [stacktraceFrames]);
-
const chronologicalBreadcrumbs = useMemo(() => {
return [
...(firstException && breadcrumbs
@@ -475,9 +696,7 @@ export const ExceptionSubpanel = ({
{
category: 'exception',
timestamp: new Date(logData?.timestamp ?? 0).getTime() / 1000,
- message: [firstException.type, firstException.value]
- .filter(Boolean)
- .join(': '),
+ message: getExceptionTitle(firstException),
},
]
: []),
@@ -485,16 +704,6 @@ export const ExceptionSubpanel = ({
];
}, [breadcrumbs, firstException, logData?.timestamp]);
- const {
- handleToggleMoreRows: handleStacktraceToggleMoreRows,
- hiddenRowsCount: stacktraceHiddenRowsCount,
- visibleRows: stacktraceVisibleRows,
- isExpanded: stacktraceExpanded,
- } = useShowMoreRows({
- rows: stacktraceFrames,
- maxRows: Math.max(5, firstFrameIndex + 1),
- });
-
const {
handleToggleMoreRows: handleBreadcrumbToggleMoreRows,
hiddenRowsCount: breadcrumbHiddenRowsCount,
@@ -504,118 +713,20 @@ export const ExceptionSubpanel = ({
rows: chronologicalBreadcrumbs,
});
- // TODO: show all frames (stackable)
return (
-
-
- {firstException.type && (
-
- {firstException.type}
-
- )}
-
- {firstException.type && firstException.value && ': '}
- {firstException.value}
-
-
-
- {firstException.mechanism && (
-
-
- true
- ) : (
- false
- )
- }
- />
-
- {firstException.mechanism?.data?.function ? (
-
- ) : null}
- {firstException.mechanism?.data?.handler ? (
-
- ) : null}
- {firstException.mechanism?.data?.target ? (
-
- ) : null}
-
- )}
- {/* {shouldShowSourceMapFtux && } */}
- >
- )
- }
- >
- {
- console.error(err);
- }}
- fallbackRender={() => (
-
- An error occurred while rendering stacktrace
-
- )}
- >
-
-
-
- {typeof exceptionValues[0].stacktrace === 'string' && (
-
- {exceptionValues[0].stacktrace}
-
- )}
+ {exceptionValues.map((exception, i) => (
+ 0} />
+ ))}
+ {/* {shouldShowSourceMapFtux && } */}
- {stacktraceHiddenRowsCount ? (
-
- ) : null}
-
+ {firstException?.rawStacktrace && (
+
+
+
+
+
+ )}
{breadcrumbVisibleRows.length > 0 && (
diff --git a/packages/app/src/components/__tests__/DBRowOverviewPanel.exception.test.tsx b/packages/app/src/components/__tests__/DBRowOverviewPanel.exception.test.tsx
index adcb5ecc2..117addb11 100644
--- a/packages/app/src/components/__tests__/DBRowOverviewPanel.exception.test.tsx
+++ b/packages/app/src/components/__tests__/DBRowOverviewPanel.exception.test.tsx
@@ -34,8 +34,38 @@ function renderRow(row: Record) {
renderWithMantine();
}
-const exceptionShown = () =>
- mockExceptionSubpanel.mock.calls[0]?.[0].exceptionValues[0];
+const exceptionValuesShown = () =>
+ mockExceptionSubpanel.mock.calls[0]?.[0].exceptionValues;
+const exceptionShown = () => exceptionValuesShown()[0];
+
+const CHROME_STACK = [
+ 'Error: Checkout failed',
+ ' at checkout (https://example.clickhouse.com/assets/app.js:120:11)',
+ 'Caused by: TypeError: Failed to fetch',
+ ' at fetchCart (https://example.clickhouse.com/assets/app.js:42:9)',
+].join('\n');
+
+const JAVA_STACK = [
+ 'java.lang.IllegalStateException: Checkout failed',
+ '\tat com.example.Checkout.run(Checkout.java:120)',
+ 'Caused by: java.io.IOException: Failed to fetch',
+ '\tat com.example.Cart.fetch(Cart.java:42)',
+].join('\n');
+
+const exceptionRow = (
+ language: string | undefined,
+ stacktrace: string,
+ key = 'exception.stacktrace',
+) => ({
+ ...(language != null && {
+ __hdx_resource_attributes: { 'telemetry.sdk.language': language },
+ }),
+ __hdx_event_attributes: {
+ 'exception.type': 'Error',
+ 'exception.message': 'Checkout failed',
+ [key]: stacktrace,
+ },
+});
describe('RowOverviewPanel exception section', () => {
beforeEach(() => jest.clearAllMocks());
@@ -76,6 +106,103 @@ describe('RowOverviewPanel exception section', () => {
expect(exceptionShown()).toMatchObject({ type: 'FromEvent' });
});
+ it('parses a JavaScript stack into frames and Caused by sections', () => {
+ renderRow(exceptionRow('webjs', CHROME_STACK));
+
+ expect(exceptionValuesShown()).toEqual([
+ {
+ type: 'Error',
+ value: 'Checkout failed',
+ mechanism: undefined,
+ rawStacktrace: CHROME_STACK,
+ stacktrace: {
+ frames: [
+ {
+ filename: 'https://example.clickhouse.com/assets/app.js',
+ function: 'checkout',
+ lineno: 120,
+ colno: 11,
+ in_app: true,
+ },
+ ],
+ },
+ },
+ {
+ type: 'TypeError',
+ value: 'Failed to fetch',
+ stacktrace: {
+ frames: [
+ {
+ filename: 'https://example.clickhouse.com/assets/app.js',
+ function: 'fetchCart',
+ lineno: 42,
+ colno: 9,
+ in_app: true,
+ },
+ ],
+ },
+ },
+ ]);
+ });
+
+ it('parses a Node.js exception span event', () => {
+ renderRow({
+ __hdx_resource_attributes: { 'telemetry.sdk.language': 'nodejs' },
+ __hdx_events_exception_attributes: {
+ 'exception.type': 'Error',
+ 'exception.stacktrace':
+ 'Error: boom\n at handler (/srv/app/dist/server.js:12:9)',
+ },
+ });
+
+ expect(exceptionShown().stacktrace.frames).toEqual([
+ expect.objectContaining({ function: 'handler', lineno: 12, colno: 9 }),
+ ]);
+ });
+
+ it.each([
+ ['a java row with a Java stack', 'java', JAVA_STACK],
+ ['a python row', 'python', CHROME_STACK],
+ ['a row without resource attributes', undefined, CHROME_STACK],
+ ['a webjs row without frames', 'webjs', 'Error: Checkout failed'],
+ ])('keeps the raw text for %s', (_, language, stacktrace) => {
+ renderRow(exceptionRow(language, stacktrace));
+
+ expect(exceptionValuesShown()).toHaveLength(1);
+ expect(exceptionShown().stacktrace).toBe(stacktrace);
+ expect(exceptionShown().rawStacktrace).toBeUndefined();
+ });
+
+ it('keeps JSON parsed_stacktrace rows unchanged', () => {
+ const parsedStacktrace = {
+ frames: [
+ {
+ filename: 'https://example.clickhouse.com/assets/app.js',
+ function: 'checkout',
+ lineno: 120,
+ colno: 11,
+ in_app: true,
+ },
+ ],
+ };
+ renderRow(
+ exceptionRow(
+ 'webjs',
+ JSON.stringify(parsedStacktrace),
+ 'exception.parsed_stacktrace',
+ ),
+ );
+
+ expect(exceptionValuesShown()).toEqual([
+ {
+ type: 'Error',
+ value: 'Checkout failed',
+ mechanism: undefined,
+ stacktrace: parsedStacktrace,
+ },
+ ]);
+ });
+
it('hides the section without exception attributes', () => {
renderRow({
__hdx_event_attributes: { 'http.request.method': 'GET' },
diff --git a/packages/app/src/components/__tests__/ExceptionSubpanel.test.tsx b/packages/app/src/components/__tests__/ExceptionSubpanel.test.tsx
index 3b90aafde..38d1c2f54 100644
--- a/packages/app/src/components/__tests__/ExceptionSubpanel.test.tsx
+++ b/packages/app/src/components/__tests__/ExceptionSubpanel.test.tsx
@@ -1,5 +1,6 @@
import React from 'react';
-import { screen } from '@testing-library/react';
+import { screen, within } from '@testing-library/react';
+import userEvent from '@testing-library/user-event';
import { ExceptionSubpanel } from '@/components/ExceptionSubpanel';
@@ -76,3 +77,371 @@ describe('ExceptionSubpanel breadcrumbs', () => {
expect(badgeStates()).toEqual([[level, state]]);
});
});
+
+const APP = 'https://example.clickhouse.com/assets/app.js';
+const FRAMEWORK =
+ 'https://example.clickhouse.com/_next/static/chunks/framework-2c79e2a64abdb08b.js';
+const RAW_STACK = [
+ 'Error: Checkout failed',
+ ` at checkout (${APP}:120:11)`,
+ ` at Object.Ub (${FRAMEWORK}:9:16994)`,
+ 'Caused by: TypeError: Failed to fetch',
+ ` at fetchCart (${APP}:42:9)`,
+].join('\n');
+
+const PARSED_EXCEPTION_VALUES = [
+ {
+ type: 'Error',
+ value: 'Checkout failed',
+ rawStacktrace: RAW_STACK,
+ stacktrace: {
+ frames: [
+ {
+ filename: FRAMEWORK,
+ function: 'Object.Ub',
+ lineno: 9,
+ colno: 16994,
+ in_app: false,
+ },
+ {
+ filename: APP,
+ function: 'checkout',
+ lineno: 120,
+ colno: 11,
+ in_app: true,
+ },
+ ],
+ },
+ },
+ {
+ type: 'TypeError',
+ value: 'Failed to fetch',
+ stacktrace: {
+ frames: [
+ {
+ filename: APP,
+ function: 'fetchCart',
+ lineno: 42,
+ colno: 9,
+ in_app: true,
+ },
+ ],
+ },
+ },
+];
+
+const rowTexts = () =>
+ screen.getAllByRole('row').map(row => row.textContent?.trim());
+
+describe('ExceptionSubpanel stack frames', () => {
+ it('renders the thrown exception with its frames, newest first', () => {
+ renderWithMantine(
+ ,
+ );
+
+ expect(rowTexts()).toEqual([
+ `${APP}:120:11 in checkout`,
+ `${FRAMEWORK}:9:16994 in Object.Ub`,
+ ]);
+ expect(
+ screen
+ .getAllByLabelText('Library or framework code')
+ .map(icon => icon.closest('tr')?.textContent?.trim()),
+ ).toEqual([`${FRAMEWORK}:9:16994 in Object.Ub`]);
+ expect(screen.getByText('Error').closest('div')?.textContent).toBe(
+ 'Error: Checkout failed',
+ );
+ expect(screen.getByText('Error').closest('button')).toBeNull();
+ });
+
+ it('collapses each cause to its title until expanded', async () => {
+ renderWithMantine(
+ ,
+ );
+ const causes = () =>
+ screen
+ .getAllByRole('button', { name: /^Caused by: / })
+ .map(cause => [cause.textContent, cause.getAttribute('aria-expanded')]);
+
+ expect(causes()).toEqual([
+ ['Caused by: TypeError: Failed to fetch', 'false'],
+ ['Caused by: Error: Socket closed', 'false'],
+ ]);
+ expect(rowTexts()).toEqual([
+ `${APP}:120:11 in checkout`,
+ `${FRAMEWORK}:9:16994 in Object.Ub`,
+ ]);
+
+ await userEvent.click(
+ screen.getByRole('button', { name: /Failed to fetch/ }),
+ );
+
+ expect(causes()).toEqual([
+ ['Caused by: TypeError: Failed to fetch', 'true'],
+ ['Caused by: Error: Socket closed', 'false'],
+ ]);
+ expect(rowTexts()).toEqual([
+ `${APP}:120:11 in checkout`,
+ `${FRAMEWORK}:9:16994 in Object.Ub`,
+ `${APP}:42:9 in fetchCart`,
+ ]);
+
+ await userEvent.click(
+ screen.getByRole('button', { name: /Failed to fetch/ }),
+ );
+
+ expect(rowTexts()).not.toContain(`${APP}:42:9 in fetchCart`);
+ });
+
+ it('renders a section without frames as its title alone', () => {
+ const { container } = renderWithMantine(
+ ,
+ );
+
+ // A cause's unparsed text is not shown. Causes only come from a parsed
+ // stack, whose full text is in "Raw stack trace".
+ expect(
+ Array.from(container.querySelectorAll('.panelSectionWrapper'), section =>
+ Array.from(section.children, child => child.textContent),
+ ),
+ ).toEqual([
+ ['Error: Checkout failed'],
+ ['Caused by: TypeError: Failed to fetch'],
+ ['Caused by: Error: Socket closed'],
+ ]);
+ expect(
+ screen.queryByRole('button', { name: /Caused by:/ }),
+ ).not.toBeInTheDocument();
+ });
+
+ it('keeps the mechanism of a section without frames', () => {
+ renderWithMantine(
+ ,
+ );
+
+ expect(screen.getByText('onerror')).toBeInTheDocument();
+ expect(screen.getByText('false')).toBeInTheDocument();
+ });
+
+ it('does not render the frames of a collapsed cause', () => {
+ renderWithMantine(
+ ,
+ );
+
+ // `hidden` also counts rows mounted but hidden by CSS, which would still
+ // run their source map lookups.
+ expect(screen.queryAllByRole('row', { hidden: true })).toHaveLength(0);
+ });
+
+ it('puts a collapsed cause on one line and its full message on hover', async () => {
+ renderWithMantine(
+ ,
+ );
+ const cause = screen.getByRole('button', { name: /^Caused by: / });
+ const title = () => within(cause).getByText('Error').parentElement;
+
+ expect(cause.textContent).toBe(
+ 'Caused by: Error: Row 12 is invalid: price must be a number',
+ );
+ expect(title()).toHaveAttribute('data-truncate');
+ expect(title()).toHaveAttribute(
+ 'title',
+ 'Error: Row 12 is invalid:\r\n price must be a number',
+ );
+
+ await userEvent.click(cause);
+
+ expect(cause.textContent).toBe(
+ 'Caused by: Error: Row 12 is invalid:\r\n price must be a number',
+ );
+ expect(title()).not.toHaveAttribute('data-truncate');
+ expect(title()).not.toHaveAttribute('title');
+ expect(title()?.style.whiteSpace).toBe('pre-wrap');
+ });
+
+ it('shows the message of an open section with its line breaks', () => {
+ const longToken = `https://react.dev/errors/418?args[]=${'x'.repeat(400)}`;
+ renderWithMantine(
+ ,
+ );
+ const title = (type: string) => screen.getByText(type).parentElement;
+
+ expect(screen.getByText('Error').closest('div')?.textContent).toBe(
+ 'Error: Checkout failed\nfor cart 42',
+ );
+ expect(screen.getByText('TypeError').closest('div')?.textContent).toBe(
+ `Caused by: TypeError: Failed to fetch\n${longToken}`,
+ );
+ for (const type of ['Error', 'TypeError']) {
+ expect(title(type)?.style.whiteSpace).toBe('pre-wrap');
+ expect(title(type)?.style.overflowWrap).toBe('anywhere');
+ expect(title(type)).not.toHaveAttribute('data-truncate');
+ expect(title(type)).not.toHaveAttribute('data-line-clamp');
+ }
+ });
+
+ it('keeps the raw stack trace behind a collapsed section', async () => {
+ renderWithMantine(
+ ,
+ );
+ const rawText = () => screen.queryByText(/Caused by: TypeError/);
+
+ expect(rawText()).not.toBeInTheDocument();
+ await userEvent.click(screen.getByText('Raw stack trace'));
+ expect(rawText()?.textContent).toBe(RAW_STACK);
+ });
+
+ it('omits the line and column of a frame that has none', () => {
+ renderWithMantine(
+ ,
+ );
+
+ expect(rowTexts()).toEqual([
+ `${APP}:37 in checkout`,
+ '[native code] in forEach',
+ ]);
+ });
+
+ it.each(['', '?'])(
+ 'omits "in" for a frame whose function is %p',
+ functionName => {
+ renderWithMantine(
+ ,
+ );
+
+ expect(screen.getByRole('row').textContent).toBe(`${APP}:5:10`);
+ },
+ );
+
+ it('shows an unparsed stack as text', () => {
+ renderWithMantine(
+ ,
+ );
+
+ expect(screen.getByText(/Caused by: TypeError/).textContent).toBe(
+ RAW_STACK,
+ );
+ expect(screen.queryByRole('row')).not.toBeInTheDocument();
+ expect(screen.queryByText('Raw stack trace')).not.toBeInTheDocument();
+ });
+});
diff --git a/packages/app/src/types.ts b/packages/app/src/types.ts
index 9c549f77c..f5387c1d3 100644
--- a/packages/app/src/types.ts
+++ b/packages/app/src/types.ts
@@ -58,8 +58,8 @@ export type StacktraceFrame = {
filename: string;
function: string;
module?: string;
- lineno: number;
- colno: number;
+ lineno?: number;
+ colno?: number;
in_app: boolean;
context_line?: string;
pre_context?: string[];
diff --git a/packages/app/src/utils/__tests__/javascriptStack.test.ts b/packages/app/src/utils/__tests__/javascriptStack.test.ts
new file mode 100644
index 000000000..3340ce5cf
--- /dev/null
+++ b/packages/app/src/utils/__tests__/javascriptStack.test.ts
@@ -0,0 +1,441 @@
+import {
+ isJavaScriptRuntime,
+ parseJavaScriptException,
+} from '@/utils/javascriptStack';
+
+const APP = 'https://app.example.com/assets/app.js';
+
+const framesOf = (stack: string) => parseJavaScriptException(stack)?.frames;
+
+describe('parseJavaScriptException', () => {
+ it('parses a Chrome stack oldest first, keeping URLs and 1-based columns', () => {
+ const stack = [
+ "TypeError: Cannot read properties of undefined (reading 'id')",
+ ' at c (https://app.example.com/_next/static/chunks/pages/checkout-3f2a.js:1:2817)',
+ ' at Object.Ub (https://app.example.com/_next/static/chunks/framework-2c79e2a64abdb08b.js:9:16994)',
+ ` at ${APP}:5:10`,
+ ].join('\n');
+
+ expect(parseJavaScriptException(stack)).toEqual({
+ frames: [
+ { filename: APP, function: '?', lineno: 5, colno: 10, in_app: true },
+ {
+ filename:
+ 'https://app.example.com/_next/static/chunks/framework-2c79e2a64abdb08b.js',
+ function: 'Object.Ub',
+ lineno: 9,
+ colno: 16994,
+ in_app: false,
+ },
+ {
+ filename:
+ 'https://app.example.com/_next/static/chunks/pages/checkout-3f2a.js',
+ function: 'c',
+ lineno: 1,
+ colno: 2817,
+ in_app: true,
+ },
+ ],
+ causes: [],
+ });
+ });
+
+ it('parses Chrome eval and async frames', () => {
+ const stack = [
+ 'Error: bad',
+ ' at eval (eval at aha (http://localhost:5000/app.js:37:5), :1:1)',
+ ` at async submit (${APP}:130:5)`,
+ ' at Object.callback (async http://localhost:5000/app.js:25:7)',
+ ].join('\n');
+
+ expect(framesOf(stack)).toEqual([
+ {
+ filename: 'http://localhost:5000/app.js',
+ function: 'Object.callback',
+ lineno: 25,
+ colno: 7,
+ in_app: true,
+ },
+ {
+ filename: APP,
+ function: 'async submit',
+ lineno: 130,
+ colno: 5,
+ in_app: true,
+ },
+ {
+ filename: 'http://localhost:5000/app.js',
+ function: 'eval',
+ lineno: 37,
+ colno: 5,
+ in_app: true,
+ },
+ ]);
+ });
+
+ it('parses a Node stack and marks library and runtime frames', () => {
+ const stack = [
+ 'Error: boom',
+ ' at handler (/srv/app/dist/server.js:12:9)',
+ ' at Layer.handle (/srv/app/node_modules/express/lib/router/layer.js:95:5)',
+ ' at process.processTicksAndRejections (node:internal/process/task_queues:95:5)',
+ ' at async Promise.all (index 0)',
+ ].join('\n');
+
+ expect(framesOf(stack)).toEqual([
+ {
+ filename: 'index 0',
+ function: 'async Promise.all',
+ in_app: false,
+ },
+ {
+ filename: 'node:internal/process/task_queues',
+ function: 'process.processTicksAndRejections',
+ lineno: 95,
+ colno: 5,
+ in_app: false,
+ },
+ {
+ filename: '/srv/app/node_modules/express/lib/router/layer.js',
+ function: 'Layer.handle',
+ lineno: 95,
+ colno: 5,
+ in_app: false,
+ },
+ {
+ filename: '/srv/app/dist/server.js',
+ function: 'handler',
+ lineno: 12,
+ colno: 9,
+ in_app: true,
+ },
+ ]);
+ });
+
+ it('parses a Firefox stack, including async and eval frames', () => {
+ const stack = [
+ 'aha@http://localhost:5000/app.js line 37 > eval:1:1',
+ `checkout@${APP}:120:11`,
+ `async*submit@${APP}:130:5`,
+ `@${APP}:150:1`,
+ ].join('\n');
+
+ expect(framesOf(stack)).toEqual([
+ { filename: APP, function: '?', lineno: 150, colno: 1, in_app: true },
+ {
+ filename: APP,
+ function: 'async*submit',
+ lineno: 130,
+ colno: 5,
+ in_app: true,
+ },
+ {
+ filename: APP,
+ function: 'checkout',
+ lineno: 120,
+ colno: 11,
+ in_app: true,
+ },
+ {
+ filename: 'http://localhost:5000/app.js',
+ function: 'aha',
+ lineno: 37,
+ in_app: true,
+ },
+ ]);
+ });
+
+ it('parses a Safari stack, dropping eval code frames', () => {
+ const stack = [
+ 'forEach@[native code]',
+ `checkout@${APP}:120:11`,
+ 'eval code@',
+ `global code@${APP}:150:1`,
+ ].join('\n');
+
+ expect(framesOf(stack)).toEqual([
+ {
+ filename: APP,
+ function: 'global code',
+ lineno: 150,
+ colno: 1,
+ in_app: true,
+ },
+ {
+ filename: APP,
+ function: 'checkout',
+ lineno: 120,
+ colno: 11,
+ in_app: true,
+ },
+ { filename: '[native code]', function: 'forEach', in_app: false },
+ ]);
+ });
+
+ it('splits Chrome Caused by sections into causes with their own frames', () => {
+ const stack = [
+ 'Error: Checkout failed',
+ ` at checkout (${APP}:120:11)`,
+ 'Caused by: TypeError: Failed to fetch',
+ ` at fetchCart (${APP}:42:9)`,
+ 'Caused by: Error',
+ ` at request (${APP}:10:3)`,
+ ].join('\n');
+
+ expect(parseJavaScriptException(stack)).toEqual({
+ frames: [
+ {
+ filename: APP,
+ function: 'checkout',
+ lineno: 120,
+ colno: 11,
+ in_app: true,
+ },
+ ],
+ causes: [
+ {
+ type: 'TypeError',
+ value: 'Failed to fetch',
+ frames: [
+ {
+ filename: APP,
+ function: 'fetchCart',
+ lineno: 42,
+ colno: 9,
+ in_app: true,
+ },
+ ],
+ },
+ {
+ type: 'Error',
+ value: '',
+ frames: [
+ {
+ filename: APP,
+ function: 'request',
+ lineno: 10,
+ colno: 3,
+ in_app: true,
+ },
+ ],
+ },
+ ],
+ });
+ });
+
+ it('splits a Firefox stack whose cause has a Name: message line', () => {
+ const stack = [
+ `checkout@${APP}:120:11`,
+ 'Caused by: TypeError: NetworkError when attempting to fetch resource.',
+ `fetchCart@${APP}:42:9`,
+ ].join('\n');
+
+ expect(parseJavaScriptException(stack)).toEqual({
+ frames: [
+ {
+ filename: APP,
+ function: 'checkout',
+ lineno: 120,
+ colno: 11,
+ in_app: true,
+ },
+ ],
+ causes: [
+ {
+ type: 'TypeError',
+ value: 'NetworkError when attempting to fetch resource.',
+ frames: [
+ {
+ filename: APP,
+ function: 'fetchCart',
+ lineno: 42,
+ colno: 9,
+ in_app: true,
+ },
+ ],
+ },
+ ],
+ });
+ });
+
+ it('titles a cause without a Name: prefix by its whole line', () => {
+ const stack = [
+ ` at checkout (${APP}:120:11)`,
+ 'Caused by: request timed out after 30s',
+ ].join('\n');
+
+ expect(parseJavaScriptException(stack)?.causes).toEqual([
+ { type: '', value: 'request timed out after 30s', frames: [] },
+ ]);
+ });
+
+ it('does not read URLs in a multi-line message as frames', () => {
+ const stack = [
+ 'Error: Checkout failed',
+ 'https://api.example.com/cart returned 500',
+ ` at checkout (${APP}:120:11)`,
+ 'Caused by: Error: Request failed',
+ 'https://api.example.com/cart/items returned 503',
+ ` at fetchCart (${APP}:42:9)`,
+ ].join('\n');
+
+ const parsed = parseJavaScriptException(stack);
+
+ expect(parsed?.frames.map(frame => frame.function)).toEqual(['checkout']);
+ expect(parsed?.causes).toEqual([
+ expect.objectContaining({
+ type: 'Error',
+ value:
+ 'Request failed\nhttps://api.example.com/cart/items returned 503',
+ frames: [expect.objectContaining({ function: 'fetchCart' })],
+ }),
+ ]);
+ });
+
+ it('does not read URLs in a multi-line Firefox cause message as frames', () => {
+ const stack = [
+ `checkout@${APP}:120:11`,
+ 'Caused by: GraphQLError: Validation failed:',
+ ' - field "email" is required',
+ ' - see https://docs.example.com/errors#validation',
+ `fetchCart@${APP}:42:9`,
+ ].join('\n');
+
+ expect(parseJavaScriptException(stack)?.causes).toEqual([
+ {
+ type: 'GraphQLError',
+ value: [
+ 'Validation failed:',
+ ' - field "email" is required',
+ ' - see https://docs.example.com/errors#validation',
+ ].join('\n'),
+ frames: [expect.objectContaining({ function: 'fetchCart' })],
+ },
+ ]);
+ });
+
+ // V8 prints `new Error('\nRow 12 is invalid:')` as a line of `Error: `
+ // followed by the message's lines.
+ it.each([
+ [
+ 'a message that starts with a line break',
+ ['Caused by: Error: ', 'Row 12 is invalid:', ' sku is required'],
+ { type: 'Error', value: 'Row 12 is invalid:\n sku is required' },
+ ],
+ [
+ 'a whitespace-only message',
+ ['Caused by: Error: ', ''],
+ { type: 'Error', value: '' },
+ ],
+ [
+ 'an empty Caused by line',
+ ['Caused by: ', 'TypeError: Failed to fetch'],
+ { type: 'TypeError', value: 'Failed to fetch' },
+ ],
+ [
+ 'a message without a Name: prefix',
+ ['Caused by: request timed out after 30s', ''],
+ { type: '', value: 'request timed out after 30s' },
+ ],
+ ])('trims the blank lines around %s', (_, causeLines, title) => {
+ const stack = [
+ ` at checkout (${APP}:120:11)`,
+ ...causeLines,
+ ` at fetchCart (${APP}:42:9)`,
+ ].join('\n');
+
+ expect(parseJavaScriptException(stack)?.causes).toEqual([
+ {
+ ...title,
+ frames: [expect.objectContaining({ function: 'fetchCart' })],
+ },
+ ]);
+ });
+
+ it('keeps a URL line of a frameless Firefox cause in its title', () => {
+ const stack = [
+ `checkout@${APP}:1:2`,
+ 'Caused by: TypeError: bad',
+ 'see https://docs.example.com/x',
+ ].join('\n');
+
+ expect(parseJavaScriptException(stack)?.causes).toEqual([
+ {
+ type: 'TypeError',
+ value: 'bad\nsee https://docs.example.com/x',
+ frames: [],
+ },
+ ]);
+ });
+
+ it('skips lines with more than 32 parentheses', () => {
+ const stack = [
+ `checkout@${APP}:120:11`,
+ `submit${'()'.repeat(17)}@${APP}:130:5`,
+ ].join('\n');
+
+ expect(framesOf(stack)).toEqual([
+ expect.objectContaining({ function: 'checkout' }),
+ ]);
+ });
+
+ it.each([
+ ['an error line alone', 'Error: x'],
+ [
+ 'frameless toString() text with a URL',
+ 'HttpException: GET https://api.example.com/x failed\nCaused by: {"code":1}',
+ ],
+ [
+ 'a stack over 64 KB',
+ `Error: x\n${` at fn (${APP}:1:2)\n`.repeat(2000)}`,
+ ],
+ ['empty text', ''],
+ ['a Caused by without frames', 'Error: x\nCaused by: Error: y'],
+ ])('returns undefined for %s', (_, stack) => {
+ expect(parseJavaScriptException(stack)).toBeUndefined();
+ });
+
+ it.each([
+ ['/srv/app/node_modules/react-dom/cjs/react-dom.js', false],
+ ['http://localhost:5173/node_modules/.vite/deps/react-dom.js', false],
+ ['node:internal/process/task_queues', false],
+ ['internal/modules/cjs/loader.js', false],
+ [
+ 'https://app.example.com/_next/static/chunks/framework-2c79e2a64abdb08b.js',
+ false,
+ ],
+ [
+ 'https://app.example.com/_next/static/chunks/webpack-59c5c889f52620d6.js',
+ false,
+ ],
+ ['https://app.example.com/_next/static/chunks/main-app-5f1c09ab.js', false],
+ ['webpack:///webpack/bootstrap', false],
+ ['https://app.example.com/static/js/runtime-main.4c1f2b9d.js', false],
+ ['https://app.example.com/webpack-runtime-8a2b3c.js', false],
+ [
+ 'https://app.example.com/_next/static/chunks/pages/checkout-3f2a.js',
+ true,
+ ],
+ ['https://app.example.com/_next/static/chunks/main-feature.js', true],
+ ])('sets in_app for %s to %s', (filename, inApp) => {
+ expect(framesOf(`Error: x\n at fn (${filename}:1:2)`)).toEqual([
+ expect.objectContaining({ filename, in_app: inApp }),
+ ]);
+ });
+});
+
+describe('isJavaScriptRuntime', () => {
+ it.each([
+ ['webjs', true],
+ ['nodejs', true],
+ ['java', false],
+ ['python', false],
+ [undefined, false],
+ ])('treats telemetry.sdk.language %s as %s', (language, expected) => {
+ expect(
+ isJavaScriptRuntime(
+ language == null ? {} : { 'telemetry.sdk.language': language },
+ ),
+ ).toBe(expected);
+ });
+});
diff --git a/packages/app/src/utils/javascriptStack.ts b/packages/app/src/utils/javascriptStack.ts
new file mode 100644
index 000000000..be1d32ad6
--- /dev/null
+++ b/packages/app/src/utils/javascriptStack.ts
@@ -0,0 +1,124 @@
+import type { StacktraceFrame } from '@/types';
+import { defaultStackParser } from '@/utils/sentryStackParser';
+import type { Attributes } from '@/utils/spanAttributeReaders';
+
+const JAVASCRIPT_SDK_LANGUAGES = new Set(['webjs', 'nodejs']);
+
+// Parsing runs during render on client-controlled text, and a crafted 1 KB line
+// still costs the line parsers ~1.5 ms, so longer stacks are shown as text.
+const MAX_PARSED_STACK_LENGTH = 64 * 1024;
+
+const CAUSED_BY_REGEX = /^\s*Caused by: /;
+const V8_FRAME_REGEX = /^\s*at /;
+const ERROR_TITLE_REGEX = /^([^\s:]+)(?:: ([\s\S]*))?$/;
+
+// Library, runtime and bundler frames. The old browser SDK deleted Next.js
+// framework chunks outright; marking them keeps the frames, and the panel
+// opens on the app's newest in-app frame instead of them.
+const NOT_IN_APP_FILENAME_PATTERNS = [
+ /node_modules/,
+ /^(node:|internal\/)/,
+ /\/framework-[0-9a-f]+\.js/,
+ /\/_next\/static\/chunks\/(main|main-app|webpack|polyfills)(-[0-9a-f]+)?\.js/,
+ /(^|\/)webpack\/(runtime|bootstrap)/,
+ /\/(webpack-runtime|runtime[~-]main)[^/]*\.js/,
+];
+
+type JavaScriptCause = {
+ type: string;
+ value: string;
+ frames: StacktraceFrame[];
+};
+
+export type ParsedJavaScriptException = {
+ frames: StacktraceFrame[];
+ causes: JavaScriptCause[];
+};
+
+export function isJavaScriptRuntime(resourceAttributes: Attributes): boolean {
+ const language = resourceAttributes['telemetry.sdk.language'];
+ return typeof language === 'string' && JAVASCRIPT_SDK_LANGUAGES.has(language);
+}
+
+// Frames without a line number are native, anonymous or `data:` URI frames:
+// never app code, and nothing a source map can resolve.
+function isInApp({ filename, lineno }: StacktraceFrame): boolean {
+ return (
+ lineno !== undefined &&
+ !NOT_IN_APP_FILENAME_PATTERNS.some(pattern => pattern.test(filename))
+ );
+}
+
+// The `Name: message` header is every line before the first frame, and a
+// multi-line message can contain URL lines the Gecko parser would read as
+// frames. V8 frames start with `at `. Firefox and Safari frames always contain
+// `@` (`fn@url:1:2`, `@url:1:2`, `fn@[native code]`); their main stack has no
+// header, but `Caused by:` sections and frameless `toString()` text do.
+function getHeaderLength(lines: string[]): number {
+ const v8FrameIndex = lines.findIndex(line => V8_FRAME_REGEX.test(line));
+ if (v8FrameIndex !== -1) {
+ return v8FrameIndex;
+ }
+ const geckoFrameIndex = lines.findIndex(line => line.includes('@'));
+ return geckoFrameIndex === -1 ? lines.length : geckoFrameIndex;
+}
+
+function parseSection(lines: string[]) {
+ const headerLength = getHeaderLength(lines);
+ const frames = defaultStackParser(lines.join('\n'), headerLength).map(
+ frame => ({ ...frame, in_app: isInApp(frame) }),
+ );
+ return { header: lines.slice(0, headerLength), frames };
+}
+
+// The title is the `Caused by:` line joined with the header lines below it, so
+// a message that starts with a line break (`Error: \nDetails`) would keep it
+// and open with a blank line. Only the start is trimmed before matching: a
+// whitespace-only message leaves `Error: `, whose space the regex needs.
+function parseCauseTitle(
+ title: string,
+): Pick {
+ const text = title.trimStart();
+ const match = ERROR_TITLE_REGEX.exec(text);
+ return match
+ ? { type: match[1], value: (match[2] ?? '').trim() }
+ : { type: '', value: text.trimEnd() };
+}
+
+/**
+ * Parses an `Error.stack` string from a JavaScript SDK, which appends each
+ * error in the `cause` chain as a Java-style `Caused by: Name: message`
+ * section. Frames are oldest first. Returns undefined when no section has a
+ * frame or the text is too long to parse, so the caller can show it instead.
+ */
+export function parseJavaScriptException(
+ stack: string,
+): ParsedJavaScriptException | undefined {
+ if (stack.length > MAX_PARSED_STACK_LENGTH) {
+ return undefined;
+ }
+ const sections: { title?: string; lines: string[] }[] = [{ lines: [] }];
+ for (const line of stack.split('\n')) {
+ if (CAUSED_BY_REGEX.test(line)) {
+ sections.push({ title: line.replace(CAUSED_BY_REGEX, ''), lines: [] });
+ } else {
+ sections[sections.length - 1].lines.push(line);
+ }
+ }
+
+ const [main, ...causes] = sections.map(({ title, lines }) => ({
+ title,
+ ...parseSection(lines),
+ }));
+ if (![main, ...causes].some(section => section.frames.length > 0)) {
+ return undefined;
+ }
+
+ return {
+ frames: main.frames,
+ causes: causes.map(({ title, header, frames }) => ({
+ ...parseCauseTitle([title, ...header].join('\n')),
+ frames,
+ })),
+ };
+}
diff --git a/packages/app/src/utils/sentryStackParser.ts b/packages/app/src/utils/sentryStackParser.ts
new file mode 100644
index 000000000..37aad8ab0
--- /dev/null
+++ b/packages/app/src/utils/sentryStackParser.ts
@@ -0,0 +1,330 @@
+// Parses JavaScript `Error.stack` text into frames: Sentry's Chrome (V8) and
+// Gecko (Firefox, Safari) line parsers and `createStackParser`, ported from
+// sentry-javascript 11.10.0:
+// https://github.com/getsentry/sentry-javascript/blob/11.10.0/packages/browser/src/stack-parsers.ts
+// https://github.com/getsentry/sentry-javascript/blob/11.10.0/packages/core/src/utils/stacktrace.ts
+//
+// Changes from upstream:
+// - Typed against HyperDX's `StacktraceFrame`, without upstream's `exec()`
+// result casts.
+// - Only the Chrome and Gecko line parsers (upstream's
+// `defaultStackLineParsers`); WinJS and Opera are dropped.
+// - `createStackParser` drops the unused `framesToPop` parameter.
+// - The final reverse no longer strips `sentryWrapped` and
+// `captureException`/`captureMessage` frames. Those target Sentry's own SDK
+// and would drop an app's own `captureException` frame.
+// - The `data:` URI frame sets `in_app` like every other frame.
+// - `createStackParser` skips lines with more than 32 parentheses.
+//
+// MIT License
+//
+// Copyright (c) 2019 Functional Software, Inc. dba Sentry
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is furnished to do
+// so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// The line parsers were originally forked from https://github.com/csnover/TraceKit,
+// and were largely re-written as part of raven-js. Upstream's TraceKit notice:
+//
+// Copyright (c) 2013 Onur Can Cakmak onur.cakmak@gmail.com and all TraceKit contributors.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this
+// software and associated documentation files(the 'Software'), to deal in the Software
+// without restriction, including without limitation the rights to use, copy, modify,
+// merge, publish, distribute, sublicense, and / or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to the following
+// conditions:
+//
+// The above copyright notice and this permission notice shall be included in all copies
+// or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+// PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+// CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
+// OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import type { StacktraceFrame } from '@/types';
+
+type StackLineParserFn = (line: string) => StacktraceFrame | undefined;
+type StackLineParser = [priority: number, parse: StackLineParserFn];
+
+const STACKTRACE_FRAME_LIMIT = 50;
+const MAX_LINE_PARENTHESES = 32;
+export const UNKNOWN_FUNCTION = '?';
+// Used to sanitize webpack (error: *) wrapped stack errors
+const WEBPACK_ERROR_REGEXP = /\(error: (.*)\)/;
+
+const CHROME_PRIORITY = 30;
+const GECKO_PRIORITY = 50;
+
+function createFrame(
+ filename: string,
+ func: string,
+ lineno?: number,
+ colno?: number,
+): StacktraceFrame {
+ const frame: StacktraceFrame = {
+ filename,
+ function: func === '' ? UNKNOWN_FUNCTION : func,
+ in_app: true, // All browser frames are considered in_app
+ };
+
+ if (lineno !== undefined) {
+ frame.lineno = lineno;
+ }
+
+ if (colno !== undefined) {
+ frame.colno = colno;
+ }
+
+ return frame;
+}
+
+// This regex matches frames that have no function name (ie. are at the top level of a module).
+// For example "at http://localhost:5000//script.js:1:126"
+// Frames _with_ function names usually look as follows: "at commitLayoutEffects (react-dom.development.js:23426:1)"
+const chromeRegexNoFnName = /^\s*at (\S+?)(?::(\d+))(?::(\d+))\s*$/i;
+
+// This regex matches all the frames that have a function name.
+const chromeRegex =
+ /^\s*at (?:(.+?\)(?: \[.+\])?|.*?) ?\((?:address at )?)?(?:async )?((?:|[-a-z]+:|.*bundle|\/)?.*?)(?::(\d+))?(?::(\d+))?\)?\s*$/i;
+
+const chromeEvalRegex = /\((\S*)(?::(\d+))(?::(\d+))\)/;
+
+// Matches stack frames with data URIs instead of filename so we can still get the function name
+// Example: "at dynamicFn (data:application/javascript,export function dynamicFn() {..."
+const chromeDataUriRegex = /at (.+?) ?\(data:(.+?),/;
+
+// Chromium based browsers: Chrome, Brave, new Opera, new Edge
+const chromeStackParserFn: StackLineParserFn = line => {
+ const dataUriMatch = line.match(chromeDataUriRegex);
+ if (dataUriMatch) {
+ return {
+ filename: ``,
+ function: dataUriMatch[1],
+ in_app: true,
+ };
+ }
+
+ // If the stack line has no function name, we need to parse it differently
+ const noFnParts = chromeRegexNoFnName.exec(line);
+
+ if (noFnParts) {
+ const [, filename, line, col] = noFnParts;
+ return createFrame(filename, UNKNOWN_FUNCTION, +line, +col);
+ }
+
+ const parts = chromeRegex.exec(line);
+
+ if (parts) {
+ const isEval = parts[2]?.indexOf('eval') === 0; // start of line
+
+ if (isEval) {
+ const subMatch = chromeEvalRegex.exec(parts[2]);
+
+ if (subMatch) {
+ // throw out eval line/column and use top-most line/column number
+ parts[2] = subMatch[1]; // url
+ parts[3] = subMatch[2]; // line
+ parts[4] = subMatch[3]; // column
+ }
+ }
+
+ const [func, filename] = extractSafariExtensionDetails(
+ parts[1] || UNKNOWN_FUNCTION,
+ parts[2],
+ );
+
+ return createFrame(
+ filename,
+ func,
+ parts[3] ? +parts[3] : undefined,
+ parts[4] ? +parts[4] : undefined,
+ );
+ }
+
+ return;
+};
+
+const chromeStackLineParser: StackLineParser = [
+ CHROME_PRIORITY,
+ chromeStackParserFn,
+];
+
+// gecko regex: `(?:bundle|\d+\.js)`: `bundle` is for react native, `\d+\.js` also but specifically for ram bundles because it
+// generates filenames without a prefix like `file://` the filenames in the stacktrace are just 42.js
+// We need this specific case for now because we want no other regex to match.
+const geckoREgex =
+ /^\s*(.*?)(?:\((.*?)\))?(?:^|@)?((?:[-a-z]+)?:\/.*?|\[native code\]|[^@]*(?:bundle|\d+\.js)|\/[\w\-. /=]+)(?::(\d+))?(?::(\d+))?\s*$/i;
+const geckoEvalRegex = /(\S+) line (\d+)(?: > eval line \d+)* > eval/i;
+
+const gecko: StackLineParserFn = line => {
+ const parts = geckoREgex.exec(line);
+
+ if (parts) {
+ const isEval = parts[3] && parts[3].indexOf(' > eval') > -1;
+ if (isEval) {
+ const subMatch = geckoEvalRegex.exec(parts[3]);
+
+ if (subMatch) {
+ // throw out eval line/column and use top-most line number
+ parts[1] = parts[1] || 'eval';
+ parts[3] = subMatch[1];
+ parts[4] = subMatch[2];
+ parts[5] = ''; // no column when eval
+ }
+ }
+
+ let filename = parts[3];
+ let func = parts[1] || UNKNOWN_FUNCTION;
+ [func, filename] = extractSafariExtensionDetails(func, filename);
+
+ return createFrame(
+ filename,
+ func,
+ parts[4] ? +parts[4] : undefined,
+ parts[5] ? +parts[5] : undefined,
+ );
+ }
+
+ return;
+};
+
+const geckoStackLineParser: StackLineParser = [GECKO_PRIORITY, gecko];
+
+/**
+ * Safari web extensions, starting version unknown, can produce "frames-only" stacktraces.
+ * What it means, is that instead of format like:
+ *
+ * Error: wat
+ * at function@url:row:col
+ * at function@url:row:col
+ * at function@url:row:col
+ *
+ * it produces something like:
+ *
+ * function@url:row:col
+ * function@url:row:col
+ * function@url:row:col
+ *
+ * Because of that, it won't be captured by `chrome` RegExp and will fall into `Gecko` branch.
+ * This function is extracted so that we can use it in both places without duplicating the logic.
+ * Unfortunately "just" changing RegExp is too complicated now and making it pass all tests
+ * and fix this case seems like an impossible, or at least way too time-consuming task.
+ */
+const extractSafariExtensionDetails = (
+ func: string,
+ filename: string,
+): [string, string] => {
+ const isSafariExtension = func.indexOf('safari-extension') !== -1;
+ const isSafariWebExtension = func.indexOf('safari-web-extension') !== -1;
+
+ return isSafariExtension || isSafariWebExtension
+ ? [
+ func.indexOf('@') !== -1 ? func.split('@')[0] : UNKNOWN_FUNCTION,
+ isSafariExtension
+ ? `safari-extension:${filename}`
+ : `safari-web-extension:${filename}`,
+ ]
+ : [func, filename];
+};
+
+function createStackParser(...parsers: StackLineParser[]) {
+ const sortedParsers = parsers.sort((a, b) => a[0] - b[0]).map(p => p[1]);
+
+ return (stack: string, skipFirstLines: number = 0): StacktraceFrame[] => {
+ const frames: StacktraceFrame[] = [];
+ const lines = stack.split('\n');
+
+ for (let i = skipFirstLines; i < lines.length; i++) {
+ let line = lines[i];
+ // Truncate lines over 1kb because many of the regular expressions use
+ // backtracking which results in run time that increases exponentially
+ // with input size. Huge strings can result in hangs/Denial of Service:
+ // https://github.com/getsentry/sentry-javascript/issues/2286
+ if (line.length > 1024) {
+ line = line.slice(0, 1024);
+ }
+
+ // Upstream parses the page's own `Error.stack`; this parses stored text
+ // anyone can send. The Gecko regex backtracks in proportion to the
+ // number of `(` times `)` in a line, which the 1 KB cut doesn't bound
+ // (~90 ms for one line). Real frames have a handful.
+ if (line.replace(/[^()]/g, '').length > MAX_LINE_PARENTHESES) {
+ continue;
+ }
+
+ // https://github.com/getsentry/sentry-javascript/issues/5459
+ // Remove webpack (error: *) wrappers
+ const cleanedLine = WEBPACK_ERROR_REGEXP.test(line)
+ ? line.replace(WEBPACK_ERROR_REGEXP, '$1')
+ : line;
+
+ // https://github.com/getsentry/sentry-javascript/issues/7813
+ // Skip Error: lines
+ // Using includes() instead of a regex to avoid O(n²) backtracking on long lines
+ // https://github.com/getsentry/sentry-javascript/issues/20052
+ if (cleanedLine.includes('Error: ')) {
+ continue;
+ }
+
+ for (const parser of sortedParsers) {
+ const frame = parser(cleanedLine);
+
+ if (frame) {
+ frames.push(frame);
+ break;
+ }
+ }
+
+ if (frames.length >= STACKTRACE_FRAME_LIMIT) {
+ break;
+ }
+ }
+
+ return reverseFrames(frames);
+ };
+}
+
+/**
+ * Enforces a limit of max number of frames. Assumes stack input is ordered
+ * from top to bottom and returns the reverse representation so call site of
+ * the function that caused the crash is the last frame in the array.
+ */
+function reverseFrames(stack: ReadonlyArray) {
+ if (!stack.length) {
+ return [];
+ }
+
+ const localStack = Array.from(stack).reverse();
+ const lastFrame = localStack[localStack.length - 1];
+
+ return localStack.slice(0, STACKTRACE_FRAME_LIMIT).map(frame => ({
+ ...frame,
+ filename: frame.filename || lastFrame.filename,
+ function: frame.function || UNKNOWN_FUNCTION,
+ }));
+}
+
+/** Frames oldest first, every frame `in_app: true`, at most 50. */
+export const defaultStackParser = createStackParser(
+ chromeStackLineParser,
+ geckoStackLineParser,
+);
diff --git a/packages/app/styles/LogSidePanel.module.scss b/packages/app/styles/LogSidePanel.module.scss
index 6a3e3d1c4..74c3c6f90 100644
--- a/packages/app/styles/LogSidePanel.module.scss
+++ b/packages/app/styles/LogSidePanel.module.scss
@@ -64,6 +64,10 @@
.panelSectionWrapperTitle {
padding: 18px 24px;
border-bottom: 1px solid var(--color-border);
+
+ &:last-child {
+ border-bottom: none;
+ }
}
.kbdShortcuts {
From 7de661b6afd2c968e7640f3113785d4099e0c212 Mon Sep 17 00:00:00 2001
From: Spencer Torres
Date: Wed, 4 Nov 2026 13:34:25 -0800
Subject: [PATCH 030/103] feat: browser event summary in row side panel (#3442)
---
.changeset/browser-event-row-summary.md | 5 +
.../src/components/BrowserEventSubpanel.tsx | 78 +++++++
.../app/src/components/DBRowDataPanel.tsx | 75 +++++-
.../app/src/components/DBRowOverviewPanel.tsx | 64 ++++-
.../app/src/components/DBRowSidePanel.tsx | 8 +
.../src/components/DBRowSidePanelHeader.tsx | 7 +-
.../__tests__/DBRowDataPanel.test.ts | 162 +++++++++++++
.../DBRowOverviewPanel.browserEvent.test.tsx | 218 ++++++++++++++++++
.../DBRowSidePanel.browserEvent.test.tsx | 169 ++++++++++++++
.../__tests__/DBRowSidePanelHeader.test.tsx | 14 ++
.../src/utils/__tests__/browserEvents.test.ts | 31 +++
packages/app/src/utils/browserEvents.ts | 65 +++++-
.../app/tests/e2e/features/sessions.spec.ts | 18 ++
packages/app/tests/e2e/seed-otel-sessions.ts | 34 ++-
14 files changed, 932 insertions(+), 16 deletions(-)
create mode 100644 .changeset/browser-event-row-summary.md
create mode 100644 packages/app/src/components/BrowserEventSubpanel.tsx
create mode 100644 packages/app/src/components/__tests__/DBRowOverviewPanel.browserEvent.test.tsx
create mode 100644 packages/app/src/components/__tests__/DBRowSidePanel.browserEvent.test.tsx
diff --git a/.changeset/browser-event-row-summary.md b/.changeset/browser-event-row-summary.md
new file mode 100644
index 000000000..0b4e69838
--- /dev/null
+++ b/.changeset/browser-event-row-summary.md
@@ -0,0 +1,5 @@
+---
+'@hyperdx/app': patch
+---
+
+feat: Row side panel summarizes clicks, navigations and console messages from OpenTelemetry browser SDKs, and navigation rows no longer show an HTTP Request section
diff --git a/packages/app/src/components/BrowserEventSubpanel.tsx b/packages/app/src/components/BrowserEventSubpanel.tsx
new file mode 100644
index 000000000..2c7bcbb11
--- /dev/null
+++ b/packages/app/src/components/BrowserEventSubpanel.tsx
@@ -0,0 +1,78 @@
+import {
+ BrowserEventKind,
+ CONSOLE_METHOD_ATTRIBUTE,
+ firstLine,
+ getElementAttributes,
+} from '@/utils/browserEvents';
+import {
+ Attributes,
+ firstSetString,
+ isSet,
+} from '@/utils/spanAttributeReaders';
+
+import { RequestDetailRow, RequestDetailsTable } from './RequestDetailsTable';
+
+const PAGE_URL_KEY = 'browser.document.url.full';
+
+function read(attributes: Attributes, key: string): string | undefined {
+ return firstSetString(attributes, [key]);
+}
+
+// Map columns hold the flag as 'true'/'false', JSON columns as a Bool.
+function yesNo(value: unknown): string | undefined {
+ if (!isSet(value)) {
+ return undefined;
+ }
+ return String(value) === 'true' ? 'Yes' : 'No';
+}
+
+function getClickRows(attributes: Attributes): RequestDetailRow[] {
+ return [
+ ['Element', read(attributes, 'browser.tag_name')?.toLowerCase()],
+ ['CSS selector', read(attributes, 'browser.css_selector')],
+ ...getElementAttributes(attributes).map(
+ ([name, value]): RequestDetailRow => [`data-otel-${name}`, value],
+ ),
+ ['Page URL', read(attributes, PAGE_URL_KEY)],
+ ];
+}
+
+function getNavigationRows(attributes: Attributes): RequestDetailRow[] {
+ return [
+ ['URL', read(attributes, 'url.full')],
+ ['Navigation type', read(attributes, 'browser.navigation.type')],
+ ['Same document', yesNo(attributes['browser.navigation.same_document'])],
+ ['Hash change', yesNo(attributes['browser.navigation.hash_change'])],
+ ];
+}
+
+function getConsoleRows(
+ attributes: Attributes,
+ body: string | undefined,
+): RequestDetailRow[] {
+ return [
+ ['Method', read(attributes, CONSOLE_METHOD_ATTRIBUTE)],
+ // The SDK sends the console message as the body, never as an attribute.
+ ['Message', body ? firstLine(body) : undefined],
+ ['Page URL', read(attributes, PAGE_URL_KEY)],
+ ];
+}
+
+export function BrowserEventSubpanel({
+ kind,
+ attributes,
+ body,
+}: {
+ kind: BrowserEventKind;
+ /** The row's flattened event attributes. */
+ attributes: Attributes;
+ body: string | undefined;
+}) {
+ const rows =
+ kind === 'click'
+ ? getClickRows(attributes)
+ : kind === 'navigation'
+ ? getNavigationRows(attributes)
+ : getConsoleRows(attributes, body);
+ return ;
+}
diff --git a/packages/app/src/components/DBRowDataPanel.tsx b/packages/app/src/components/DBRowDataPanel.tsx
index ddf77f48f..35053b2b2 100644
--- a/packages/app/src/components/DBRowDataPanel.tsx
+++ b/packages/app/src/components/DBRowDataPanel.tsx
@@ -1,6 +1,7 @@
import { useMemo } from 'react';
import { flatten } from 'flat';
import type { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse';
+import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import {
isLogSource,
isTraceSource,
@@ -15,6 +16,7 @@ import {
getSelectAllColumnsQuerySettings,
useSelectAllColumnsQuerySettings,
} from '@/hooks/useMaterializedAliasColumnsOption';
+import { useColumns } from '@/hooks/useMetadata';
import { WithClause } from '@/hooks/useRowWhere';
import {
getDisplayedTimestampValueExpression,
@@ -43,6 +45,7 @@ export enum ROW_DATA_ALIASES {
DURATION_MS = '__hdx_duration_ms',
SPAN_KIND = '__hdx_span_kind',
SPAN_LINKS = '__hdx_span_links',
+ EVENT_NAME = '__hdx_event_name',
}
export function useRowData({
@@ -97,6 +100,46 @@ export function useRowData({
// select that instead of `*` when fetching full row data.
const knownColumns = getKnownColumnsList(source);
+ // A bare column comes back from `*` or the Known Columns List whenever the
+ // table has it, so it is read from the row instead of selected: older OTel
+ // log tables have no EventName column, and selecting it would fail the whole
+ // lookup.
+ const eventNameExpr = isLogSource(source)
+ ? source.eventNameExpression?.trim()
+ : undefined;
+ const eventNameColumn =
+ eventNameExpr && /^[A-Za-z_][A-Za-z0-9_]*$/.test(eventNameExpr)
+ ? eventNameExpr
+ : undefined;
+
+ // `*` leaves out ALIAS and MATERIALIZED columns, so the table's columns
+ // decide whether a bare column is selected after all. renderChartConfig
+ // already reads these cached columns before the lookup runs, so waiting on
+ // them adds no query. A Known Columns List names its columns, so it returns
+ // such a column whenever it lists it.
+ const checksEventNameColumn =
+ rowId != null && eventNameColumn != null && knownColumns == null;
+ const { data: columns, isLoading: isColumnsLoading } = useColumns(
+ tcFromSource(source),
+ {
+ enabled: checksEventNameColumn,
+ // A failed DESCRIBE only costs the event name, so the lookup does not
+ // wait out retries.
+ retry: false,
+ },
+ );
+ const isEventNameColumnPending = checksEventNameColumn && isColumnsLoading;
+ // A disabled query still returns columns another page has cached.
+ const eventNameColumnKind = checksEventNameColumn
+ ? columns?.find(column => column.name === eventNameColumn)?.default_type
+ : undefined;
+ const selectsEventNameColumn =
+ eventNameColumnKind === 'ALIAS' || eventNameColumnKind === 'MATERIALIZED';
+ const selectedEventNameExpr =
+ eventNameColumn == null || selectsEventNameColumn
+ ? eventNameExpr
+ : undefined;
+
const baseConfig = {
source: source.id,
connection: source.connection,
@@ -168,6 +211,14 @@ export function useRowData({
},
]
: []),
+ ...(selectedEventNameExpr
+ ? [
+ {
+ valueExpression: selectedEventNameExpr,
+ alias: ROW_DATA_ALIASES.EVENT_NAME,
+ },
+ ]
+ : []),
...(source.kind === SourceKind.Trace && source.spanEventsValueExpression
? [
{
@@ -214,7 +265,15 @@ export function useRowData({
const additionalQuerySettings = useSelectAllColumnsQuerySettings(source);
- const baseQueryKey = ['row_side_panel', rowId, aliasWith, source];
+ // This key replaces the config-derived one, so it has to carry the column
+ // check's effect on the select.
+ const baseQueryKey = [
+ 'row_side_panel',
+ rowId,
+ aliasWith,
+ source,
+ selectsEventNameColumn,
+ ];
// Both halves of the filter are needed for `renderChartConfig` to emit one, so
// a source with no usable timestamp expression can't be bounded at all.
const hasWindow = dateRange != null && timestampValueExpr != null;
@@ -228,7 +287,7 @@ export function useRowData({
},
{
queryKey: [...baseQueryKey, dateRange],
- enabled: rowId != null && hasWindow,
+ enabled: rowId != null && hasWindow && !isEventNameColumnPending,
additionalQuerySettings,
},
);
@@ -249,7 +308,7 @@ export function useRowData({
// often resolve from cache instead of scanning.
const fallbackResult = useQueriedChartConfig(baseConfig, {
queryKey: [...baseQueryKey, undefined],
- enabled: rowId != null && isFallbackActive,
+ enabled: rowId != null && isFallbackActive && !isEventNameColumnPending,
additionalQuerySettings,
});
@@ -259,7 +318,9 @@ export function useRowData({
// report loading until it settles rather than briefly claiming the row is
// absent.
const isLoading =
- queryResult.isLoading || (isBoundedEmpty && queryResult.isPending);
+ isEventNameColumnPending ||
+ queryResult.isLoading ||
+ (isBoundedEmpty && queryResult.isPending);
// Normalize resource, event and span-event exception attributes to always use flat keys for both JSON and Map columns
const normalizedData = useMemo(() => {
@@ -288,11 +349,15 @@ export function useRowData({
);
}
+ if (eventNameColumn != null && !selectsEventNameColumn) {
+ normalizedRow[ROW_DATA_ALIASES.EVENT_NAME] = row[eventNameColumn];
+ }
+
return {
...queryResult.data,
data: [normalizedRow],
};
- }, [queryResult.data]);
+ }, [queryResult.data, eventNameColumn, selectsEventNameColumn]);
return {
...queryResult,
diff --git a/packages/app/src/components/DBRowOverviewPanel.tsx b/packages/app/src/components/DBRowOverviewPanel.tsx
index ebad17c2e..49d650965 100644
--- a/packages/app/src/components/DBRowOverviewPanel.tsx
+++ b/packages/app/src/components/DBRowOverviewPanel.tsx
@@ -1,11 +1,16 @@
import { use, useCallback, useMemo } from 'react';
import isString from 'lodash/isString';
import pickBy from 'lodash/pickBy';
-import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types';
+import {
+ isLogSource,
+ SourceKind,
+ TSource,
+} from '@hyperdx/common-utils/dist/types';
import { Accordion, Box, Flex, Text } from '@mantine/core';
import { WithClause } from '@/hooks/useRowWhere';
import { getEventBody } from '@/source';
+import { getBrowserEventKind, isBrowserEventRow } from '@/utils/browserEvents';
import { getHighlightedAttributesFromData } from '@/utils/highlightedAttributes';
import {
isJavaScriptRuntime,
@@ -15,6 +20,7 @@ import { resolveRowTimestampAnchor } from '@/utils/rowTimestamps';
import { getRequestPanelTitle } from '@/utils/rpcSpanAttributes';
import { isSet } from '@/utils/spanAttributeReaders';
+import { BrowserEventSubpanel } from './BrowserEventSubpanel';
import {
getJSONColumnNames,
getMapColumnNames,
@@ -98,6 +104,7 @@ export function RowOverviewPanel({
// assume OTel column names for now
const topLevelAttributeKeys = [
'ServiceName',
+ 'EventName',
'SpanName',
'Duration',
'SeverityText',
@@ -110,7 +117,16 @@ export function RowOverviewPanel({
'ScopeName',
'ScopeVersion',
];
- const topLevelAttributes = pickBy(firstRow, (value, key) => {
+ // `*` leaves out an ALIAS or MATERIALIZED EventName, so `useRowData` only
+ // returns it under its alias.
+ const rowWithEventName =
+ isLogSource(source) &&
+ source.eventNameExpression?.trim() === 'EventName' &&
+ firstRow?.EventName == null &&
+ firstRow?.__hdx_event_name != null
+ ? { ...firstRow, EventName: firstRow.__hdx_event_name }
+ : firstRow;
+ const topLevelAttributes = pickBy(rowWithEventName, (value, key) => {
if (value === '') {
return false;
}
@@ -151,6 +167,24 @@ export function RowOverviewPanel({
[flattenedEventAttributes],
);
+ // Only log sources select the event name, and gating here keeps every trace
+ // row out whatever attributes it carries.
+ const { browserEventKind, isBrowserRow } = useMemo(() => {
+ if (!isLogSource(source)) {
+ return { browserEventKind: undefined, isBrowserRow: false };
+ }
+
+ const row = {
+ eventName: firstRow?.__hdx_event_name,
+ attributes: flattenedEventAttributes,
+ };
+
+ return {
+ browserEventKind: getBrowserEventKind(row),
+ isBrowserRow: isBrowserEventRow(row),
+ };
+ }, [source, firstRow?.__hdx_event_name, flattenedEventAttributes]);
+
const exceptionAttributes = useMemo(() => {
const spanEventAttributes = firstRow?.__hdx_events_exception_attributes;
if (
@@ -271,6 +305,7 @@ export function RowOverviewPanel({
// In that case suppress the body paper entirely instead of
// rendering an "[Empty]" placeholder.
bodyConfigured={mainContentColumn !== undefined}
+ hideEmptyBody={isBrowserRow}
severityText={firstRow?.__hdx_severity_text}
rowData={firstRow}
/>
@@ -280,6 +315,7 @@ export function RowOverviewPanel({
mt="sm"
defaultValue={[
'exception',
+ 'browserEvent',
'spanEvents',
'spanLinks',
'linkedFrom',
@@ -311,7 +347,29 @@ export function RowOverviewPanel({
)}
- {requestPanelTitle != null && (
+ {browserEventKind != null && (
+
+
+
+ Browser event
+
+
+
+
+
+
+
+
+ )}
+
+ {requestPanelTitle != null && browserEventKind == null && (
diff --git a/packages/app/src/components/DBRowSidePanel.tsx b/packages/app/src/components/DBRowSidePanel.tsx
index 161224a67..b7fccb964 100644
--- a/packages/app/src/components/DBRowSidePanel.tsx
+++ b/packages/app/src/components/DBRowSidePanel.tsx
@@ -52,6 +52,7 @@ import TabBar from '@/TabBar';
import { SearchConfig } from '@/types';
import { FormatTime } from '@/useFormatTime';
import { formatDistanceToNowStrictShort } from '@/utils';
+import { isBrowserEventRow } from '@/utils/browserEvents';
import { getHighlightedAttributesFromData } from '@/utils/highlightedAttributes';
import {
getRowLookupWindow,
@@ -1109,6 +1110,13 @@ export const DBRowSidePanelInner = ({
mainContent={mainContent}
mainContentHeader={mainContentColumn}
bodyShownInTitle={isBodyShownInTitle}
+ hideEmptyBody={
+ isLogSource(source) &&
+ isBrowserEventRow({
+ eventName: normalizedRow?.[ROW_DATA_ALIASES.EVENT_NAME],
+ attributes: normalizedRow?.[ROW_DATA_ALIASES.EVENT_ATTRIBUTES],
+ })
+ }
severityText={severityText}
rowData={normalizedRow}
/>
diff --git a/packages/app/src/components/DBRowSidePanelHeader.tsx b/packages/app/src/components/DBRowSidePanelHeader.tsx
index 8995a9ab7..684dc9d09 100644
--- a/packages/app/src/components/DBRowSidePanelHeader.tsx
+++ b/packages/app/src/components/DBRowSidePanelHeader.tsx
@@ -28,6 +28,8 @@ export default function DBRowSidePanelHeader({
// trace span's name, for example), so the body paper is omitted rather
// than repeating the title directly below it.
bodyShownInTitle = false,
+ // Rows that never carry a body, such as browser events, skip the empty state.
+ hideEmptyBody = false,
severityText,
rowData,
}: {
@@ -35,6 +37,7 @@ export default function DBRowSidePanelHeader({
mainContentHeader?: string;
bodyConfigured?: boolean;
bodyShownInTitle?: boolean;
+ hideEmptyBody?: boolean;
attributes?: HighlightedAttribute[];
severityText?: string;
rowData?: Record;
@@ -90,7 +93,9 @@ export default function DBRowSidePanelHeader({
return (
<>
- {!bodyConfigured || bodyShownInTitle ? null : mainContent ? (
+ {!bodyConfigured ||
+ bodyShownInTitle ||
+ (hideEmptyBody && !mainContent) ? null : mainContent ? (
({
useQueriedChartConfig: jest.fn(),
}));
+type ColumnsResult = {
+ data?: { name: string; default_type: string }[];
+ isLoading: boolean;
+};
+let mockColumns: ColumnsResult;
+const mockUseColumns = jest.fn(
+ (_table: unknown, _options: { enabled?: boolean }) => mockColumns,
+);
+jest.mock('@/hooks/useMetadata', () => ({
+ useColumns: (table: unknown, options: { enabled?: boolean }) =>
+ mockUseColumns(table, options),
+}));
+
const mockUseQueriedChartConfig = useQueriedChartConfig as jest.Mock;
const showMaterializedAliasColumns = () =>
localStorage.setItem('hdx-row-show-materialized-alias-columns', 'true');
@@ -32,6 +45,7 @@ describe('DBRowDataPanel', () => {
beforeEach(() => {
jest.clearAllMocks();
localStorage.clear();
+ mockColumns = { data: undefined, isLoading: false };
mockUseQueriedChartConfig.mockReturnValue({
data: {
data: [],
@@ -126,6 +140,154 @@ describe('DBRowDataPanel', () => {
expect(config.select).not.toContainEqual({ valueExpression: '*' });
});
+ describe('event name', () => {
+ const eventNameAlias = expect.objectContaining({
+ alias: '__hdx_event_name',
+ });
+ const eventNameColumn = (default_type: string) => [
+ { name: 'Body', default_type: '' },
+ { name: 'EventName', default_type },
+ ];
+ const lookUp = (
+ overrides: Partial,
+ row: Record = {},
+ ) => {
+ mockUseQueriedChartConfig.mockReturnValue({
+ data: { data: [row], meta: [], rows: 1, isComplete: true },
+ isLoading: false,
+ isError: false,
+ isSuccess: true,
+ });
+ const { result } = renderHook(() =>
+ useRowData({ source: { ...source, ...overrides }, rowId: "id='a'" }),
+ );
+ const [[{ select }, { queryKey }]] = mockUseQueriedChartConfig.mock.calls;
+ return {
+ select,
+ queryKey,
+ row: result.current.data?.data[0],
+ checksColumns: mockUseColumns.mock.calls[0][1].enabled,
+ };
+ };
+
+ it('reads a plain column from the row instead of selecting it', () => {
+ mockColumns = { data: eventNameColumn(''), isLoading: false };
+ const { select, row, checksColumns } = lookUp(
+ { eventNameExpression: 'EventName' },
+ { EventName: 'browser.navigation' },
+ );
+ expect(checksColumns).toBe(true);
+ expect(select).not.toContainEqual(eventNameAlias);
+ expect(row?.__hdx_event_name).toBe('browser.navigation');
+ });
+
+ it.each(['ALIAS', 'MATERIALIZED'])(
+ 'selects a %s column, which `*` leaves out',
+ kind => {
+ mockColumns = { data: eventNameColumn(kind), isLoading: false };
+ const { select, row } = lookUp(
+ { eventNameExpression: 'EventName' },
+ { __hdx_event_name: 'browser.resource_timing' },
+ );
+ expect(select).toContainEqual({
+ valueExpression: 'EventName',
+ alias: '__hdx_event_name',
+ });
+ expect(row?.__hdx_event_name).toBe('browser.resource_timing');
+ },
+ );
+
+ it('keys the lookup by whether it selects the column', () => {
+ mockColumns = { data: eventNameColumn(''), isLoading: false };
+ const plain = lookUp({ eventNameExpression: 'EventName' }).queryKey;
+ mockUseQueriedChartConfig.mockClear();
+ mockColumns = { data: eventNameColumn('ALIAS'), isLoading: false };
+ const alias = lookUp({ eventNameExpression: 'EventName' }).queryKey;
+ expect(alias).not.toEqual(plain);
+ });
+
+ it('leaves the event name unset when the table has no such column', () => {
+ mockColumns = {
+ data: [{ name: 'Body', default_type: '' }],
+ isLoading: false,
+ };
+ const { select, row } = lookUp({ eventNameExpression: 'EventName' });
+ expect(select).not.toContainEqual(eventNameAlias);
+ expect(row?.__hdx_event_name).toBeUndefined();
+ });
+
+ it('reads the row when the table columns fail to load', () => {
+ const { select, row } = lookUp(
+ { eventNameExpression: 'EventName' },
+ { EventName: 'browser.navigation' },
+ );
+ expect(select).not.toContainEqual(eventNameAlias);
+ expect(row?.__hdx_event_name).toBe('browser.navigation');
+ });
+
+ it.each<{ lookup: string; dateRange?: [Date, Date] }>([
+ { lookup: 'unbounded' },
+ { lookup: 'bounded', dateRange: [new Date(0), new Date(1000)] },
+ ])(
+ 'waits for the table columns before the $lookup lookup',
+ ({ dateRange }) => {
+ mockColumns = { data: undefined, isLoading: true };
+ const { result } = renderHook(() =>
+ useRowData({
+ source: { ...source, eventNameExpression: 'EventName' },
+ rowId: "id='a'",
+ dateRange,
+ }),
+ );
+ for (const [, options] of mockUseQueriedChartConfig.mock.calls) {
+ expect(options.enabled).toBe(false);
+ }
+ expect(result.current.isLoading).toBe(true);
+ },
+ );
+
+ // Columns another page cached must not add a select that the Known
+ // Columns List leaves out.
+ it('selects nothing that a Known Columns List leaves out', () => {
+ mockColumns = { data: eventNameColumn('ALIAS'), isLoading: false };
+ const { select, row, checksColumns } = lookUp({
+ eventNameExpression: 'EventName',
+ knownColumnsListExpression: 'Timestamp, Body',
+ });
+ expect(checksColumns).toBe(false);
+ expect(select).not.toContainEqual(eventNameAlias);
+ expect(row?.__hdx_event_name).toBeUndefined();
+ });
+
+ it('reads a column that a Known Columns List names from the row', () => {
+ const { row } = lookUp(
+ {
+ eventNameExpression: 'EventName',
+ knownColumnsListExpression: 'Timestamp, Body, EventName',
+ },
+ { EventName: 'browser.navigation' },
+ );
+ expect(row?.__hdx_event_name).toBe('browser.navigation');
+ });
+
+ it('selects any other expression under the alias', () => {
+ const { select, checksColumns } = lookUp({
+ eventNameExpression: "LogAttributes['event.name']",
+ });
+ expect(checksColumns).toBe(false);
+ expect(select).toContainEqual({
+ valueExpression: "LogAttributes['event.name']",
+ alias: '__hdx_event_name',
+ });
+ });
+
+ it('reads no event name when the expression is unset', () => {
+ const { select, row } = lookUp({}, { EventName: 'browser.navigation' });
+ expect(select).not.toContainEqual(eventNameAlias);
+ expect(row?.__hdx_event_name).toBeUndefined();
+ });
+ });
+
it('adds no query settings while the viewer option is off', () => {
renderHook(() => useRowData({ source, rowId: "id='abc123'" }));
diff --git a/packages/app/src/components/__tests__/DBRowOverviewPanel.browserEvent.test.tsx b/packages/app/src/components/__tests__/DBRowOverviewPanel.browserEvent.test.tsx
new file mode 100644
index 000000000..7e96d162b
--- /dev/null
+++ b/packages/app/src/components/__tests__/DBRowOverviewPanel.browserEvent.test.tsx
@@ -0,0 +1,218 @@
+import React from 'react';
+import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types';
+import { screen, within } from '@testing-library/react';
+
+import { RowOverviewPanel } from '@/components/DBRowOverviewPanel';
+
+let mockRow: Record;
+jest.mock('../DBRowDataPanel', () => ({
+ useRowData: () => ({ data: { data: [mockRow] } }),
+ getJSONColumnNames: () => [],
+ getMapColumnNames: () => [],
+}));
+
+jest.mock('../NetworkPropertyPanel', () => ({
+ NetworkPropertySubpanel: () => null,
+}));
+
+jest.mock('../ExceptionSubpanel', () => ({ ExceptionSubpanel: () => null }));
+
+const mockJsonViewer = jest.fn((_props: { data: unknown }) => null);
+jest.mock('../DBRowJsonViewer', () => ({
+ DBRowJsonViewer: (props: { data: unknown }) => mockJsonViewer(props),
+}));
+
+jest.mock('../linkedSpans', () => ({
+ useReverseSpanLinks: () => ({ links: [] }),
+ useLinkedSpanDetails: () => ({ details: new Map() }),
+}));
+
+// `any`, not `as`, keeps this partial fixture off the lint warning budget.
+const asSource = (value: any): TSource => value;
+const LOG_SOURCE = asSource({
+ kind: SourceKind.Log,
+ eventAttributesExpression: 'LogAttributes',
+ bodyExpression: 'Body',
+});
+const TRACE_SOURCE = asSource({
+ kind: SourceKind.Trace,
+ eventAttributesExpression: 'SpanAttributes',
+});
+
+const PAGE_URL = 'http://localhost:5173/';
+const SELECTOR =
+ '#root > main > section.card.p-4:nth-child(2) > button.btn.btn-primary';
+const CLICK_ATTRIBUTES = {
+ 'browser.tag_name': 'BUTTON',
+ 'browser.css_selector': SELECTOR,
+ 'browser.document.url.full': PAGE_URL,
+ 'browser.element.attributes.testid': 'checkout',
+};
+
+function renderRow(
+ attributes: Record,
+ row: Record,
+ source = LOG_SOURCE,
+) {
+ mockRow = { __hdx_body: '', __hdx_event_attributes: attributes, ...row };
+ renderWithMantine();
+}
+
+const section = () => screen.getByTestId('browser-event-section');
+const expectDetail = (label: string, value: string) =>
+ expect(
+ within(section()).getByRole('row', { name: `${label} ${value}` }),
+ ).toBeInTheDocument();
+const noBodyPaper = () => screen.queryByText('No body for this event.');
+
+describe('RowOverviewPanel browser event section', () => {
+ it('summarizes a click and hides the empty body', () => {
+ renderRow(CLICK_ATTRIBUTES, {
+ __hdx_event_name: 'browser.user_action.click',
+ });
+
+ expect(within(section()).getByText('Browser event')).toBeInTheDocument();
+ expectDetail('Element', 'button');
+ expectDetail('CSS selector', SELECTOR);
+ expectDetail('data-otel-testid', 'checkout');
+ expectDetail('Page URL', PAGE_URL);
+ expect(noBodyPaper()).not.toBeInTheDocument();
+ });
+
+ it('shows a navigation instead of an HTTP request', () => {
+ renderRow(
+ {
+ 'url.full': `${PAGE_URL}checkout`,
+ 'browser.navigation.same_document': 'false',
+ 'browser.navigation.hash_change': 'false',
+ },
+ { __hdx_event_name: 'browser.navigation' },
+ );
+
+ expectDetail('URL', `${PAGE_URL}checkout`);
+ expectDetail('Same document', 'No');
+ expectDetail('Hash change', 'No');
+ expect(screen.queryByText('HTTP Request')).not.toBeInTheDocument();
+ });
+
+ it('reads navigation flags from a JSON column', () => {
+ renderRow(
+ {
+ 'browser.navigation.same_document': true,
+ 'browser.navigation.hash_change': false,
+ 'browser.navigation.type': 'push',
+ },
+ { __hdx_event_name: 'browser.navigation' },
+ );
+
+ expectDetail('Navigation type', 'push');
+ expectDetail('Same document', 'Yes');
+ expectDetail('Hash change', 'No');
+ });
+
+ it('shows the console method and the first line of the message', () => {
+ renderRow(
+ { 'browser.console.method': 'log' },
+ {
+ __hdx_event_name: 'browser.console',
+ __hdx_body: 'Checkout started\nstep 2',
+ },
+ );
+
+ expectDetail('Method', 'log');
+ expectDetail('Message', 'Checkout started');
+ expect(screen.getByText('Checkout started step 2')).toBeInTheDocument();
+ });
+
+ it('finds a click by its attributes when the row has no event name', () => {
+ renderRow(CLICK_ATTRIBUTES, {});
+
+ expectDetail('Element', 'button');
+ expect(noBodyPaper()).not.toBeInTheDocument();
+ });
+
+ it('keeps the HTTP request on a resource timing row', () => {
+ renderRow(
+ { 'url.full': `${PAGE_URL}assets/app.js` },
+ { __hdx_event_name: 'browser.resource_timing' },
+ );
+
+ expect(screen.getByText('HTTP Request')).toBeInTheDocument();
+ expect(
+ screen.queryByTestId('browser-event-section'),
+ ).not.toBeInTheDocument();
+ expect(noBodyPaper()).not.toBeInTheDocument();
+ });
+
+ it('keeps the exception section and the empty body on an exception', () => {
+ renderRow(
+ { 'exception.type': 'TypeError', 'exception.message': 'x is undefined' },
+ { __hdx_event_name: 'exception' },
+ );
+
+ expect(screen.getByText('Exception')).toBeInTheDocument();
+ expect(
+ screen.queryByTestId('browser-event-section'),
+ ).not.toBeInTheDocument();
+ expect(noBodyPaper()).toBeInTheDocument();
+ });
+
+ it('shows no section on a trace row', () => {
+ renderRow(CLICK_ATTRIBUTES, {}, TRACE_SOURCE);
+
+ expect(
+ screen.queryByTestId('browser-event-section'),
+ ).not.toBeInTheDocument();
+ });
+});
+
+describe('RowOverviewPanel top level attributes', () => {
+ const topLevelData = () =>
+ mockJsonViewer.mock.calls.map(([props]) => props.data);
+
+ beforeEach(() => mockJsonViewer.mockClear());
+
+ it('lists the EventName column', () => {
+ renderRow(CLICK_ATTRIBUTES, {
+ ServiceName: 'web',
+ EventName: 'browser.user_action.click',
+ });
+
+ expect(topLevelData()).toContainEqual({
+ ServiceName: 'web',
+ EventName: 'browser.user_action.click',
+ });
+ });
+
+ it('leaves out an empty EventName', () => {
+ renderRow({}, { ServiceName: 'api', EventName: '' });
+
+ expect(topLevelData()).toContainEqual({ ServiceName: 'api' });
+ });
+
+ it('lists an ALIAS EventName from the row alias', () => {
+ renderRow(
+ CLICK_ATTRIBUTES,
+ { ServiceName: 'web', __hdx_event_name: 'browser.user_action.click' },
+ asSource({ ...LOG_SOURCE, eventNameExpression: 'EventName' }),
+ );
+
+ expect(topLevelData()).toContainEqual({
+ ServiceName: 'web',
+ EventName: 'browser.user_action.click',
+ });
+ });
+
+ it('does not list another event name expression as EventName', () => {
+ renderRow(
+ CLICK_ATTRIBUTES,
+ { ServiceName: 'web', __hdx_event_name: 'browser.user_action.click' },
+ asSource({
+ ...LOG_SOURCE,
+ eventNameExpression: "LogAttributes['event.name']",
+ }),
+ );
+
+ expect(topLevelData()).toContainEqual({ ServiceName: 'web' });
+ });
+});
diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.browserEvent.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.browserEvent.test.tsx
new file mode 100644
index 000000000..22aca215f
--- /dev/null
+++ b/packages/app/src/components/__tests__/DBRowSidePanel.browserEvent.test.tsx
@@ -0,0 +1,169 @@
+import React from 'react';
+import { TSource } from '@hyperdx/common-utils/dist/types';
+import { MantineProvider } from '@mantine/core';
+import { render } from '@testing-library/react';
+
+jest.mock('nuqs', () => {
+ const actual = jest.requireActual('nuqs');
+ return {
+ ...actual,
+ useQueryState: (_key: string, parser?: { defaultValue?: unknown }) => [
+ parser && 'defaultValue' in parser ? parser.defaultValue : null,
+ jest.fn(),
+ ],
+ };
+});
+
+const mockRow: { current: Record } = { current: {} };
+jest.mock('../DBRowDataPanel', () => ({
+ __esModule: true,
+ useRowData: () => ({
+ data: { data: [mockRow.current] },
+ isLoading: false,
+ isSuccess: true,
+ isError: false,
+ error: null,
+ }),
+ ROW_DATA_ALIASES: {
+ DURATION_MS: '__hdx_duration',
+ SPAN_KIND: '__hdx_span_kind',
+ SERVICE_NAME: '__hdx_service_name',
+ SEVERITY_TEXT: '__hdx_severity_text',
+ EVENT_NAME: '__hdx_event_name',
+ EVENT_ATTRIBUTES: '__hdx_event_attributes',
+ },
+ rowHasK8sContext: () => false,
+ RowDataPanel: () => null,
+}));
+
+const mockHeaderProps: { current: { hideEmptyBody?: boolean } } = {
+ current: {},
+};
+jest.mock('../DBRowSidePanelHeader', () => ({
+ __esModule: true,
+ default: (props: { hideEmptyBody?: boolean }) => {
+ mockHeaderProps.current = props;
+ return null;
+ },
+}));
+
+jest.mock('@/source', () => ({
+ __esModule: true,
+ getEventBody: () => '__hdx_body',
+ useSource: () => ({ data: undefined }),
+}));
+jest.mock('../DBSessionPanel', () => ({
+ __esModule: true,
+ useSessionId: () => ({ rumSessionId: undefined, rumServiceName: undefined }),
+ DBSessionPanel: () => null,
+}));
+jest.mock('@/utils/highlightedAttributes', () => ({
+ __esModule: true,
+ getHighlightedAttributesFromData: () => [],
+}));
+jest.mock('../DBTracePanel', () => ({ __esModule: true, default: () => null }));
+jest.mock('../ContextSidePanel', () => ({
+ __esModule: true,
+ default: () => null,
+}));
+jest.mock('../DBInfraPanel', () => ({ __esModule: true, default: () => null }));
+jest.mock('../DBRowOverviewPanel', () => ({
+ __esModule: true,
+ RowOverviewPanel: () => null,
+}));
+jest.mock('../DBRowSidePanelErrorState', () => ({
+ __esModule: true,
+ DBRowSidePanelErrorState: () => null,
+}));
+jest.mock('../SidePanelBreadcrumbs', () => ({
+ __esModule: true,
+ default: () => null,
+}));
+jest.mock('../LogLevel', () => ({ __esModule: true, default: () => null }));
+jest.mock('../ServiceMap/ServiceMapSidePanel', () => ({
+ __esModule: true,
+ default: () => null,
+}));
+jest.mock('../TimelineChart/utils', () => ({
+ __esModule: true,
+ renderMs: () => '',
+}));
+jest.mock('../DrawerUtils', () => ({
+ __esModule: true,
+ DrawerFullWidthToggle: () => null,
+ INITIAL_DRAWER_WIDTH_PERCENT: 50,
+}));
+jest.mock('@/LogSidePanelElements', () => ({
+ __esModule: true,
+ KeyboardShortcutsModal: () => null,
+}));
+jest.mock('@/TabBar', () => ({ __esModule: true, default: () => null }));
+jest.mock('@/useFormatTime', () => ({
+ __esModule: true,
+ FormatTime: () => null,
+}));
+
+// Imported after the mock factories above.
+import { DBRowSidePanelInner } from '@/components/DBRowSidePanel';
+import useSidePanelStack from '@/hooks/useSidePanelStack';
+import { makeLogSource, makeTraceSource } from '@/llm/__fixtures__/sources';
+
+const LOG_SOURCE = makeLogSource();
+const TRACE_SOURCE = makeTraceSource();
+
+function Harness({ source }: { source: TSource }) {
+ const sidePanelStack = useSidePanelStack({ initialRowId: 'row-1' });
+ return (
+
+ );
+}
+
+function hideEmptyBodyFor(source: TSource, row: Record) {
+ mockRow.current = { __hdx_body: '', ...row };
+ render(
+
+
+ ,
+ );
+ return mockHeaderProps.current.hideEmptyBody;
+}
+
+describe('DBRowSidePanelInner, empty body on browser event rows', () => {
+ beforeEach(() => {
+ mockHeaderProps.current = {};
+ });
+
+ // Only the event name identifies a resource timing row; it has no detection
+ // keys for the attribute fallback to find.
+ it('hides it on a log row the event name marks as a browser event', () => {
+ expect(
+ hideEmptyBodyFor(LOG_SOURCE, {
+ __hdx_event_name: 'browser.resource_timing',
+ __hdx_event_attributes: { 'url.full': 'https://example.com/app.js' },
+ }),
+ ).toBe(true);
+ });
+
+ it('keeps it on a log row with no browser event name or keys', () => {
+ expect(
+ hideEmptyBodyFor(LOG_SOURCE, {
+ __hdx_event_name: '',
+ __hdx_event_attributes: { 'http.route': '/api' },
+ }),
+ ).toBe(false);
+ });
+
+ it('keeps it on a trace row that carries click keys', () => {
+ expect(
+ hideEmptyBodyFor(TRACE_SOURCE, {
+ __hdx_event_attributes: { 'browser.tag_name': 'BUTTON' },
+ }),
+ ).toBe(false);
+ });
+});
diff --git a/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx
index 9b49e56c3..1a1253929 100644
--- a/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx
+++ b/packages/app/src/components/__tests__/DBRowSidePanelHeader.test.tsx
@@ -89,6 +89,20 @@ describe('DBRowSidePanelHeader: body section (HDX-4373)', () => {
expect(screen.queryByText('No body for this event.')).toBeInTheDocument();
});
+ it('drops the empty-state paper when hideEmptyBody is set', () => {
+ renderWithMantine();
+ expect(
+ screen.queryByText('No body for this event.'),
+ ).not.toBeInTheDocument();
+ });
+
+ it('still renders a non-empty body when hideEmptyBody is set', () => {
+ renderWithMantine(
+ ,
+ );
+ expect(screen.getByText('checkout failed')).toBeInTheDocument();
+ });
+
it('omits the body paper when the panel title already shows the body', () => {
renderWithMantine(
{
},
);
});
+
+describe('getBrowserEventKind', () => {
+ it.each([
+ [CLICK, {}, 'click'],
+ [NAVIGATION, {}, 'navigation'],
+ [CONSOLE, {}, 'console'],
+ [EXCEPTION, { 'exception.type': 'TypeError' }, undefined],
+ [undefined, { 'browser.tag_name': 'BUTTON' }, 'click'],
+ ['', { 'browser.console.method': 'log' }, 'console'],
+ // JSON columns keep the Bool, Map columns hold the string.
+ [undefined, { 'browser.navigation.same_document': false }, 'navigation'],
+ [undefined, { 'browser.navigation.same_document': 'false' }, 'navigation'],
+ [undefined, { 'url.full': 'http://localhost:5173/' }, undefined],
+ [undefined, undefined, undefined],
+ ])('reads event %p with %p as %p', (eventName, attributes, kind) => {
+ expect(getBrowserEventKind({ eventName, attributes })).toBe(kind);
+ });
+});
+
+describe('isBrowserEventRow', () => {
+ it.each([
+ ['browser.web_vital', {}, true],
+ [EXCEPTION, {}, false],
+ [undefined, { 'browser.tag_name': 'BUTTON' }, true],
+ [undefined, { 'service.name': 'api' }, false],
+ ])('reads event %p with %p as %p', (eventName, attributes, expected) => {
+ expect(isBrowserEventRow({ eventName, attributes })).toBe(expected);
+ });
+});
diff --git a/packages/app/src/utils/browserEvents.ts b/packages/app/src/utils/browserEvents.ts
index 6591d35d1..f023d5804 100644
--- a/packages/app/src/utils/browserEvents.ts
+++ b/packages/app/src/utils/browserEvents.ts
@@ -38,7 +38,7 @@ function joinWords(...words: Array): string {
return words.filter(Boolean).join(' ');
}
-function firstLine(text: string): string {
+export function firstLine(text: string): string {
return text.split('\n', 1)[0].trim();
}
@@ -51,7 +51,9 @@ function setPairs(entries: [string, unknown][]): [string, string][] {
// The SDK sends data-otel-* attributes, prefix removed, as one map. The
// collector flattens it into `browser.element.attributes.` keys; without
// that step a Map column holds it as a JSON string.
-function getElementAttributes(attributes: Attributes): [string, string][] {
+export function getElementAttributes(
+ attributes: Attributes,
+): [string, string][] {
const prefix = `${ELEMENT_ATTRIBUTES_KEY}.`;
const flatPairs = setPairs(
Object.entries(attributes)
@@ -153,6 +155,12 @@ function getExceptionLabel(
};
}
+function toAttributes(attributes: unknown): Attributes {
+ return isRecord(attributes)
+ ? flatten(attributes)
+ : {};
+}
+
export function getBrowserEventLabel({
eventName,
body,
@@ -163,9 +171,7 @@ export function getBrowserEventLabel({
/** Flat keys from a Map column or nested objects from a JSON column. */
attributes: unknown;
}): BrowserEventLabel {
- const flat: Attributes = isRecord(attributes)
- ? flatten(attributes)
- : {};
+ const flat = toAttributes(attributes);
const text = typeof body === 'string' ? body : '';
switch (eventName) {
case BROWSER_EVENT_NAMES.CLICK:
@@ -186,3 +192,52 @@ export function getBrowserEventLabel({
};
}
}
+
+export type BrowserEventKind = 'click' | 'navigation' | 'console';
+
+type BrowserEventRow = {
+ eventName: unknown;
+ /** Flat keys from a Map column or nested objects from a JSON column. */
+ attributes: unknown;
+};
+
+const KIND_BY_EVENT_NAME = new Map([
+ [BROWSER_EVENT_NAMES.CLICK, 'click'],
+ [BROWSER_EVENT_NAMES.NAVIGATION, 'navigation'],
+ [BROWSER_EVENT_NAMES.CONSOLE, 'console'],
+]);
+
+// A row has no event name when its source predates the Event Name Expression
+// field, or when its table (or Known Columns List) has no such column. The SDK
+// always sends these keys with their kind, and only with it.
+const KIND_ATTRIBUTE_KEYS: [BrowserEventKind, string[]][] = [
+ ['click', ['browser.tag_name', 'browser.css_selector']],
+ [
+ 'navigation',
+ [
+ 'browser.navigation.same_document',
+ 'browser.navigation.hash_change',
+ 'browser.navigation.type',
+ ],
+ ],
+ ['console', [CONSOLE_METHOD_ATTRIBUTE]],
+];
+
+export function getBrowserEventKind({
+ eventName,
+ attributes,
+}: BrowserEventRow): BrowserEventKind | undefined {
+ if (isSet(eventName)) {
+ return KIND_BY_EVENT_NAME.get(String(eventName));
+ }
+ const flat = toAttributes(attributes);
+ return KIND_ATTRIBUTE_KEYS.find(([, keys]) =>
+ keys.some(key => isSet(flat[key])),
+ )?.[0];
+}
+
+export function isBrowserEventRow(row: BrowserEventRow): boolean {
+ return isSet(row.eventName)
+ ? String(row.eventName).startsWith('browser.')
+ : getBrowserEventKind(row) != null;
+}
diff --git a/packages/app/tests/e2e/features/sessions.spec.ts b/packages/app/tests/e2e/features/sessions.spec.ts
index 6e84754f1..806af2c16 100644
--- a/packages/app/tests/e2e/features/sessions.spec.ts
+++ b/packages/app/tests/e2e/features/sessions.spec.ts
@@ -93,6 +93,7 @@ test.describe('Client Sessions Functionality', { tag: ['@sessions'] }, () => {
}) => {
const panel = sessionsPage.sessionSidePanel;
const rows = sessionsPage.getSessionEventRows();
+ const section = panel.getByTestId('browser-event-section');
await test.step('Highlighted shows the console error but not the click', async () => {
await page.goto('/search');
@@ -116,6 +117,14 @@ test.describe('Client Sessions Functionality', { tag: ['@sessions'] }, () => {
await click.getByText('Clicked button').click();
await expect(panel.getByTestId('side-panel-tab-overview')).toBeVisible();
await expect(page.getByTestId('row-side-panel')).toHaveCount(0);
+ // The Log Attributes viewer below repeats these values verbatim.
+ await expect(
+ section.getByText('data-otel-testid', { exact: true }),
+ ).toBeVisible();
+ await expect(
+ section.getByText('#root > main > button.checkout', { exact: true }),
+ ).toBeVisible();
+ await expect(panel.getByText('No body for this event.')).toHaveCount(0);
});
await test.step('Escape returns to the event list', async () => {
@@ -123,6 +132,15 @@ test.describe('Client Sessions Functionality', { tag: ['@sessions'] }, () => {
await expect(rows.first()).toBeVisible();
});
+ await test.step('A navigation opens as a browser event, not an HTTP request', async () => {
+ const navigation = rows.filter({ hasText: 'Navigated' });
+ await navigation.getByText('Navigated').click();
+ await expect(section.getByText('push', { exact: true })).toBeVisible();
+ await expect(panel.getByText('HTTP Request')).toHaveCount(0);
+ await page.keyboard.press('Escape');
+ await expect(rows.first()).toBeVisible();
+ });
+
await test.step('A session keyed on rum.sessionId shows no log events', async () => {
const logQueries: string[] = [];
// query_id marks ClickHouse queries; the app's own telemetry exports
diff --git a/packages/app/tests/e2e/seed-otel-sessions.ts b/packages/app/tests/e2e/seed-otel-sessions.ts
index 74ede603f..ffd57684c 100644
--- a/packages/app/tests/e2e/seed-otel-sessions.ts
+++ b/packages/app/tests/e2e/seed-otel-sessions.ts
@@ -23,7 +23,35 @@ function replayRows(
{
type: 2,
data: {
- node: { type: 0, childNodes: [], id: 1 },
+ // rrweb inserts its style rules into , so replay throws without one.
+ node: {
+ type: 0,
+ id: 1,
+ childNodes: [
+ {
+ type: 2,
+ tagName: 'html',
+ attributes: {},
+ id: 2,
+ childNodes: [
+ {
+ type: 2,
+ tagName: 'head',
+ attributes: {},
+ childNodes: [],
+ id: 3,
+ },
+ {
+ type: 2,
+ tagName: 'body',
+ attributes: {},
+ childNodes: [],
+ id: 4,
+ },
+ ],
+ },
+ ],
+ },
initialOffset: { left: 0, top: 0 },
},
},
@@ -74,7 +102,8 @@ const RESOURCE_TIMING = {
/**
* OTel SDK log events around the +500 ms span: the click 3 ms before it that
- * triggers its trace, its request timing, and a console error.
+ * triggers its trace, its request timing, a same-document navigation, and a
+ * console error.
*/
export function generateOtelSessionLogEvents(seedRef: number): string {
const ns = (offsetMs: number) =>
@@ -89,6 +118,7 @@ export function generateOtelSessionLogEvents(seedRef: number): string {
return [
click(newSdk),
`('${ns(520)}', '${newSdk}-trace', '${newSdk}-span', 0, '', 9, '${SERVICE}', '', '', ${resource}, '', '@opentelemetry/browser-instrumentation/resource-timing', '', {}, {'session.id':'${newSdk}','browser.document.url.full':'${PAGE_URL}','url.full':'${PAGE_URL}api/error',${timing}}, 'browser.resource_timing')`,
+ `('${ns(700)}', '', '', 0, '', 9, '${SERVICE}', '', '', ${resource}, '', '', '', {}, {'session.id':'${newSdk}','url.full':'${PAGE_URL}checkout','browser.document.url.full':'${PAGE_URL}checkout','browser.navigation.same_document':'true','browser.navigation.hash_change':'false','browser.navigation.type':'push'}, 'browser.navigation')`,
`('${ns(800)}', '', '', 0, 'error', 17, '${SERVICE}', 'Error: checkout failed', '', ${resource}, '', '', '', {}, {'session.id':'${newSdk}','browser.console.method':'error','browser.document.url.full':'${PAGE_URL}'}, 'browser.console')`,
click(bothKeys),
].join(',\n');
From a13c80efe4a630663e455f09b5e02ab88edd90d4 Mon Sep 17 00:00:00 2001
From: Baris Can Vural
Date: Thu, 5 Nov 2026 01:07:58 +0300
Subject: [PATCH 031/103] feat(mcp): limit, having and a default order for
clickstack_table (#3558)
Closes #3555
## What
MCP surface only; dashboard tiles and the app don't change.
- **`limit`** (optional, positive integer): SQL `LIMIT`, applied after ordering. No default, so a call without it returns the same rows as today.
- **`having`** (optional): post-aggregation SQL `HAVING`. It can use a select item's alias (`"Errors > 10"`) or repeat the aggregate (`"count() > 100"`).
- **Default order:** with a `groupBy` and no `orderBy`, a `table` result is ordered by the first select item, descending. Today the order is whatever ClickHouse returns, so response trimming keeps arbitrary groups. An explicit `orderBy` is used exactly as before.
How:
- The default lives in the tool (`defaultOrderBy` in `packages/api/src/mcp/tools/query/table.ts`), next to `resolveOrderBy`, not in `renderChartConfig`. It uses the first item's alias when it has one, otherwise its position (`ORDER BY 1 DESC`): unaliased select items have no name to refer to, and the rendered `SELECT` lists the select items before the `groupBy` columns. It is skipped for metric selects (their SQL is built from CTEs, and `increase` already orders its top-N groups) and for the `number`/`pie`/`bar` shapes.
- `having` and `limit` reach the chart config through two new optional `runConfigTile` options, set only by `clickstack_table`. The table tile schema has no `limit`, and pie/bar tiles have no `having`, so passing them through the tile would drop them for some shapes; the options apply to every shape and leave the tile schemas alone. Other callers of `runConfigTile` don't pass them, so nothing changes for them.
## Tests
- `query.test.ts`: `defaultOrderBy` (position vs alias, quoted alias, no default without `groupBy`, for other shapes, or for metric selects).
- `queryTool.int.test.ts`, against ClickHouse with 3 services of 1/5/3 rows: default order with an alias and by position, explicit `orderBy` unchanged, `limit` after ordering, `having` on an alias.
- API MCP unit tests: 487 pass. `queryTool.int.test.ts`: 95 pass. `tsc --noEmit`, eslint (no new warnings) and prettier are clean.
Not included: unknown input keys are still stripped silently; that's #3543.
---
.changeset/mcp-table-limit-having-order.md | 5 +
packages/api/src/mcp/__tests__/query.test.ts | 38 +++++-
.../src/mcp/__tests__/queryTool.int.test.ts | 126 ++++++++++++++++++
packages/api/src/mcp/tools/query/helpers.ts | 18 +++
packages/api/src/mcp/tools/query/table.ts | 67 +++++++++-
5 files changed, 250 insertions(+), 4 deletions(-)
create mode 100644 .changeset/mcp-table-limit-having-order.md
diff --git a/.changeset/mcp-table-limit-having-order.md b/.changeset/mcp-table-limit-having-order.md
new file mode 100644
index 000000000..b03351d00
--- /dev/null
+++ b/.changeset/mcp-table-limit-having-order.md
@@ -0,0 +1,5 @@
+---
+'@hyperdx/api': patch
+---
+
+feat(mcp): `clickstack_table` takes optional `limit` and `having`, and orders grouped results by the first select item, descending, when no `orderBy` is given
diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts
index 8d4ea7a42..60a089c37 100644
--- a/packages/api/src/mcp/__tests__/query.test.ts
+++ b/packages/api/src/mcp/__tests__/query.test.ts
@@ -31,7 +31,7 @@ import {
whereLanguageSchema,
whereSchema,
} from '@/mcp/tools/query/schemas';
-import { resolveOrderBy } from '@/mcp/tools/query/table';
+import { defaultOrderBy, resolveOrderBy } from '@/mcp/tools/query/table';
import { getErrorCategory } from '@/mcp/utils/errors';
describe('parseTimeRange', () => {
@@ -1430,3 +1430,39 @@ describe('where / whereLanguage descriptions', () => {
);
});
});
+
+describe('defaultOrderBy', () => {
+ it('orders a grouped table by the first select item, descending', () => {
+ expect(defaultOrderBy([{}, {}], 'SpanName', 'table')).toBe(
+ '1 DESC, SpanName ASC',
+ );
+ });
+
+ it('uses the first select item alias when it has one', () => {
+ expect(defaultOrderBy([{ alias: 'Errors' }], 'SpanName', 'table')).toBe(
+ 'Errors DESC, SpanName ASC',
+ );
+ expect(
+ defaultOrderBy([{ alias: 'P95 Latency' }], 'SpanName', 'table'),
+ ).toBe('"P95 Latency" DESC, SpanName ASC');
+ });
+
+ it('quotes an alias with surrounding spaces as SELECT renders it', () => {
+ expect(defaultOrderBy([{ alias: ' Errors' }], 'SpanName', 'table')).toBe(
+ '" Errors" DESC, SpanName ASC',
+ );
+ expect(defaultOrderBy([{ alias: ' ' }], 'SpanName', 'table')).toBe(
+ '1 DESC, SpanName ASC',
+ );
+ });
+
+ it('adds no order without a groupBy, for other shapes, or for metric selects', () => {
+ expect(defaultOrderBy([{}], undefined, 'table')).toBeUndefined();
+ expect(defaultOrderBy([{}], ' ', 'table')).toBeUndefined();
+ expect(defaultOrderBy([{}], 'SpanName', 'pie')).toBeUndefined();
+ expect(defaultOrderBy([{}], 'SpanName', 'number')).toBeUndefined();
+ expect(
+ defaultOrderBy([{ metricType: 'gauge' }], 'SpanName', 'table'),
+ ).toBeUndefined();
+ });
+});
diff --git a/packages/api/src/mcp/__tests__/queryTool.int.test.ts b/packages/api/src/mcp/__tests__/queryTool.int.test.ts
index c481324a4..28765758a 100644
--- a/packages/api/src/mcp/__tests__/queryTool.int.test.ts
+++ b/packages/api/src/mcp/__tests__/queryTool.int.test.ts
@@ -364,6 +364,132 @@ describe('MCP Query Tools', () => {
expect(result.isError).toBeFalsy();
expect(result.content).toHaveLength(1);
});
+
+ describe('having, limit and the default order', () => {
+ const now = Date.now();
+ const tableArgs = () => ({
+ sourceId: logSource._id.toString(),
+ select: [{ aggFn: 'count', alias: 'n' }],
+ groupBy: 'ServiceName',
+ startTime: new Date(now - 60 * 60 * 1000).toISOString(),
+ endTime: new Date(now + 60 * 1000).toISOString(),
+ });
+ const rows = (result: Awaited>) => {
+ const parsed: { result: { data: { ServiceName: string }[] } } =
+ JSON.parse(getFirstText(result));
+ return parsed.result.data;
+ };
+
+ beforeEach(async () => {
+ // a: 1 row, b: 5 rows, c: 3 rows
+ const logs: Parameters[0] = [];
+ for (const [service, count] of [
+ ['a', 1],
+ ['b', 5],
+ ['c', 3],
+ ] as const) {
+ for (let i = 0; i < count; i++) {
+ logs.push({
+ Body: `event ${i}`,
+ ServiceName: service,
+ SeverityText: 'INFO',
+ Timestamp: new Date(now - 60 * 1000 + i),
+ });
+ }
+ }
+ await bulkInsertLogs(logs);
+ });
+
+ it('orders by the first select item, descending, when orderBy is omitted', async () => {
+ const result = await callTool(client, 'clickstack_table', tableArgs());
+ expect(result.isError).toBeFalsy();
+ expect(rows(result).map(r => r.ServiceName)).toEqual(['b', 'c', 'a']);
+ });
+
+ it('orders by position when the first select item has no alias', async () => {
+ const result = await callTool(client, 'clickstack_table', {
+ ...tableArgs(),
+ select: [{ aggFn: 'count' }],
+ });
+ expect(result.isError).toBeFalsy();
+ expect(rows(result).map(r => r.ServiceName)).toEqual(['b', 'c', 'a']);
+ });
+
+ it('keeps an explicit orderBy', async () => {
+ const result = await callTool(client, 'clickstack_table', {
+ ...tableArgs(),
+ orderBy: 'ServiceName ASC',
+ });
+ expect(rows(result).map(r => r.ServiceName)).toEqual(['a', 'b', 'c']);
+ });
+
+ it('applies limit after ordering', async () => {
+ const result = await callTool(client, 'clickstack_table', {
+ ...tableArgs(),
+ limit: 2,
+ });
+ expect(rows(result).map(r => r.ServiceName)).toEqual(['b', 'c']);
+ });
+
+ it('applies having to the groups', async () => {
+ const result = await callTool(client, 'clickstack_table', {
+ ...tableArgs(),
+ having: 'n > 1',
+ });
+ expect(rows(result).map(r => r.ServiceName)).toEqual(['b', 'c']);
+ });
+
+ it('orders by a quoted alias with a space, by default and explicitly', async () => {
+ const select = [{ aggFn: 'count', alias: 'Line Count' }];
+ const byDefault = await callTool(client, 'clickstack_table', {
+ ...tableArgs(),
+ select,
+ });
+ expect(byDefault.isError).toBeFalsy();
+ expect(rows(byDefault).map(r => r.ServiceName)).toEqual([
+ 'b',
+ 'c',
+ 'a',
+ ]);
+ const explicit = await callTool(client, 'clickstack_table', {
+ ...tableArgs(),
+ select,
+ orderBy: '"Line Count" ASC',
+ });
+ expect(explicit.isError).toBeFalsy();
+ expect(rows(explicit).map(r => r.ServiceName)).toEqual(['a', 'c', 'b']);
+ });
+
+ it.each(['pie', 'bar'] as const)(
+ 'keeps the largest groups with limit on shape "%s"',
+ async shape => {
+ const result = await callTool(client, 'clickstack_table', {
+ ...tableArgs(),
+ shape,
+ limit: 2,
+ });
+ expect(result.isError).toBeFalsy();
+ expect(rows(result).map(r => r.ServiceName)).toEqual(['b', 'c']);
+ },
+ );
+
+ it.each(['pie', 'bar'] as const)(
+ 'applies having on shape "%s"',
+ async shape => {
+ const result = await callTool(client, 'clickstack_table', {
+ ...tableArgs(),
+ shape,
+ having: 'n > 1',
+ });
+ expect(result.isError).toBeFalsy();
+ expect(
+ rows(result)
+ .map(r => r.ServiceName)
+ .sort(),
+ ).toEqual(['b', 'c']);
+ },
+ );
+ });
});
// ─── clickstack_search ─────────────────────────────────────────────────────────
diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts
index dd3dde0d4..0ea157aef 100644
--- a/packages/api/src/mcp/tools/query/helpers.ts
+++ b/packages/api/src/mcp/tools/query/helpers.ts
@@ -423,6 +423,10 @@ export async function runConfigTile(
variables?: ChartVariable[];
/** Serialized size the result is trimmed to. Defaults to trimToolResponse's limit. */
maxResponseChars?: number;
+ /** Post-aggregation SQL HAVING condition for builder tiles (MCP table tool). */
+ having?: string;
+ /** SQL LIMIT for builder tiles that aren't searches (MCP table tool). */
+ limit?: number;
/** Source lookup for raw SQL tiles, so a batch can share one per-call cache. */
lookupSource?: (sourceId: string) => ReturnType;
},
@@ -583,9 +587,23 @@ export async function runConfigTile(
? { granularity: options?.granularity ?? 'auto' }
: {};
+ // having/limit come from the MCP table tool's own inputs, not from the
+ // tile, so dashboard tiles and the app keep their current behaviour.
+ const tableToolOverrides = isSearch
+ ? {}
+ : {
+ ...(options?.having
+ ? { having: options.having, havingLanguage: 'sql' as const }
+ : {}),
+ ...(options?.limit
+ ? { limit: { limit: options.limit, offset: 0 } }
+ : {}),
+ };
+
const chartConfig = {
...builderConfig,
...searchOverrides,
+ ...tableToolOverrides,
...metricSelectOverrides,
...granularityOverride,
from: {
diff --git a/packages/api/src/mcp/tools/query/table.ts b/packages/api/src/mcp/tools/query/table.ts
index 6c757bd33..5c1a1f035 100644
--- a/packages/api/src/mcp/tools/query/table.ts
+++ b/packages/api/src/mcp/tools/query/table.ts
@@ -59,7 +59,32 @@ const tableSchema = z.object({
),
whereLanguage: whereLanguageSchema,
groupBy: groupBySchema,
- orderBy: orderBySchema,
+ having: z
+ .string()
+ .trim()
+ .max(10000)
+ .optional()
+ .describe(
+ 'Post-aggregation SQL HAVING condition, applied after grouping. ' +
+ 'Give the select item an alias and reference it, e.g. alias "Errors" and having "Errors > 10". ' +
+ 'A bare aggregate such as count() ignores the top-level where.',
+ ),
+ orderBy: orderBySchema.describe(
+ 'Sort results by this column. For shape "table" on a non-metric source with groupBy set, ' +
+ 'defaults to the first select item descending, then the groupBy columns. ' +
+ 'When ordering by an alias that contains spaces or special characters, ' +
+ `wrap the alias in quotes: e.g. '"P95 Latency" DESC'.`,
+ ),
+ limit: z
+ .number()
+ .int()
+ .positive()
+ .max(10000)
+ .optional()
+ .describe(
+ 'Maximum number of rows (SQL LIMIT), applied after ordering. Omit to return every group. ' +
+ 'Metric sources get no default order, so set orderBy as well, or the kept groups are arbitrary.',
+ ),
startTime: startTimeSchema,
endTime: endTimeSchema,
});
@@ -196,6 +221,38 @@ export function resolveOrderBy(
return orderBy;
}
+/**
+ * Default ORDER BY for grouped tables when the caller sets none: the first
+ * select item, descending, so trimmed or limited results keep the largest
+ * groups. MCP only; dashboard tiles and the app are unchanged.
+ *
+ * Unaliased select items have no name to refer to, so the first one is
+ * referenced by position (`1`): the rendered SELECT lists the select items
+ * first, then the groupBy columns. Metric sources are skipped: their SQL is
+ * built from CTEs, and `increase` already orders its top-N groups.
+ *
+ * The groupBy columns follow as a tiebreak, so a `limit` keeps the same groups
+ * on every call, as convertToCategoricalChartConfig does for pie/bar.
+ *
+ * @internal Exported for testing only.
+ */
+export function defaultOrderBy(
+ selectItems: { alias?: string; metricType?: string }[],
+ groupBy: string | undefined,
+ displayType: 'table' | 'number' | 'pie' | 'bar',
+): string | undefined {
+ if (displayType !== 'table' || !groupBy?.trim()) return undefined;
+ if (selectItems.length === 0 || selectItems.some(s => s.metricType)) {
+ return undefined;
+ }
+ // Quote the alias exactly as SELECT renders it; trim only to test emptiness.
+ const alias = selectItems[0].alias;
+ const first = alias?.trim()
+ ? `${quoteAliasForOrderBy(alias)} DESC`
+ : '1 DESC';
+ return `${first}, ${groupBy.trim()} ASC`;
+}
+
// ─── Tool registration ───────────────────────────────────────────────────────
export function registerTable({ context, registerTool }: ToolRegistrar) {
@@ -270,12 +327,15 @@ export function registerTable({ context, registerTool }: ToolRegistrar) {
const { items: selectItems, warnings: mergeWarnings } =
mergeWhereIntoSelectItems(select, input.where, input.whereLanguage);
+ const groupBy = displayType === 'number' ? undefined : input.groupBy;
const tile = buildTile('MCP Table', 12, 4, {
displayType,
sourceId: input.sourceId,
select: selectItems,
- groupBy: displayType === 'number' ? undefined : input.groupBy,
- orderBy: resolveOrderBy(input.orderBy, selectItems),
+ groupBy,
+ orderBy:
+ resolveOrderBy(input.orderBy, selectItems) ??
+ defaultOrderBy(selectItems, groupBy, displayType),
});
const result = await runConfigTile(
@@ -283,6 +343,7 @@ export function registerTable({ context, registerTool }: ToolRegistrar) {
tile,
startDate,
endDate,
+ { having: input.having, limit: input.limit },
);
// Surface language-mismatch warnings so the agent knows the top-level
From 5584f84bbf513467ba1f52ca8906cd5285e2ca17 Mon Sep 17 00:00:00 2001
From: Brandon Pereira
Date: Wed, 4 Nov 2026 16:26:22 -0600
Subject: [PATCH 032/103] fix(mcp): make event_patterns drill-down snippets
executable (#3426)
## Summary
`clickstack_event_patterns` returned a `whereSnippet` per pattern that agents were told to pass to `clickstack_search`, but it often matched nothing. A template like `Sending <*> rows` became `Body:"Sending rows"` (a phrase that never occurs), and map-field body expressions produced Lucene that failed to parse. Agents hit dead ends on the main drill-down path.
Changes:
- **Executable drill-down.** `whereSnippet` is now a SQL `match()` predicate (`whereLanguage: "sql"`) built from the template: literal tokens in order, one non-whitespace token per `<*>`, whitespace matching Drain's JS `\s` tokenization. It already ANDs in the caller's `where` (rendered to SQL, dashboard variables expanded) and the source's table filter. If the caller filter can't be rendered as SQL, `summary.drilldown.includesFilters` is `false` with a warning. Templates over 4000 chars get `drilldownUnavailable` instead of an empty snippet that would match everything.
- **Source table filter.** Pattern mining (`event_patterns`, `emerging_signals`, dashboard pattern tiles) now applies the log source's `tableFilterExpression`. Counts can drop for sources that set one.
- **`bodyExpression`** also accepts `JSONExtractString(, '', ...)`. It is rebuilt from validated args, so the injection allowlist still holds.
- **Sampling transparency.** Response reports `sampleCoverage`, and `summary.trend` gives the real bucket interval and boundaries (granularity rounding means fewer or wider buckets than requested). Tool description and usage text explain that counts are extrapolated and a 0 bucket doesn't mean no events.
- **`minePatterns` template bug.** Each cluster now reports its own template. Before, it re-matched the first member's body and could pick up a different same-length cluster's template that other members didn't fit, so the drill-down missed some of the pattern's own events.
### How to reproduce
**Starting point:** `yarn dev`, MCP client connected to the local API, a log source with data.
**On `main`:**
1. Call `clickstack_event_patterns` on a log source.
2. Pass a pattern containing `<*>` as `where` to `clickstack_search`. It returns no rows, or a parse error when `bodyExpression` is a map field.
**On this branch:**
1. Same call. Each pattern has a SQL `whereSnippet` and `whereLanguage: "sql"`.
2. Pass it to `clickstack_search` with `whereLanguage: "sql"`. It returns that pattern's events.
Covered by new unit tests (`patternDrilldown.test.ts`, `mine-patterns.test.ts`, `query.test.ts`) and integration tests in `queryTool.int.test.ts` / `queryTile.int.test.ts` that run each snippet through search.
### References
HDX-5748
---
.changeset/mcp-event-patterns-drilldown.md | 10 +
packages/api/src/fixtures.ts | 1 +
.../dashboards/queryTile.int.test.ts | 26 ++
packages/api/src/mcp/__tests__/query.test.ts | 55 +++
.../src/mcp/__tests__/queryTool.int.test.ts | 396 +++++++++++++++++-
.../api/src/mcp/prompts/dashboards/content.ts | 31 +-
.../mcp/tools/query/__tests__/denoise.test.ts | 108 +++++
.../query/__tests__/patternDrilldown.test.ts | 179 ++++++++
packages/api/src/mcp/tools/query/denoise.ts | 74 +++-
.../api/src/mcp/tools/query/eventPatterns.ts | 58 ++-
packages/api/src/mcp/tools/query/helpers.ts | 62 ++-
.../src/mcp/tools/query/patternDrilldown.ts | 108 +++++
.../src/mcp/tools/query/runEventPatterns.ts | 239 ++++++++---
.../src/__tests__/mine-patterns.test.ts | 113 +++++
.../src/core/renderChartConfig.ts | 8 +-
.../src/core/searchChartConfig.ts | 17 +-
packages/common-utils/src/drain/drain.ts | 5 +
.../common-utils/src/drain/mine-patterns.ts | 62 ++-
18 files changed, 1401 insertions(+), 151 deletions(-)
create mode 100644 .changeset/mcp-event-patterns-drilldown.md
create mode 100644 packages/api/src/mcp/tools/query/__tests__/denoise.test.ts
create mode 100644 packages/api/src/mcp/tools/query/__tests__/patternDrilldown.test.ts
create mode 100644 packages/api/src/mcp/tools/query/patternDrilldown.ts
diff --git a/.changeset/mcp-event-patterns-drilldown.md b/.changeset/mcp-event-patterns-drilldown.md
new file mode 100644
index 000000000..edda3830e
--- /dev/null
+++ b/.changeset/mcp-event-patterns-drilldown.md
@@ -0,0 +1,10 @@
+---
+'@hyperdx/api': patch
+'@hyperdx/common-utils': patch
+---
+
+fix: make clickstack_event_patterns drill-down snippets executable
+
+Each pattern's `whereSnippet` is now a SQL predicate (`whereLanguage: "sql"`) that keeps the template's token order and variable positions and includes the caller's filter and the source table filter, so passing it to `clickstack_search` returns the pattern's events. Before, a template like `Sending <*> rows` produced `Body:"Sending rows"`, which matched nothing, and map fields produced Lucene that failed to parse. `bodyExpression` now also accepts `JSONExtractString(, '', ...)` to mine a field inside JSON bodies, and the tool reports sample coverage and the actual trend bucket interval and boundaries.
+
+`clickstack_event_patterns`, `clickstack_emerging_signals` and dashboard event-pattern tiles now apply the log source's `tableFilterExpression`, so their counts can drop for sources that set one. `minePatterns` now reports each cluster's own template; before, it could report the template of a different same-length cluster that some members did not fit.
diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts
index 47d2edafd..26df966c5 100644
--- a/packages/api/src/fixtures.ts
+++ b/packages/api/src/fixtures.ts
@@ -434,6 +434,7 @@ export const bulkInsertLogs = async (
ServiceName: string;
SeverityText: string;
Timestamp: Date;
+ LogAttributes?: Record;
}[],
) => {
if (!config.IS_CI) {
diff --git a/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts b/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts
index d2c2ab73a..48de1b8f6 100644
--- a/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts
+++ b/packages/api/src/mcp/__tests__/dashboards/queryTile.int.test.ts
@@ -891,6 +891,30 @@ describe('MCP Dashboard Tools - clickstack_query_tile', () => {
};
};
+ // clickstack_search has no variable context, so each snippet must carry
+ // the expanded selection and return only the selected service's rows.
+ const expectDrilldownServices = async (text: string, service: string) => {
+ const { patterns } = JSON.parse(text);
+ expect(patterns.length).toBeGreaterThan(0);
+ for (const p of patterns) {
+ expect(p.whereSnippet).not.toContain('$__filter');
+ expect(p.whereSnippet).not.toContain('$service');
+ const search = await callTool(ctx.client!, 'clickstack_search', {
+ sourceId: logSourceId,
+ startTime,
+ endTime,
+ where: p.whereSnippet,
+ whereLanguage: 'sql',
+ columns: 'ServiceName',
+ });
+ expect(search.isError).toBeFalsy();
+ const rows: { ServiceName: string }[] = JSON.parse(getFirstText(search))
+ .result.data;
+ expect(rows.length).toBeGreaterThan(0);
+ for (const r of rows) expect(r.ServiceName).toBe(service);
+ }
+ };
+
it('expands $__filter to a no-op on an event_patterns tile with nothing selected', async () => {
const dashboard = await saveDashboard([eventPatternsTile()]);
@@ -931,6 +955,7 @@ describe('MCP Dashboard Tools - clickstack_query_tile', () => {
expect(sampledCount).toBe(2);
expect(bodies.some(b => b.includes('checkout'))).toBe(true);
expect(bodies.some(b => b.includes('payments'))).toBe(false);
+ await expectDrilldownServices(getFirstText(result), 'checkout');
});
it('expands a Lucene variable reference on an event_patterns tile', async () => {
@@ -953,6 +978,7 @@ describe('MCP Dashboard Tools - clickstack_query_tile', () => {
expect(totalCount).toBe(1);
expect(bodies.some(b => b.includes('checkout'))).toBe(false);
expect(bodies.some(b => b.includes('payments'))).toBe(true);
+ await expectDrilldownServices(getFirstText(result), 'payments');
});
it('reports an event_patterns macro naming a variable the dashboard does not declare', async () => {
diff --git a/packages/api/src/mcp/__tests__/query.test.ts b/packages/api/src/mcp/__tests__/query.test.ts
index 60a089c37..1fb8106a5 100644
--- a/packages/api/src/mcp/__tests__/query.test.ts
+++ b/packages/api/src/mcp/__tests__/query.test.ts
@@ -21,6 +21,7 @@ import {
INCREASE_TOP_N_CAP,
isServerError,
mergeWhereIntoSelectItems,
+ normalizeBodyExpression,
parseTimeRange,
} from '@/mcp/tools/query/helpers';
import {
@@ -109,6 +110,60 @@ describe('parseTimeRange', () => {
// ─── mergeWhereIntoSelectItems ───────────────────────────────────────────────
+describe('normalizeBodyExpression', () => {
+ it.each([
+ ['Body', 'Body'],
+ ['SpanName', 'SpanName'],
+ ["SpanAttributes['http.url']", "SpanAttributes['http.url']"],
+ [
+ "JSONExtractString(Body, 'message')",
+ "JSONExtractString(Body, 'message')",
+ ],
+ [
+ "JSONExtractString(Body,'error', 'message')",
+ "JSONExtractString(Body, 'error', 'message')",
+ ],
+ [
+ "JSONExtractString(Body, 'app.user.currency')",
+ "JSONExtractString(Body, 'app.user.currency')",
+ ],
+ [
+ "JSONExtractString(LogAttributes['object'], 'regarding', 'kind')",
+ "JSONExtractString(LogAttributes['object'], 'regarding', 'kind')",
+ ],
+ [
+ String.raw`JSONExtractString(Body, 'it\'s, ok')`,
+ String.raw`JSONExtractString(Body, 'it\'s, ok')`,
+ ],
+ [
+ "JSONExtractString(Body, 'it''s, ok')",
+ "JSONExtractString(Body, 'it''s, ok')",
+ ],
+ ])('accepts %s', (expr, expected) => {
+ expect(normalizeBodyExpression(expr)).toBe(expected);
+ });
+
+ it.each([
+ 'Body) OR (1=1',
+ 'lower(Body)',
+ "JSONExtract(Body, 'message', 'String')",
+ 'JSONExtractString(Body)',
+ 'JSONExtractString(Body, message)',
+ "JSONExtractString(lower(Body), 'message')",
+ "JSONExtractString(Body, 'm') OR 1=1",
+ "JSONExtractString(Body, 'm'), Body",
+ "JSONExtractString(Body, 'm' || 'x')",
+ "JSONExtractString(Body, 'm')) OR (lower(Body)",
+ "JSONExtractString(LogAttributes['a'], 'm') OR (1=1",
+ "JSONExtractString(Body, 'm', (SELECT 1))",
+ // A stray quote in the column would pair with a key's quotes.
+ "JSONExtractString(Body', ' OR 1=1 --')",
+ "JSONExtractString(Body, 'm'' OR 1=1)",
+ ])('rejects %s', expr => {
+ expect(normalizeBodyExpression(expr)).toBeNull();
+ });
+});
+
describe('mergeWhereIntoSelectItems', () => {
it('should return items unchanged when topWhere is empty', () => {
const items = [{ where: 'level:error', whereLanguage: 'lucene' as const }];
diff --git a/packages/api/src/mcp/__tests__/queryTool.int.test.ts b/packages/api/src/mcp/__tests__/queryTool.int.test.ts
index 28765758a..7b7cec82a 100644
--- a/packages/api/src/mcp/__tests__/queryTool.int.test.ts
+++ b/packages/api/src/mcp/__tests__/queryTool.int.test.ts
@@ -18,9 +18,10 @@ import {
getServer,
seedExponentialHistogramMetric,
} from '@/fixtures';
+import * as patternDrilldown from '@/mcp/tools/query/patternDrilldown';
import { McpContext } from '@/mcp/tools/types';
import Connection from '@/models/connection';
-import { Source, type SourceDocument } from '@/models/source';
+import { LogSource, Source, type SourceDocument } from '@/models/source';
import Team from '@/models/team';
import { callTool, createTestClient, getFirstText } from './mcpTestUtils';
@@ -816,6 +817,399 @@ describe('MCP Query Tools', () => {
expect(hasNonZeroTrend).toBe(true);
});
});
+
+ describe('drill-down, extraction and sampling', () => {
+ const now = new Date();
+ const base = new Date(now.getTime() - 5 * 60 * 1000);
+ const window = {
+ startTime: new Date(now.getTime() - 10 * 60 * 1000).toISOString(),
+ endTime: new Date(now.getTime() + 60 * 1000).toISOString(),
+ };
+
+ const tricky = [
+ ...['k1', 'k2', 'k3'].map(
+ k =>
+ String.raw`Lookup "${k}" in C:\data\idx it's (a|b)*[c]+ $1 ^x {n} .?`,
+ ),
+ ...['a1', 'b2'].map(id => `Zahlung für ${id} fehlgeschlagen ✓ 日本`),
+ // Drain splits on these; RE2's \s alone would not.
+ ...['1', '2'].map(n => `\uFEFFReset ${n}\u00A0done`),
+ ...['1\u202F234', '5\u202F678'].map(n => `Paid ${n} EUR`),
+ ];
+
+ beforeEach(async () => {
+ let t = 0;
+ const at = () => new Date(base.getTime() + t++ * 1000);
+ const logs: Parameters[0] = [];
+ for (const n of [10, 20, 30, 40, 50, 60]) {
+ logs.push({
+ Body: `Sending ${n} rows`,
+ ServiceName: 'svc-a',
+ SeverityText: 'INFO',
+ Timestamp: at(),
+ });
+ }
+ for (const n of [1, 2, 3]) {
+ logs.push({
+ Body: `Sending ${n} more rows`,
+ ServiceName: 'svc-a',
+ SeverityText: 'INFO',
+ Timestamp: at(),
+ });
+ }
+ for (const n of [70, 80, 90, 100]) {
+ logs.push({
+ Body: `Sending ${n} rows`,
+ ServiceName: 'svc-b',
+ SeverityText: 'INFO',
+ Timestamp: at(),
+ });
+ }
+ for (const body of tricky) {
+ logs.push({
+ Body: body,
+ ServiceName: 'tricky',
+ SeverityText: 'WARN',
+ Timestamp: at(),
+ });
+ }
+ for (const s of [5, 15, 30]) {
+ logs.push({
+ Body: 'request failed',
+ ServiceName: 'attrs',
+ SeverityText: 'ERROR',
+ Timestamp: at(),
+ LogAttributes: { 'exception.message': `Timeout after ${s} s` },
+ });
+ }
+ logs.push({
+ Body: 'request failed',
+ ServiceName: 'attrs',
+ SeverityText: 'ERROR',
+ Timestamp: at(),
+ });
+ for (const id of [123, 456, 789]) {
+ logs.push({
+ Body: JSON.stringify({
+ level: 'error',
+ message: `Payment ${id} declined by issuer`,
+ }),
+ ServiceName: 'json',
+ SeverityText: 'ERROR',
+ Timestamp: at(),
+ });
+ }
+ for (const body of [
+ JSON.stringify({ message: 5 }),
+ JSON.stringify({ message: null }),
+ JSON.stringify({ message: { nested: 'x' } }),
+ JSON.stringify({ other: 'x' }),
+ JSON.stringify({ message: '' }),
+ 'not json at all',
+ ]) {
+ logs.push({
+ Body: body,
+ ServiceName: 'json',
+ SeverityText: 'ERROR',
+ Timestamp: at(),
+ });
+ }
+ await bulkInsertLogs(logs);
+ });
+
+ type MinedPattern = {
+ pattern: string;
+ sampleCount: number;
+ estimatedCount: number;
+ whereSnippet: string;
+ whereLanguage: string;
+ drilldownUnavailable?: string;
+ samples: string[];
+ trend?: { time: string; count: number }[];
+ };
+ type MinedOutput = {
+ summary: {
+ totalCount: number;
+ sampledCount: number;
+ sampleCoverage: number;
+ sampleMultiplier: number;
+ bodyColumn: string;
+ timeRange: { start: string; end: string };
+ trend?: {
+ requestedBuckets: number;
+ bucketCount: number;
+ bucketInterval: string;
+ firstBucketStart: string;
+ lastBucketEnd: string;
+ };
+ drilldown: {
+ tool: string;
+ sourceId: string;
+ startTime: string;
+ endTime: string;
+ includesFilters: boolean;
+ warning?: string;
+ };
+ };
+ patterns: MinedPattern[];
+ };
+
+ const mine = async (
+ args: Record,
+ ): Promise => {
+ const result = await callTool(client, 'clickstack_event_patterns', {
+ sourceId: logSource._id.toString(),
+ ...window,
+ ...args,
+ });
+ expect(result.isError).toBeFalsy();
+ return JSON.parse(getFirstText(result));
+ };
+
+ const drill = async (output: MinedOutput, pattern: MinedPattern) => {
+ expect(pattern.whereLanguage).toBe('sql');
+ expect(pattern.whereSnippet).not.toBe('');
+ const { drilldown } = output.summary;
+ expect(drilldown.tool).toBe('clickstack_search');
+ const result = await callTool(client, 'clickstack_search', {
+ sourceId: drilldown.sourceId,
+ startTime: drilldown.startTime,
+ endTime: drilldown.endTime,
+ where: pattern.whereSnippet,
+ whereLanguage: pattern.whereLanguage,
+ columns: 'Body, ServiceName',
+ maxResults: 200,
+ });
+ expect(result.isError).toBeFalsy();
+ const parsed: {
+ result: { data: { Body: string; ServiceName: string }[] };
+ } = JSON.parse(getFirstText(result));
+ return parsed.result.data;
+ };
+
+ const findPattern = (output: MinedOutput, template: string) => {
+ const p = output.patterns.find(x => x.pattern === template);
+ if (!p) throw new Error(`pattern not found: ${template}`);
+ return p;
+ };
+
+ it('drills into a template with a mid-message variable and keeps a Lucene filter', async () => {
+ const output = await mine({ where: 'ServiceName:svc-a' });
+ expect(output.summary.drilldown.sourceId).toBe(
+ logSource._id.toString(),
+ );
+
+ const rows = await drill(
+ output,
+ findPattern(output, 'Sending <*> rows'),
+ );
+ expect(rows).toHaveLength(6);
+ for (const r of rows) {
+ expect(r.Body).toMatch(/^Sending \d+ rows$/);
+ expect(r.ServiceName).toBe('svc-a');
+ }
+
+ const more = await drill(
+ output,
+ findPattern(output, 'Sending <*> more rows'),
+ );
+ expect(more).toHaveLength(3);
+ for (const r of more) expect(r.Body).toMatch(/^Sending \d more rows$/);
+ });
+
+ it('keeps a SQL filter in the drill-down', async () => {
+ const output = await mine({
+ where: "ServiceName = 'svc-b'",
+ whereLanguage: 'sql',
+ });
+ const rows = await drill(
+ output,
+ findPattern(output, 'Sending <*> rows'),
+ );
+ expect(rows).toHaveLength(4);
+ for (const r of rows) expect(r.ServiceName).toBe('svc-b');
+ });
+
+ it('builds predicates that parse and match messages with quotes, backslashes, regex characters and Unicode', async () => {
+ const output = await mine({ where: 'ServiceName:tricky' });
+ expect(output.summary.totalCount).toBe(tricky.length);
+ expect(output.patterns.length).toBeGreaterThan(0);
+
+ let drilled = 0;
+ for (const p of output.patterns) {
+ const rows = await drill(output, p);
+ // The sample covers every row and the fixture shapes do not overlap,
+ // so the drill-down must return exactly the cluster's members.
+ expect(rows).toHaveLength(p.sampleCount);
+ for (const r of rows) expect(tricky).toContain(r.Body);
+ drilled += rows.length;
+ }
+ expect(drilled).toBe(tricky.length);
+ });
+
+ it('mines a map attribute and drills into it with SQL, not Lucene bracket syntax', async () => {
+ const output = await mine({
+ bodyExpression: "LogAttributes['exception.message']",
+ where: "LogAttributes['exception.message'] != ''",
+ whereLanguage: 'sql',
+ });
+ expect(output.summary.totalCount).toBe(3);
+ const p = findPattern(output, 'Timeout after <*> s');
+ expect(p.whereSnippet).toContain(
+ "match(LogAttributes['exception.message'],",
+ );
+ expect(p.whereSnippet).not.toContain("']:");
+
+ const rows = await drill(output, p);
+ expect(rows).toHaveLength(3);
+ for (const r of rows) expect(r.ServiceName).toBe('attrs');
+ });
+
+ it('mines a field inside JSON bodies with JSONExtractString and drills with the same expression', async () => {
+ const expr = "JSONExtractString(Body, 'message')";
+ const output = await mine({
+ bodyExpression: expr,
+ where: `ServiceName = 'json' AND ${expr} != ''`,
+ whereLanguage: 'sql',
+ });
+ expect(output.summary.bodyColumn).toBe(expr);
+ // Non-JSON bodies and missing / null / empty values extract to '' and
+ // are filtered out. How numbers and objects extract varies by
+ // ClickHouse version, so only the payment strings are asserted.
+ expect(output.summary.totalCount).toBeGreaterThanOrEqual(3);
+ const allSamples = output.patterns.flatMap(x => x.samples);
+ expect(allSamples).not.toContain('');
+
+ const p = findPattern(output, 'Payment <*> declined by issuer');
+ expect(p.sampleCount).toBe(3);
+ expect(p.whereSnippet).toContain(`match(${expr},`);
+ const rows = await drill(output, p);
+ expect(rows).toHaveLength(3);
+ for (const r of rows) {
+ expect(JSON.parse(r.Body).message).toMatch(/^Payment \d+ declined/);
+ }
+ });
+
+ it('rejects function calls in bodyExpression other than JSONExtractString over a column', async () => {
+ for (const bodyExpression of [
+ 'lower(Body)',
+ "JSONExtractString(lower(Body), 'message')",
+ "JSONExtractString(Body, 'message') OR (1=1",
+ ]) {
+ const result = await callTool(client, 'clickstack_event_patterns', {
+ sourceId: logSource._id.toString(),
+ bodyExpression,
+ ...window,
+ });
+ expect(result.isError).toBe(true);
+ expect(getFirstText(result)).toContain('bodyExpression');
+ }
+ });
+
+ it('applies the source table filter to mining and to the drill-down', async () => {
+ // The field lives on the Log discriminator; Source.updateOne drops it.
+ await LogSource.updateOne(
+ { _id: logSource._id },
+ { $set: { tableFilterExpression: "ServiceName != 'svc-b'" } },
+ );
+ const output = await mine({});
+ const p = findPattern(output, 'Sending <*> rows');
+ expect(p.sampleCount).toBe(6);
+ expect(p.whereSnippet).toContain("ServiceName != 'svc-b'");
+ const rows = await drill(output, p);
+ expect(rows).toHaveLength(6);
+ for (const r of rows) expect(r.ServiceName).toBe('svc-a');
+ expect(output.summary.drilldown.includesFilters).toBe(true);
+ expect(output.summary.drilldown.warning).toBeUndefined();
+ });
+
+ it('keeps the table filter and warns when the where filter cannot be rendered as SQL', async () => {
+ await LogSource.updateOne(
+ { _id: logSource._id },
+ { $set: { tableFilterExpression: "ServiceName != 'svc-b'" } },
+ );
+ const spy = jest
+ .spyOn(patternDrilldown, 'userWhereAsSql')
+ .mockRejectedValueOnce(new Error('cannot render'));
+ try {
+ const output = await mine({ where: 'SeverityText:INFO' });
+ expect(output.summary.drilldown.includesFilters).toBe(false);
+ expect(output.summary.drilldown.warning).toContain('table filter');
+ const p = findPattern(output, 'Sending <*> rows');
+ expect(p.whereSnippet).toContain("ServiceName != 'svc-b'");
+ const rows = await drill(output, p);
+ expect(rows).toHaveLength(6);
+ for (const r of rows) expect(r.ServiceName).toBe('svc-a');
+ } finally {
+ spy.mockRestore();
+ }
+ });
+
+ it('marks templates too long for a predicate as drilldownUnavailable', async () => {
+ await bulkInsertLogs([
+ {
+ Body: 'x'.repeat(4001),
+ ServiceName: 'long',
+ SeverityText: 'INFO',
+ Timestamp: base,
+ },
+ ]);
+ const output = await mine({ where: 'ServiceName:long' });
+ expect(output.patterns).toHaveLength(1);
+ const [p] = output.patterns;
+ expect(p.whereSnippet).toBe('');
+ expect(p.drilldownUnavailable).toContain('longer than 4000');
+ });
+
+ it('separates sampled counts from estimates', async () => {
+ const output = await mine({
+ where: 'ServiceName:svc-a',
+ sampleSize: 3,
+ });
+ const { summary } = output;
+ expect(summary.totalCount).toBe(9);
+ expect(summary.sampledCount).toBe(3);
+ expect(summary.sampleCoverage).toBeCloseTo(3 / 9, 3);
+ expect(summary.sampleMultiplier).toBe(3);
+ const sampledTotal = output.patterns.reduce(
+ (n, p) => n + p.sampleCount,
+ 0,
+ );
+ expect(sampledTotal).toBe(3);
+ for (const p of output.patterns) {
+ expect(p.estimatedCount).toBe(p.sampleCount * 3);
+ }
+ });
+
+ it('reports the effective trend buckets, and omits trends when trendBuckets is 0', async () => {
+ const withTrend = await mine({
+ where: 'ServiceName:svc-a',
+ trendBuckets: 24,
+ });
+ const { trend } = withTrend.summary;
+ if (!trend) throw new Error('summary.trend missing');
+ expect(trend.requestedBuckets).toBe(24);
+ expect(trend.bucketCount).toBeLessThanOrEqual(24);
+ expect(typeof trend.bucketInterval).toBe('string');
+ expect(new Date(trend.firstBucketStart).getTime()).toBeLessThanOrEqual(
+ new Date(withTrend.summary.timeRange.start).getTime(),
+ );
+ expect(new Date(trend.lastBucketEnd).getTime()).toBeGreaterThanOrEqual(
+ new Date(withTrend.summary.timeRange.end).getTime(),
+ );
+ for (const p of withTrend.patterns) {
+ expect(p.trend).toHaveLength(trend.bucketCount);
+ }
+
+ const noTrend = await mine({
+ where: 'ServiceName:svc-a',
+ trendBuckets: 0,
+ });
+ expect(noTrend.summary.trend).toBeUndefined();
+ for (const p of noTrend.patterns) expect(p.trend).toBeUndefined();
+ expect(noTrend.patterns.length).toBeGreaterThan(0);
+ });
+ });
});
// ─── clickstack_sql ────────────────────────────────────────────────────────────
diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts
index 159e391db..1fb3bb2f6 100644
--- a/packages/api/src/mcp/prompts/dashboards/content.ts
+++ b/packages/api/src/mcp/prompts/dashboards/content.ts
@@ -1467,33 +1467,10 @@ containers[i].tabs[j].id). Read the path to know which input to fix.
== EVENT PATTERN MINING ==
-Use clickstack_event_patterns when asked to find common log messages, recurring
-patterns, noisy services, or top event types. This is the right choice whenever
-the question is about "most common logs", "top patterns", "what is generating
-the most log volume", or similar pattern-discovery questions.
-
-It samples random events and clusters them with the Drain algorithm, returning:
- - pattern templates (with <*> wildcards for variable parts)
- - estimated counts (scaled from the sample to the full time range)
- - time-bucketed trends
- - a whereSnippet for each pattern to drill into matching raw events
-
-Required parameters:
- - sourceId (call clickstack_list_sources first)
- - startTime / endTime for the time window
-
-Optional parameters:
- - where: filter to specific services or severity levels before mining
- - sampleSize: increase for more accurate patterns (default 10000, max 25000)
- - bodyExpression: column to mine (auto-detected: Body for logs, SpanName for traces)
-
-Example: find top patterns for production services over the last 4 hours:
- clickstack_event_patterns({
- sourceId: "",
- startTime: "<4 hours ago ISO>",
- endTime: "",
- where: "service.name:prod-*"
- })
+Use clickstack_event_patterns for pattern-discovery questions: most common
+logs, top patterns, noisy services, what generates the most log volume. Its
+tool description covers parameters, examples, sampling limits, and how to
+drill into a pattern with clickstack_search.
== COMMON MISTAKES ==
diff --git a/packages/api/src/mcp/tools/query/__tests__/denoise.test.ts b/packages/api/src/mcp/tools/query/__tests__/denoise.test.ts
new file mode 100644
index 000000000..20393f6d4
--- /dev/null
+++ b/packages/api/src/mcp/tools/query/__tests__/denoise.test.ts
@@ -0,0 +1,108 @@
+import { minePatterns } from '@hyperdx/common-utils/dist/drain';
+
+import { buildNoisyBodyMatcher } from '@/mcp/tools/query/denoise';
+
+function lcg(seed: number) {
+ let s = seed;
+ return () => {
+ s = (s * 1103515245 + 12345) % 2 ** 31;
+ return s / 2 ** 31;
+ };
+}
+
+const wildcards = (template: string) =>
+ template.split(/\s+/).filter(t => t === '<*>').length;
+
+describe('buildNoisyBodyMatcher', () => {
+ const startDate = new Date('2026-01-01T00:00:00Z');
+ const endDate = new Date('2026-01-01T01:00:00Z');
+
+ it('removes members of a noisy cluster unless a non-noisy template fits them more specifically', () => {
+ // Short random bodies over a small vocabulary produce same-length
+ // clusters that overlap, where `match` sends a member elsewhere.
+ const words = ['a', 'b', 'c', 'ok', 'fail', 'x1', 'y2', '42', 'retry'];
+ for (let seed = 1; seed <= 50; seed++) {
+ const rand = lcg(seed);
+ const bodies = Array.from({ length: 300 }, () =>
+ Array.from(
+ { length: 2 + Math.floor(rand() * 3) },
+ () => words[Math.floor(rand() * words.length)],
+ ).join(' '),
+ );
+ const { patterns } = minePatterns(
+ bodies.map(body => ({ body })),
+ {
+ totalCount: bodies.length,
+ startDate,
+ endDate,
+ trendBuckets: 0,
+ maxSamples: bodies.length,
+ getBody: row => row.body,
+ getTimestamp: () => startDate.getTime(),
+ },
+ );
+ const templates = patterns.map(p => p.pattern);
+ for (const noisy of patterns) {
+ const isNoisy = buildNoisyBodyMatcher(
+ templates,
+ new Set([noisy.pattern]),
+ );
+ const own = wildcards(noisy.pattern);
+ for (const { body } of noisy.samples) {
+ const tokens = body.split(' ');
+ const moreSpecific = templates.some(t => {
+ const tt = t.split(' ');
+ return (
+ t !== noisy.pattern &&
+ wildcards(t) < own &&
+ tt.length === tokens.length &&
+ tt.every((tok, i) => tok === '<*>' || tok === tokens.at(i))
+ );
+ });
+ if (isNoisy(body) === moreSpecific) {
+ throw new Error(
+ `seed ${seed}: "${body}" in noisy "${noisy.pattern}" ` +
+ `removed=${isNoisy(body)}`,
+ );
+ }
+ }
+ }
+ }
+ });
+
+ it('keeps rows of a specific cluster when a broad template is noisy', () => {
+ const isNoisy = buildNoisyBodyMatcher(
+ ['<*> <*>', 'user login'],
+ new Set(['<*> <*>']),
+ );
+ expect(isNoisy('foo bar')).toBe(true);
+ expect(isNoisy('user login')).toBe(false);
+ });
+
+ it('keeps bodies that do not fit a noisy template', () => {
+ const templates = [
+ 'heartbeat ok <*>',
+ 'GET <*> <*> ok 200',
+ 'GET user <*> <*> <*>',
+ 'user <*> logged in',
+ ];
+ const isNoisy = buildNoisyBodyMatcher(
+ templates,
+ new Set(['heartbeat ok <*>', 'GET <*> <*> ok 200']),
+ );
+ expect(isNoisy('heartbeat ok 999')).toBe(true);
+ expect(isNoisy(' GET api a ok 200 ')).toBe(true);
+ expect(isNoisy('user 7 logged in')).toBe(false);
+ expect(isNoisy('heartbeat ok')).toBe(false);
+ expect(isNoisy('GET user p q r')).toBe(false);
+ expect(isNoisy('something unseen')).toBe(false);
+ });
+
+ it('prefers the noisy template on a tie', () => {
+ const isNoisy = buildNoisyBodyMatcher(
+ ['a <*> c', 'a b <*>'],
+ new Set(['a b <*>']),
+ );
+ expect(isNoisy('a b c')).toBe(true);
+ });
+});
diff --git a/packages/api/src/mcp/tools/query/__tests__/patternDrilldown.test.ts b/packages/api/src/mcp/tools/query/__tests__/patternDrilldown.test.ts
new file mode 100644
index 000000000..ce69e41d6
--- /dev/null
+++ b/packages/api/src/mcp/tools/query/__tests__/patternDrilldown.test.ts
@@ -0,0 +1,179 @@
+import { minePatterns } from '@hyperdx/common-utils/dist/drain';
+
+import {
+ andSql,
+ buildPatternPredicate,
+ MAX_PREDICATE_TEMPLATE_LENGTH,
+ templateToRegex,
+} from '@/mcp/tools/query/patternDrilldown';
+
+// RE2 spells code points \x{HHHH} and categories \pZ; JS (with the u flag)
+// spells them \u{HHHH} and \p{Z}. ClickHouse behaviour is covered by
+// integration tests.
+function toJsRegex(re2: string): RegExp {
+ return RegExp(
+ re2
+ .replace(/\\x\{([0-9A-Fa-f]+)\}/g, '\\u{$1}')
+ .replace(/(? {
+ it('keeps the variable position between literal tokens', () => {
+ const t = 'Sending <*> rows';
+ expect(matches(t, 'Sending 10 rows')).toBe(true);
+ expect(matches(t, 'Sending 20 rows')).toBe(true);
+
+ // The old Lucene snippet (Body:"Sending rows") required these two words
+ // to be adjacent, so it matched none of the real messages above. The new
+ // predicate must not broaden either: each of these differs in shape.
+ expect(matches(t, 'Sending rows')).toBe(false);
+ expect(matches(t, 'Sending 10 more rows')).toBe(false);
+ expect(matches(t, 'Sending 10 rows now')).toBe(false);
+ expect(matches(t, 'Resending 10 rows')).toBe(false);
+ expect(matches(t, 'Sending 10 rowsX')).toBe(false);
+ expect(matches(t, 'rows 10 Sending')).toBe(false);
+ });
+
+ it('matches whole tokens only', () => {
+ expect(matches('user <*> logged in', 'user bob logged in')).toBe(true);
+ expect(matches('user <*> logged in', 'superuser bob logged in')).toBe(
+ false,
+ );
+ expect(matches('user <*> logged in', 'user bob logged inside')).toBe(false);
+ expect(matches('a <*>', 'ab 1')).toBe(false);
+ });
+
+ it('handles repeated variables', () => {
+ const t = 'copy <*> to <*> from <*>';
+ expect(matches(t, 'copy a.txt to /tmp from host-1')).toBe(true);
+ expect(matches(t, 'copy a.txt to /tmp from')).toBe(false);
+ expect(matches(t, 'copy a.txt b.txt to /tmp from host-1')).toBe(false);
+ });
+
+ it('treats a template made only of variables as a token-count match', () => {
+ expect(matches('<*> <*>', 'alpha beta')).toBe(true);
+ expect(matches('<*> <*>', 'alpha')).toBe(false);
+ expect(matches('<*> <*>', 'alpha beta gamma')).toBe(false);
+ });
+
+ it('escapes regex metacharacters, quotes and backslashes in literals', () => {
+ const literal = String.raw`path=C:\temp "quoted" it's (x)*[y]+ $1 ^a|b {n} .?`;
+ const t = `${literal} <*>`;
+ expect(matches(t, `${literal} 42`)).toBe(true);
+ // `.` must be literal, not "any character".
+ expect(matches(t, `${literal.replace('.?', 'x?')} 42`)).toBe(false);
+ // `*` and `+` must be literal, not quantifiers.
+ expect(matches(t, `${literal.replace('(x)*', '(x)(x)')} 42`)).toBe(false);
+ expect(matches(t, `${literal.replace('[y]+', '[y]]')} 42`)).toBe(false);
+ });
+
+ it('matches Unicode tokens separated by spaces, tabs and newlines', () => {
+ const t = 'café <*> 日本';
+ expect(matches(t, 'café x 日本')).toBe(true);
+ expect(matches(t, ' café\n\tx 日本 \n')).toBe(true);
+ expect(matches(t, 'cafe x 日本')).toBe(false);
+ // U+200B (zero-width space) is not whitespace to Drain, so it is part of
+ // the token and the value has only two tokens.
+ expect(matches(t, 'café\u200Bx 日本')).toBe(false);
+ });
+
+ it('treats the Unicode whitespace Drain splits on as separators', () => {
+ expect(matches('Connection <*> reset', '\uFEFFConnection 42 reset')).toBe(
+ true,
+ );
+ expect(matches('<*> <*> EUR', '1\u202F234 EUR')).toBe(true);
+ expect(matches('a <*> b', 'a\u00A0x\u3000b')).toBe(true);
+ expect(matches('a <*> b', 'a x\u00A0y b')).toBe(false);
+ });
+
+ it('a variable spans one token, even a multi-byte one', () => {
+ expect(matches('id <*> ok', 'id 日本語 ok')).toBe(true);
+ expect(matches('id <*> ok', 'id 日本 語 ok')).toBe(false);
+ });
+
+ it('turns an empty template into a whitespace-only match', () => {
+ expect(matches('', '')).toBe(true);
+ expect(matches('', ' \n')).toBe(true);
+ expect(matches('', 'x')).toBe(false);
+ });
+
+ it('matches every sampled member of every Drain cluster, and nothing from other shapes', () => {
+ const messages = [
+ ...[10, 20, 30, 40].map(n => `Sending ${n} rows`),
+ ...[1, 2, 3].map(n => `Sending ${n} more rows`),
+ ...['alice', 'bob', 'carol'].map(
+ u => `User ${u} logged in from 10.0.0.${u.length}`,
+ ),
+ ...['a', 'b'].map(k => `cache miss for key "${k}\\x" (retry)`),
+ 'café ünïcode 日本 message',
+ ];
+ const { patterns } = minePatterns(
+ messages.map(body => ({ body })),
+ {
+ totalCount: messages.length,
+ startDate: new Date(0),
+ endDate: new Date(1000),
+ trendBuckets: 0,
+ maxSamples: messages.length,
+ getBody: r => r.body,
+ getTimestamp: () => 0,
+ },
+ );
+ expect(patterns.length).toBeGreaterThan(1);
+
+ for (const p of patterns) {
+ const re = toJsRegex(templateToRegex(p.pattern));
+ const members = p.samples.map(s => s.body);
+ expect(members).toHaveLength(p.sampleCount);
+ for (const m of members) expect(re.test(m)).toBe(true);
+ // Members of other clusters with a different shape must not match.
+ const others = patterns
+ .filter(o => o !== p)
+ .flatMap(o => o.samples.map(s => s.body));
+ for (const o of others) {
+ if (re.test(o)) {
+ // Only allowed when that cluster's template is a specialization
+ // of this one (same shape); none of these fixtures overlap.
+ throw new Error(`"${p.pattern}" unexpectedly matched "${o}"`);
+ }
+ }
+ }
+ });
+});
+
+describe('buildPatternPredicate', () => {
+ it('wraps the selected value expression in match() with an escaped literal', () => {
+ const predicate = buildPatternPredicate(
+ "LogAttributes['exception.type']",
+ String.raw`it's C:\x <*>`,
+ );
+ expect(predicate).toMatch(/^match\(LogAttributes\['exception\.type'\], '/);
+ // Quote doubled for SQL, backslash doubled for SQL on top of the regex escape.
+ expect(predicate).toContain("it''s");
+ expect(predicate).toContain(String.raw`C:\\\\x`);
+ });
+
+ it('returns null for templates too long to turn into a predicate', () => {
+ const long = 'x'.repeat(MAX_PREDICATE_TEMPLATE_LENGTH + 1);
+ expect(buildPatternPredicate('Body', long)).toBeNull();
+ const atLimit = 'x'.repeat(MAX_PREDICATE_TEMPLATE_LENGTH);
+ expect(buildPatternPredicate('Body', atLimit)).not.toBeNull();
+ });
+});
+
+describe('andSql', () => {
+ it('skips blanks and parenthesizes when combining', () => {
+ expect(andSql()).toBe('');
+ expect(andSql('', undefined, ' ')).toBe('');
+ expect(andSql('a = 1')).toBe('a = 1');
+ expect(andSql('a = 1 OR b = 2', '', 'c = 3')).toBe(
+ '(a = 1 OR b = 2) AND (c = 3)',
+ );
+ });
+});
diff --git a/packages/api/src/mcp/tools/query/denoise.ts b/packages/api/src/mcp/tools/query/denoise.ts
index 182aa5448..231cad240 100644
--- a/packages/api/src/mcp/tools/query/denoise.ts
+++ b/packages/api/src/mcp/tools/query/denoise.ts
@@ -5,8 +5,6 @@ import {
DENOISE_SAMPLE_SIZE,
flattenBody,
minePatterns,
- TemplateMiner,
- TemplateMinerConfig,
} from '@hyperdx/common-utils/dist/drain';
import type { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types';
import { DisplayType } from '@hyperdx/common-utils/dist/types';
@@ -18,6 +16,8 @@ import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout';
import { resolveBodyExpression } from './helpers';
+const PATTERN_WILDCARD = '<*>';
+
// ─── Types ───────────────────────────────────────────────────────────────────
export interface DenoiseResult {
@@ -204,9 +204,6 @@ export async function denoiseSearchResults(
}
// ── Identify noisy patterns (>10% of sampled events) ──
- // Key by template string rather than cluster ID so we are not coupled to
- // the auto-incrementing IDs generated inside minePatterns(). The matching
- // miner below produces its own IDs; comparing template strings is stable.
const sampledRowCount = sampleRows.length;
const noisyTemplates = new Set();
const removedPatterns: DenoiseResult['removedPatterns'] = [];
@@ -226,16 +223,6 @@ export async function denoiseSearchResults(
return { rows, removedPatterns: [] };
}
- // ── Build a miner trained on the same sample for row matching ──
- const drainConfig = new TemplateMinerConfig();
- const miner = new TemplateMiner(drainConfig);
- for (const row of sampleRows) {
- const raw = row.__hdx_pattern_body;
- const bodyText = flattenBody(raw != null ? String(raw) : '');
- miner.addLogMessage(bodyText);
- }
-
- // ── Match each result row and filter out noisy ones ──
const bodyColumnKey = findBodyColumnKey(rows[0], bodyColumn);
if (!bodyColumnKey) {
return {
@@ -245,13 +232,14 @@ export async function denoiseSearchResults(
};
}
+ const isNoisy = buildNoisyBodyMatcher(
+ patterns.map(p => p.pattern),
+ noisyTemplates,
+ );
const filteredRows = rows.filter(row => {
const bodyValue = row[bodyColumnKey];
if (bodyValue == null) return true; // Keep rows with no body
- const bodyText = flattenBody(String(bodyValue));
- const match = miner.match(bodyText, 'fallback');
- if (!match) return true; // No pattern match — keep the row
- return !noisyTemplates.has(match.getTemplate());
+ return !isNoisy(String(bodyValue));
});
return {
@@ -262,6 +250,54 @@ export async function denoiseSearchResults(
// ─── Helpers ─────────────────────────────────────────────────────────────────
+/**
+ * Return a predicate that is true when the most specific template a body
+ * fits (fewest <*>) is noisy. Fitting means same token count, same literal
+ * tokens, any token at each <*>. Taking the most specific fit keeps rows of
+ * a rare `user login` cluster when a broad `<*> <*>` cluster is noisy; ties
+ * go to the noisy template. Re-running Drain's `match` instead can land a
+ * member on a different same-length cluster than the one it was counted in.
+ */
+export function buildNoisyBodyMatcher(
+ templates: string[],
+ noisyTemplates: Set,
+): (body: string) => boolean {
+ const candidates = templates.map(template => {
+ const tokens = tokenize(template);
+ return {
+ tokens,
+ wildcards: tokens.filter(t => t === PATTERN_WILDCARD).length,
+ noisy: noisyTemplates.has(template),
+ };
+ });
+ return body => {
+ const tokens = tokenize(flattenBody(body));
+ let best: (typeof candidates)[number] | undefined;
+ for (const c of candidates) {
+ const fits =
+ c.tokens.length === tokens.length &&
+ c.tokens.every(
+ (tok, i) => tok === PATTERN_WILDCARD || tok === tokens.at(i),
+ );
+ if (!fits) continue;
+ if (
+ best == null ||
+ c.wildcards < best.wildcards ||
+ (c.wildcards === best.wildcards && c.noisy)
+ ) {
+ best = c;
+ }
+ }
+ return best?.noisy ?? false;
+ };
+}
+
+// Drain tokenizes with trim + split(/\s+/).
+function tokenize(text: string): string[] {
+ const trimmed = text.trim();
+ return trimmed === '' ? [] : trimmed.split(/\s+/);
+}
+
/**
* Find the key in a result row that corresponds to the body column expression.
* ClickHouse may return the column under its expression name or a simplified alias.
diff --git a/packages/api/src/mcp/tools/query/eventPatterns.ts b/packages/api/src/mcp/tools/query/eventPatterns.ts
index fc27683a1..6e09c127c 100644
--- a/packages/api/src/mcp/tools/query/eventPatterns.ts
+++ b/packages/api/src/mcp/tools/query/eventPatterns.ts
@@ -30,15 +30,20 @@ const eventPatternsSchema = z.object({
.default(10_000)
.describe(
'Random rows to sample for pattern mining. ' +
- 'Higher values produce more accurate patterns but take longer.',
+ 'Higher values produce more accurate patterns but take longer. ' +
+ 'Patterns rarer than about 1/sampleSize of matching rows may not appear at all.',
),
bodyExpression: z
.string()
.optional()
.describe(
- 'Column expression to mine patterns from. ' +
+ 'Expression to mine patterns from: a column or map reference, or JSONExtractString ' +
+ 'over one with string-literal keys to mine a field inside a JSON body. ' +
'Auto-detected from the source if omitted (Body for logs, SpanName for traces). ' +
- 'Example: "Body", "SpanName", "SpanAttributes[\'http.url\']"',
+ 'Examples: "Body", "SpanName", "LogAttributes[\'exception.type\']", ' +
+ "\"JSONExtractString(Body, 'error', 'message')\". Other function calls are rejected. " +
+ "Rows where the value is '' (missing key, invalid JSON) are mined as an empty " +
+ 'pattern; add where " != \'\'" (whereLanguage:"sql") to exclude them.',
),
topN: z
.number()
@@ -54,9 +59,12 @@ const eventPatternsSchema = z.object({
.optional()
.default(24)
.describe(
- 'Number of time buckets for per-pattern trend sparklines. ' +
- 'Compare bucket counts at start vs end to spot patterns that started or stopped mid-window. ' +
- 'Set to 0 to disable trend computation (smaller response).',
+ 'Maximum number of time buckets for per-pattern trends. The bucket width is rounded ' +
+ 'up to a standard interval (15s, 1m, 5m, 15m, 30m, 1h, ...) and aligned to it, so ' +
+ 'fewer buckets can come back and the first can start before startTime; ' +
+ 'summary.trend reports the actual interval and boundaries. ' +
+ 'Trend counts are extrapolated from the sample, so sparse buckets read 0 even when ' +
+ 'events occurred. Set to 0 to disable trends (smaller response).',
),
startTime: startTimeSchema,
endTime: endTimeSchema,
@@ -76,24 +84,28 @@ export function registerEventPatterns({
title: 'Event Pattern Mining',
annotations: READ_ONLY_TOOL,
description:
- 'Discover the most common log messages and event patterns. ' +
- 'Samples random events, clusters them using the Drain algorithm, and returns ' +
- 'patterns sorted by frequency with estimated counts and time trends.\n\n' +
- 'PREFER THIS TOOL over clickstack_search or clickstack_table when the goal is to ' +
- 'understand what kinds of messages, errors, or events exist — e.g. "sample logs", ' +
- '"what errors are happening", "show me common messages", "what does this service log". ' +
- 'It returns frequency-ranked patterns instead of raw rows, giving a much better overview.\n\n' +
- 'Also use when asked about "top patterns", "common logs", "noisy services", ' +
- '"recurring messages", or log noise analysis.\n\n' +
- 'Each pattern includes a "whereSnippet" — use it as the "where" parameter in ' +
- 'a follow-up clickstack_search call to browse matching raw events.\n\n' +
+ 'Discover what kinds of messages, errors, or events exist. Samples random events, ' +
+ 'clusters them into templates (Drain), and returns patterns ranked by frequency with ' +
+ 'sampled and estimated counts, optional trends, and raw samples. Use for "what errors ' +
+ 'are happening", common or noisy logs, recurring messages.\n\n' +
+ 'Examples:\n' +
+ ' - "What kinds of errors is checkout producing?" → where:"ServiceName:checkout AND ' +
+ 'SeverityText:error", topN:5, sampleSize:1000 (trends on by default)\n' +
+ ' - "What messages dominate this service?" → where:"ServiceName:api", trendBuckets:0\n' +
+ ' - Patterns in a map attribute → bodyExpression:"LogAttributes[\'exception.type\']", ' +
+ 'where:"LogAttributes[\'exception.type\'] != \'\'", whereLanguage:"sql"\n' +
+ ' - A field inside JSON bodies → bodyExpression:"JSONExtractString(Body, \'message\')", ' +
+ 'where:"JSONExtractString(Body, \'message\') != \'\'", whereLanguage:"sql"\n\n' +
+ 'Use instead: exact counts or breakdowns by a known field → clickstack_table / ' +
+ 'clickstack_timeseries; raw rows → clickstack_search; patterns new or gone vs an ' +
+ 'earlier window → clickstack_emerging_signals.\n\n' +
+ 'Counts come from a sample: estimatedCount is an extrapolation, and a pattern with ' +
+ '0 sampled events may still exist.\n\n' +
+ 'Drill down: call clickstack_search with the same sourceId and time range, ' +
+ 'where = a pattern\'s whereSnippet, whereLanguage = "sql". whereSnippet already ' +
+ 'includes your where filter unless summary.drilldown.includesFilters is false.\n\n' +
'Requires sourceId — call clickstack_list_sources then clickstack_describe_source first.\n\n' +
- PREFER_BUILDER_OVER_SQL_NUDGE +
- '\n\n' +
- 'When to use which tool:\n' +
- ' - clickstack_event_patterns: clustering / recurring shapes / noise analysis\n' +
- ' - clickstack_search: raw individual rows\n' +
- ' - clickstack_table: aggregated metrics / counts / top-N',
+ PREFER_BUILDER_OVER_SQL_NUDGE,
inputSchema: eventPatternsSchema,
},
async input => {
diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts
index 0ea157aef..ec94c8f95 100644
--- a/packages/api/src/mcp/tools/query/helpers.ts
+++ b/packages/api/src/mcp/tools/query/helpers.ts
@@ -76,9 +76,67 @@ export function resolveBodyExpression(
return multiExpr.length === 1 ? expression : multiExpr[0];
}
-/** Reject bodyExpression values containing SQL-unsafe characters. */
+/**
+ * A bodyExpression made only of word characters, `.`, `'`, `:`, `[`, `]` and
+ * `-`. No spaces, parentheses, commas or operators, so it cannot carry a
+ * function call or a second condition.
+ *
+ * Accepts: `Body`, `SpanAttributes['http.url']`
+ * Rejects: `Body) OR (1=1`, `lower(Body)`
+ */
// eslint-disable-next-line no-useless-escape
-export const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/;
+const SAFE_BODY_EXPR_CHARS = /^[\w.':\[\]\-]+$/;
+
+/**
+ * The column argument of JSONExtractString: an identifier (letters, digits,
+ * `_`, `.`; not starting with a digit), optionally followed by one map key in
+ * single quotes. The key may not contain `'` or `\`.
+ *
+ * Accepts: `Body`, `ResourceAttributes.service`, `LogAttributes['object']`
+ * Rejects: `lower(Body)`, `Body'`, `LogAttributes['a'']`
+ */
+const COLUMN_OR_MAP_REF = /^[A-Za-z_][\w.]*$|^[A-Za-z_][\w.]*\['[^'\\]*'\]$/;
+
+/**
+ * One complete single-quoted SQL string. Inside the quotes, any character
+ * other than `'` and `\`, a backslash escape (`\'`, `\\`), or a doubled
+ * quote (`''`).
+ *
+ * Accepts: `'message'`, `'it\'s'`, `'it''s'`
+ * Rejects: `message`, `'a' || 'b'`, `'m'' OR 1=1` (unterminated)
+ */
+const SQL_STRING_LITERAL = /^'([^'\\]|\\.|'')*'$/;
+
+/**
+ * A whole expression of the form `JSONExtractString(...)`. Captures the raw
+ * argument list (which may span lines, hence the `s` flag) for
+ * `normalizeBodyExpression` to split and validate argument by argument.
+ */
+const JSON_EXTRACT_STRING_CALL = /^JSONExtractString\((.*)\)$/s;
+
+/**
+ * Return a caller-supplied bodyExpression in a form that is safe to embed in
+ * SQL, or null when it is not allowed. Accepts a column or map reference, or
+ * JSONExtractString(, '', ...), the form the app
+ * generates for JSON fields. The JSONExtractString form is rebuilt from its
+ * validated arguments, so nothing else reaches the SQL.
+ */
+export function normalizeBodyExpression(expression: string): string | null {
+ const trimmed = expression.trim();
+ if (SAFE_BODY_EXPR_CHARS.test(trimmed)) return trimmed;
+ const call = JSON_EXTRACT_STRING_CALL.exec(trimmed);
+ if (!call) return null;
+ const [column, ...keys] = splitAndTrimWithBracket(call[1]);
+ if (
+ column == null ||
+ keys.length === 0 ||
+ !COLUMN_OR_MAP_REF.test(column) ||
+ !keys.every(k => SQL_STRING_LITERAL.test(k))
+ ) {
+ return null;
+ }
+ return `JSONExtractString(${[column, ...keys].join(', ')})`;
+}
// ─── Safety limits ───────────────────────────────────────────────────────────
diff --git a/packages/api/src/mcp/tools/query/patternDrilldown.ts b/packages/api/src/mcp/tools/query/patternDrilldown.ts
new file mode 100644
index 000000000..5e85cb6c0
--- /dev/null
+++ b/packages/api/src/mcp/tools/query/patternDrilldown.ts
@@ -0,0 +1,108 @@
+import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
+import { renderWhereExpressionStr } from '@hyperdx/common-utils/dist/core/renderChartConfig';
+import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils';
+import type {
+ ChartVariable,
+ UseTextIndex,
+} from '@hyperdx/common-utils/dist/types';
+import { substituteVariables } from '@hyperdx/common-utils/dist/variables';
+
+const PATTERN_WILDCARD = '<*>';
+
+/**
+ * Templates longer than this get no predicate: the regex (and the SQL that
+ * embeds it) would be unwieldy, and very long templates are usually a
+ * single unparameterized payload rather than a reusable shape.
+ */
+export const MAX_PREDICATE_TEMPLATE_LENGTH = 4000;
+
+export const PATTERN_MATCH_SEMANTICS =
+ 'whereSnippet matches values with the same whitespace-separated token count as the ' +
+ 'template, the same literal tokens (case-sensitive) in the same positions, and exactly ' +
+ 'one non-whitespace token at each <*>. Every sampled member of the cluster matches. ' +
+ 'It can also match unsampled events whose shape fits, including ones Drain would have ' +
+ 'put in a more specific cluster, so the drill-down is a superset of cluster membership.';
+
+// Drain tokenizes with JS `\s`, which includes Unicode spaces and BOM; RE2's
+// `\s` is ASCII-only. JS `\s` = RE2 `\s` + \v + \pZ (Zs/Zl/Zp) + BOM. `\pZ`
+// keeps the snippet short; spelling out the code points repeats ~100 chars
+// between every token.
+const JS_WHITESPACE = '\\s\\x{0B}\\pZ\\x{FEFF}';
+const WS = `[${JS_WHITESPACE}]`;
+const NON_WS = `[^${JS_WHITESPACE}]`;
+
+function escapeRe2Literal(token: string): string {
+ return token.replace(/[\\^$.|?*+()[\]{}]/g, '\\$&');
+}
+
+/** Anchored RE2 regex that a value matches when it fits the Drain template. */
+export function templateToRegex(template: string): string {
+ const tokens = template.split(/\s+/).filter(t => t.length > 0);
+ const body = tokens
+ .map(t => (t === PATTERN_WILDCARD ? `${NON_WS}+` : escapeRe2Literal(t)))
+ .join(`${WS}+`);
+ return `^${WS}*${body}${WS}*$`;
+}
+
+/**
+ * SQL predicate selecting events whose `valueExpression` fits the template,
+ * or null when the template is too long to turn into a predicate.
+ */
+export function buildPatternPredicate(
+ valueExpression: string,
+ template: string,
+): string | null {
+ if (template.length > MAX_PREDICATE_TEMPLATE_LENGTH) return null;
+ return `match(${valueExpression}, '${escapeSqlString(templateToRegex(template))}')`;
+}
+
+/** AND together SQL conditions, skipping blanks. */
+export function andSql(...conditions: Array): string {
+ const parts = conditions.filter(
+ (c): c is string => c != null && c.trim() !== '',
+ );
+ if (parts.length <= 1) return parts[0] ?? '';
+ return parts.map(c => `(${c})`).join(' AND ');
+}
+
+/**
+ * Render the caller's filter as SQL so it can be ANDed with a pattern
+ * predicate in one clickstack_search `where`. Uses the same rendering as the
+ * mining query. Dashboard variables are expanded first because the search
+ * tool has no variable context.
+ */
+export async function userWhereAsSql({
+ where,
+ whereLanguage,
+ variables,
+ metadata,
+ databaseName,
+ tableName,
+ connectionId,
+ implicitColumnExpression,
+ useTextIndexForImplicitColumn,
+}: {
+ where: string;
+ whereLanguage: 'lucene' | 'sql';
+ variables?: ChartVariable[];
+ metadata: Metadata;
+ databaseName: string;
+ tableName: string;
+ connectionId: string;
+ implicitColumnExpression?: string;
+ useTextIndexForImplicitColumn?: UseTextIndex;
+}): Promise {
+ if (!where.trim()) return '';
+ const expanded = variables
+ ? substituteVariables(where, { variables, inputLanguage: whereLanguage })
+ : where;
+ return renderWhereExpressionStr({
+ condition: expanded,
+ language: whereLanguage,
+ metadata,
+ from: { databaseName, tableName },
+ connectionId,
+ implicitColumnExpression,
+ useTextIndexForImplicitColumn,
+ });
+}
diff --git a/packages/api/src/mcp/tools/query/runEventPatterns.ts b/packages/api/src/mcp/tools/query/runEventPatterns.ts
index b9a2464c4..cd2ad6f30 100644
--- a/packages/api/src/mcp/tools/query/runEventPatterns.ts
+++ b/packages/api/src/mcp/tools/query/runEventPatterns.ts
@@ -1,6 +1,9 @@
import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
+import { getSourceTableFilters } from '@hyperdx/common-utils/dist/core/searchChartConfig';
import {
+ convertGranularityToSeconds,
getFirstTimestampValueExpression,
+ type Granularity,
splitAndTrimWithBracket,
} from '@hyperdx/common-utils/dist/core/utils';
import { minePatterns } from '@hyperdx/common-utils/dist/drain';
@@ -9,6 +12,7 @@ import type {
ChartVariable,
} from '@hyperdx/common-utils/dist/types';
import { DisplayType } from '@hyperdx/common-utils/dist/types';
+import { trace } from '@opentelemetry/api';
import { ClickhouseClient } from '@/clickhouse';
import { getConnectionById } from '@/controllers/connection';
@@ -19,13 +23,30 @@ import {
mcpUserError,
} from '@/mcp/utils/errors';
import { MCP_QUERY_MAX_EXECUTION_SEC } from '@/mcp/utils/timeout';
+import { getCounter } from '@/utils/instrumentation';
+import logger from '@/utils/logger';
import { trimToolResponse } from '@/utils/trimToolResponse';
import {
clickHouseErrorResult,
+ normalizeBodyExpression,
resolveBodyExpression,
- SAFE_BODY_EXPR_CHARS,
} from './helpers';
+import {
+ andSql,
+ buildPatternPredicate,
+ MAX_PREDICATE_TEMPLATE_LENGTH,
+ PATTERN_MATCH_SEMANTICS,
+ userWhereAsSql,
+} from './patternDrilldown';
+
+const drilldownWhereFailedCounter = getCounter(
+ 'hyperdx.mcp.event_patterns.drilldown_where_failed',
+ {
+ description:
+ 'Count of event pattern requests whose where filter could not be rendered as SQL for the drill-down.',
+ },
+);
// ─── Reusable window mining ──────────────────────────────────────────────────
@@ -50,6 +71,8 @@ export async function mineWindowPatterns(
abortSignal?: AbortSignal;
/** Dashboard variables and their selected values. Omitted outside a dashboard context. */
variables?: ChartVariable[];
+ /** Also render every row filter as one SQL condition for drill-down. */
+ includeDrilldownWhere?: boolean;
},
): Promise<
| { error: McpErrorResult }
@@ -59,6 +82,15 @@ export async function mineWindowPatterns(
totalCount: number;
sampledCount: number;
bodyColumn: string;
+ trendGranularity?: Granularity;
+ trendBucketStarts: number[];
+ /**
+ * Row filters as one SQL condition ('' when there are none). Undefined
+ * when not requested. The source table filter is always included; the
+ * caller filter only when `includesUserFilter` is true (it could not be
+ * rendered as SQL otherwise).
+ */
+ drilldown?: { where: string; includesUserFilter: boolean };
}
> {
const sampleSize = options?.sampleSize ?? 10_000;
@@ -85,22 +117,26 @@ export async function mineWindowPatterns(
};
}
- // Sanitize caller-supplied bodyExpression: must be a single column reference
- // matching the documented format (e.g. "Body", "SpanAttributes['http.url']").
- // The allowlist rejects injection attempts like "Body) OR (1=1".
+ // Sanitize caller-supplied bodyExpression: a single column/map reference or
+ // JSONExtractString over one. The allowlist rejects injection attempts like
+ // "Body) OR (1=1".
let bodyColumn: string | undefined;
if (options?.bodyExpression) {
const parts = splitAndTrimWithBracket(options.bodyExpression);
- if (parts.length !== 1 || !SAFE_BODY_EXPR_CHARS.test(parts[0])) {
+ const normalized =
+ parts.length === 1 ? normalizeBodyExpression(parts[0]) : null;
+ if (normalized == null) {
return {
error: mcpUserError(
- 'bodyExpression must be a single column expression ' +
- '(e.g. "Body", "SpanName", "SpanAttributes[\'http.url\']"). ' +
- 'Multiple expressions, function calls, or sub-queries are not allowed.',
+ 'bodyExpression must be a single column or map reference ' +
+ '(e.g. "Body", "SpanName", "SpanAttributes[\'http.url\']"), or ' +
+ 'JSONExtractString over one with string-literal keys ' +
+ "(e.g. \"JSONExtractString(Body, 'error', 'message')\"). " +
+ 'Other function calls, multiple expressions, or sub-queries are not allowed.',
),
};
}
- bodyColumn = parts[0];
+ bodyColumn = normalized;
} else {
bodyColumn = resolveBodyExpression(source);
}
@@ -114,6 +150,11 @@ export async function mineWindowPatterns(
};
}
+ const filters = getSourceTableFilters(source);
+ const tableFilterSql = andSql(
+ ...filters.map(f => ('condition' in f ? f.condition : undefined)),
+ );
+
const clickhouseClient = new ClickhouseClient({
host: connection.host,
username: connection.username,
@@ -145,6 +186,7 @@ export async function mineWindowPatterns(
},
where: options?.where ?? '',
whereLanguage: options?.whereLanguage ?? ('lucene' as const),
+ filters,
connection: source.connection.toString(),
timestampValueExpression: source.timestampValueExpression,
implicitColumnExpression: implicitColumn,
@@ -165,6 +207,7 @@ export async function mineWindowPatterns(
},
where: options?.where ?? '',
whereLanguage: options?.whereLanguage ?? ('lucene' as const),
+ filters,
connection: source.connection.toString(),
timestampValueExpression: source.timestampValueExpression,
implicitColumnExpression: implicitColumn,
@@ -174,14 +217,50 @@ export async function mineWindowPatterns(
variables: options?.variables,
} satisfies ChartConfigWithDateRange;
- // The client is constructed fresh per call (not pooled) and is only used for
- // the two queries below — the Drain mining afterward never touches it. Close
- // it in a finally so the underlying HTTP agent's sockets are released whether
- // the queries succeed or throw, rather than leaking one client per call
- // (emerging_signals opens two per invocation).
+ // Drill-down is optional, so a caller filter that cannot be rendered as SQL
+ // resolves to null instead of failing the mining. Once the queries fail the
+ // client is closed under the rendering, so its failure is not reported.
+ let queriesFailed = false;
+ const userWherePromise: Promise =
+ options?.includeDrilldownWhere
+ ? userWhereAsSql({
+ where: options.where ?? '',
+ whereLanguage: options.whereLanguage ?? 'lucene',
+ variables: options.variables,
+ metadata,
+ databaseName: source.from.databaseName,
+ tableName: source.from.tableName,
+ connectionId: source.connection.toString(),
+ implicitColumnExpression: implicitColumn,
+ useTextIndexForImplicitColumn,
+ }).catch(err => {
+ if (queriesFailed || options.abortSignal?.aborted) return null;
+ trace.getActiveSpan()?.setAttributes({
+ 'mcp.event_patterns.drilldown_where_failed': true,
+ 'mcp.event_patterns.drilldown_where_error':
+ err instanceof Error ? err.message : String(err),
+ });
+ drilldownWhereFailedCounter.add(1, {
+ where_language: options.whereLanguage ?? 'lucene',
+ });
+ logger.warn(
+ { err, teamId, sourceId },
+ 'event patterns: could not render where as SQL for drill-down',
+ );
+ return null;
+ })
+ : Promise.resolve(undefined);
+
+ // The client is constructed fresh per call (not pooled) and serves the two
+ // queries below plus the metadata lookups of the drill-down rendering — the
+ // Drain mining afterward never touches it. Close it in a finally so the
+ // underlying HTTP agent's sockets are released whether the queries succeed
+ // or throw, rather than leaking one client per call (emerging_signals opens
+ // two per invocation).
let sampleResult, countResult;
+ let userWhereSql: string | null | undefined;
try {
- [sampleResult, countResult] = await Promise.all([
+ [sampleResult, countResult, userWhereSql] = await Promise.all([
clickhouseClient.queryChartConfig({
config: sampleConfig,
metadata,
@@ -204,8 +283,10 @@ export async function mineWindowPatterns(
abort_signal: options?.abortSignal,
},
}),
+ userWherePromise,
]);
} catch (err) {
+ queriesFailed = true;
return { error: clickHouseErrorResult(err) };
} finally {
await clickhouseClient.close().catch(() => {});
@@ -213,25 +294,22 @@ export async function mineWindowPatterns(
const sampleRows = sampleResult.data ?? [];
const totalCount = Number(countResult.data?.[0]?.total ?? 0);
- if (sampleRows.length === 0) {
- return {
- patterns: [],
- sampleMultiplier: 1,
- totalCount,
- sampledCount: 0,
- bodyColumn,
- };
- }
+ const drilldown =
+ userWhereSql === undefined
+ ? undefined
+ : {
+ where: andSql(tableFilterSql, userWhereSql ?? undefined),
+ includesUserFilter: userWhereSql !== null,
+ };
// Wrap the synchronous Drain mining in try/catch. Without this a thrown
// Drain error escapes the helper — and since callers (emerging_signals)
// await it inside Promise.all and only inspect the returned {error} union
// AFTER the await, an unwrapped throw becomes an unhandled rejection / 500
// instead of a clean MCP error result.
- let patterns: ReturnType['patterns'];
- let sampleMultiplier: number;
+ let mined: ReturnType;
try {
- ({ patterns, sampleMultiplier } = minePatterns(sampleRows, {
+ mined = minePatterns(sampleRows, {
totalCount,
startDate,
endDate,
@@ -245,18 +323,21 @@ export async function mineWindowPatterns(
const tsRaw = row.__hdx_pattern_ts;
return tsRaw != null ? new Date(String(tsRaw)).getTime() : null;
},
- }));
+ });
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
return { error: mcpServerError(`Pattern mining failed: ${message}`) };
}
return {
- patterns,
- sampleMultiplier,
totalCount,
- sampledCount: sampleRows.length,
bodyColumn,
+ drilldown,
+ patterns: mined.patterns,
+ sampleMultiplier: mined.sampleMultiplier,
+ sampledCount: sampleRows.length,
+ trendGranularity: mined.trendGranularity,
+ trendBucketStarts: mined.trendBucketStarts,
};
}
@@ -318,6 +399,7 @@ export async function runEventPatterns(
trendBuckets,
abortSignal: options?.abortSignal,
variables: options?.variables,
+ includeDrilldownWhere: true,
});
if ('error' in mined) return mined.error;
@@ -327,8 +409,16 @@ export async function runEventPatterns(
totalCount,
sampledCount,
bodyColumn,
+ trendGranularity,
+ trendBucketStarts,
+ drilldown,
} = mined;
+ const timeRange = {
+ start: startDate.toISOString(),
+ end: endDate.toISOString(),
+ };
+
if (sampledCount === 0) {
return {
content: [
@@ -343,10 +433,7 @@ export async function runEventPatterns(
clusterCount: 0,
patternsReturned: 0,
bodyColumn,
- timeRange: {
- start: startDate.toISOString(),
- end: endDate.toISOString(),
- },
+ timeRange,
},
patterns: [],
hint: 'No data found in the queried time range. Try setting startTime to a wider window (e.g. 24 hours ago) or check that filters match existing data.',
@@ -360,23 +447,12 @@ export async function runEventPatterns(
}
// ── Format response ──
- // Convert trend timestamps to ISO strings, extract sample body texts,
- // and build a whereSnippet per pattern so the agent can drill into
- // matching events via a follow-up clickstack_search query.
const slicedPatterns = rawPatterns.slice(0, topN);
const patterns = slicedPatterns.map((p, i) => {
- // Build a Lucene-compatible where clause from the pattern's literal
- // (non-<*>) tokens. This lets agents chain: pattern → search.
- // Escape \ and " (the phrase-query metachars) in each token.
- const literalTokens = p.pattern
- .split(/\s+/)
- .filter(t => t !== '<*>' && t.length > 0)
- .map(t => t.replace(/[\\"]/g, '\\$&'));
+ const predicate = buildPatternPredicate(bodyColumn, p.pattern);
const whereSnippet =
- literalTokens.length > 0
- ? `${bodyColumn}:"${literalTokens.join(' ')}"`
- : '';
+ predicate == null ? '' : andSql(drilldown?.where, predicate);
const shareOfTotal = sampledCount > 0 ? p.sampleCount / sampledCount : 0;
@@ -396,6 +472,14 @@ export async function runEventPatterns(
sampleCount: p.sampleCount,
shareOfTotal: Math.round(shareOfTotal * 10000) / 10000,
whereSnippet,
+ whereLanguage: 'sql' as const,
+ ...(predicate == null
+ ? {
+ drilldownUnavailable:
+ `Template is longer than ${MAX_PREDICATE_TEMPLATE_LENGTH} characters, so ` +
+ 'whereSnippet is empty. Do not search with it; it would match every row.',
+ }
+ : {}),
...(formattedTrend ? { trend: formattedTrend } : {}),
samples: p.samples.map(row => {
const raw = row.__hdx_pattern_body;
@@ -404,27 +488,70 @@ export async function runEventPatterns(
};
});
+ const trendSummary =
+ trendGranularity && trendBucketStarts.length > 0
+ ? {
+ requestedBuckets: trendBuckets,
+ bucketCount: trendBucketStarts.length,
+ bucketInterval: trendGranularity,
+ firstBucketStart: new Date(trendBucketStarts[0]).toISOString(),
+ lastBucketEnd: new Date(
+ trendBucketStarts[trendBucketStarts.length - 1] +
+ convertGranularityToSeconds(trendGranularity) * 1000,
+ ).toISOString(),
+ }
+ : undefined;
+
const output = {
summary: {
totalCount,
sampledCount,
+ sampleCoverage:
+ totalCount > 0
+ ? Math.round(Math.min(sampledCount / totalCount, 1) * 10000) / 10000
+ : 1,
sampleMultiplier: Math.round(sampleMultiplier * 100) / 100,
clusterCount: rawPatterns.length,
patternsReturned: patterns.length,
bodyColumn,
- timeRange: {
- start: startDate.toISOString(),
- end: endDate.toISOString(),
+ timeRange,
+ ...(trendSummary ? { trend: trendSummary } : {}),
+ drilldown: {
+ tool: 'clickstack_search',
+ sourceId,
+ startTime: timeRange.start,
+ endTime: timeRange.end,
+ whereLanguage: 'sql',
+ includesFilters: drilldown?.includesUserFilter ?? false,
+ ...(drilldown?.includesUserFilter
+ ? {}
+ : {
+ warning:
+ 'The where filter could not be rendered as SQL, so whereSnippet holds only ' +
+ 'the pattern predicate and the source table filter. AND your filter onto it ' +
+ '(in SQL) before searching.',
+ }),
+ matchSemantics: PATTERN_MATCH_SEMANTICS,
},
},
patterns,
usage:
- 'shareOfTotal is the fraction of sampled rows matching this pattern. ' +
- 'estimatedCount = sampleCount * sampleMultiplier. ' +
- (trendBuckets > 0
- ? 'trend.count is similarly extrapolated from sample bucket counts. '
+ 'totalCount is exact. sampleCount is the number of sampled rows in the pattern. ' +
+ 'estimatedCount = sampleCount * sampleMultiplier, an extrapolation, not a count; ' +
+ 'use clickstack_table for exact counts. shareOfTotal = sampleCount / sampledCount. ' +
+ (trendSummary
+ ? 'trend.count = round(sampled rows in the bucket * sampleMultiplier). Buckets are ' +
+ 'summary.trend.bucketInterval wide and aligned to it, so there can be fewer ' +
+ 'than requested and the first/last can extend past timeRange (partial buckets). ' +
+ 'A 0 bucket means no SAMPLED row fell in it, not that none occurred: when ' +
+ 'sampleMultiplier is N, a bucket with fewer than about N real events often ' +
+ 'shows 0. Compare trends only for patterns with many sampled rows. '
: '') +
- 'Use whereSnippet as the "where" parameter in a clickstack_search call to browse matching raw events.',
+ 'To browse raw events of a pattern, call clickstack_search with the sourceId, ' +
+ 'startTime and endTime from summary.drilldown, where = whereSnippet, and ' +
+ 'whereLanguage = "sql". whereSnippet already includes the source table filter and, ' +
+ 'unless summary.drilldown.includesFilters is false, your where filter, so do not ' +
+ 'add them again. Skip patterns that have drilldownUnavailable.',
};
const { data: trimmedOutput, isTrimmed } = trimToolResponse(output);
diff --git a/packages/common-utils/src/__tests__/mine-patterns.test.ts b/packages/common-utils/src/__tests__/mine-patterns.test.ts
index 4553d9244..1aa39020a 100644
--- a/packages/common-utils/src/__tests__/mine-patterns.test.ts
+++ b/packages/common-utils/src/__tests__/mine-patterns.test.ts
@@ -264,6 +264,66 @@ describe('minePatterns', () => {
// The bucket containing the event should have count = 1 * 100 = 100
expect(nonZeroBuckets[0].count).toBe(100);
});
+
+ it('rounds the bucket width up, so fewer buckets than requested can come back', () => {
+ const start = new Date('2024-01-01T00:00:00Z');
+ const end = new Date('2024-01-01T01:00:00Z');
+
+ const result = minePatterns(
+ [{ body: 'event', ts: start.toISOString() }],
+ {
+ totalCount: 1,
+ startDate: start,
+ endDate: end,
+ trendBuckets: 24,
+ ...defaultCallbacks,
+ },
+ );
+
+ // 3600s / 24 = 150s, rounded up to the 5 minute interval.
+ expect(result.trendGranularity).toBe('5 minute');
+ expect(result.trendBucketStarts).toHaveLength(12);
+ expect(result.patterns[0].trend.map(b => b.ts)).toEqual(
+ result.trendBucketStarts,
+ );
+ });
+
+ it('aligns the first bucket to the interval, before an unaligned start', () => {
+ const start = new Date('2024-01-01T00:07:00Z');
+ const end = new Date('2024-01-01T01:07:00Z');
+
+ const result = minePatterns(
+ [{ body: 'event', ts: start.toISOString() }],
+ {
+ totalCount: 1,
+ startDate: start,
+ endDate: end,
+ trendBuckets: 4,
+ ...defaultCallbacks,
+ },
+ );
+
+ expect(result.trendGranularity).toBe('15 minute');
+ expect(result.trendBucketStarts[0]).toBe(
+ new Date('2024-01-01T00:00:00Z').getTime(),
+ );
+ expect(result.trendBucketStarts[0]).toBeLessThan(start.getTime());
+ });
+
+ it('skips trend computation when trendBuckets is 0', () => {
+ const result = minePatterns([makeRow('event')], {
+ totalCount: 1,
+ startDate: baseDate,
+ endDate,
+ trendBuckets: 0,
+ ...defaultCallbacks,
+ });
+
+ expect(result.trendGranularity).toBeUndefined();
+ expect(result.trendBucketStarts).toEqual([]);
+ expect(result.patterns[0].trend).toEqual([]);
+ expect(result.patterns[0].sampleCount).toBe(1);
+ });
});
describe('NaN/null timestamp handling', () => {
@@ -390,3 +450,56 @@ describe('minePatterns', () => {
});
});
});
+
+describe('minePatterns template fidelity', () => {
+ // Deterministic PRNG so the fuzz input is stable across runs.
+ function lcg(seed: number) {
+ let s = seed;
+ return () => {
+ s = (s * 1103515245 + 12345) % 2 ** 31;
+ return s / 2 ** 31;
+ };
+ }
+
+ function fits(template: string, body: string): boolean {
+ const t = template.split(/\s+/).filter(Boolean);
+ const b = body.split(/\s+/).filter(Boolean);
+ return (
+ t.length === b.length &&
+ t.every((tok, i) => tok === '<*>' || tok === b.at(i))
+ );
+ }
+
+ it("reports each cluster's own template, which every member fits", () => {
+ const words = ['a', 'b', 'c', 'ok', 'fail', 'x1', 'y2', '42', 'retry'];
+ for (let seed = 1; seed <= 50; seed++) {
+ const rand = lcg(seed);
+ const rows = Array.from({ length: 300 }, () => {
+ const len = 2 + Math.floor(rand() * 3);
+ const body = Array.from(
+ { length: len },
+ () => words[Math.floor(rand() * words.length)],
+ ).join(' ');
+ return makeRow(body);
+ });
+ const { patterns } = minePatterns(rows, {
+ totalCount: rows.length,
+ startDate: baseDate,
+ endDate,
+ trendBuckets: 0,
+ maxSamples: rows.length,
+ ...defaultCallbacks,
+ });
+ for (const p of patterns) {
+ for (const s of p.samples) {
+ const body = String(s.body);
+ if (!fits(p.pattern, body)) {
+ throw new Error(
+ `seed ${seed}: "${body}" does not fit "${p.pattern}"`,
+ );
+ }
+ }
+ }
+ }
+ });
+});
diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts
index f0c5af8fb..44181ced7 100644
--- a/packages/common-utils/src/core/renderChartConfig.ts
+++ b/packages/common-utils/src/core/renderChartConfig.ts
@@ -1260,7 +1260,13 @@ function renderFrom({
);
}
-async function renderWhereExpressionStr({
+/**
+ * Render a where condition as a SQL string: Lucene is serialized against the
+ * table's schema, and the result is rewritten to use materialized columns
+ * where the table has them. Used by chart rendering and by any caller that
+ * needs the same SQL outside a chart config.
+ */
+export async function renderWhereExpressionStr({
condition,
language,
metadata,
diff --git a/packages/common-utils/src/core/searchChartConfig.ts b/packages/common-utils/src/core/searchChartConfig.ts
index 37144db8a..066c0f976 100644
--- a/packages/common-utils/src/core/searchChartConfig.ts
+++ b/packages/common-utils/src/core/searchChartConfig.ts
@@ -14,6 +14,18 @@ import {
TSource,
} from '@/types';
+/**
+ * The source's `tableFilterExpression` as a SQL filter, or `[]` when it has
+ * none. Log sources are the only kind that carries the field. Every path
+ * that queries a source's rows should apply this so they all see the same
+ * row set (see `buildSearchChartConfig`).
+ */
+export function getSourceTableFilters(source: TSource): Filter[] {
+ return isLogSource(source) && source.tableFilterExpression
+ ? [{ type: 'sql', condition: source.tableFilterExpression }]
+ : [];
+}
+
/**
* Return type for `buildSearchChartConfig`.
*
@@ -131,10 +143,7 @@ export function buildSearchChartConfig(
// For hard isolation, configure a ClickHouse ROW POLICY at the DB level
// instead. Existing values are still honored here for backward
// compatibility; new sources should not set the field.
- const tableFilter: Filter[] =
- isLogSource(source) && source.tableFilterExpression != null
- ? [{ type: 'sql', condition: source.tableFilterExpression }]
- : [];
+ const tableFilter = getSourceTableFilters(source);
const userFilters: Filter[] = input.filters ?? [];
const mergedFilters: Filter[] = [...tableFilter, ...userFilters];
diff --git a/packages/common-utils/src/drain/drain.ts b/packages/common-utils/src/drain/drain.ts
index ead441df6..661cf89e1 100644
--- a/packages/common-utils/src/drain/drain.ts
+++ b/packages/common-utils/src/drain/drain.ts
@@ -65,6 +65,11 @@ export class Drain {
return total;
}
+ /** Look up a cluster by ID without touching LRU order. */
+ getCluster(id: number): LogCluster | undefined {
+ return this.clusterPeek(id);
+ }
+
private clusterPeek(id: number): LogCluster | undefined {
if (this.unlimitedStore) return this.unlimitedStore.get(id);
return this.limitedStore!.peek(id);
diff --git a/packages/common-utils/src/drain/mine-patterns.ts b/packages/common-utils/src/drain/mine-patterns.ts
index 83a33397e..f5f663962 100644
--- a/packages/common-utils/src/drain/mine-patterns.ts
+++ b/packages/common-utils/src/drain/mine-patterns.ts
@@ -1,5 +1,6 @@
import {
convertDateRangeToGranularityString,
+ Granularity,
timeBucketByGranularity,
toStartOfInterval,
} from '@/core/utils';
@@ -53,6 +54,18 @@ export interface PatternGroup> {
export interface MinePatternResult> {
patterns: PatternGroup[];
sampleMultiplier: number;
+ /**
+ * Bucket width actually used for `trend`. Rounded up to a standard
+ * interval, so there can be fewer buckets than requested. Undefined when
+ * trends are disabled (trendBuckets <= 0).
+ */
+ trendGranularity?: Granularity;
+ /**
+ * Start of every trend bucket in ms. Buckets align to the interval, so the
+ * first one can start before `startDate` and the last can end after
+ * `endDate`.
+ */
+ trendBucketStarts: number[];
}
export interface MinePatternOptions> {
@@ -62,7 +75,7 @@ export interface MinePatternOptions> {
startDate: Date;
/** End of the query time range. */
endDate: Date;
- /** Max number of trend time buckets to generate. Default: 24. */
+ /** Max number of trend time buckets to generate. Default: 24. 0 disables trends. */
trendBuckets?: number;
/** Max number of sample rows to keep per pattern. Default: 5. */
maxSamples?: number;
@@ -102,26 +115,29 @@ export function minePatterns>(
getTimestamp,
} = options;
+ const granularity =
+ trendBuckets > 0
+ ? convertDateRangeToGranularityString([startDate, endDate], trendBuckets)
+ : undefined;
+ const allBuckets = granularity
+ ? timeBucketByGranularity(startDate, endDate, granularity).map(d =>
+ d.getTime(),
+ )
+ : [];
+
if (rows.length === 0) {
- return { patterns: [], sampleMultiplier: 1 };
+ return {
+ patterns: [],
+ sampleMultiplier: 1,
+ trendGranularity: granularity,
+ trendBucketStarts: allBuckets,
+ };
}
// ── Set up Drain miner ──
const drainConfig = new TemplateMinerConfig();
const miner = new TemplateMiner(drainConfig);
- // ── Compute time buckets using shared helpers ──
- const granularity = convertDateRangeToGranularityString(
- [startDate, endDate],
- trendBuckets,
- );
- const allBucketDates = timeBucketByGranularity(
- startDate,
- endDate,
- granularity,
- );
- const allBuckets = allBucketDates.map(d => d.getTime());
-
// ── Process each row through Drain ──
const clustered: Array<{
clusterId: number;
@@ -147,7 +163,9 @@ export function minePatterns>(
>();
for (const { clusterId, row, bodyText, tsMs } of clustered) {
- const bucket = toStartOfInterval(new Date(tsMs), granularity).getTime();
+ const bucket = granularity
+ ? toStartOfInterval(new Date(tsMs), granularity).getTime()
+ : 0;
const existing = groups.get(clusterId);
if (existing) {
if (existing.samples.length < maxSamples) {
@@ -158,12 +176,15 @@ export function minePatterns>(
(existing.bucketCounts.get(bucket) ?? 0) + 1,
);
} else {
- const match = miner.match(bodyText, 'fallback');
+ // Read the template from the row's own cluster: re-matching the body can
+ // land on a different same-length cluster whose template other members
+ // of this one do not fit.
+ const cluster = miner.drain.getCluster(clusterId);
const bucketCounts = new Map();
bucketCounts.set(bucket, 1);
groups.set(clusterId, {
samples: [row],
- template: match?.getTemplate() ?? bodyText,
+ template: cluster?.getTemplate() ?? bodyText,
bucketCounts,
});
}
@@ -200,5 +221,10 @@ export function minePatterns>(
patterns.sort((a, b) => b.estimatedCount - a.estimatedCount);
- return { patterns, sampleMultiplier };
+ return {
+ patterns,
+ sampleMultiplier,
+ trendGranularity: granularity,
+ trendBucketStarts: allBuckets,
+ };
}
From 349fc2141642a5f2c5207440e562f217282c1948 Mon Sep 17 00:00:00 2001
From: Brandon Pereira
Date: Wed, 4 Nov 2026 17:02:52 -0600
Subject: [PATCH 033/103] feat(mcp): add clickstack_describe_table tool (#3523)
## Summary
Adds the MCP tool `clickstack_describe_table`. We see significant `clickstack_sql` usage for inspecting non-source tables and want an off-the-shelf tool.
- **Input:** `connectionId`, `database`, optional `table`.
- **With `table`:** returns the columns (name, type, default kind and expression, comment, whether the column is in the sorting or primary key), the engine, sorting/primary/partition keys, data-skipping indices, approximate total rows and bytes, and an optional row sample. `sampleRows` defaults to 0 and is capped at 5. The sample goes through `trimToolResponse`. If the sample query fails, the response still includes the schema and adds `sampleRowsError`.
- **Without `table`:** lists the tables in the database (name, engine, approximate rows). Uses the same `limit` and opaque `nextCursor` paging as `clickstack_list_metrics`.
- Read-only. It queries `system.tables`, `system.columns` and `system.data_skipping_indices` with bound parameters, and the sample uses `Identifier` params. It uses the MCP ClickHouse settings (readonly=2, `max_execution_time`), the request timeout and `runWithTimeout`. The connection lookup is scoped to the team.
- A missing database or table, or a permission error, comes back as a user error with a hint. Network and infra failures stay server errors through `clickHouseErrorResult`.
- The tool is registered with the sources tools, so it gets tracing from `registerTool`. It also sets `mcp.describe_table.*` span attributes.
- The `clickstack_sql` description now tells agents to call `clickstack_describe_table` instead of querying `system.columns` / `system.tables`. `MCP.md` is updated.
### How to reproduce
**Starting point:** `yarn dev` (the tool needs the API), with an MCP client connected to the local MCP server.
**On `main`:**
1. List the tools. `clickstack_describe_table` is not there.
**On this branch:**
1. Call `clickstack_list_sources` and copy a `connectionId`.
2. Call `clickstack_describe_table` with that `connectionId` and `database: "default"`. It returns the tables, with a `nextCursor` if there are more than `limit`.
3. Call it again with `table: "otel_logs"` and `sampleRows: 2`. It returns columns, keys, indices, size and 2 sample rows.
4. Call it with `table: "does_not_exist"`. It returns a user error that says the table was not found.
Tests: `src/mcp/__tests__/describeTable.test.ts` (unit) and `src/mcp/__tests__/describeTable.int.test.ts` (integration).
### How to test on Vercel preview
N/A. This is not a UI change.
Fixes HDX-5746
---
.changeset/mcp-describe-table.md | 5 +
MCP.md | 1 +
.../mcp/__tests__/describeTable.int.test.ts | 338 +++++++++
.../src/mcp/__tests__/describeTable.test.ts | 649 ++++++++++++++++++
packages/api/src/mcp/tools/query/sql.ts | 7 +-
.../src/mcp/tools/sources/describeTable.ts | 380 ++++++++++
.../mcp/tools/sources/describeTableQueries.ts | 341 +++++++++
.../mcp/tools/sources/describeTableSchema.ts | 83 +++
packages/api/src/mcp/tools/sources/index.ts | 2 +
9 files changed, 1803 insertions(+), 3 deletions(-)
create mode 100644 .changeset/mcp-describe-table.md
create mode 100644 packages/api/src/mcp/__tests__/describeTable.int.test.ts
create mode 100644 packages/api/src/mcp/__tests__/describeTable.test.ts
create mode 100644 packages/api/src/mcp/tools/sources/describeTable.ts
create mode 100644 packages/api/src/mcp/tools/sources/describeTableQueries.ts
create mode 100644 packages/api/src/mcp/tools/sources/describeTableSchema.ts
diff --git a/.changeset/mcp-describe-table.md b/.changeset/mcp-describe-table.md
new file mode 100644
index 000000000..bd4e02299
--- /dev/null
+++ b/.changeset/mcp-describe-table.md
@@ -0,0 +1,5 @@
+---
+'@hyperdx/api': minor
+---
+
+feat(mcp): add `clickstack_describe_table` to list the tables in a database and describe a table that is not a registered source (columns, keys, engine, skipping indices, approximate size, optional row sample)
diff --git a/MCP.md b/MCP.md
index 2ece9e1d3..724e75463 100644
--- a/MCP.md
+++ b/MCP.md
@@ -122,6 +122,7 @@ with:
| ----------------------------- | -------------------------------------------------------------------------------------------- |
| `clickstack_list_sources` | List all data sources and connections as a lightweight catalog (IDs, names, kinds) |
| `clickstack_describe_source` | Full column schema, attribute keys, sampled low-cardinality values, and a round-trippable `config` block (the exact shape `clickstack_save_source` accepts, incl. correlation IDs) for a single source; for metric sources also returns a per-kind metric-name sample |
+| `clickstack_describe_table` | Inspect a ClickHouse table that is not a registered source: columns, keys, engine, skipping indices, approximate size, and an optional row sample. Without a table, lists the tables in a database (paginated) |
| `clickstack_save_source` | Create (omit id) or update (provide id) a data source (log, trace, session, metric, or promql) so shipped telemetry becomes queryable. To clone a source, read its `config` from `clickstack_describe_source`, drop `id`, and pass it here |
| `clickstack_delete_source` | Permanently delete a data source by ID |
| `clickstack_list_metrics` | Paginated catalog of metric names on a metric source with optional kind / namePattern / time-window filters |
diff --git a/packages/api/src/mcp/__tests__/describeTable.int.test.ts b/packages/api/src/mcp/__tests__/describeTable.int.test.ts
new file mode 100644
index 000000000..c1c4344ab
--- /dev/null
+++ b/packages/api/src/mcp/__tests__/describeTable.int.test.ts
@@ -0,0 +1,338 @@
+import { SourceKind } from '@hyperdx/common-utils/dist/types';
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import mongoose from 'mongoose';
+
+import * as config from '@/config';
+import {
+ getLoggedInAgent,
+ getServer,
+ getTestFixtureClickHouseClient,
+} from '@/fixtures';
+import { McpContext } from '@/mcp/tools/types';
+import Connection from '@/models/connection';
+import { Source } from '@/models/source';
+
+import { callTool, createTestClient, getFirstText } from './mcpTestUtils';
+
+const DATABASE = 'mcp_describe_table_test';
+// Kept apart so the Distributed table does not change DATABASE's listing.
+const DIST_DATABASE = 'mcp_describe_table_dist_test';
+
+describe('clickstack_describe_table', () => {
+ const server = getServer();
+ let connection: any;
+ let client: Client;
+
+ beforeAll(async () => {
+ await server.start();
+ const ch = await getTestFixtureClickHouseClient();
+ await ch.command({ query: `DROP DATABASE IF EXISTS ${DATABASE}` });
+ await ch.command({ query: `DROP DATABASE IF EXISTS ${DIST_DATABASE}` });
+ await ch.command({ query: `CREATE DATABASE ${DATABASE}` });
+ await ch.command({ query: `CREATE DATABASE ${DIST_DATABASE}` });
+ await ch.command({
+ query: `
+ CREATE TABLE ${DATABASE}.events (
+ Timestamp DateTime64(9),
+ ServiceName LowCardinality(String) COMMENT 'emitting service',
+ Body String,
+ Day Date MATERIALIZED toDate(Timestamp),
+ INDEX idx_body Body TYPE tokenbf_v1(8192, 3, 0) GRANULARITY 1
+ )
+ ENGINE = MergeTree
+ PARTITION BY toDate(Timestamp)
+ PRIMARY KEY ServiceName
+ ORDER BY (ServiceName, Timestamp)
+ COMMENT 'test events'
+ `,
+ });
+ await ch.command({
+ query: `CREATE VIEW ${DATABASE}.events_view AS SELECT ServiceName FROM ${DATABASE}.events`,
+ });
+ await ch.command({
+ query: `CREATE TABLE ${DIST_DATABASE}.events AS ${DATABASE}.events ENGINE = Distributed(hdx_cluster, ${DATABASE}, events)`,
+ });
+ await ch.insert({
+ table: `${DATABASE}.events`,
+ values: [
+ { Timestamp: '2024-01-01 00:00:00', ServiceName: 'api', Body: 'a' },
+ { Timestamp: '2024-01-01 00:00:01', ServiceName: 'web', Body: 'b' },
+ { Timestamp: '2024-01-01 00:00:02', ServiceName: 'api', Body: 'c' },
+ ],
+ format: 'JSONEachRow',
+ });
+ });
+
+ beforeEach(async () => {
+ const { team, user } = await getLoggedInAgent(server);
+ connection = await Connection.create({
+ team: team._id,
+ name: 'Default',
+ host: config.CLICKHOUSE_HOST,
+ username: config.CLICKHOUSE_USER,
+ password: config.CLICKHOUSE_PASSWORD,
+ });
+ const context: McpContext = {
+ teamId: team._id.toString(),
+ userId: user._id.toString(),
+ };
+ client = await createTestClient(context);
+ });
+
+ afterEach(async () => {
+ await client?.close();
+ await server.clearDBs();
+ });
+
+ afterAll(async () => {
+ const ch = await getTestFixtureClickHouseClient();
+ await ch.command({ query: `DROP DATABASE IF EXISTS ${DIST_DATABASE}` });
+ await ch.command({ query: `DROP DATABASE IF EXISTS ${DATABASE}` });
+ await server.stop();
+ });
+
+ const call = (args: Record) =>
+ callTool(client, 'clickstack_describe_table', {
+ connectionId: connection._id.toString(),
+ ...args,
+ });
+
+ it('describes columns, keys, indices, and size of a table', async () => {
+ const result = await call({ database: DATABASE, table: 'events' });
+ expect(result.isError).toBeFalsy();
+ const output = JSON.parse(getFirstText(result));
+
+ expect(output.table).toMatchObject({
+ database: DATABASE,
+ name: 'events',
+ engine: 'MergeTree',
+ sortingKey: 'ServiceName, Timestamp',
+ primaryKey: 'ServiceName',
+ partitionKey: 'toDate(Timestamp)',
+ comment: 'test events',
+ approxTotalRows: 3,
+ });
+ expect(output.table.approxTotalBytes).toBeGreaterThan(0);
+
+ expect(output.columns).toEqual([
+ {
+ name: 'Timestamp',
+ type: 'DateTime64(9)',
+ inSortingKey: true,
+ inPrimaryKey: false,
+ },
+ {
+ name: 'ServiceName',
+ type: 'LowCardinality(String)',
+ comment: 'emitting service',
+ inSortingKey: true,
+ inPrimaryKey: true,
+ },
+ {
+ name: 'Body',
+ type: 'String',
+ inSortingKey: false,
+ inPrimaryKey: false,
+ },
+ {
+ name: 'Day',
+ type: 'Date',
+ defaultKind: 'MATERIALIZED',
+ defaultExpression: 'toDate(Timestamp)',
+ inSortingKey: false,
+ inPrimaryKey: false,
+ },
+ ]);
+ expect(output.dataSkippingIndices).toEqual([
+ { name: 'idx_body', type: 'tokenbf_v1', expression: 'Body' },
+ ]);
+ expect(output.sampleRows).toBeUndefined();
+ });
+
+ it('reads keys and indices of a Distributed table from its local table', async () => {
+ const result = await call({ database: DIST_DATABASE, table: 'events' });
+ expect(result.isError).toBeFalsy();
+ const output = JSON.parse(getFirstText(result));
+ expect(output.table).toMatchObject({
+ engine: 'Distributed',
+ localTable: {
+ cluster: 'hdx_cluster',
+ database: DATABASE,
+ table: 'events',
+ engine: 'MergeTree',
+ },
+ sortingKey: 'ServiceName, Timestamp',
+ primaryKey: 'ServiceName',
+ partitionKey: 'toDate(Timestamp)',
+ });
+ expect(
+ output.columns.find((c: any) => c.name === 'ServiceName'),
+ ).toMatchObject({
+ inSortingKey: true,
+ inPrimaryKey: true,
+ });
+ expect(output.dataSkippingIndices).toEqual([
+ { name: 'idx_body', type: 'tokenbf_v1', expression: 'Body' },
+ ]);
+ });
+
+ it('reflects an index added after a previous describe', async () => {
+ await call({ database: DATABASE, table: 'events' });
+ const ch = await getTestFixtureClickHouseClient();
+ await ch.command({
+ query: `ALTER TABLE ${DATABASE}.events ADD INDEX idx_svc ServiceName TYPE set(10) GRANULARITY 1`,
+ });
+ try {
+ const output = JSON.parse(
+ getFirstText(await call({ database: DATABASE, table: 'events' })),
+ );
+ expect(
+ output.dataSkippingIndices.map((i: { name: string }) => i.name),
+ ).toContain('idx_svc');
+ } finally {
+ await ch.command({
+ query: `ALTER TABLE ${DATABASE}.events DROP INDEX idx_svc`,
+ });
+ }
+ });
+
+ it('points to clickstack_describe_source when the table backs a source', async () => {
+ const source = await Source.create({
+ kind: SourceKind.Log,
+ team: connection.team,
+ from: { databaseName: DATABASE, tableName: 'events' },
+ timestampValueExpression: 'Timestamp',
+ connection: connection._id,
+ name: 'Events',
+ });
+ const metricSource = await Source.create({
+ kind: SourceKind.Metric,
+ team: connection.team,
+ from: { databaseName: DATABASE, tableName: '' },
+ metricTables: { gauge: 'events' },
+ timestampValueExpression: 'Timestamp',
+ resourceAttributesExpression: 'ResourceAttributes',
+ connection: connection._id,
+ name: 'Event metrics',
+ });
+
+ const output = JSON.parse(
+ getFirstText(await call({ database: DATABASE, table: 'events' })),
+ );
+ expect(output.sources).toEqual(
+ expect.arrayContaining([
+ { id: source._id.toString(), name: 'Events', kind: SourceKind.Log },
+ {
+ id: metricSource._id.toString(),
+ name: 'Event metrics',
+ kind: SourceKind.Metric,
+ },
+ ]),
+ );
+ expect(output.nextSteps.describeSource).toMatch(
+ /clickstack_describe_source/,
+ );
+ expect(output.nextSteps.query).toBeUndefined();
+ expect(output.columns.length).toBeGreaterThan(0);
+ });
+
+ it('does not match a source on another connection', async () => {
+ const otherConnection = await Connection.create({
+ team: connection.team,
+ name: 'Other',
+ host: config.CLICKHOUSE_HOST,
+ username: config.CLICKHOUSE_USER,
+ password: config.CLICKHOUSE_PASSWORD,
+ });
+ await Source.create({
+ kind: SourceKind.Log,
+ team: connection.team,
+ from: { databaseName: DATABASE, tableName: 'events' },
+ timestampValueExpression: 'Timestamp',
+ connection: otherConnection._id,
+ name: 'Events',
+ });
+
+ const output = JSON.parse(
+ getFirstText(await call({ database: DATABASE, table: 'events' })),
+ );
+ expect(output.sources).toBeUndefined();
+ expect(output.nextSteps.query).toMatch(/clickstack_save_source/);
+ });
+
+ it('returns a capped sample of rows when asked', async () => {
+ const result = await call({
+ database: DATABASE,
+ table: 'events',
+ sampleRows: 2,
+ });
+ expect(result.isError).toBeFalsy();
+ const output = JSON.parse(getFirstText(result));
+ expect(output.sampleRows).toHaveLength(2);
+ expect(output.sampleRows[0]).toHaveProperty('ServiceName');
+ });
+
+ it('rejects more than 5 sample rows', async () => {
+ const result = await call({
+ database: DATABASE,
+ table: 'events',
+ sampleRows: 50,
+ });
+ expect(result.isError).toBe(true);
+ });
+
+ it('lists tables in a database across pages', async () => {
+ const first = JSON.parse(
+ getFirstText(await call({ database: DATABASE, limit: 1 })),
+ );
+ expect(first.tables).toEqual([
+ { name: 'events', engine: 'MergeTree', approxTotalRows: 3 },
+ ]);
+ expect(first.nextCursor).toEqual(expect.any(String));
+
+ const second = JSON.parse(
+ getFirstText(
+ await call({ database: DATABASE, limit: 1, cursor: first.nextCursor }),
+ ),
+ );
+ expect(second.tables).toEqual([
+ { name: 'events_view', engine: 'View', approxTotalRows: null },
+ ]);
+ expect(second.nextCursor).toBeUndefined();
+ });
+
+ it('returns user errors for a missing table or database', async () => {
+ const missingTable = await call({ database: DATABASE, table: 'nope' });
+ expect(missingTable.isError).toBe(true);
+ expect(getFirstText(missingTable)).toMatch(/not found/);
+
+ const missingDb = await call({ database: 'mcp_no_such_db' });
+ expect(missingDb.isError).toBe(true);
+ expect(getFirstText(missingDb)).toMatch(/not found/);
+ });
+
+ it('treats database and table names as values, not SQL', async () => {
+ const result = await call({
+ database: DATABASE,
+ table: "events' OR '1'='1",
+ });
+ expect(result.isError).toBe(true);
+ expect(getFirstText(result)).toMatch(/not found/);
+ });
+
+ it("does not use another team's connection", async () => {
+ const otherConnection = await Connection.create({
+ team: new mongoose.Types.ObjectId(),
+ name: 'Other',
+ host: config.CLICKHOUSE_HOST,
+ username: config.CLICKHOUSE_USER,
+ password: config.CLICKHOUSE_PASSWORD,
+ });
+ const result = await callTool(client, 'clickstack_describe_table', {
+ connectionId: otherConnection._id.toString(),
+ database: DATABASE,
+ });
+ expect(result.isError).toBe(true);
+ expect(getFirstText(result)).toMatch(/Connection .* not found/);
+ });
+});
diff --git a/packages/api/src/mcp/__tests__/describeTable.test.ts b/packages/api/src/mcp/__tests__/describeTable.test.ts
new file mode 100644
index 000000000..bb7ef19e2
--- /dev/null
+++ b/packages/api/src/mcp/__tests__/describeTable.test.ts
@@ -0,0 +1,649 @@
+const mockQuery = jest.fn();
+const mockGetConnectionById = jest.fn();
+
+jest.mock('@/models/source', () => ({}));
+jest.mock('@/controllers/sources', () => ({}));
+jest.mock('@/controllers/connection', () => ({
+ getConnectionById: (...args: unknown[]) => mockGetConnectionById(...args),
+}));
+jest.mock('@/clickhouse', () => ({
+ ClickhouseClient: jest.fn(() => ({ query: mockQuery })),
+}));
+
+import { ClickHouseError } from '@clickhouse/client-common';
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+
+import { registerDescribeTable } from '@/mcp/tools/sources/describeTable';
+import {
+ shapeTableDescription,
+ shapeTableListEntry,
+} from '@/mcp/tools/sources/describeTableQueries';
+import {
+ describeTableSchema,
+ encodeTableListCursor,
+ validateDescribeTableInput,
+} from '@/mcp/tools/sources/describeTableSchema';
+import type { RegisterToolFn, ToolResult } from '@/mcp/tools/types';
+import { getErrorCategory, type McpErrorResult } from '@/mcp/utils/errors';
+
+const CONNECTION_ID = '65f5e4a3b9e77c001a567890';
+
+type QueryArgs = {
+ query: string;
+ query_params: Record;
+ clickhouse_settings: Record;
+};
+
+const queryCalls = (): QueryArgs[] =>
+ mockQuery.mock.calls.map(([args]: [QueryArgs]) => args);
+
+function respondWith(routes: Record) {
+ mockQuery.mockImplementation(async ({ query: sql }: QueryArgs) => {
+ const rows =
+ Object.entries(routes).find(([fragment]) =>
+ sql.includes(fragment),
+ )?.[1] ?? [];
+ if (rows instanceof Error) throw rows;
+ return { json: async () => ({ data: rows }) };
+ });
+}
+
+function getHandler() {
+ let handler:
+ | ((args: Record) => Promise)
+ | undefined;
+ const registerTool: RegisterToolFn = (_name, config, h) => {
+ handler = args => h(config.inputSchema.parse(args));
+ };
+ registerDescribeTable({
+ server: new McpServer({ name: 'test', version: '1.0.0' }),
+ context: { teamId: 'team-1', userId: 'user-1' },
+ registerTool,
+ });
+ return handler!;
+}
+
+function errorCategory(result: ToolResult) {
+ const isMcpError = (r: ToolResult): r is ToolResult & McpErrorResult =>
+ r.isError === true;
+ return isMcpError(result) ? getErrorCategory(result) : undefined;
+}
+
+const parseText = (result: ToolResult) => JSON.parse(result.content[0].text);
+
+const TABLE_ROW = {
+ name: 'events',
+ engine: 'MergeTree',
+ sorting_key: 'ServiceName, Timestamp',
+ primary_key: 'ServiceName',
+ partition_key: 'toDate(Timestamp)',
+ comment: '',
+ total_rows: '1200',
+ total_bytes: '34567',
+};
+
+const COLUMN_ROWS = [
+ {
+ name: 'Timestamp',
+ type: 'DateTime64(9)',
+ default_kind: '',
+ default_expression: '',
+ comment: '',
+ is_in_sorting_key: 1,
+ is_in_primary_key: 0,
+ },
+ {
+ name: 'Day',
+ type: 'Date',
+ default_kind: 'MATERIALIZED',
+ default_expression: 'toDate(Timestamp)',
+ comment: 'partition day',
+ is_in_sorting_key: 0,
+ is_in_primary_key: 0,
+ },
+];
+
+describe('clickstack_describe_table', () => {
+ beforeEach(() => {
+ mockQuery.mockReset();
+ mockGetConnectionById.mockReset().mockResolvedValue({
+ host: 'http://localhost:8123',
+ username: 'default',
+ password: '',
+ });
+ });
+
+ describe('input validation', () => {
+ it('caps sampleRows at 5 and defaults it to 0', () => {
+ expect(
+ describeTableSchema.safeParse({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 't',
+ sampleRows: 6,
+ }).success,
+ ).toBe(false);
+ const parsed = describeTableSchema.parse({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ });
+ expect(parsed.sampleRows).toBe(0);
+ expect(parsed.limit).toBe(50);
+ });
+
+ it('rejects an empty database name', () => {
+ expect(
+ describeTableSchema.safeParse({
+ connectionId: CONNECTION_ID,
+ database: '',
+ }).success,
+ ).toBe(false);
+ });
+
+ it('rejects sampleRows without a table and cursor with a table', () => {
+ expect(
+ validateDescribeTableInput({ sampleRows: 2, cursor: undefined }),
+ ).toMatch(/sampleRows requires table/);
+ expect(
+ validateDescribeTableInput({ table: 't', sampleRows: 0, cursor: 'x' }),
+ ).toMatch(/cursor only applies when listing/);
+ expect(
+ validateDescribeTableInput({ table: 't', sampleRows: 3 }),
+ ).toBeNull();
+ });
+
+ it('returns a user error for a malformed connectionId without a lookup', async () => {
+ const result = await getHandler()({
+ connectionId: 'not-an-id',
+ database: 'default',
+ });
+ expect(result.isError).toBe(true);
+ expect(mockGetConnectionById).not.toHaveBeenCalled();
+ });
+
+ it('returns a user error when the connection is not in the team', async () => {
+ mockGetConnectionById.mockResolvedValue(null);
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ });
+ expect(result.isError).toBe(true);
+ expect(mockGetConnectionById).toHaveBeenCalledWith(
+ 'team-1',
+ CONNECTION_ID,
+ true,
+ );
+ });
+ });
+
+ describe('output shaping', () => {
+ it('shapes a table description from system table rows', () => {
+ expect(
+ shapeTableDescription({
+ database: 'default',
+ table: { ...TABLE_ROW, total_rows: null, total_bytes: null },
+ keys: TABLE_ROW,
+ columns: COLUMN_ROWS,
+ indices: [{ name: 'idx_day', type: 'minmax', expression: 'Day' }],
+ }),
+ ).toEqual({
+ table: {
+ database: 'default',
+ name: 'events',
+ engine: 'MergeTree',
+ sortingKey: 'ServiceName, Timestamp',
+ primaryKey: 'ServiceName',
+ partitionKey: 'toDate(Timestamp)',
+ approxTotalRows: null,
+ approxTotalBytes: null,
+ },
+ columns: [
+ {
+ name: 'Timestamp',
+ type: 'DateTime64(9)',
+ inSortingKey: true,
+ inPrimaryKey: false,
+ },
+ {
+ name: 'Day',
+ type: 'Date',
+ defaultKind: 'MATERIALIZED',
+ defaultExpression: 'toDate(Timestamp)',
+ comment: 'partition day',
+ inSortingKey: false,
+ inPrimaryKey: false,
+ },
+ ],
+ dataSkippingIndices: [
+ { name: 'idx_day', type: 'minmax', expression: 'Day' },
+ ],
+ });
+ });
+
+ it('converts quoted 64-bit row counts to numbers', () => {
+ expect(
+ shapeTableListEntry({
+ name: 'events',
+ engine: 'MergeTree',
+ total_rows: '42',
+ }),
+ ).toEqual({ name: 'events', engine: 'MergeTree', approxTotalRows: 42 });
+ });
+
+ it('binds database and table names as query parameters', async () => {
+ respondWith({ 'FROM system.tables': [TABLE_ROW] });
+ await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: "evil'db",
+ table: 'events`; DROP TABLE x',
+ sampleRows: 1,
+ });
+ const calls = queryCalls();
+ expect(calls.some(c => c.query.includes(':Identifier}'))).toBe(true);
+ for (const { query: sql, query_params } of calls) {
+ expect(sql).not.toContain('evil');
+ expect(sql).not.toContain('DROP');
+ expect(Object.values(query_params)).toEqual(
+ expect.arrayContaining(["evil'db", 'events`; DROP TABLE x']),
+ );
+ }
+ });
+
+ it('pages the table list with an opaque cursor', async () => {
+ respondWith({
+ 'FROM system.tables': [
+ { name: 'a', engine: 'MergeTree', total_rows: '1' },
+ { name: 'b', engine: 'View', total_rows: null },
+ { name: 'c', engine: 'MergeTree', total_rows: '3' },
+ ],
+ });
+ const output = parseText(
+ await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ limit: 2,
+ }),
+ );
+ expect(output.tables).toEqual([
+ { name: 'a', engine: 'MergeTree', approxTotalRows: 1 },
+ { name: 'b', engine: 'View', approxTotalRows: null },
+ ]);
+ expect(output.nextCursor).toBe(encodeTableListCursor({ lastName: 'b' }));
+ });
+
+ it('rejects a malformed cursor', async () => {
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ cursor: 'garbage',
+ });
+ expect(result.isError).toBe(true);
+ expect(result.content[0].text).toMatch(/Invalid cursor/);
+ });
+
+ it('reports a missing database as a user error', async () => {
+ respondWith({});
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'nope',
+ });
+ expect(result.isError).toBe(true);
+ expect(errorCategory(result)).toBe('user');
+ expect(result.content[0].text).toMatch(/Database "nope" not found/);
+ });
+
+ it('reports a missing table as a user error', async () => {
+ respondWith({});
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'nope',
+ });
+ expect(result.isError).toBe(true);
+ expect(errorCategory(result)).toBe('user');
+ expect(result.content[0].text).toMatch(/default\.nope" not found/);
+ });
+
+ it('maps permission errors to user errors with a hint', async () => {
+ respondWith({
+ 'FROM system.tables': [TABLE_ROW],
+ 'FROM system.columns': new Error(
+ 'Code: 497. DB::Exception: default: Not enough privileges. (ACCESS_DENIED)',
+ ),
+ });
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ });
+ expect(result.isError).toBe(true);
+ expect(errorCategory(result)).toBe('user');
+ expect(result.content[0].text).toMatch(/lacks the privileges/);
+ });
+
+ it('skips sampling by default', async () => {
+ respondWith({ 'FROM system.tables': [TABLE_ROW] });
+ const output = parseText(
+ await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ }),
+ );
+ expect(output.sampleRows).toBeUndefined();
+ expect(queryCalls().some(q => q.query.includes(':Identifier}'))).toBe(
+ false,
+ );
+ });
+
+ it('trims oversized sample rows', async () => {
+ respondWith({
+ 'FROM system.tables': [TABLE_ROW],
+ ':Identifier}': [{ Body: 'x'.repeat(50_000) }, { Body: 'short' }],
+ });
+ const output = parseText(
+ await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ sampleRows: 2,
+ }),
+ );
+ expect(output.sampleRows).toHaveLength(2);
+ expect(output.sampleRows[0].__hdx_trimmed).toBe(true);
+ expect(output.sampleRows[1]).toEqual({ Body: 'short' });
+ });
+
+ it('keeps the schema when sampling fails', async () => {
+ respondWith({
+ 'FROM system.tables': [TABLE_ROW],
+ ':Identifier}': new Error('ACCESS_DENIED'),
+ });
+ const output = parseText(
+ await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ sampleRows: 1,
+ }),
+ );
+ expect(output.table.name).toBe('events');
+ expect(output.sampleRowsError).toBe('ACCESS_DENIED');
+ expect(output.sampleRowsHint).toMatch(/schema above is complete/);
+ });
+
+ it('lists an existing empty database without an error', async () => {
+ respondWith({ 'FROM system.databases': [{ name: 'empty' }] });
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'empty',
+ });
+ expect(result.isError).toBeFalsy();
+ expect(parseText(result).tables).toEqual([]);
+ });
+
+ it('reports a missing database when a cursor is replayed against it', async () => {
+ respondWith({});
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'dropped',
+ cursor: encodeTableListCursor({ lastName: 'a' }),
+ });
+ expect(result.isError).toBe(true);
+ expect(result.content[0].text).toMatch(/Database "dropped" not found/);
+ });
+
+ it('reads keys, indices, and key columns from the local table of a Distributed table', async () => {
+ respondWith({
+ 'FROM system.tables': [
+ {
+ ...TABLE_ROW,
+ engine: 'Distributed',
+ engine_full:
+ "Distributed('prod', 'default', 'events_local', rand())",
+ sorting_key: '',
+ primary_key: '',
+ partition_key: '',
+ total_rows: null,
+ },
+ ],
+ ', system.tables)': [{ ...TABLE_ROW, engine: 'ReplicatedMergeTree' }],
+ ', system.data_skipping_indices)': [
+ { name: 'idx', type: 'minmax', expression: 'Day' },
+ ],
+ ', system.columns)': [
+ { name: 'Timestamp', is_in_sorting_key: 1, is_in_primary_key: 1 },
+ ],
+ 'FROM system.columns': COLUMN_ROWS.map(c => ({
+ ...c,
+ is_in_sorting_key: 0,
+ is_in_primary_key: 0,
+ })),
+ });
+
+ const output = parseText(
+ await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ }),
+ );
+ expect(output.table).toMatchObject({
+ engine: 'Distributed',
+ localTable: {
+ cluster: 'prod',
+ database: 'default',
+ table: 'events_local',
+ engine: 'ReplicatedMergeTree',
+ },
+ sortingKey: 'ServiceName, Timestamp',
+ primaryKey: 'ServiceName',
+ });
+ expect(output.columns[0]).toMatchObject({
+ name: 'Timestamp',
+ inSortingKey: true,
+ inPrimaryKey: true,
+ });
+ expect(output.dataSkippingIndices).toEqual([
+ { name: 'idx', type: 'minmax', expression: 'Day' },
+ ]);
+ const localQueries = queryCalls().filter(c =>
+ c.query.includes('cluster('),
+ );
+ expect(localQueries).toHaveLength(3);
+ for (const { query_params } of localQueries) {
+ expect(Object.values(query_params)).toEqual(
+ expect.arrayContaining(['prod', 'events_local']),
+ );
+ }
+ });
+
+ it('falls back to the Distributed table when cluster() is not granted', async () => {
+ const denied = new Error(
+ 'Code: 497. DB::Exception: Not enough privileges. To execute this query, it is necessary to have the grant REMOTE ON *.*. (ACCESS_DENIED)',
+ );
+ respondWith({
+ 'FROM system.tables': [
+ {
+ ...TABLE_ROW,
+ engine: 'Distributed',
+ engine_full:
+ "Distributed('prod', 'default', 'events_local', rand())",
+ sorting_key: '',
+ primary_key: '',
+ partition_key: '',
+ },
+ ],
+ ', system.tables)': denied,
+ ', system.data_skipping_indices)': denied,
+ ', system.columns)': denied,
+ 'FROM system.columns': COLUMN_ROWS,
+ });
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ });
+ expect(result.isError).toBeFalsy();
+ const output = parseText(result);
+ expect(output.table).toMatchObject({
+ engine: 'Distributed',
+ sortingKey: '',
+ });
+ expect(output.columns).toHaveLength(2);
+ expect(output.localTableHint).toMatch(/REMOTE grant/);
+ expect(output.table.localTable).toEqual({
+ cluster: 'prod',
+ database: 'default',
+ table: 'events_local',
+ });
+ });
+
+ it('falls back to the Distributed table when its cluster is unknown', async () => {
+ const unknownCluster = new Error(
+ "Code: 701. DB::Exception: Requested cluster 'prod' not found. (CLUSTER_DOESNT_EXIST)",
+ );
+ respondWith({
+ 'FROM system.tables': [
+ {
+ ...TABLE_ROW,
+ engine: 'Distributed',
+ engine_full:
+ "Distributed('prod', 'default', 'events_local', rand())",
+ },
+ ],
+ ', system.tables)': unknownCluster,
+ ', system.data_skipping_indices)': unknownCluster,
+ ', system.columns)': unknownCluster,
+ 'FROM system.columns': COLUMN_ROWS,
+ });
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ });
+ expect(result.isError).toBeFalsy();
+ const output = parseText(result);
+ expect(output.columns).toHaveLength(2);
+ expect(output.localTableHint).toMatch(/unknown cluster/);
+ expect(output.dataSkippingIndicesHint).toBeDefined();
+ });
+
+ it('keeps the schema when the sample times out', async () => {
+ const timeout = new Error('query failed', {
+ cause: new ClickHouseError({
+ message: 'Timeout exceeded: elapsed 30s',
+ code: '159',
+ type: 'TIMEOUT_EXCEEDED',
+ }),
+ });
+ respondWith({
+ 'FROM system.tables': [TABLE_ROW],
+ ':Identifier}': timeout,
+ });
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ sampleRows: 1,
+ });
+ expect(result.isError).toBeFalsy();
+ expect(parseText(result).sampleRowsError).toBeDefined();
+ });
+
+ it('still describes the table when skip indices are not readable', async () => {
+ respondWith({
+ 'FROM system.tables': [TABLE_ROW],
+ 'FROM system.data_skipping_indices': new Error(
+ 'Code: 497. DB::Exception: Not enough privileges. (ACCESS_DENIED)',
+ ),
+ });
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ });
+ expect(result.isError).toBeFalsy();
+ const output = parseText(result);
+ expect(output.dataSkippingIndices).toEqual([]);
+ expect(output.dataSkippingIndicesHint).toMatch(/Could not read/);
+ });
+
+ it('reports a sample over the byte cap as sampleRowsError', async () => {
+ respondWith({
+ 'FROM system.tables': [TABLE_ROW],
+ ':Identifier}': new Error(
+ 'Code: 396. DB::Exception: Limit for result exceeded, max bytes: 976.56 KiB. (TOO_MANY_ROWS_OR_BYTES)',
+ ),
+ });
+ const output = parseText(
+ await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ sampleRows: 5,
+ }),
+ );
+ expect(output.columns).toBeDefined();
+ expect(output.sampleRowsError).toMatch(/TOO_MANY_ROWS_OR_BYTES/);
+ });
+
+ it('keeps several medium sample rows within the sample budget', async () => {
+ const columns = Array.from({ length: 200 }, (_, i) => ({
+ ...COLUMN_ROWS[0],
+ name: `Column_${i}`,
+ }));
+ respondWith({
+ 'FROM system.tables': [TABLE_ROW],
+ 'FROM system.columns': columns,
+ ':Identifier}': Array.from({ length: 5 }, () => ({
+ Body: 'x'.repeat(9_900),
+ })),
+ });
+ const output = parseText(
+ await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ sampleRows: 5,
+ }),
+ );
+ expect(JSON.stringify(output.sampleRows).length).toBeLessThan(11_000);
+ expect(output.columns).toHaveLength(200);
+ expect(output.note).toMatch(/trimmed/);
+ });
+
+ it('caps the bytes ClickHouse returns for the sample', async () => {
+ respondWith({ 'FROM system.tables': [TABLE_ROW] });
+ await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ sampleRows: 1,
+ });
+ const sampleCall = queryCalls().find(c =>
+ c.query.includes(':Identifier}'),
+ );
+ expect(sampleCall?.clickhouse_settings.max_result_bytes).toBeDefined();
+ // Below the 30s tool budget, so a slow sample fails on its own.
+ expect(sampleCall?.clickhouse_settings.max_execution_time).toBe(10);
+ });
+
+ it('reports an infrastructure failure while sampling as a server error', async () => {
+ respondWith({
+ 'FROM system.tables': [TABLE_ROW],
+ ':Identifier}': Object.assign(new Error('connect ECONNREFUSED'), {
+ code: 'ECONNREFUSED',
+ }),
+ });
+ const result = await getHandler()({
+ connectionId: CONNECTION_ID,
+ database: 'default',
+ table: 'events',
+ sampleRows: 1,
+ });
+ expect(result.isError).toBe(true);
+ expect(errorCategory(result)).toBe('server');
+ });
+ });
+});
diff --git a/packages/api/src/mcp/tools/query/sql.ts b/packages/api/src/mcp/tools/query/sql.ts
index fd68dbc88..82a33f7a8 100644
--- a/packages/api/src/mcp/tools/query/sql.ts
+++ b/packages/api/src/mcp/tools/query/sql.ts
@@ -109,11 +109,12 @@ export function registerSql({ context, registerTool }: ToolRegistrar) {
'ONLY use raw SQL when the query genuinely cannot be expressed by a builder tool — ' +
`i.e. it requires ${SQL_FALLBACK_CRITERIA}. ` +
'A single-table aggregation, top-N, time-series, or row browse is ALWAYS a ' +
- 'builder-tool job, never raw SQL. If you are unsure, try the builder tool first and only ' +
- 'fall back to SQL if it cannot express what you need.\n\n' +
+ 'builder-tool job, never raw SQL.\n\n' +
'Requires connectionId — call clickstack_list_sources to find connections. Pass sourceId ' +
'too when querying a source, so $__sourceTable resolves to its table. ' +
- 'Call clickstack_describe_source to discover column names before writing SQL.\n\n' +
+ 'Before writing SQL, get column names from clickstack_describe_source, or ' +
+ 'clickstack_describe_table for tables that are not sources (instead of ' +
+ 'querying system.columns / system.tables).\n\n' +
'Results are always returned as table rows — for time-series semantics, ' +
'include a time column and ORDER BY it in your SQL.\n\n' +
'To run several independent queries at once (e.g. comparing sources), pass ' +
diff --git a/packages/api/src/mcp/tools/sources/describeTable.ts b/packages/api/src/mcp/tools/sources/describeTable.ts
new file mode 100644
index 000000000..63290a75f
--- /dev/null
+++ b/packages/api/src/mcp/tools/sources/describeTable.ts
@@ -0,0 +1,380 @@
+import { quoteIdentifierIfNeeded } from '@hyperdx/common-utils/dist/core/metadata';
+import { SourceKind } from '@hyperdx/common-utils/dist/types';
+import { trace } from '@opentelemetry/api';
+
+import { ClickhouseClient } from '@/clickhouse';
+import { getConnectionById } from '@/controllers/connection';
+import { READ_ONLY_TOOL } from '@/mcp/tools/annotations';
+import {
+ clickHouseErrorResult,
+ isQueryOutOfTime,
+ isServerError,
+ MCP_REQUEST_TIMEOUT,
+} from '@/mcp/tools/query/helpers';
+import type { ToolRegistrar, ToolResult } from '@/mcp/tools/types';
+import {
+ mcpServerError,
+ mcpUserError,
+ sanitizeFetchError,
+ validateObjectId,
+} from '@/mcp/utils/errors';
+import { MCP_TOOL_TIMEOUT_MS, runWithTimeout } from '@/mcp/utils/timeout';
+import { Source } from '@/models/source';
+import logger from '@/utils/logger';
+import { trimToolResponse } from '@/utils/trimToolResponse';
+
+import {
+ databaseExists,
+ fetchSampleRows,
+ fetchTableDetails,
+ fetchTablePage,
+ isPrivilegeError,
+ shapeTableDescription,
+ shapeTableListEntry,
+ type TableQueryContext,
+} from './describeTableQueries';
+import {
+ decodeTableListCursor,
+ type DescribeTableInput,
+ describeTableSchema,
+ encodeTableListCursor,
+ MAX_SAMPLE_ROWS,
+ validateDescribeTableInput,
+} from './describeTableSchema';
+
+// Sample rows can hold arbitrarily large values; keep them a small share of
+// the response so they never crowd out the schema.
+const SAMPLE_ROWS_MAX_CHARS = 10_000;
+const RESPONSE_MAX_CHARS = 50_000;
+
+function jsonResult(payload: unknown): ToolResult {
+ return { content: [{ type: 'text', text: JSON.stringify(payload) }] };
+}
+
+/**
+ * Not-found and permission failures are the caller's to fix, so they go back
+ * as user errors with a hint; only infrastructure failures stay server errors.
+ */
+function tableQueryErrorResult(e: unknown, prefix: string) {
+ const message = e instanceof Error ? e.message : String(e);
+ let suffix: string | undefined;
+ if (isPrivilegeError(e)) {
+ suffix =
+ "The ClickHouse user on this connection lacks the privileges needed. Ask the connection's administrator to grant them, or use another connection.";
+ } else if (/UNKNOWN_(TABLE|DATABASE)|does not exist/i.test(message)) {
+ suffix =
+ 'Call clickstack_describe_table with only database to list the tables it contains.';
+ }
+ return clickHouseErrorResult(e, { prefix, suffix });
+}
+
+async function listTables(
+ ctx: TableQueryContext,
+ input: DescribeTableInput,
+): Promise {
+ const cursor = input.cursor ? decodeTableListCursor(input.cursor) : null;
+ if (input.cursor && !cursor) {
+ return mcpUserError(
+ 'Invalid cursor. Omit cursor to start over, or pass the exact `nextCursor` value returned by a previous call.',
+ );
+ }
+
+ let rows;
+ try {
+ rows = await fetchTablePage(ctx, {
+ database: input.database,
+ afterName: cursor?.lastName,
+ limit: input.limit,
+ });
+ } catch (e) {
+ return tableQueryErrorResult(e, 'Failed to list tables');
+ }
+
+ // system.tables returns nothing for a missing database rather than erroring.
+ if (rows.length === 0) {
+ try {
+ if (!(await databaseExists(ctx, input.database))) {
+ return mcpUserError(
+ `Database "${input.database}" not found or not visible to this connection's ClickHouse user.`,
+ );
+ }
+ } catch (e) {
+ return tableQueryErrorResult(e, 'Failed to look up database');
+ }
+ }
+
+ const page = rows.slice(0, input.limit);
+ const nextCursor =
+ rows.length > input.limit
+ ? encodeTableListCursor({ lastName: page[page.length - 1].name })
+ : undefined;
+
+ trace
+ .getActiveSpan()
+ ?.setAttribute('mcp.describe_table.table_count', page.length);
+
+ return jsonResult({
+ database: input.database,
+ tables: page.map(shapeTableListEntry),
+ ...(nextCursor && { nextCursor }),
+ nextSteps: {
+ describe:
+ 'Call clickstack_describe_table again with table set to get its columns, keys, and indices.',
+ },
+ });
+}
+
+function trimSampleRows(rows: Record[]) {
+ const perRowBudget = Math.floor(
+ SAMPLE_ROWS_MAX_CHARS / Math.max(rows.length, 1),
+ );
+ let isTrimmed = false;
+ const data = rows.map(row => {
+ const trimmed = trimToolResponse(row, perRowBudget);
+ isTrimmed ||= trimmed.isTrimmed;
+ return trimmed.data;
+ });
+ return { data, isTrimmed };
+}
+
+async function sampleTableRows(
+ ctx: TableQueryContext,
+ args: { database: string; table: string; limit: number },
+) {
+ try {
+ const rows = trimSampleRows(await fetchSampleRows(ctx, args));
+ return { sampleRows: rows.data, isTrimmed: rows.isTrimmed };
+ } catch (e) {
+ // A slow View or Distributed table can time out on SELECT *; only a
+ // connection failure should cost the schema already fetched.
+ if (isServerError(e) && !isQueryOutOfTime(e)) throw e;
+ // The schema is still useful when the user can read system tables but
+ // not the table itself.
+ return {
+ sampleRowsError: sanitizeFetchError(e),
+ sampleRowsHint:
+ 'Only the row sample failed; the schema above is complete. Retry without sampleRows, ' +
+ "or check that the connection's ClickHouse user can SELECT from this table.",
+ isTrimmed: false,
+ };
+ }
+}
+
+/** Sources on this connection that read the table, so callers can be sent to clickstack_describe_source. */
+async function findSourcesForTable(args: {
+ teamId: string;
+ connectionId: string;
+ database: string;
+ table: string;
+}) {
+ try {
+ const sources = await Source.find({
+ team: args.teamId,
+ connection: args.connectionId,
+ 'from.databaseName': args.database,
+ kind: { $ne: SourceKind.Promql },
+ }).lean();
+ return sources
+ .filter(
+ s =>
+ s.from?.tableName === args.table ||
+ ('metricTables' in s &&
+ Object.values(s.metricTables ?? {}).includes(args.table)),
+ )
+ .map(s => ({ id: s._id.toString(), name: s.name, kind: s.kind }));
+ } catch (e) {
+ // The schema is still worth returning without the source hint.
+ logger.warn(
+ { err: e, teamId: args.teamId, connectionId: args.connectionId },
+ 'clickstack_describe_table source lookup failed',
+ );
+ return [];
+ }
+}
+
+async function describeTable(
+ ctx: TableQueryContext,
+ input: DescribeTableInput & { table: string; teamId: string },
+): Promise {
+ const { database, table } = input;
+ let details;
+ let sources;
+ try {
+ [details, sources] = await Promise.all([
+ fetchTableDetails(ctx, { database, table }),
+ findSourcesForTable({
+ teamId: input.teamId,
+ connectionId: ctx.connectionId,
+ database,
+ table,
+ }),
+ ]);
+ } catch (e) {
+ return tableQueryErrorResult(e, 'Failed to describe table');
+ }
+
+ if (!details) {
+ return mcpUserError(
+ `Table "${database}.${table}" not found or not visible to this connection's ClickHouse user. ` +
+ 'Call clickstack_describe_table with only database to list its tables.',
+ );
+ }
+
+ const { indicesUnavailable, localTableUnavailable, ...shapeInput } = details;
+ const { columns, ...description } = shapeTableDescription({
+ database,
+ ...shapeInput,
+ });
+
+ let sample = {};
+ let sampleTrimmed = false;
+ if (input.sampleRows > 0) {
+ try {
+ const { isTrimmed, ...result } = await sampleTableRows(ctx, {
+ database,
+ table,
+ limit: input.sampleRows,
+ });
+ sample = result;
+ sampleTrimmed = isTrimmed;
+ } catch (e) {
+ return clickHouseErrorResult(e, 'Failed to sample rows');
+ }
+ }
+
+ trace.getActiveSpan()?.setAttributes({
+ 'mcp.describe_table.column_count': columns.length,
+ 'mcp.describe_table.index_count': description.dataSkippingIndices.length,
+ 'mcp.describe_table.engine': description.table.engine,
+ 'mcp.describe_table.source_count': sources.length,
+ });
+
+ const qualifiedName = `${quoteIdentifierIfNeeded(database)}.${quoteIdentifierIfNeeded(table)}`;
+ const nextSteps =
+ sources.length > 0
+ ? {
+ describeSource:
+ `This table backs ${sources.length === 1 ? 'a source' : 'sources'}. ` +
+ `Call clickstack_describe_source with sourceId "${sources[0].id}" for attribute keys and sampled values, ` +
+ 'then query it with the builder tools (clickstack_table, clickstack_timeseries, clickstack_search) rather than clickstack_sql.',
+ }
+ : {
+ query:
+ `Query it with clickstack_sql using connectionId "${ctx.connectionId}" and ` +
+ `FROM ${qualifiedName}. ` +
+ 'To query it with the builder tools, register it as a source with clickstack_save_source.',
+ };
+
+ const rest = {
+ ...description,
+ ...(sources.length > 0 && { sources }),
+ ...(localTableUnavailable && {
+ localTableHint:
+ 'Could not read the local table through cluster() (missing REMOTE grant, unknown cluster, or unreachable shard), so keys and key flags come from the Distributed table itself and may be empty.',
+ }),
+ ...(indicesUnavailable && {
+ dataSkippingIndicesHint:
+ 'Could not read system.data_skipping_indices for this table (missing grant, or the local table was unreachable), so indices are not listed.',
+ }),
+ ...sample,
+ nextSteps,
+ };
+ // Trim only the column list, against what the rest of the response leaves,
+ // so a large sample cannot push columns out.
+ const trimmedColumns = trimToolResponse(
+ columns,
+ RESPONSE_MAX_CHARS - JSON.stringify(rest).length,
+ );
+
+ return jsonResult({
+ ...rest,
+ columns: trimmedColumns.data,
+ ...((sampleTrimmed || trimmedColumns.isTrimmed) && {
+ note: 'Result was trimmed for context size. Some columns or sample values may be omitted.',
+ }),
+ });
+}
+
+export function registerDescribeTable({
+ context,
+ registerTool,
+}: ToolRegistrar): void {
+ const teamId = context.teamId.toString();
+
+ registerTool(
+ 'clickstack_describe_table',
+ {
+ title: 'Describe ClickHouse Table',
+ annotations: READ_ONLY_TOOL,
+ description:
+ 'Inspect ClickHouse tables that are NOT registered as sources. For a source, ' +
+ 'call clickstack_describe_source instead.\n\n' +
+ 'Use this instead of querying system.tables / system.columns with clickstack_sql.\n\n' +
+ 'With table set, returns:\n' +
+ '- table: engine, sorting key, primary key, partition key, approximate total rows and bytes\n' +
+ '- columns[]: name, type, default kind/expression, comment, and whether it is in the sorting/primary key\n' +
+ '- dataSkippingIndices[]: name, type, expression\n' +
+ `- sampleRows: up to ${MAX_SAMPLE_ROWS} rows when sampleRows is set (off by default)\n\n` +
+ 'Without table, lists the tables in the database (name, engine, approximate rows), ' +
+ 'paginated. Pass the returned `nextCursor` back unchanged to fetch the next page.',
+ inputSchema: describeTableSchema,
+ },
+ async rawInput => {
+ const input = describeTableSchema.parse(rawInput);
+
+ const invalid = validateDescribeTableInput(input);
+ if (invalid) return mcpUserError(invalid);
+ const invalidId = validateObjectId(input.connectionId, 'connectionId');
+ if (invalidId) return invalidId;
+
+ trace.getActiveSpan()?.setAttributes({
+ 'mcp.describe_table.mode': input.table ? 'describe' : 'list',
+ 'mcp.describe_table.connection_id': input.connectionId,
+ 'mcp.describe_table.database': input.database,
+ 'mcp.describe_table.sample_rows': input.sampleRows,
+ ...(input.table && { 'mcp.describe_table.table': input.table }),
+ });
+
+ const connection = await getConnectionById(
+ teamId,
+ input.connectionId,
+ true,
+ );
+ if (!connection) {
+ return mcpUserError(
+ `Connection "${input.connectionId}" not found. Call clickstack_list_sources to see available connections.`,
+ );
+ }
+
+ const clickhouseClient = new ClickhouseClient({
+ host: connection.host,
+ username: connection.username,
+ password: connection.password,
+ requestTimeout: MCP_REQUEST_TIMEOUT,
+ });
+
+ const outcome = await runWithTimeout(
+ signal => {
+ const ctx = {
+ clickhouseClient,
+ connectionId: input.connectionId,
+ signal,
+ };
+ return input.table
+ ? describeTable(ctx, { ...input, table: input.table, teamId })
+ : listTables(ctx, input);
+ },
+ { timeoutMs: MCP_TOOL_TIMEOUT_MS },
+ );
+ if (!outcome.timedOut) return outcome.value;
+
+ logger.warn(
+ { teamId, connectionId: input.connectionId, database: input.database },
+ 'clickstack_describe_table timed out',
+ );
+ return mcpServerError(
+ 'clickstack_describe_table timed out. The ClickHouse server may be under load; try again.',
+ );
+ },
+ );
+}
diff --git a/packages/api/src/mcp/tools/sources/describeTableQueries.ts b/packages/api/src/mcp/tools/sources/describeTableQueries.ts
new file mode 100644
index 000000000..85b00dd8c
--- /dev/null
+++ b/packages/api/src/mcp/tools/sources/describeTableQueries.ts
@@ -0,0 +1,341 @@
+import type { ClickHouseSettings } from '@clickhouse/client-common';
+import {
+ type ChSql,
+ chSql,
+ tableExpr,
+} from '@hyperdx/common-utils/dist/clickhouse';
+import type {
+ SkipIndexMetadata,
+ TableMetadata,
+} from '@hyperdx/common-utils/dist/core/metadata';
+import { getDistributedTableArgs } from '@hyperdx/common-utils/dist/core/utils';
+
+import type { ClickhouseClient } from '@/clickhouse';
+import { MCP_CLICKHOUSE_SETTINGS } from '@/mcp/tools/query/helpers';
+import logger from '@/utils/logger';
+
+export type TableQueryContext = {
+ clickhouseClient: Pick;
+ connectionId: string;
+ signal: AbortSignal;
+};
+
+async function queryRows(
+ { clickhouseClient, connectionId, signal }: TableQueryContext,
+ sql: ChSql,
+ settings?: ClickHouseSettings,
+): Promise {
+ const response = await clickhouseClient.query<'JSON'>({
+ query: sql.sql,
+ query_params: sql.params,
+ format: 'JSON',
+ connectionId,
+ clickhouse_settings: { ...MCP_CLICKHOUSE_SETTINGS, ...settings },
+ abort_signal: signal,
+ });
+ const result = (await response.json()) as { data: T[] };
+ return result.data;
+}
+
+/** ClickHouse returns 64-bit integers as strings, and NULL for views. */
+function toApproxCount(value: string | number | null): number | null {
+ if (value == null) return null;
+ const n = Number(value);
+ return Number.isFinite(n) ? n : null;
+}
+
+export type SystemTableRow = {
+ name: string;
+ engine: string;
+ total_rows: string | number | null;
+};
+
+// total_rows / total_bytes are NULL for views, unlike TableMetadata's typing.
+type TableInfo = Pick & {
+ total_rows: string | null;
+ total_bytes: string | null;
+};
+
+type TableKeys = Pick<
+ TableMetadata,
+ 'engine' | 'sorting_key' | 'primary_key' | 'partition_key'
+>;
+
+type ColumnKeyFlags = {
+ name: string;
+ is_in_sorting_key: number;
+ is_in_primary_key: number;
+};
+
+export type SystemColumnRow = ColumnKeyFlags & {
+ type: string;
+ default_kind: string;
+ default_expression: string;
+ comment: string;
+};
+
+type DistributedTarget = { cluster: string; database: string; table: string };
+
+export function shapeTableListEntry(row: SystemTableRow) {
+ return {
+ name: row.name,
+ engine: row.engine,
+ approxTotalRows: toApproxCount(row.total_rows),
+ };
+}
+
+/**
+ * `keys` comes from the local table when `table` is Distributed, since a
+ * Distributed table has no keys or indices of its own.
+ */
+export function shapeTableDescription({
+ database,
+ table,
+ keys,
+ distributedTarget,
+ localTableEngine,
+ columns,
+ indices,
+}: {
+ database: string;
+ table: TableInfo;
+ keys: TableKeys;
+ distributedTarget?: DistributedTarget;
+ /** Unset when the local table could not be read. */
+ localTableEngine?: string;
+ columns: SystemColumnRow[];
+ indices: Pick[];
+}) {
+ return {
+ table: {
+ database,
+ name: table.name,
+ engine: table.engine,
+ ...(distributedTarget && {
+ localTable: {
+ ...distributedTarget,
+ ...(localTableEngine && { engine: localTableEngine }),
+ },
+ }),
+ sortingKey: keys.sorting_key,
+ primaryKey: keys.primary_key,
+ partitionKey: keys.partition_key,
+ ...(table.comment && { comment: table.comment }),
+ approxTotalRows: toApproxCount(table.total_rows),
+ approxTotalBytes: toApproxCount(table.total_bytes),
+ },
+ columns: columns.map(c => ({
+ name: c.name,
+ type: c.type,
+ ...(c.default_kind && {
+ defaultKind: c.default_kind,
+ defaultExpression: c.default_expression,
+ }),
+ ...(c.comment && { comment: c.comment }),
+ inSortingKey: Boolean(Number(c.is_in_sorting_key)),
+ inPrimaryKey: Boolean(Number(c.is_in_primary_key)),
+ })),
+ dataSkippingIndices: indices.map(i => ({
+ name: i.name,
+ type: i.type,
+ expression: i.expression,
+ })),
+ };
+}
+
+/** One extra row past `limit` signals that another page exists. */
+export function fetchTablePage(
+ ctx: TableQueryContext,
+ {
+ database,
+ afterName,
+ limit,
+ }: { database: string; afterName: string | undefined; limit: number },
+) {
+ return queryRows(
+ ctx,
+ chSql`
+ SELECT name, engine, total_rows
+ FROM system.tables
+ WHERE database = ${{ String: database }}
+ AND NOT is_temporary
+ ${afterName !== undefined ? chSql`AND name > ${{ String: afterName }}` : ''}
+ ORDER BY name ASC
+ LIMIT ${{ Int32: limit + 1 }}
+ `,
+ );
+}
+
+export async function databaseExists(ctx: TableQueryContext, database: string) {
+ const rows = await queryRows<{ name: string }>(
+ ctx,
+ chSql`SELECT name FROM system.databases WHERE name = ${{ String: database }}`,
+ );
+ return rows.length > 0;
+}
+
+type SystemTableName = 'tables' | 'columns' | 'data_skipping_indices';
+
+/** A system table, read across the cluster when `cluster` is set. */
+function systemTable(name: SystemTableName, cluster?: string): ChSql {
+ return cluster
+ ? chSql`cluster(${{ String: cluster }}, system.${name})`
+ : chSql`system.${name}`;
+}
+
+export function isPrivilegeError(e: unknown) {
+ const message = e instanceof Error ? e.message : String(e);
+ return /ACCESS_DENIED|NOT_ENOUGH_PRIVILEGES|Not enough privileges/i.test(
+ message,
+ );
+}
+
+/** Resolves to null instead of rejecting when the user lacks a grant. */
+function nullOnPrivilegeError(query: Promise): Promise {
+ return query.catch((e: unknown) => {
+ if (isPrivilegeError(e)) return null;
+ throw e;
+ });
+}
+
+/**
+ * Resolves to null on any failure. The cluster() lookups can fail for reasons
+ * besides grants (unknown cluster, unreachable shard), and the Distributed
+ * table's own schema is still worth returning.
+ */
+function nullOnFailure(query: Promise): Promise {
+ return query.catch((e: unknown) => {
+ logger.warn({ error: e }, 'Distributed local-table lookup failed');
+ return null;
+ });
+}
+
+/**
+ * Returns undefined when the table does not exist or is not visible. Every
+ * lookup is uncached so the result reflects ALTERs made since the last call.
+ */
+export async function fetchTableDetails(
+ ctx: TableQueryContext,
+ { database, table }: { database: string; table: string },
+) {
+ const rows = await queryRows(
+ ctx,
+ chSql`
+ SELECT * FROM system.tables
+ WHERE database = ${{ String: database }} AND name = ${{ String: table }}
+ LIMIT 1
+ `,
+ );
+ const info = rows.at(0);
+ if (!info) return undefined;
+
+ // A Distributed table has no keys or indices of its own; read them from
+ // the local table it routes to.
+ const distributedTarget =
+ info.engine === 'Distributed' ? getDistributedTableArgs(info) : undefined;
+ const keyTable = distributedTarget ?? { database, table, cluster: undefined };
+
+ const [localKeys, indices, columns, localKeyFlags] = await Promise.all([
+ distributedTarget
+ ? nullOnFailure(
+ queryRows(
+ ctx,
+ chSql`
+ SELECT engine, sorting_key, primary_key, partition_key
+ FROM ${systemTable('tables', keyTable.cluster)}
+ WHERE database = ${{ String: keyTable.database }}
+ AND name = ${{ String: keyTable.table }}
+ LIMIT 1
+ `,
+ ),
+ )
+ : undefined,
+ // Indices and the local-table lookups are optional detail, and cluster()
+ // needs the REMOTE grant; a missing grant should not hide the schema.
+ (distributedTarget ? nullOnFailure : nullOnPrivilegeError)(
+ queryRows>(
+ ctx,
+ chSql`
+ SELECT name, type, expr AS expression
+ FROM ${systemTable('data_skipping_indices', keyTable.cluster)}
+ WHERE database = ${{ String: keyTable.database }}
+ AND table = ${{ String: keyTable.table }}
+ LIMIT 1 BY name
+ `,
+ ),
+ ),
+ queryRows(
+ ctx,
+ chSql`
+ SELECT name, type, default_kind, default_expression, comment,
+ is_in_sorting_key, is_in_primary_key
+ FROM system.columns
+ WHERE database = ${{ String: database }} AND table = ${{ String: table }}
+ ORDER BY position
+ `,
+ ),
+ distributedTarget
+ ? nullOnFailure(
+ queryRows(
+ ctx,
+ chSql`
+ SELECT name, is_in_sorting_key, is_in_primary_key
+ FROM ${systemTable('columns', keyTable.cluster)}
+ WHERE database = ${{ String: keyTable.database }}
+ AND table = ${{ String: keyTable.table }}
+ LIMIT 1 BY name
+ `,
+ ),
+ )
+ : undefined,
+ ]);
+
+ const localKeyRow = localKeys?.at(0);
+ const flagsByName = new Map(localKeyFlags?.map(f => [f.name, f]));
+ return {
+ table: info,
+ keys: localKeyRow ?? info,
+ distributedTarget,
+ localTableEngine: localKeyRow?.engine,
+ indices: indices ?? [],
+ indicesUnavailable: indices === null,
+ localTableUnavailable:
+ distributedTarget != null && (!localKeyRow || localKeyFlags === null),
+ columns: columns.map(c => {
+ const flags = flagsByName.get(c.name);
+ return flags
+ ? {
+ ...c,
+ is_in_sorting_key: flags.is_in_sorting_key,
+ is_in_primary_key: flags.is_in_primary_key,
+ }
+ : c;
+ }),
+ };
+}
+
+// Five rows of wide String columns can still be large. Past this cap
+// ClickHouse fails the query, which surfaces as sampleRowsError, so nothing
+// unbounded is buffered.
+const SAMPLE_MAX_RESULT_BYTES = 1_000_000;
+// Well inside the tool's budget, which the schema queries have already used
+// part of, so a slow sample fails on its own instead of timing out the call.
+const SAMPLE_MAX_EXECUTION_SEC = 10;
+
+export function fetchSampleRows(
+ ctx: TableQueryContext,
+ {
+ database,
+ table,
+ limit,
+ }: { database: string; table: string; limit: number },
+) {
+ return queryRows>(
+ ctx,
+ chSql`SELECT * FROM ${tableExpr({ database, table })} LIMIT ${{ Int32: limit }}`,
+ {
+ max_result_bytes: SAMPLE_MAX_RESULT_BYTES.toString(),
+ max_execution_time: SAMPLE_MAX_EXECUTION_SEC,
+ },
+ );
+}
diff --git a/packages/api/src/mcp/tools/sources/describeTableSchema.ts b/packages/api/src/mcp/tools/sources/describeTableSchema.ts
new file mode 100644
index 000000000..4e70b85cf
--- /dev/null
+++ b/packages/api/src/mcp/tools/sources/describeTableSchema.ts
@@ -0,0 +1,83 @@
+import { z } from 'zod';
+
+import {
+ decodeCursor as decodeCursorPayload,
+ encodeCursor as encodeCursorPayload,
+} from '@/utils/pagination';
+
+const DEFAULT_TABLE_LIST_LIMIT = 50;
+const MAX_TABLE_LIST_LIMIT = 500;
+export const MAX_SAMPLE_ROWS = 5;
+
+const tableListCursorSchema = z.object({ lastName: z.string() });
+
+export type TableListCursor = z.infer;
+
+export function encodeTableListCursor(payload: TableListCursor): string {
+ return encodeCursorPayload(payload);
+}
+
+export function decodeTableListCursor(raw: string): TableListCursor | null {
+ return decodeCursorPayload(raw, tableListCursorSchema);
+}
+
+export const describeTableSchema = z.object({
+ connectionId: z
+ .string()
+ .describe(
+ 'Connection to inspect. Call clickstack_list_sources to find connection IDs.',
+ ),
+ database: z.string().min(1).describe('ClickHouse database name.'),
+ table: z
+ .string()
+ .min(1)
+ .optional()
+ .describe(
+ 'Table to describe. Omit to list the tables in the database instead.',
+ ),
+ sampleRows: z
+ .number()
+ .int()
+ .min(0)
+ .max(MAX_SAMPLE_ROWS)
+ .optional()
+ .default(0)
+ .describe(
+ `Number of rows to sample from the table (0-${MAX_SAMPLE_ROWS}). Only valid with table. Default: 0.`,
+ ),
+ limit: z
+ .number()
+ .int()
+ .min(1)
+ .max(MAX_TABLE_LIST_LIMIT)
+ .optional()
+ .default(DEFAULT_TABLE_LIST_LIMIT)
+ .describe(
+ 'Maximum tables returned per page when listing. Use cursor to fetch further pages.',
+ ),
+ cursor: z
+ .string()
+ .optional()
+ .describe(
+ 'Opaque pagination cursor returned by a previous listing call as `nextCursor`. ' +
+ 'Pass it back unchanged to get the next page.',
+ ),
+});
+
+export type DescribeTableInput = z.infer;
+
+/**
+ * Checks the option combinations a flat Zod object cannot express. Returns an
+ * error message, or null when the input is consistent.
+ */
+export function validateDescribeTableInput(
+ input: Pick,
+): string | null {
+ if (input.table == null && input.sampleRows > 0) {
+ return 'sampleRows requires table. Pass a table name, or omit sampleRows to list tables.';
+ }
+ if (input.table != null && input.cursor) {
+ return 'cursor only applies when listing tables. Omit table to page through the table list, or omit cursor to describe the table.';
+ }
+ return null;
+}
diff --git a/packages/api/src/mcp/tools/sources/index.ts b/packages/api/src/mcp/tools/sources/index.ts
index 2efea649a..2c36fc9d0 100644
--- a/packages/api/src/mcp/tools/sources/index.ts
+++ b/packages/api/src/mcp/tools/sources/index.ts
@@ -3,6 +3,7 @@ import type { ToolDefinition, ToolRegistrar } from '@/mcp/tools/types';
import { registerDeleteSource } from './deleteSource';
import { registerDescribeMetric } from './describeMetric';
import { registerDescribeSource } from './describeSource';
+import { registerDescribeTable } from './describeTable';
import { registerListMetrics } from './listMetrics';
import { registerListSources } from './listSources';
import { registerSaveSource } from './saveSource';
@@ -10,6 +11,7 @@ import { registerSaveSource } from './saveSource';
const sourcesTools: ToolDefinition = (registrar: ToolRegistrar) => {
registerListSources(registrar);
registerDescribeSource(registrar);
+ registerDescribeTable(registrar);
registerSaveSource(registrar);
registerDeleteSource(registrar);
registerListMetrics(registrar);
From 702181002a09bcc3c57ad1c16333869e6299abe2 Mon Sep 17 00:00:00 2001
From: Spencer Torres