From 1f51945e8180ea03e4f00ca0f5631db3993b42d2 Mon Sep 17 00:00:00 2001 From: archat-hash Date: Wed, 7 Oct 2026 10:14:19 +0200 Subject: [PATCH 1/4] fix(app): let the Database query drawer use the span attribute skip index Opening a database statement filtered on the coalesced attribute lookup alone. No skip index covers that expression, so every tile in the drawer read each granule in the selected window, which is what times out on a busy source. Prefilter with an expression an index does cover, in the form the table supports: `AttributeItems` where the schema has that column, otherwise `mapValues(Attributes)`. The prefilter matches a superset of the spans and the coalesced comparison still decides, so results are unchanged. It also helps where neither is indexed, by short-circuiting the coalesce. Measured on 4M spans, one statement, caches off: granules rows read bytes CPU before 499/499 4,001,873 191.8MiB 2945ms after 26/499 212,992 22.4MiB 256ms The older schema reached the same ratios through its mapValues index. Both were confirmed end to end against ClickHouse 26.8 and 26.1. JSON attribute columns keep the previous condition, and statements are now escaped, so one containing a quote no longer breaks the query. Co-Authored-By: Claude Opus 5 --- .changeset/db-statement-index-prefilter.md | 5 + .../src/__tests__/serviceDashboard.test.ts | 141 ++++++++++++++++++ .../ServiceDashboardDbQuerySidePanel.tsx | 16 +- packages/app/src/serviceDashboard.ts | 106 +++++++++++-- 4 files changed, 253 insertions(+), 15 deletions(-) create mode 100644 .changeset/db-statement-index-prefilter.md diff --git a/.changeset/db-statement-index-prefilter.md b/.changeset/db-statement-index-prefilter.md new file mode 100644 index 0000000000..1b89ff782e --- /dev/null +++ b/.changeset/db-statement-index-prefilter.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +fix: Let the Services > Database query drawer use the span attribute skip index instead of scanning the whole trace window diff --git a/packages/app/src/__tests__/serviceDashboard.test.ts b/packages/app/src/__tests__/serviceDashboard.test.ts index 5a5821337e..726aa35b52 100644 --- a/packages/app/src/__tests__/serviceDashboard.test.ts +++ b/packages/app/src/__tests__/serviceDashboard.test.ts @@ -1,4 +1,5 @@ import type { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; +import { parseQuery } from '@hyperdx/common-utils/dist/filters'; import type { TTraceSource } from '@hyperdx/common-utils/dist/types'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { renderHook } from '@testing-library/react'; @@ -7,6 +8,7 @@ import * as metadataModule from '@/hooks/useMetadata'; import { getExpressions, makeCoalescedFieldsAccessQuery, + makeDbStatementCondition, useServiceDashboardExpressions, } from '@/serviceDashboard'; @@ -146,6 +148,145 @@ describe('Service Dashboard', () => { }); }); + describe('dbStatementIndexHint', () => { + const attributesColumn = { + name: 'SpanAttributes', + type: 'Map(LowCardinality(String), String)', + } as ColumnMeta; + const itemsColumn = { name: 'SpanAttributeItems' } as ColumnMeta; + + it('should prefer the attribute items column when the table has one', () => { + expect( + getExpressions(mockSource, [attributesColumn, itemsColumn], []) + .dbStatementIndexHint, + ).toEqual({ kind: 'items', column: 'SpanAttributeItems' }); + }); + + it('should fall back to the attribute map, which older schemas index by value', () => { + expect( + getExpressions(mockSource, [attributesColumn], []).dbStatementIndexHint, + ).toEqual({ kind: 'mapValues', column: 'SpanAttributes' }); + }); + + it('should be undefined when the attribute field is not a map', () => { + expect( + getExpressions( + mockSource, + [{ name: 'SpanAttributes', type: 'String' } as ColumnMeta], + [], + ).dbStatementIndexHint, + ).toBeUndefined(); + }); + + it('should be undefined for JSON attribute columns', () => { + expect( + getExpressions( + mockSource, + [attributesColumn, itemsColumn], + ['SpanAttributes'], + ).dbStatementIndexHint, + ).toBeUndefined(); + }); + + it('should not mistake the attribute field itself for an items column', () => { + expect( + getExpressions( + { ...mockSource, eventAttributesExpression: 'Attrs' }, + [{ name: 'Attrs', type: 'Map(String, String)' } as ColumnMeta], + [], + ).dbStatementIndexHint, + ).toEqual({ kind: 'mapValues', column: 'Attrs' }); + }); + }); + + describe('makeDbStatementCondition', () => { + const dbStatement = + "coalesce(nullif(SpanAttributes['db.query.text'], ''), nullif(SpanAttributes['db.statement'], ''))"; + const equality = `${dbStatement} IN ('SELECT 1')`; + + it('should only compare the statement when nothing is indexed', () => { + expect( + makeDbStatementCondition({ + expressions: { dbStatement, dbStatementIndexHint: undefined }, + statement: 'SELECT 1', + }), + ).toBe(equality); + }); + + it('should prefilter on the items column, which newer schemas index', () => { + expect( + makeDbStatementCondition({ + expressions: { + dbStatement, + dbStatementIndexHint: { + kind: 'items', + column: 'SpanAttributeItems', + }, + }, + statement: 'SELECT 1', + }), + ).toBe( + "(has(SpanAttributeItems, 'db.query.text=SELECT 1') OR " + + "has(SpanAttributeItems, 'db.statement=SELECT 1')) " + + `AND ${equality}`, + ); + }); + + it('should prefilter on the attribute values, which older schemas index', () => { + expect( + makeDbStatementCondition({ + expressions: { + dbStatement, + dbStatementIndexHint: { + kind: 'mapValues', + column: 'SpanAttributes', + }, + }, + statement: 'SELECT 1', + }), + ).toBe( + `(has(mapValues(SpanAttributes), 'SELECT 1')) AND ${equality}`, + ); + }); + + it.each([ + ['items' as const, 'SpanAttributeItems'], + ['mapValues' as const, 'SpanAttributes'], + ])('should escape quotes in the %s prefilter', (kind, column) => { + const condition = makeDbStatementCondition({ + expressions: { dbStatement, dbStatementIndexHint: { kind, column } }, + statement: "SELECT 'a'", + }); + + expect(condition).toContain("SELECT ''a''"); + expect(condition).not.toContain("SELECT 'a'"); + }); + + it.each([ + [undefined], + [{ kind: 'items' as const, column: 'SpanAttributeItems' }], + [{ kind: 'mapValues' as const, column: 'SpanAttributes' }], + ])( + 'should be read back by the search page as the same filter (%p)', + dbStatementIndexHint => { + const statement = "SELECT 'a' FROM t WHERE b IN (1) AND c = ?"; + + const { filters } = parseQuery([ + { + type: 'sql', + condition: makeDbStatementCondition({ + expressions: { dbStatement, dbStatementIndexHint }, + statement, + }), + }, + ]); + + expect(Object.keys(filters)).toEqual([dbStatement]); + expect([...filters[dbStatement].included]).toEqual([statement]); + }, + ); + }); + describe('useServiceDashboardExpressions', () => { const mockColumns: ColumnMeta[] = [ { name: 'Duration', type: 'UInt64' }, diff --git a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx index 3a22294ca8..3106fd8ca9 100644 --- a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx +++ b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx @@ -16,7 +16,10 @@ import { ChartCard } from '@/components/charts/ChartCard'; import { DBTimeChart } from '@/components/DBTimeChart'; import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils'; import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile'; -import { useServiceDashboardExpressions } from '@/serviceDashboard'; +import { + makeDbStatementCondition, + useServiceDashboardExpressions, +} from '@/serviceDashboard'; import { useSource } from '@/source'; import { useZIndex, ZIndexContext } from '@/zIndex'; @@ -46,16 +49,23 @@ export default function ServiceDashboardDbQuerySidePanel({ const drawerZIndex = contextZIndex + 10; const dbQueryFilters = useMemo(() => { + if (!expressions || !dbQuery) { + return []; + } + const filters: Filter[] = [ { type: 'sql', - condition: `${expressions?.dbStatement} IN ('${dbQuery}')`, + condition: makeDbStatementCondition({ + expressions, + statement: dbQuery, + }), }, ]; if (service) { filters.push({ type: 'sql', - condition: `${expressions?.service} IN ('${service}')`, + condition: `${expressions.service} IN ('${service}')`, }); } return filters; diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts index edf2056625..41d7c2df09 100644 --- a/packages/app/src/serviceDashboard.ts +++ b/packages/app/src/serviceDashboard.ts @@ -1,12 +1,27 @@ import { useMemo } from 'react'; import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types'; import { useColumns, useJsonColumns } from './hooks/useMetadata'; const COALESCE_FIELDS_LIMIT = 100; +// Ordered from highest to lowest precedence +const DB_STATEMENT_KEYS = ['db.query.text', 'db.statement']; + +/** + * Where a skip index can be made to match a database statement. The OTel + * schemas index span attributes two ways depending on the ClickHouse version: + * newer ones an `AttributeItems` column of `key=value` strings, older + * ones `mapValues(Attributes)`. + */ +type DbStatementIndexHint = { + kind: 'items' | 'mapValues'; + column: string; +}; + // Helper function to format field access based on column type function formatFieldAccess( field: string, @@ -89,18 +104,9 @@ function getDefaults({ isAttributeFieldJSON?: boolean; } = {}) { const dbStatement = makeCoalescedFieldsAccessQuery( - [ - formatFieldAccess( - spanAttributeField, - 'db.query.text', - isAttributeFieldJSON, - ), - formatFieldAccess( - spanAttributeField, - 'db.statement', - isAttributeFieldJSON, - ), - ], + DB_STATEMENT_KEYS.map(key => + formatFieldAccess(spanAttributeField, key, isAttributeFieldJSON), + ), isAttributeFieldJSON, ); @@ -138,6 +144,77 @@ function getDefaults({ const ENDPOINT_MATERIALIZED_COLUMN_NAME = 'endpoint'; +function getAttributeItemsColumn( + attributesField: string, + columns: ColumnMeta[], +): string | undefined { + const itemsColumn = attributesField.replace(/Attributes$/, 'AttributeItems'); + const exists = + itemsColumn !== attributesField && + columns.some(column => column.name === itemsColumn); + return exists ? itemsColumn : undefined; +} + +function getDbStatementIndexHint( + attributesField: string, + isAttributeFieldJSON: boolean, + columns: ColumnMeta[], +): DbStatementIndexHint | undefined { + if (isAttributeFieldJSON) { + return undefined; + } + + const itemsColumn = getAttributeItemsColumn(attributesField, columns); + if (itemsColumn) { + return { kind: 'items', column: itemsColumn }; + } + + const isMap = columns.some( + column => + column.name === attributesField && column.type.startsWith('Map('), + ); + return isMap ? { kind: 'mapValues', column: attributesField } : undefined; +} + +function makeIndexPrefilter( + hint: DbStatementIndexHint, + statement: string, +): string { + if (hint.kind === 'mapValues') { + return `has(mapValues(${hint.column}), '${escapeSqlString(statement)}')`; + } + return DB_STATEMENT_KEYS.map( + key => + `has(${hint.column}, '${escapeSqlString(`${key}=${statement}`)}')`, + ).join(' OR '); +} + +/** + * Condition selecting the spans of a single database statement. + * + * No skip index covers the coalesced attribute lookup, so on its own it makes + * ClickHouse read every granule in the time range. The prefilter matches a + * superset of those spans in a form an index does cover; the coalesced + * comparison still decides the match, so results are unchanged. It is also + * cheaper where no index applies, since it short-circuits the coalesce. + */ +export function makeDbStatementCondition({ + expressions, + statement, +}: { + expressions: Pick< + ReturnType, + 'dbStatement' | 'dbStatementIndexHint' + >; + statement: string; +}): string { + const equality = `${expressions.dbStatement} IN ('${escapeSqlString(statement)}')`; + const hint = expressions.dbStatementIndexHint; + return hint + ? `(${makeIndexPrefilter(hint, statement)}) AND ${equality}` + : equality; +} + export function getExpressions( source: TTraceSource, columns: ColumnMeta[], @@ -172,6 +249,11 @@ export function getExpressions( // Database dbStatement: defaults.dbStatement, + dbStatementIndexHint: getDbStatementIndexHint( + spanAttributeField, + isAttributeFieldJSON, + columns, + ), }; const auxExpressions = { From 35235fc41cbb6142f3203514a49618983a42039b Mon Sep 17 00:00:00 2001 From: archat-hash Date: Mon, 12 Oct 2026 19:15:56 +0200 Subject: [PATCH 2/4] fix(app): delegate the statement prefilter rewrite to the query renderer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback: picking the items column by name skipped the checks the existing KV-items path already makes — server version and direct_read support, ALIAS vs MATERIALIZED, a matching text index — and hardcoded `=` as the separator, so a column built with another separator silently matched nothing. Emit the prefilter as plain map comparisons instead. `rewriteSqlFilterWithKvItems` already runs over every sql filter and rewrites that shape onto the validated items column with the separator read from its DDL, and schemas without such a column keep the comparisons, which their mapValues index serves. The hint type and the three helpers that sniffed the schema are gone. Also escape the service name, which the drawer still interpolated raw. Confirmed against ClickHouse 26.8: the drawer's queries arrive rewritten to `has(SpanAttributeItems, concat('db.query.text', '=', …))` and read 9 of 492 granules. Co-Authored-By: Claude Opus 5 --- .../src/__tests__/serviceDashboard.test.ts | 149 ++++++++---------- .../ServiceDashboardDbQuerySidePanel.tsx | 3 +- packages/app/src/serviceDashboard.ts | 92 +++-------- 3 files changed, 89 insertions(+), 155 deletions(-) diff --git a/packages/app/src/__tests__/serviceDashboard.test.ts b/packages/app/src/__tests__/serviceDashboard.test.ts index 726aa35b52..5a9e32237b 100644 --- a/packages/app/src/__tests__/serviceDashboard.test.ts +++ b/packages/app/src/__tests__/serviceDashboard.test.ts @@ -1,4 +1,5 @@ import type { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; +import { rewriteSqlFilterWithKvItems } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { parseQuery } from '@hyperdx/common-utils/dist/filters'; import type { TTraceSource } from '@hyperdx/common-utils/dist/types'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; @@ -148,140 +149,114 @@ describe('Service Dashboard', () => { }); }); - describe('dbStatementIndexHint', () => { - const attributesColumn = { - name: 'SpanAttributes', - type: 'Map(LowCardinality(String), String)', - } as ColumnMeta; - const itemsColumn = { name: 'SpanAttributeItems' } as ColumnMeta; - - it('should prefer the attribute items column when the table has one', () => { - expect( - getExpressions(mockSource, [attributesColumn, itemsColumn], []) - .dbStatementIndexHint, - ).toEqual({ kind: 'items', column: 'SpanAttributeItems' }); - }); - - it('should fall back to the attribute map, which older schemas index by value', () => { + describe('dbStatementPrefilterFields', () => { + it('should expose the map accessors the renderer can rewrite', () => { expect( - getExpressions(mockSource, [attributesColumn], []).dbStatementIndexHint, - ).toEqual({ kind: 'mapValues', column: 'SpanAttributes' }); + getExpressions(mockSource, [], []).dbStatementPrefilterFields, + ).toEqual([ + "SpanAttributes['db.query.text']", + "SpanAttributes['db.statement']", + ]); }); - it('should be undefined when the attribute field is not a map', () => { + it('should be empty for JSON attribute columns', () => { expect( - getExpressions( - mockSource, - [{ name: 'SpanAttributes', type: 'String' } as ColumnMeta], - [], - ).dbStatementIndexHint, - ).toBeUndefined(); - }); - - it('should be undefined for JSON attribute columns', () => { - expect( - getExpressions( - mockSource, - [attributesColumn, itemsColumn], - ['SpanAttributes'], - ).dbStatementIndexHint, - ).toBeUndefined(); - }); - - it('should not mistake the attribute field itself for an items column', () => { - expect( - getExpressions( - { ...mockSource, eventAttributesExpression: 'Attrs' }, - [{ name: 'Attrs', type: 'Map(String, String)' } as ColumnMeta], - [], - ).dbStatementIndexHint, - ).toEqual({ kind: 'mapValues', column: 'Attrs' }); + getExpressions(mockSource, [], ['SpanAttributes']) + .dbStatementPrefilterFields, + ).toEqual([]); }); }); describe('makeDbStatementCondition', () => { const dbStatement = "coalesce(nullif(SpanAttributes['db.query.text'], ''), nullif(SpanAttributes['db.statement'], ''))"; + const prefilterFields = [ + "SpanAttributes['db.query.text']", + "SpanAttributes['db.statement']", + ]; const equality = `${dbStatement} IN ('SELECT 1')`; - it('should only compare the statement when nothing is indexed', () => { + it('should only compare the statement when there is nothing to prefilter on', () => { expect( makeDbStatementCondition({ - expressions: { dbStatement, dbStatementIndexHint: undefined }, + expressions: { dbStatement, dbStatementPrefilterFields: [] }, statement: 'SELECT 1', }), ).toBe(equality); }); - it('should prefilter on the items column, which newer schemas index', () => { + it('should prefilter on each attribute the statement can come from', () => { expect( makeDbStatementCondition({ - expressions: { - dbStatement, - dbStatementIndexHint: { - kind: 'items', - column: 'SpanAttributeItems', - }, - }, + expressions: { dbStatement, dbStatementPrefilterFields: prefilterFields }, statement: 'SELECT 1', }), ).toBe( - "(has(SpanAttributeItems, 'db.query.text=SELECT 1') OR " + - "has(SpanAttributeItems, 'db.statement=SELECT 1')) " + + "(SpanAttributes['db.query.text'] IN ('SELECT 1') OR " + + "SpanAttributes['db.statement'] IN ('SELECT 1')) " + `AND ${equality}`, ); }); - it('should prefilter on the attribute values, which older schemas index', () => { - expect( - makeDbStatementCondition({ - expressions: { - dbStatement, - dbStatementIndexHint: { - kind: 'mapValues', - column: 'SpanAttributes', - }, - }, - statement: 'SELECT 1', - }), - ).toBe( - `(has(mapValues(SpanAttributes), 'SELECT 1')) AND ${equality}`, - ); + it('should escape quotes everywhere the statement appears', () => { + const condition = makeDbStatementCondition({ + expressions: { dbStatement, dbStatementPrefilterFields: prefilterFields }, + statement: "SELECT 'a'", + }); + + expect(condition.match(/SELECT ''a''/g)).toHaveLength(3); + expect(condition).not.toContain("IN ('SELECT 'a'')"); }); - it.each([ - ['items' as const, 'SpanAttributeItems'], - ['mapValues' as const, 'SpanAttributes'], - ])('should escape quotes in the %s prefilter', (kind, column) => { + it('should be rewritten onto the items column by the query renderer', () => { const condition = makeDbStatementCondition({ - expressions: { dbStatement, dbStatementIndexHint: { kind, column } }, - statement: "SELECT 'a'", + expressions: { dbStatement, dbStatementPrefilterFields: prefilterFields }, + statement: 'SELECT 1', }); - expect(condition).toContain("SELECT ''a''"); - expect(condition).not.toContain("SELECT 'a'"); + const rewritten = rewriteSqlFilterWithKvItems( + condition, + new Map([ + [ + 'SpanAttributes', + { + kv: { + columnName: 'SpanAttributeItems', + indexName: 'idx_span_attr_items', + separator: ':', + useHasAny: false, + mapColumn: 'SpanAttributes', + }, + }, + ], + ]), + ); + + expect(rewritten).toContain( + "has(`SpanAttributeItems`, concat('db.query.text', ':', 'SELECT 1'))", + ); + expect(rewritten).toContain( + "has(`SpanAttributeItems`, concat('db.statement', ':', 'SELECT 1'))", + ); + expect(rewritten).toContain(dbStatement); }); - it.each([ - [undefined], - [{ kind: 'items' as const, column: 'SpanAttributeItems' }], - [{ kind: 'mapValues' as const, column: 'SpanAttributes' }], - ])( + it.each([[[]], [prefilterFields]])( 'should be read back by the search page as the same filter (%p)', - dbStatementIndexHint => { + dbStatementPrefilterFields => { const statement = "SELECT 'a' FROM t WHERE b IN (1) AND c = ?"; const { filters } = parseQuery([ { type: 'sql', condition: makeDbStatementCondition({ - expressions: { dbStatement, dbStatementIndexHint }, + expressions: { dbStatement, dbStatementPrefilterFields }, statement, }), }, ]); - expect(Object.keys(filters)).toEqual([dbStatement]); + expect(filters[dbStatement]).toBeDefined(); expect([...filters[dbStatement].included]).toEqual([statement]); }, ); diff --git a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx index 3106fd8ca9..72a4440cd1 100644 --- a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx +++ b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx @@ -1,6 +1,7 @@ import { useCallback, useMemo } from 'react'; import { pick } from 'lodash'; import { parseAsString, useQueryState } from 'nuqs'; +import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; import { DisplayType, type Filter, @@ -65,7 +66,7 @@ export default function ServiceDashboardDbQuerySidePanel({ if (service) { filters.push({ type: 'sql', - condition: `${expressions.service} IN ('${service}')`, + condition: `${expressions.service} IN ('${escapeSqlString(service)}')`, }); } return filters; diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts index 41d7c2df09..8a24eee78f 100644 --- a/packages/app/src/serviceDashboard.ts +++ b/packages/app/src/serviceDashboard.ts @@ -11,16 +11,6 @@ const COALESCE_FIELDS_LIMIT = 100; // Ordered from highest to lowest precedence const DB_STATEMENT_KEYS = ['db.query.text', 'db.statement']; -/** - * Where a skip index can be made to match a database statement. The OTel - * schemas index span attributes two ways depending on the ClickHouse version: - * newer ones an `AttributeItems` column of `key=value` strings, older - * ones `mapValues(Attributes)`. - */ -type DbStatementIndexHint = { - kind: 'items' | 'mapValues'; - column: string; -}; // Helper function to format field access based on column type function formatFieldAccess( @@ -110,6 +100,15 @@ function getDefaults({ isAttributeFieldJSON, ); + // Plain map accessors: the shape `rewriteSqlFilterWithKvItems` rewrites onto + // an indexed items column. JSON columns have no such column, and their + // accessors are not map subscripts, so they get no prefilter. + const dbStatementPrefilterFields = isAttributeFieldJSON + ? [] + : DB_STATEMENT_KEYS.map(key => + formatFieldAccess(spanAttributeField, key, false), + ); + return { duration: 'Duration', durationPrecision: 9, @@ -139,64 +138,25 @@ function getDefaults({ isAttributeFieldJSON, ), dbStatement, + dbStatementPrefilterFields, }; } const ENDPOINT_MATERIALIZED_COLUMN_NAME = 'endpoint'; -function getAttributeItemsColumn( - attributesField: string, - columns: ColumnMeta[], -): string | undefined { - const itemsColumn = attributesField.replace(/Attributes$/, 'AttributeItems'); - const exists = - itemsColumn !== attributesField && - columns.some(column => column.name === itemsColumn); - return exists ? itemsColumn : undefined; -} - -function getDbStatementIndexHint( - attributesField: string, - isAttributeFieldJSON: boolean, - columns: ColumnMeta[], -): DbStatementIndexHint | undefined { - if (isAttributeFieldJSON) { - return undefined; - } - - const itemsColumn = getAttributeItemsColumn(attributesField, columns); - if (itemsColumn) { - return { kind: 'items', column: itemsColumn }; - } - - const isMap = columns.some( - column => - column.name === attributesField && column.type.startsWith('Map('), - ); - return isMap ? { kind: 'mapValues', column: attributesField } : undefined; -} - -function makeIndexPrefilter( - hint: DbStatementIndexHint, - statement: string, -): string { - if (hint.kind === 'mapValues') { - return `has(mapValues(${hint.column}), '${escapeSqlString(statement)}')`; - } - return DB_STATEMENT_KEYS.map( - key => - `has(${hint.column}, '${escapeSqlString(`${key}=${statement}`)}')`, - ).join(' OR '); +function inList(expression: string, value: string): string { + return `${expression} IN ('${escapeSqlString(value)}')`; } /** * Condition selecting the spans of a single database statement. * * No skip index covers the coalesced attribute lookup, so on its own it makes - * ClickHouse read every granule in the time range. The prefilter matches a - * superset of those spans in a form an index does cover; the coalesced - * comparison still decides the match, so results are unchanged. It is also - * cheaper where no index applies, since it short-circuits the coalesce. + * ClickHouse read every granule in the time range. The prefilter compares the + * same keys directly, which matches a superset of those spans while the + * coalesced comparison still decides; `rewriteSqlFilterWithKvItems` then + * rewrites each comparison onto the schema's indexed items column, and older + * schemas serve them from their `mapValues` index as written. */ export function makeDbStatementCondition({ expressions, @@ -204,14 +164,16 @@ export function makeDbStatementCondition({ }: { expressions: Pick< ReturnType, - 'dbStatement' | 'dbStatementIndexHint' + 'dbStatement' | 'dbStatementPrefilterFields' >; statement: string; }): string { - const equality = `${expressions.dbStatement} IN ('${escapeSqlString(statement)}')`; - const hint = expressions.dbStatementIndexHint; - return hint - ? `(${makeIndexPrefilter(hint, statement)}) AND ${equality}` + const equality = inList(expressions.dbStatement, statement); + const prefilter = expressions.dbStatementPrefilterFields.map(field => + inList(field, statement), + ); + return prefilter.length + ? `(${prefilter.join(' OR ')}) AND ${equality}` : equality; } @@ -249,11 +211,7 @@ export function getExpressions( // Database dbStatement: defaults.dbStatement, - dbStatementIndexHint: getDbStatementIndexHint( - spanAttributeField, - isAttributeFieldJSON, - columns, - ), + dbStatementPrefilterFields: defaults.dbStatementPrefilterFields, }; const auxExpressions = { From 46cb39b50547029074a41693fea1f05d6bca141d Mon Sep 17 00:00:00 2001 From: archat-hash Date: Mon, 12 Oct 2026 20:10:24 +0200 Subject: [PATCH 3/4] refactor(app): build the service filter with the shared inList helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback: the escaped `IN ('…')` was spelled out twice after the last commit. Export the helper and use it for the service filter too, so quoting and escaping stay in one place. Co-Authored-By: Claude Opus 5 --- .../app/src/components/ServiceDashboardDbQuerySidePanel.tsx | 4 ++-- packages/app/src/serviceDashboard.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx index 72a4440cd1..a580f7819b 100644 --- a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx +++ b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx @@ -1,7 +1,6 @@ import { useCallback, useMemo } from 'react'; import { pick } from 'lodash'; import { parseAsString, useQueryState } from 'nuqs'; -import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; import { DisplayType, type Filter, @@ -18,6 +17,7 @@ import { DBTimeChart } from '@/components/DBTimeChart'; import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils'; import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile'; import { + inList, makeDbStatementCondition, useServiceDashboardExpressions, } from '@/serviceDashboard'; @@ -66,7 +66,7 @@ export default function ServiceDashboardDbQuerySidePanel({ if (service) { filters.push({ type: 'sql', - condition: `${expressions.service} IN ('${escapeSqlString(service)}')`, + condition: inList(expressions.service, service), }); } return filters; diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts index 8a24eee78f..ff593657d1 100644 --- a/packages/app/src/serviceDashboard.ts +++ b/packages/app/src/serviceDashboard.ts @@ -11,7 +11,6 @@ const COALESCE_FIELDS_LIMIT = 100; // Ordered from highest to lowest precedence const DB_STATEMENT_KEYS = ['db.query.text', 'db.statement']; - // Helper function to format field access based on column type function formatFieldAccess( field: string, @@ -144,7 +143,8 @@ function getDefaults({ const ENDPOINT_MATERIALIZED_COLUMN_NAME = 'endpoint'; -function inList(expression: string, value: string): string { +/** A quoted, escaped `IN` comparison — the only place either is spelled out. */ +export function inList(expression: string, value: string): string { return `${expression} IN ('${escapeSqlString(value)}')`; } From d60acf2333367102fa6d8679d6ef5bcd24b81b9c Mon Sep 17 00:00:00 2001 From: archat-hash Date: Tue, 13 Oct 2026 06:31:13 +0200 Subject: [PATCH 4/4] test(app): cover escaping through the kv-items rewrite Review feedback: no test drove a statement with quotes or backslashes through rewriteSqlFilterWithKvItems, and the round-trip assertion allowed extra facets instead of pinning the single one the search sidebar gets. Drop the comment on inList, which claimed the IN comparison is spelled out nowhere else while a sibling panel still interpolates its own. Co-Authored-By: Claude Opus 5 --- .../src/__tests__/serviceDashboard.test.ts | 72 ++++++++++++------- packages/app/src/serviceDashboard.ts | 1 - 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/packages/app/src/__tests__/serviceDashboard.test.ts b/packages/app/src/__tests__/serviceDashboard.test.ts index 5a9e32237b..a6c3866f61 100644 --- a/packages/app/src/__tests__/serviceDashboard.test.ts +++ b/packages/app/src/__tests__/serviceDashboard.test.ts @@ -188,7 +188,10 @@ describe('Service Dashboard', () => { it('should prefilter on each attribute the statement can come from', () => { expect( makeDbStatementCondition({ - expressions: { dbStatement, dbStatementPrefilterFields: prefilterFields }, + expressions: { + dbStatement, + dbStatementPrefilterFields: prefilterFields, + }, statement: 'SELECT 1', }), ).toBe( @@ -200,7 +203,10 @@ describe('Service Dashboard', () => { it('should escape quotes everywhere the statement appears', () => { const condition = makeDbStatementCondition({ - expressions: { dbStatement, dbStatementPrefilterFields: prefilterFields }, + expressions: { + dbStatement, + dbStatementPrefilterFields: prefilterFields, + }, statement: "SELECT 'a'", }); @@ -208,30 +214,36 @@ describe('Service Dashboard', () => { expect(condition).not.toContain("IN ('SELECT 'a'')"); }); - it('should be rewritten onto the items column by the query renderer', () => { - const condition = makeDbStatementCondition({ - expressions: { dbStatement, dbStatementPrefilterFields: prefilterFields }, - statement: 'SELECT 1', - }); + const kvItemsLookup = new Map([ + [ + 'SpanAttributes', + { + kv: { + columnName: 'SpanAttributeItems', + indexName: 'idx_span_attr_items', + separator: ':', + useHasAny: false, + mapColumn: 'SpanAttributes', + }, + }, + ], + ]); - const rewritten = rewriteSqlFilterWithKvItems( - condition, - new Map([ - [ - 'SpanAttributes', - { - kv: { - columnName: 'SpanAttributeItems', - indexName: 'idx_span_attr_items', - separator: ':', - useHasAny: false, - mapColumn: 'SpanAttributes', - }, - }, - ], - ]), + const rewrite = (statement: string) => + rewriteSqlFilterWithKvItems( + makeDbStatementCondition({ + expressions: { + dbStatement, + dbStatementPrefilterFields: prefilterFields, + }, + statement, + }), + kvItemsLookup, ); + it('should be rewritten onto the items column by the query renderer', () => { + const rewritten = rewrite('SELECT 1'); + expect(rewritten).toContain( "has(`SpanAttributeItems`, concat('db.query.text', ':', 'SELECT 1'))", ); @@ -241,6 +253,18 @@ describe('Service Dashboard', () => { expect(rewritten).toContain(dbStatement); }); + it.each([["SELECT 'a' FROM t"], ["SELECT 'a\\b'"], ['{"q":{"_id":"?"}}']])( + 'should survive the rewrite with its escaping intact (%j)', + statement => { + const rewritten = rewrite(statement); + + const quoteCount = rewritten.split("'").length - 1; + + expect(rewritten).toContain('has(`SpanAttributeItems`'); + expect(quoteCount % 2).toBe(0); + }, + ); + it.each([[[]], [prefilterFields]])( 'should be read back by the search page as the same filter (%p)', dbStatementPrefilterFields => { @@ -256,7 +280,7 @@ describe('Service Dashboard', () => { }, ]); - expect(filters[dbStatement]).toBeDefined(); + expect(Object.keys(filters)).toEqual([dbStatement]); expect([...filters[dbStatement].included]).toEqual([statement]); }, ); diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts index ff593657d1..d48221e43c 100644 --- a/packages/app/src/serviceDashboard.ts +++ b/packages/app/src/serviceDashboard.ts @@ -143,7 +143,6 @@ function getDefaults({ const ENDPOINT_MATERIALIZED_COLUMN_NAME = 'endpoint'; -/** A quoted, escaped `IN` comparison — the only place either is spelled out. */ export function inList(expression: string, value: string): string { return `${expression} IN ('${escapeSqlString(value)}')`; }