diff --git a/.changeset/db-statement-index-prefilter.md b/.changeset/db-statement-index-prefilter.md new file mode 100644 index 0000000000..1b89ff782e --- /dev/null +++ b/.changeset/db-statement-index-prefilter.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +fix: Let the Services > Database query drawer use the span attribute skip index instead of scanning the whole trace window diff --git a/packages/app/src/__tests__/serviceDashboard.test.ts b/packages/app/src/__tests__/serviceDashboard.test.ts index 5a5821337e..a6c3866f61 100644 --- a/packages/app/src/__tests__/serviceDashboard.test.ts +++ b/packages/app/src/__tests__/serviceDashboard.test.ts @@ -1,4 +1,6 @@ import type { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; +import { rewriteSqlFilterWithKvItems } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { parseQuery } from '@hyperdx/common-utils/dist/filters'; import type { TTraceSource } from '@hyperdx/common-utils/dist/types'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; import { renderHook } from '@testing-library/react'; @@ -7,6 +9,7 @@ import * as metadataModule from '@/hooks/useMetadata'; import { getExpressions, makeCoalescedFieldsAccessQuery, + makeDbStatementCondition, useServiceDashboardExpressions, } from '@/serviceDashboard'; @@ -146,6 +149,143 @@ describe('Service Dashboard', () => { }); }); + describe('dbStatementPrefilterFields', () => { + it('should expose the map accessors the renderer can rewrite', () => { + expect( + getExpressions(mockSource, [], []).dbStatementPrefilterFields, + ).toEqual([ + "SpanAttributes['db.query.text']", + "SpanAttributes['db.statement']", + ]); + }); + + it('should be empty for JSON attribute columns', () => { + expect( + getExpressions(mockSource, [], ['SpanAttributes']) + .dbStatementPrefilterFields, + ).toEqual([]); + }); + }); + + describe('makeDbStatementCondition', () => { + const dbStatement = + "coalesce(nullif(SpanAttributes['db.query.text'], ''), nullif(SpanAttributes['db.statement'], ''))"; + const prefilterFields = [ + "SpanAttributes['db.query.text']", + "SpanAttributes['db.statement']", + ]; + const equality = `${dbStatement} IN ('SELECT 1')`; + + it('should only compare the statement when there is nothing to prefilter on', () => { + expect( + makeDbStatementCondition({ + expressions: { dbStatement, dbStatementPrefilterFields: [] }, + statement: 'SELECT 1', + }), + ).toBe(equality); + }); + + it('should prefilter on each attribute the statement can come from', () => { + expect( + makeDbStatementCondition({ + expressions: { + dbStatement, + dbStatementPrefilterFields: prefilterFields, + }, + statement: 'SELECT 1', + }), + ).toBe( + "(SpanAttributes['db.query.text'] IN ('SELECT 1') OR " + + "SpanAttributes['db.statement'] IN ('SELECT 1')) " + + `AND ${equality}`, + ); + }); + + it('should escape quotes everywhere the statement appears', () => { + const condition = makeDbStatementCondition({ + expressions: { + dbStatement, + dbStatementPrefilterFields: prefilterFields, + }, + statement: "SELECT 'a'", + }); + + expect(condition.match(/SELECT ''a''/g)).toHaveLength(3); + expect(condition).not.toContain("IN ('SELECT 'a'')"); + }); + + const kvItemsLookup = new Map([ + [ + 'SpanAttributes', + { + kv: { + columnName: 'SpanAttributeItems', + indexName: 'idx_span_attr_items', + separator: ':', + useHasAny: false, + mapColumn: 'SpanAttributes', + }, + }, + ], + ]); + + const rewrite = (statement: string) => + rewriteSqlFilterWithKvItems( + makeDbStatementCondition({ + expressions: { + dbStatement, + dbStatementPrefilterFields: prefilterFields, + }, + statement, + }), + kvItemsLookup, + ); + + it('should be rewritten onto the items column by the query renderer', () => { + const rewritten = rewrite('SELECT 1'); + + expect(rewritten).toContain( + "has(`SpanAttributeItems`, concat('db.query.text', ':', 'SELECT 1'))", + ); + expect(rewritten).toContain( + "has(`SpanAttributeItems`, concat('db.statement', ':', 'SELECT 1'))", + ); + expect(rewritten).toContain(dbStatement); + }); + + it.each([["SELECT 'a' FROM t"], ["SELECT 'a\\b'"], ['{"q":{"_id":"?"}}']])( + 'should survive the rewrite with its escaping intact (%j)', + statement => { + const rewritten = rewrite(statement); + + const quoteCount = rewritten.split("'").length - 1; + + expect(rewritten).toContain('has(`SpanAttributeItems`'); + expect(quoteCount % 2).toBe(0); + }, + ); + + it.each([[[]], [prefilterFields]])( + 'should be read back by the search page as the same filter (%p)', + dbStatementPrefilterFields => { + const statement = "SELECT 'a' FROM t WHERE b IN (1) AND c = ?"; + + const { filters } = parseQuery([ + { + type: 'sql', + condition: makeDbStatementCondition({ + expressions: { dbStatement, dbStatementPrefilterFields }, + statement, + }), + }, + ]); + + expect(Object.keys(filters)).toEqual([dbStatement]); + expect([...filters[dbStatement].included]).toEqual([statement]); + }, + ); + }); + describe('useServiceDashboardExpressions', () => { const mockColumns: ColumnMeta[] = [ { name: 'Duration', type: 'UInt64' }, diff --git a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx index 3a22294ca8..a580f7819b 100644 --- a/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx +++ b/packages/app/src/components/ServiceDashboardDbQuerySidePanel.tsx @@ -16,7 +16,11 @@ import { ChartCard } from '@/components/charts/ChartCard'; import { DBTimeChart } from '@/components/DBTimeChart'; import { DrawerBody, DrawerHeader } from '@/components/DrawerUtils'; import SlowestEventsTile from '@/components/ServiceDashboardSlowestEventsTile'; -import { useServiceDashboardExpressions } from '@/serviceDashboard'; +import { + inList, + makeDbStatementCondition, + useServiceDashboardExpressions, +} from '@/serviceDashboard'; import { useSource } from '@/source'; import { useZIndex, ZIndexContext } from '@/zIndex'; @@ -46,16 +50,23 @@ export default function ServiceDashboardDbQuerySidePanel({ const drawerZIndex = contextZIndex + 10; const dbQueryFilters = useMemo(() => { + if (!expressions || !dbQuery) { + return []; + } + const filters: Filter[] = [ { type: 'sql', - condition: `${expressions?.dbStatement} IN ('${dbQuery}')`, + condition: makeDbStatementCondition({ + expressions, + statement: dbQuery, + }), }, ]; if (service) { filters.push({ type: 'sql', - condition: `${expressions?.service} IN ('${service}')`, + condition: inList(expressions.service, service), }); } return filters; diff --git a/packages/app/src/serviceDashboard.ts b/packages/app/src/serviceDashboard.ts index edf2056625..d48221e43c 100644 --- a/packages/app/src/serviceDashboard.ts +++ b/packages/app/src/serviceDashboard.ts @@ -1,12 +1,16 @@ import { useMemo } from 'react'; import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse'; import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { escapeSqlString } from '@hyperdx/common-utils/dist/core/utils'; import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types'; import { useColumns, useJsonColumns } from './hooks/useMetadata'; const COALESCE_FIELDS_LIMIT = 100; +// Ordered from highest to lowest precedence +const DB_STATEMENT_KEYS = ['db.query.text', 'db.statement']; + // Helper function to format field access based on column type function formatFieldAccess( field: string, @@ -89,21 +93,21 @@ function getDefaults({ isAttributeFieldJSON?: boolean; } = {}) { const dbStatement = makeCoalescedFieldsAccessQuery( - [ - formatFieldAccess( - spanAttributeField, - 'db.query.text', - isAttributeFieldJSON, - ), - formatFieldAccess( - spanAttributeField, - 'db.statement', - isAttributeFieldJSON, - ), - ], + DB_STATEMENT_KEYS.map(key => + formatFieldAccess(spanAttributeField, key, isAttributeFieldJSON), + ), isAttributeFieldJSON, ); + // Plain map accessors: the shape `rewriteSqlFilterWithKvItems` rewrites onto + // an indexed items column. JSON columns have no such column, and their + // accessors are not map subscripts, so they get no prefilter. + const dbStatementPrefilterFields = isAttributeFieldJSON + ? [] + : DB_STATEMENT_KEYS.map(key => + formatFieldAccess(spanAttributeField, key, false), + ); + return { duration: 'Duration', durationPrecision: 9, @@ -133,11 +137,45 @@ function getDefaults({ isAttributeFieldJSON, ), dbStatement, + dbStatementPrefilterFields, }; } const ENDPOINT_MATERIALIZED_COLUMN_NAME = 'endpoint'; +export function inList(expression: string, value: string): string { + return `${expression} IN ('${escapeSqlString(value)}')`; +} + +/** + * Condition selecting the spans of a single database statement. + * + * No skip index covers the coalesced attribute lookup, so on its own it makes + * ClickHouse read every granule in the time range. The prefilter compares the + * same keys directly, which matches a superset of those spans while the + * coalesced comparison still decides; `rewriteSqlFilterWithKvItems` then + * rewrites each comparison onto the schema's indexed items column, and older + * schemas serve them from their `mapValues` index as written. + */ +export function makeDbStatementCondition({ + expressions, + statement, +}: { + expressions: Pick< + ReturnType, + 'dbStatement' | 'dbStatementPrefilterFields' + >; + statement: string; +}): string { + const equality = inList(expressions.dbStatement, statement); + const prefilter = expressions.dbStatementPrefilterFields.map(field => + inList(field, statement), + ); + return prefilter.length + ? `(${prefilter.join(' OR ')}) AND ${equality}` + : equality; +} + export function getExpressions( source: TTraceSource, columns: ColumnMeta[], @@ -172,6 +210,7 @@ export function getExpressions( // Database dbStatement: defaults.dbStatement, + dbStatementPrefilterFields: defaults.dbStatementPrefilterFields, }; const auxExpressions = {