From 6645b209e1c6a8f7a3fbd7e29ab210159268c73e Mon Sep 17 00:00:00 2001 From: Karl Power Date: Mon, 28 Sep 2026 15:13:02 +0200 Subject: [PATCH 1/2] fix(app): stop event deltas selection failing on Distributed tables Selecting an area in event deltas added a PartIds subquery with indexHint((_part, _part_offset) IN PartIds). On a Distributed table with two or more shards, ClickHouse rejects that as a double-distributed IN (distributed_product_mode = 'deny'). Part offsets are also only meaningful within one shard. Skip the PartIds hint on Distributed/Merge tables. It is only an index hint, so results are unchanged. Use GLOBAL IN / GLOBAL NOT IN for the AggregatedTimestamps filter on those tables, since it is a real filter and hits the same error. --- .changeset/polite-queens-help.md | 5 + packages/app/src/components/DBDeltaChart.tsx | 109 +++++++++++------- .../__tests__/DBDeltaChart.test.tsx | 91 +++++++++++++++ 3 files changed, 161 insertions(+), 44 deletions(-) create mode 100644 .changeset/polite-queens-help.md diff --git a/.changeset/polite-queens-help.md b/.changeset/polite-queens-help.md new file mode 100644 index 0000000000..ec1b70b0c0 --- /dev/null +++ b/.changeset/polite-queens-help.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +fix(app): stop event deltas selection failing on Distributed tables diff --git a/packages/app/src/components/DBDeltaChart.tsx b/packages/app/src/components/DBDeltaChart.tsx index c7e804cd05..999e7826b5 100644 --- a/packages/app/src/components/DBDeltaChart.tsx +++ b/packages/app/src/components/DBDeltaChart.tsx @@ -1,5 +1,6 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; +import { tcFromChartConfig } from '@hyperdx/common-utils/dist/core/metadata'; import { BuilderChartConfigWithDateRange, Filter, @@ -17,6 +18,7 @@ import { useElementSize } from '@mantine/hooks'; import { isAggregateFunction } from '@/ChartUtils'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import { useTableMetadata } from '@/hooks/useMetadata'; import { getFirstTimestampValueExpression } from '@/source'; import { getChartColorError, getChartColorSuccess } from '@/utils'; @@ -102,6 +104,21 @@ export default function DBDeltaChart({ config.timestampValueExpression, ); + /** Distributed (and Merge) tables need different subquery handling: + * - An `IN ` over a Distributed table is rejected under the + * default `distributed_product_mode = 'deny'`, so subqueries whose values + * are global (AggregatedTimestamps) must use GLOBAL IN. + * - The PartIds prefetch hint is skipped entirely: `_part`/`_part_offset` + * only identify rows within one shard, so the hint is meaningless there. + */ + const { data: tableMetadata, isLoading: isTableMetadataLoading } = + useTableMetadata(tcFromChartConfig(config)); + const isPointerTable = tableMetadata?.isPointerTable === true; + const canUsePartIdsHint = tableMetadata != null && !isPointerTable; + const inOperator = isPointerTable ? 'GLOBAL IN' : 'IN'; + const notInOperator = isPointerTable ? 'GLOBAL NOT IN' : 'NOT IN'; + const isSelectionQueryEnabled = hasSelection && !isTableMetadataLoading; + // Helper to build the shared AggregatedTimestamps CTE (used by both outlier and inlier queries) const buildAggregatedTimestampsCTE = () => isAggregate @@ -136,10 +153,13 @@ export default function DBDeltaChart({ } : null; + const buildAggregateTimestampCondition = (isOutlier: boolean) => + `${timestampExpr} ${isOutlier ? inOperator : notInOperator} (SELECT ${timestampExpr} FROM AggregatedTimestamps)`; + // Helper to build WITH clauses for a query (outlier or inlier) const buildWithClauses = ( isOutlier: boolean, - ): NonNullable => { + ): BuilderChartConfigWithDateRange['with'] => { const aggregatedTimestampsCTE = buildAggregatedTimestampsCTE(); // Build the SQL condition for filtering @@ -153,38 +173,40 @@ export default function DBDeltaChart({ return isOutlier ? query : `NOT (${query})`; }; - const sqlCondition = buildSqlCondition(); - const aggregateTimestampCondition = isOutlier - ? `${timestampExpr} IN (SELECT ${timestampExpr} FROM AggregatedTimestamps)` - : `${timestampExpr} NOT IN (SELECT ${timestampExpr} FROM AggregatedTimestamps)`; + const partIdsCTE = canUsePartIdsHint + ? { + name: 'PartIds', + chartConfig: { + ...config, + select: 'tuple(_part, _part_offset)', + filters: [ + ...(config.filters ?? []), + { + type: 'sql', + condition: buildSqlCondition(), + } satisfies Filter, + ...(isAggregate + ? [ + { + type: 'sql', + condition: buildAggregateTimestampCondition(isOutlier), + } satisfies Filter, + ] + : []), + ], + orderBy: [ + { ordering: 'DESC' as const, valueExpression: stableSampleExpr }, + ], + limit: { limit: SAMPLE_SIZE }, + }, + } + : undefined; - return [ + const withClauses = [ ...(aggregatedTimestampsCTE ? [aggregatedTimestampsCTE] : []), - { - name: 'PartIds', - chartConfig: { - ...config, - select: 'tuple(_part, _part_offset)', - filters: [ - ...(config.filters ?? []), - { - type: 'sql', - condition: sqlCondition, - } satisfies Filter, - ...(isAggregate - ? [ - { - type: 'sql', - condition: aggregateTimestampCondition, - } satisfies Filter, - ] - : []), - ], - orderBy: [{ ordering: 'DESC', valueExpression: stableSampleExpr }], - limit: { limit: SAMPLE_SIZE }, - }, - }, + ...(partIdsCTE ? [partIdsCTE] : []), ]; + return withClauses.length > 0 ? withClauses : undefined; }; // Helper to build filters for the main query @@ -204,29 +226,28 @@ export default function DBDeltaChart({ } }; - const sqlCondition = buildSqlCondition(); - const aggregateTimestampCondition = isOutlier - ? `${timestampExpr} IN (SELECT ${timestampExpr} FROM AggregatedTimestamps)` - : `${timestampExpr} NOT IN (SELECT ${timestampExpr} FROM AggregatedTimestamps)`; - return [ ...(config.filters ?? []), { type: 'sql', - condition: sqlCondition, + condition: buildSqlCondition(), } as { type: 'sql'; condition: string }, ...(isAggregate ? [ { type: 'sql', - condition: aggregateTimestampCondition, + condition: buildAggregateTimestampCondition(isOutlier), + } as { type: 'sql'; condition: string }, + ] + : []), + ...(canUsePartIdsHint + ? [ + { + type: 'sql', + condition: `indexHint((_part, _part_offset) IN PartIds)`, } as { type: 'sql'; condition: string }, ] : []), - { - type: 'sql', - condition: `indexHint((_part, _part_offset) IN PartIds)`, - } as { type: 'sql'; condition: string }, ]; }; @@ -243,7 +264,7 @@ export default function DBDeltaChart({ orderBy: [{ ordering: 'DESC', valueExpression: stableSampleExpr }], limit: { limit: SAMPLE_SIZE }, }, - { enabled: hasSelection }, + { enabled: isSelectionQueryEnabled }, ); const { data: inlierData, isLoading: isInlierLoading } = @@ -256,7 +277,7 @@ export default function DBDeltaChart({ orderBy: [{ ordering: 'DESC', valueExpression: stableSampleExpr }], limit: { limit: SAMPLE_SIZE }, }, - { enabled: hasSelection }, + { enabled: isSelectionQueryEnabled }, ); // When no selection exists, fetch all spans without any range filter @@ -275,7 +296,7 @@ export default function DBDeltaChart({ ); const isLoading = hasSelection - ? isOutlierLoading || isInlierLoading + ? isTableMetadataLoading || isOutlierLoading || isInlierLoading : isAllSpansLoading; const error = outlierError ?? allSpansError; diff --git a/packages/app/src/components/__tests__/DBDeltaChart.test.tsx b/packages/app/src/components/__tests__/DBDeltaChart.test.tsx index c75951ed2d..a0a40e99fa 100644 --- a/packages/app/src/components/__tests__/DBDeltaChart.test.tsx +++ b/packages/app/src/components/__tests__/DBDeltaChart.test.tsx @@ -7,11 +7,16 @@ import { screen } from '@testing-library/react'; import DBDeltaChart from '@/components/DBDeltaChart'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; +import { useTableMetadata } from '@/hooks/useMetadata'; jest.mock('@/hooks/useChartConfig', () => ({ useQueriedChartConfig: jest.fn(), })); +jest.mock('@/hooks/useMetadata', () => ({ + useTableMetadata: jest.fn(), +})); + // PropertyComparisonChart is the per-attribute renderer. Stubbing it out keeps // the legend assertions tight without dragging uPlot/element-size hooks into // the test environment. @@ -27,6 +32,7 @@ jest.mock('../PropertyComparisonChart', () => ({ })); const mockUseQueriedChartConfig = useQueriedChartConfig as jest.Mock; +const mockUseTableMetadata = useTableMetadata as jest.Mock; // Sentinel filter on baseConfig so the allSpans assertion can prove // the branch passes user-supplied filters through unchanged. Without @@ -69,6 +75,10 @@ describe('DBDeltaChart', () => { error: undefined, isLoading: false, }); + mockUseTableMetadata.mockReturnValue({ + data: { engine: 'MergeTree', isPointerTable: false }, + isLoading: false, + }); }); describe('distribution mode (no heatmap selection)', () => { @@ -180,6 +190,87 @@ describe('DBDeltaChart', () => { }); }); + describe('selection queries by table engine', () => { + const selection = { xMin: 1, xMax: 2, yMin: 1, yMax: 2 }; + const conditions = (config: BuilderChartConfigWithDateRange) => + (config.filters ?? []).map(f => ('condition' in f ? f.condition : '')); + + it('uses the PartIds prefetch hint on local MergeTree tables', () => { + renderChart(selection); + + const [[outlierConfig], [inlierConfig]] = + mockUseQueriedChartConfig.mock.calls; + for (const config of [outlierConfig, inlierConfig]) { + expect(config.with?.map((cte: { name: string }) => cte.name)).toEqual([ + 'PartIds', + ]); + expect(conditions(config)).toContain( + 'indexHint((_part, _part_offset) IN PartIds)', + ); + } + }); + + it('skips the PartIds hint on Distributed tables', () => { + mockUseTableMetadata.mockReturnValue({ + data: { engine: 'MergeTree', isPointerTable: true }, + isLoading: false, + }); + renderChart(selection); + + const [[outlierConfig], [inlierConfig]] = + mockUseQueriedChartConfig.mock.calls; + for (const config of [outlierConfig, inlierConfig]) { + expect(config.with).toBeUndefined(); + expect(conditions(config).join(' ')).not.toContain('_part'); + } + }); + + it('uses GLOBAL IN for the aggregated-timestamps subquery on Distributed tables', () => { + mockUseTableMetadata.mockReturnValue({ + data: { engine: 'MergeTree', isPointerTable: true }, + isLoading: false, + }); + renderChart({ ...selection, valueExpr: 'count()' }); + + const [[outlierConfig], [inlierConfig]] = + mockUseQueriedChartConfig.mock.calls; + expect( + outlierConfig.with?.map((cte: { name: string }) => cte.name), + ).toEqual(['AggregatedTimestamps']); + expect(conditions(outlierConfig)).toContain( + 'Timestamp GLOBAL IN (SELECT Timestamp FROM AggregatedTimestamps)', + ); + expect(conditions(inlierConfig)).toContain( + 'Timestamp GLOBAL NOT IN (SELECT Timestamp FROM AggregatedTimestamps)', + ); + }); + + it('waits for table metadata before running selection queries', () => { + mockUseTableMetadata.mockReturnValue({ + data: undefined, + isLoading: true, + }); + renderChart(selection); + + const [outlierCall, inlierCall] = mockUseQueriedChartConfig.mock.calls; + expect(outlierCall[1]).toMatchObject({ enabled: false }); + expect(inlierCall[1]).toMatchObject({ enabled: false }); + }); + + it('falls back to the hint-free query when table metadata is unavailable', () => { + mockUseTableMetadata.mockReturnValue({ + data: undefined, + isLoading: false, + }); + renderChart(selection); + + const [[outlierConfig, outlierOptions]] = + mockUseQueriedChartConfig.mock.calls; + expect(outlierOptions).toMatchObject({ enabled: true }); + expect(outlierConfig.with).toBeUndefined(); + }); + }); + describe('legendPrefix prop', () => { // The divider is a vertical separator rendered immediately after the // legendPrefix to visually divide it from the comparison legend. It's From 85116579156b9063c4aae9daff87e315216fa07a Mon Sep 17 00:00:00 2001 From: Karl Power Date: Mon, 28 Sep 2026 15:47:09 +0200 Subject: [PATCH 2/2] fix(app): use GLOBAL IN when delta chart table metadata is unavailable If the table metadata lookup failed, aggregate selections used plain IN against AggregatedTimestamps, which ClickHouse rejects on Distributed tables. Only a table confirmed local now gets plain IN and the PartIds hint; anything else uses GLOBAL IN, which also works on local tables. With the fallback correct, the lookup no longer retries, so a failure doesn't hold the selection queries back through ~7s of backoff. --- packages/app/src/components/DBDeltaChart.tsx | 14 ++++--- .../__tests__/DBDeltaChart.test.tsx | 39 +++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/packages/app/src/components/DBDeltaChart.tsx b/packages/app/src/components/DBDeltaChart.tsx index 999e7826b5..2dcf2bd0df 100644 --- a/packages/app/src/components/DBDeltaChart.tsx +++ b/packages/app/src/components/DBDeltaChart.tsx @@ -110,13 +110,17 @@ export default function DBDeltaChart({ * are global (AggregatedTimestamps) must use GLOBAL IN. * - The PartIds prefetch hint is skipped entirely: `_part`/`_part_offset` * only identify rows within one shard, so the hint is meaningless there. + * + * Only a table confirmed local gets plain IN and the hint. If the lookup + * fails, assume Distributed: GLOBAL IN also works on local tables, and + * dropping the hint doesn't change results. */ const { data: tableMetadata, isLoading: isTableMetadataLoading } = - useTableMetadata(tcFromChartConfig(config)); - const isPointerTable = tableMetadata?.isPointerTable === true; - const canUsePartIdsHint = tableMetadata != null && !isPointerTable; - const inOperator = isPointerTable ? 'GLOBAL IN' : 'IN'; - const notInOperator = isPointerTable ? 'GLOBAL NOT IN' : 'NOT IN'; + useTableMetadata(tcFromChartConfig(config), { retry: false }); + const isLocalTable = tableMetadata != null && !tableMetadata.isPointerTable; + const canUsePartIdsHint = isLocalTable; + const inOperator = isLocalTable ? 'IN' : 'GLOBAL IN'; + const notInOperator = isLocalTable ? 'NOT IN' : 'GLOBAL NOT IN'; const isSelectionQueryEnabled = hasSelection && !isTableMetadataLoading; // Helper to build the shared AggregatedTimestamps CTE (used by both outlier and inlier queries) diff --git a/packages/app/src/components/__tests__/DBDeltaChart.test.tsx b/packages/app/src/components/__tests__/DBDeltaChart.test.tsx index a0a40e99fa..6f207e2fb5 100644 --- a/packages/app/src/components/__tests__/DBDeltaChart.test.tsx +++ b/packages/app/src/components/__tests__/DBDeltaChart.test.tsx @@ -210,6 +210,19 @@ describe('DBDeltaChart', () => { } }); + it('uses plain IN for the aggregated-timestamps subquery on local MergeTree tables', () => { + renderChart({ ...selection, valueExpr: 'count()' }); + + const [[outlierConfig], [inlierConfig]] = + mockUseQueriedChartConfig.mock.calls; + expect(conditions(outlierConfig)).toContain( + 'Timestamp IN (SELECT Timestamp FROM AggregatedTimestamps)', + ); + expect(conditions(inlierConfig)).toContain( + 'Timestamp NOT IN (SELECT Timestamp FROM AggregatedTimestamps)', + ); + }); + it('skips the PartIds hint on Distributed tables', () => { mockUseTableMetadata.mockReturnValue({ data: { engine: 'MergeTree', isPointerTable: true }, @@ -269,6 +282,32 @@ describe('DBDeltaChart', () => { expect(outlierOptions).toMatchObject({ enabled: true }); expect(outlierConfig.with).toBeUndefined(); }); + + it('uses GLOBAL IN for aggregate selections when table metadata is unavailable', () => { + mockUseTableMetadata.mockReturnValue({ + data: undefined, + isLoading: false, + }); + renderChart({ ...selection, valueExpr: 'count()' }); + + const [[outlierConfig], [inlierConfig]] = + mockUseQueriedChartConfig.mock.calls; + expect(conditions(outlierConfig)).toContain( + 'Timestamp GLOBAL IN (SELECT Timestamp FROM AggregatedTimestamps)', + ); + expect(conditions(inlierConfig)).toContain( + 'Timestamp GLOBAL NOT IN (SELECT Timestamp FROM AggregatedTimestamps)', + ); + }); + + it('does not retry a failed table metadata lookup', () => { + renderChart(selection); + + expect(mockUseTableMetadata).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ retry: false }), + ); + }); }); describe('legendPrefix prop', () => {