Repository navigation
fix(app): stop event deltas selection failing on distributed tables #3235
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
6645b20
8511657
5771a7d
fbfe4f7
4aaecbe
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| "@hyperdx/app": patch | ||
| --- | ||
|
|
||
| fix(app): stop event deltas selection failing on Distributed tables |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,6 @@ | ||
| import { useCallback, useEffect, useMemo, useState } from 'react'; | ||
| import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; | ||
| import { tcFromChartConfig } from '@hyperdx/common-utils/dist/core/metadata'; | ||
| import { | ||
| BuilderChartConfigWithDateRange, | ||
| Filter, | ||
|
|
@@ -17,6 +18,7 @@ import { useElementSize } from '@mantine/hooks'; | |
|
|
||
| import { isAggregateFunction } from '@/ChartUtils'; | ||
| import { useQueriedChartConfig } from '@/hooks/useChartConfig'; | ||
| import { useTableMetadata } from '@/hooks/useMetadata'; | ||
| import { getFirstTimestampValueExpression } from '@/source'; | ||
| import { getChartColorError, getChartColorSuccess } from '@/utils'; | ||
|
|
||
|
|
@@ -102,6 +104,25 @@ export default function DBDeltaChart({ | |
| config.timestampValueExpression, | ||
| ); | ||
|
|
||
| /** Distributed (and Merge) tables need different subquery handling: | ||
| * - An `IN <subquery>` over a Distributed table is rejected under the | ||
| * default `distributed_product_mode = 'deny'`, so subqueries whose values | ||
| * are global (AggregatedTimestamps) must use GLOBAL IN. | ||
| * - The PartIds prefetch hint is skipped entirely: `_part`/`_part_offset` | ||
| * only identify rows within one shard, so the hint is meaningless there. | ||
| * | ||
| * Only a table confirmed local gets plain IN and the hint. If the lookup | ||
| * fails, assume Distributed: GLOBAL IN also works on local tables, and | ||
| * dropping the hint doesn't change results. | ||
| */ | ||
| const { data: tableMetadata, isLoading: isTableMetadataLoading } = | ||
| useTableMetadata(tcFromChartConfig(config), { retry: false }); | ||
| const isLocalTable = tableMetadata != null && !tableMetadata.isPointerTable; | ||
| const canUsePartIdsHint = isLocalTable; | ||
| const inOperator = isLocalTable ? 'IN' : 'GLOBAL IN'; | ||
| const notInOperator = isLocalTable ? 'NOT IN' : 'GLOBAL NOT IN'; | ||
| const isSelectionQueryEnabled = hasSelection && !isTableMetadataLoading; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟠 major — A failing metadata lookup holds selection queries back for about 7s while react-query retries
Nothing blocks merge automatically, but a maintainer will expect this fixed if it is a real defect in code this PR changes. If it is about surrounding code, reply and say so instead of patching. Do not widen the PR. How to respond
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. addressed in 8511657 |
||
|
|
||
| // Helper to build the shared AggregatedTimestamps CTE (used by both outlier and inlier queries) | ||
| const buildAggregatedTimestampsCTE = () => | ||
| isAggregate | ||
|
|
@@ -136,10 +157,13 @@ export default function DBDeltaChart({ | |
| } | ||
| : null; | ||
|
|
||
| const buildAggregateTimestampCondition = (isOutlier: boolean) => | ||
| `${timestampExpr} ${isOutlier ? inOperator : notInOperator} (SELECT ${timestampExpr} FROM AggregatedTimestamps)`; | ||
|
|
||
| // Helper to build WITH clauses for a query (outlier or inlier) | ||
| const buildWithClauses = ( | ||
| isOutlier: boolean, | ||
| ): NonNullable<BuilderChartConfigWithDateRange['with']> => { | ||
| ): BuilderChartConfigWithDateRange['with'] => { | ||
| const aggregatedTimestampsCTE = buildAggregatedTimestampsCTE(); | ||
|
|
||
| // Build the SQL condition for filtering | ||
|
|
@@ -153,38 +177,40 @@ export default function DBDeltaChart({ | |
| return isOutlier ? query : `NOT (${query})`; | ||
| }; | ||
|
|
||
| const sqlCondition = buildSqlCondition(); | ||
| const aggregateTimestampCondition = isOutlier | ||
| ? `${timestampExpr} IN (SELECT ${timestampExpr} FROM AggregatedTimestamps)` | ||
| : `${timestampExpr} NOT IN (SELECT ${timestampExpr} FROM AggregatedTimestamps)`; | ||
| const partIdsCTE = canUsePartIdsHint | ||
| ? { | ||
| name: 'PartIds', | ||
| chartConfig: { | ||
| ...config, | ||
| select: 'tuple(_part, _part_offset)', | ||
| filters: [ | ||
| ...(config.filters ?? []), | ||
| { | ||
| type: 'sql', | ||
| condition: buildSqlCondition(), | ||
| } satisfies Filter, | ||
| ...(isAggregate | ||
| ? [ | ||
| { | ||
| type: 'sql', | ||
| condition: buildAggregateTimestampCondition(isOutlier), | ||
| } satisfies Filter, | ||
| ] | ||
| : []), | ||
| ], | ||
| orderBy: [ | ||
| { ordering: 'DESC' as const, valueExpression: stableSampleExpr }, | ||
| ], | ||
| limit: { limit: SAMPLE_SIZE }, | ||
| }, | ||
| } | ||
| : undefined; | ||
|
|
||
| return [ | ||
| const withClauses = [ | ||
| ...(aggregatedTimestampsCTE ? [aggregatedTimestampsCTE] : []), | ||
| { | ||
| name: 'PartIds', | ||
| chartConfig: { | ||
| ...config, | ||
| select: 'tuple(_part, _part_offset)', | ||
| filters: [ | ||
| ...(config.filters ?? []), | ||
| { | ||
| type: 'sql', | ||
| condition: sqlCondition, | ||
| } satisfies Filter, | ||
| ...(isAggregate | ||
| ? [ | ||
| { | ||
| type: 'sql', | ||
| condition: aggregateTimestampCondition, | ||
| } satisfies Filter, | ||
| ] | ||
| : []), | ||
| ], | ||
| orderBy: [{ ordering: 'DESC', valueExpression: stableSampleExpr }], | ||
| limit: { limit: SAMPLE_SIZE }, | ||
| }, | ||
| }, | ||
| ...(partIdsCTE ? [partIdsCTE] : []), | ||
| ]; | ||
| return withClauses.length > 0 ? withClauses : undefined; | ||
| }; | ||
|
|
||
| // Helper to build filters for the main query | ||
|
|
@@ -204,29 +230,28 @@ export default function DBDeltaChart({ | |
| } | ||
| }; | ||
|
|
||
| const sqlCondition = buildSqlCondition(); | ||
| const aggregateTimestampCondition = isOutlier | ||
| ? `${timestampExpr} IN (SELECT ${timestampExpr} FROM AggregatedTimestamps)` | ||
| : `${timestampExpr} NOT IN (SELECT ${timestampExpr} FROM AggregatedTimestamps)`; | ||
|
|
||
| return [ | ||
| ...(config.filters ?? []), | ||
| { | ||
| type: 'sql', | ||
| condition: sqlCondition, | ||
| condition: buildSqlCondition(), | ||
| } as { type: 'sql'; condition: string }, | ||
| ...(isAggregate | ||
| ? [ | ||
| { | ||
| type: 'sql', | ||
| condition: aggregateTimestampCondition, | ||
| condition: buildAggregateTimestampCondition(isOutlier), | ||
| } as { type: 'sql'; condition: string }, | ||
| ] | ||
| : []), | ||
| ...(canUsePartIdsHint | ||
| ? [ | ||
| { | ||
| type: 'sql', | ||
| condition: `indexHint((_part, _part_offset) IN PartIds)`, | ||
| } as { type: 'sql'; condition: string }, | ||
| ] | ||
| : []), | ||
| { | ||
| type: 'sql', | ||
| condition: `indexHint((_part, _part_offset) IN PartIds)`, | ||
| } as { type: 'sql'; condition: string }, | ||
| ]; | ||
| }; | ||
|
|
||
|
|
@@ -243,7 +268,7 @@ export default function DBDeltaChart({ | |
| orderBy: [{ ordering: 'DESC', valueExpression: stableSampleExpr }], | ||
| limit: { limit: SAMPLE_SIZE }, | ||
| }, | ||
| { enabled: hasSelection }, | ||
| { enabled: isSelectionQueryEnabled }, | ||
| ); | ||
|
|
||
| const { data: inlierData, isLoading: isInlierLoading } = | ||
|
|
@@ -256,7 +281,7 @@ export default function DBDeltaChart({ | |
| orderBy: [{ ordering: 'DESC', valueExpression: stableSampleExpr }], | ||
| limit: { limit: SAMPLE_SIZE }, | ||
| }, | ||
| { enabled: hasSelection }, | ||
| { enabled: isSelectionQueryEnabled }, | ||
| ); | ||
|
|
||
| // When no selection exists, fetch all spans without any range filter | ||
|
|
@@ -275,7 +300,7 @@ export default function DBDeltaChart({ | |
| ); | ||
|
|
||
| const isLoading = hasSelection | ||
| ? isOutlierLoading || isInlierLoading | ||
| ? isTableMetadataLoading || isOutlierLoading || isInlierLoading | ||
| : isAllSpansLoading; | ||
|
|
||
| const error = outlierError ?? allSpansError; | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.