Skip to content

Commit de68052

Browse files
authored
fix: Make pattern sampling query random (#1600)
Closes HDX-2873 # Summary This PR backports a change to the event patterns sampling query from the EE repo. This change makes the event patterns sample random subset of the rows, rather than just a subset of the rows based on a LIMIT.
1 parent 64998e0 commit de68052

2 files changed

Lines changed: 61 additions & 20 deletions

File tree

‎.changeset/dry-paws-eat.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@hyperdx/app": patch
3+
---
4+
5+
fix: Make pattern sampling query random

‎packages/app/src/hooks/usePatterns.tsx‎

Lines changed: 56 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -134,26 +134,62 @@ function usePatterns({
134134
severityTextExpression?: string;
135135
enabled?: boolean;
136136
}) {
137-
const configWithPrimaryAndPartitionKey = useConfigWithPrimaryAndPartitionKey({
138-
...config,
139-
// TODO: User-configurable pattern columns and non-pattern/group by columns
140-
select: [
141-
`${bodyValueExpression} as ${PATTERN_COLUMN_ALIAS}`,
142-
`${getFirstTimestampValueExpression(config.timestampValueExpression)} as ${TIMESTAMP_COLUMN_ALIAS}`,
143-
...(severityTextExpression
144-
? [`${severityTextExpression} as ${SEVERITY_TEXT_COLUMN_ALIAS}`]
145-
: []),
146-
].join(','),
147-
// TODO: Proper sampling
148-
orderBy: [{ ordering: 'DESC', valueExpression: 'rand()' }],
149-
limit: { limit: samples },
150-
});
137+
// Create a sampling config with CTE for smart sampling
138+
const samplingConfig = useMemo(() => {
139+
// Build config with CTE for count check
140+
const configWithCTE: ChartConfigWithDateRange = {
141+
...config,
142+
// Add CTE to get table statistics for sampling
143+
with: [
144+
{
145+
name: 'tableStats',
146+
chartConfig: {
147+
...config,
148+
select: 'count() as total',
149+
groupBy: undefined,
150+
orderBy: undefined,
151+
limit: undefined,
152+
},
153+
},
154+
],
155+
// Select the required columns
156+
select: [
157+
`${bodyValueExpression} as ${PATTERN_COLUMN_ALIAS}`,
158+
`${getFirstTimestampValueExpression(config.timestampValueExpression)} as ${TIMESTAMP_COLUMN_ALIAS}`,
159+
...(severityTextExpression
160+
? [`${severityTextExpression} as ${SEVERITY_TEXT_COLUMN_ALIAS}`]
161+
: []),
162+
].join(','),
163+
// Add sampling condition as a filter
164+
filters: [
165+
...(config.filters || []),
166+
{
167+
type: 'sql',
168+
condition: `if(
169+
(SELECT total FROM tableStats) <= ${samples},
170+
1,
171+
cityHash64(${getFirstTimestampValueExpression(config.timestampValueExpression)}, rand()) % greatest(
172+
CAST((SELECT total FROM tableStats) / ${samples} AS UInt32),
173+
1
174+
) = 0
175+
)`,
176+
},
177+
],
178+
// Remove random ordering
179+
orderBy: undefined,
180+
limit: { limit: samples },
181+
};
151182

152-
const { data: sampleRows, isLoading: isSampleLoading } =
153-
useQueriedChartConfig(
154-
configWithPrimaryAndPartitionKey ?? config, // `config` satisfying type, never used due to `enabled` check
155-
{ enabled: configWithPrimaryAndPartitionKey != null && enabled },
156-
);
183+
return configWithCTE;
184+
}, [config, samples, bodyValueExpression, severityTextExpression]);
185+
186+
const configWithPrimaryAndPartitionKey =
187+
useConfigWithPrimaryAndPartitionKey(samplingConfig);
188+
189+
const { data: sampleRows, isLoading: isLoadingSampleRows } =
190+
useQueriedChartConfig(configWithPrimaryAndPartitionKey ?? samplingConfig, {
191+
enabled: configWithPrimaryAndPartitionKey != null && enabled,
192+
});
157193

158194
const { data: pyodide, isLoading: isLoadingPyodide } = usePyodide({
159195
enabled,
@@ -200,7 +236,7 @@ function usePatterns({
200236

201237
return {
202238
...query,
203-
isLoading: query.isLoading || isSampleLoading || isLoadingPyodide,
239+
isLoading: query.isLoading || isLoadingPyodide || isLoadingSampleRows,
204240
patternQueryConfig: configWithPrimaryAndPartitionKey,
205241
};
206242
}

0 commit comments

Comments
 (0)