From cb4e769198648d614a7487cbc04ad584bf110a06 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Mon, 19 Oct 2026 18:24:28 -0400 Subject: [PATCH 1/2] feat: Add group-by to the search page histogram --- .changeset/green-needles-love.md | 5 + packages/app/src/DBSearchPage.tsx | 280 +++++++++++------- .../DBSearchPage.directTrace.test.tsx | 3 + .../components/DBSearchPageFilters/hooks.ts | 17 +- .../components/DBSearchPageFilters/utils.ts | 17 +- .../SearchGroupBy/GroupByFieldList.tsx | 70 +++++ .../SearchGroupBySelect.module.scss | 41 +++ .../SearchGroupBy/SearchGroupBySelect.tsx | 214 +++++++++++++ .../__tests__/SearchGroupBySelect.test.tsx | 218 ++++++++++++++ .../__tests__/searchGroupBy.test.ts | 133 +++++++++ .../components/SearchGroupBy/searchGroupBy.ts | 97 ++++++ .../SearchGroupBy/useGroupByKeyOptions.ts | 50 ++++ packages/app/src/searchFilters.tsx | 20 +- .../search/histogram-group-by.spec.ts | 151 ++++++++++ .../app/tests/e2e/page-objects/SearchPage.ts | 81 ++++- 15 files changed, 1271 insertions(+), 126 deletions(-) create mode 100644 .changeset/green-needles-love.md create mode 100644 packages/app/src/components/SearchGroupBy/GroupByFieldList.tsx create mode 100644 packages/app/src/components/SearchGroupBy/SearchGroupBySelect.module.scss create mode 100644 packages/app/src/components/SearchGroupBy/SearchGroupBySelect.tsx create mode 100644 packages/app/src/components/SearchGroupBy/__tests__/SearchGroupBySelect.test.tsx create mode 100644 packages/app/src/components/SearchGroupBy/__tests__/searchGroupBy.test.ts create mode 100644 packages/app/src/components/SearchGroupBy/searchGroupBy.ts create mode 100644 packages/app/src/components/SearchGroupBy/useGroupByKeyOptions.ts create mode 100644 packages/app/tests/e2e/features/search/histogram-group-by.spec.ts diff --git a/.changeset/green-needles-love.md b/.changeset/green-needles-love.md new file mode 100644 index 0000000000..b65d1798ff --- /dev/null +++ b/.changeset/green-needles-love.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/app": patch +--- + +feat: Add group-by to the search page histogram diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 4a9e9e5d3f..9f5f62522c 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -3,6 +3,7 @@ import { FormEventHandler, Fragment, memo, + ReactNode, useCallback, useEffect, useMemo, @@ -101,6 +102,12 @@ import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover' import { InlineNameInput } from '@/components/InlineNameInput/InlineNameInput'; import { InputControlled } from '@/components/InputControlled'; import OnboardingModal from '@/components/OnboardingModal'; +import { + isCustomSearchGroupBy, + resolveSearchGroupBy, + retainGroupByByColumns, +} from '@/components/SearchGroupBy/searchGroupBy'; +import { SearchGroupBySelect } from '@/components/SearchGroupBy/SearchGroupBySelect'; import SearchWhereInput, { getStoredLanguage, } from '@/components/SearchInput/SearchWhereInput'; @@ -336,24 +343,29 @@ function ExpandFiltersButton({ onExpand }: { onExpand: () => void }) { function SearchResultsCountGroup({ isFilterSidebarCollapsed, onExpandFilters, - histogramTimeChartConfig, + totalCountConfig, enableParallelQueries, + children, }: { isFilterSidebarCollapsed: boolean; onExpandFilters: () => void; - histogramTimeChartConfig: BuilderChartConfigWithDateRange; + totalCountConfig: BuilderChartConfigWithDateRange; enableParallelQueries?: boolean; + children?: ReactNode; }) { return ( - - {isFilterSidebarCollapsed && ( - - )} - + + + {isFilterSidebarCollapsed && ( + + )} + + + {children} ); } @@ -898,6 +910,11 @@ function formatDroppedFiltersMessage(count: number): string { return `${count} ${noun} didn't apply to this source and ${verb} removed.`; } +function formatDroppedGroupByMessage(dropped: string[]): string { + const verb = dropped.length === 1 ? 'was' : 'were'; + return `Group by ${dropped.join(', ')} didn't apply to this source and ${verb} removed.`; +} + // This is outside as it needs to be a stable reference const queryStateMap = { source: parseAsString, @@ -1095,6 +1112,12 @@ function DBSearchPageContent() { ]).withDefault('results'), ); + // URL-only, not saved with the search: null uses the source default. + const [groupByParam, setGroupByParam] = useQueryState( + 'groupBy', + parseAsStringEncoded, + ); + const [patternColumn, setPatternColumn] = useQueryState( 'patternColumn', parseAsString, @@ -1421,6 +1444,9 @@ function DBSearchPageContent() { // effect waits for the new source's columns to load and then drops any // sidebar filters that don't apply to the new schema. const pendingFilterReconcileRef = useRef(null); + // Same, for the histogram group by. Also set when a saved search switches the + // source, since the group by isn't part of the saved search. + const pendingGroupByReconcileRef = useRef(null); const watchedSourceObj = useMemo( () => inputSourceObjs?.find(s => s.id === watchedSource), @@ -1482,6 +1508,7 @@ function DBSearchPageContent() { // bookmarked or refreshed. The submit below still runs, since that is how // a freshly loaded page pushes its defaults into the search config. if (!isCatchingUpToConfig) { + pendingGroupByReconcileRef.current = watchedSource ?? null; // If the user isn't in a saved search (or the source is different from the saved search source), reset fields if (savedSearchId == null || savedSearch?.source !== watchedSource) { setValue('select', ''); @@ -1514,29 +1541,44 @@ function DBSearchPageContent() { setSearchedConfig, ]); - const retainCompatibleFilters = useStableCallback((columns: ColumnMeta[]) => { - pendingFilterReconcileRef.current = null; + const retainCompatibleFiltersAndGroupBy = useStableCallback( + (columns: ColumnMeta[]) => { + const allowed = new Set(columns.map(c => c.name)); - const allowed = new Set(columns.map(c => c.name)); - - const dropped = searchFilters.retainFiltersByColumns(allowed); + if (pendingFilterReconcileRef.current === watchedSource) { + pendingFilterReconcileRef.current = null; + const dropped = searchFilters.retainFiltersByColumns(allowed); + if (dropped.length > 0) { + notifications.show({ + color: 'yellow', + message: formatDroppedFiltersMessage(dropped.length), + }); + } + } - if (dropped.length > 0) { - notifications.show({ - color: 'yellow', - message: formatDroppedFiltersMessage(dropped.length), - }); - } - }); + if (pendingGroupByReconcileRef.current === watchedSource) { + pendingGroupByReconcileRef.current = null; + const { kept, dropped } = retainGroupByByColumns(groupByParam, allowed); + if (dropped.length > 0) { + setGroupByParam(kept); + notifications.show({ + color: 'yellow', + message: formatDroppedGroupByMessage(dropped), + }); + } + } + }, + ); useEffect(() => { if ( - pendingFilterReconcileRef.current === watchedSource && - watchedSourceColumns + watchedSourceColumns && + (pendingFilterReconcileRef.current === watchedSource || + pendingGroupByReconcileRef.current === watchedSource) ) { - retainCompatibleFilters(watchedSourceColumns); + retainCompatibleFiltersAndGroupBy(watchedSourceColumns); } - }, [watchedSource, watchedSourceColumns, retainCompatibleFilters]); + }, [watchedSource, watchedSourceColumns, retainCompatibleFiltersAndGroupBy]); const onTableScroll = useCallback( (scrollTop: number) => { @@ -1953,18 +1995,6 @@ function DBSearchPageContent() { return undefined; } - const variableConfig: Partial< - Pick - > = {}; - switch (searchedSource?.kind) { - case SourceKind.Log: - variableConfig.groupBy = searchedSource?.severityTextExpression; - break; - case SourceKind.Trace: - variableConfig.groupBy = searchedSource?.statusCodeExpression; - break; - } - return { ...chartConfig, select: [ @@ -1990,7 +2020,7 @@ function DBSearchPageContent() { alignDateRangeToGranularity: false, // Make sure the end date is inclusive so that the histogram and table counts match dateRangeEndInclusive: true, - ...variableConfig, + groupBy: resolveSearchGroupBy(groupByParam, searchedSource), } satisfies BuilderChartConfigWithDateRange; }, [ chartConfig, @@ -1998,8 +2028,36 @@ function DBSearchPageContent() { aliasWith, searchedTimeRange, searchedConfig.select, + groupByParam, ]); + // The default severity/status grouping reads without a legend. + const isCustomGroupBy = isCustomSearchGroupBy(groupByParam, searchedSource); + + // A high-cardinality custom group by can exceed the count query's row limit + // and truncate the summed total, so totals use the default grouping. Without + // a custom group by, this is the histogram config itself so the two share one + // query. + const totalCountConfig = useMemo( + () => + isCustomGroupBy && histogramTimeChartConfig != null + ? { + ...histogramTimeChartConfig, + groupBy: resolveSearchGroupBy(null, searchedSource), + } + : histogramTimeChartConfig, + [isCustomGroupBy, histogramTimeChartConfig, searchedSource], + ); + + const histogramGroupBySelect = ( + + ); + const onFormSubmit = useCallback>( e => { e.preventDefault(); @@ -2038,9 +2096,8 @@ function DBSearchPageContent() { [onTimeRangeSelect, setIsLive], ); - // Focus a chart series into the actual search. The histogram is grouped by - // severity/status, so a focused series maps to a real column value; applying - // it as an "only" filter re-queries both the chart and the results table so + // Focus a chart series into the actual search. Applying each group-by value + // as an "only" filter re-queries both the chart and the results table so // they stay in sync (the chart-only visual focus wouldn't touch the table). const handleFocusSeries = useCallback( (groupFilters: SeriesGroupFilter[]) => { @@ -2648,7 +2705,8 @@ function DBSearchPageContent() { } > {analysisMode === 'pattern' && - histogramTimeChartConfig != null && ( + histogramTimeChartConfig != null && + totalCountConfig != null && ( + totalCountConfig={totalCountConfig} + > + {histogramGroupBySelect} + @@ -2750,67 +2810,73 @@ function DBSearchPageContent() { )} {analysisMode === 'results' && ( - {chartConfig && histogramTimeChartConfig && ( - <> - - - - - {shouldShowLiveModeHint && - denoiseResults != true && ( - - )} - + {chartConfig && + histogramTimeChartConfig && + totalCountConfig && ( + <> + + + + {histogramGroupBySelect} + + + {shouldShowLiveModeHint && + denoiseResults != true && ( + + )} + + - - - {!hasQueryError && ( - - - )} - - )} + {!hasQueryError && ( + + + + )} + + )} {hasQueryError && queryError ? ( <>
diff --git a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx index e67ec50517..5b335588ff 100644 --- a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx +++ b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx @@ -192,6 +192,9 @@ jest.mock('@/components/SearchInput/SearchWhereInput', () => ({ default: () =>
, getStoredLanguage: () => 'lucene', })); +jest.mock('@/components/SearchGroupBy/SearchGroupBySelect', () => ({ + SearchGroupBySelect: () =>
, +})); jest.mock('@/components/SearchPageActionBar', () => () =>
); jest.mock('@/components/SearchTotalCountChart', () => () =>
); jest.mock('@/components/Sources/SourceForm', () => ({ diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index 00449fed89..54267aeaf0 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -30,7 +30,11 @@ import { escapeFilterStateKeys, usePinnedFilters } from '@/searchFilters'; import { useSource } from '@/source'; import { mergePath } from '@/utils'; -import { listRequestedFacets, toQuotedClickHouseKeyExpression } from './utils'; +import { + listRequestedFacets, + sortLowCardinalityFirst, + toQuotedClickHouseKeyExpression, +} from './utils'; const INITIAL_LOAD_LIMIT = 20; @@ -120,16 +124,7 @@ function useFacets({ return []; } - const strings = allFields - .sort((a, b) => { - // First show low cardinality fields - const isLowCardinality = (type: string) => - type.includes('LowCardinality'); - return ( - (isLowCardinality(b.type) ? 1 : 0) - - (isLowCardinality(a.type) ? 1 : 0) - ); - }) + const strings = sortLowCardinalityFirst(allFields) .filter( field => field.jsType && ['string'].includes(field.jsType), // todo: add number type with sliders :D diff --git a/packages/app/src/components/DBSearchPageFilters/utils.ts b/packages/app/src/components/DBSearchPageFilters/utils.ts index 745f2730da..6d7cb1d56d 100644 --- a/packages/app/src/components/DBSearchPageFilters/utils.ts +++ b/packages/app/src/components/DBSearchPageFilters/utils.ts @@ -1,7 +1,10 @@ // Utility functions for parsing and grouping map-like field names import SqlString from 'sqlstring'; -import { parseKeyPath } from '@hyperdx/common-utils/dist/core/metadata'; +import { + type Field, + parseKeyPath, +} from '@hyperdx/common-utils/dist/core/metadata'; import type { FilterState } from '@hyperdx/common-utils/dist/filters'; import type { Facet } from '@/hooks/useMetadata'; @@ -73,6 +76,18 @@ export function cleanClickHouseExpression(key: string): string { return cleanKey; } +const isLowCardinality = (type: string) => type.includes('LowCardinality'); + +// Low-cardinality fields first: they make the most useful facets and groupings. +export function sortLowCardinalityFirst>( + fields: T[], +): T[] { + return [...fields].sort( + (a, b) => + Number(isLowCardinality(b.type)) - Number(isLowCardinality(a.type)), + ); +} + // Parse map-like field names and extract the base name and property path export function parseMapFieldName( key: string, diff --git a/packages/app/src/components/SearchGroupBy/GroupByFieldList.tsx b/packages/app/src/components/SearchGroupBy/GroupByFieldList.tsx new file mode 100644 index 0000000000..fabb1151c2 --- /dev/null +++ b/packages/app/src/components/SearchGroupBy/GroupByFieldList.tsx @@ -0,0 +1,70 @@ +import { useRef } from 'react'; +import { Checkbox, ScrollArea, Text } from '@mantine/core'; +import { useVirtualizer } from '@tanstack/react-virtual'; + +const ROW_HEIGHT_ESTIMATE = 26; + +export function GroupByFieldList({ + fields, + selected, + onToggle, +}: { + fields: string[]; + selected: string[]; + onToggle: (field: string) => void; +}) { + const viewportRef = useRef(null); + const virtualizer = useVirtualizer({ + count: fields.length, + getScrollElement: () => viewportRef.current, + estimateSize: () => ROW_HEIGHT_ESTIMATE, + overscan: 10, + getItemKey: index => fields[index] ?? index, + }); + + if (fields.length === 0) { + return ( + + No fields found + + ); + } + + return ( + +
+ {virtualizer.getVirtualItems().map(item => { + const field = fields[item.index]; + return ( +
+ onToggle(field)} + /> +
+ ); + })} +
+
+ ); +} diff --git a/packages/app/src/components/SearchGroupBy/SearchGroupBySelect.module.scss b/packages/app/src/components/SearchGroupBy/SearchGroupBySelect.module.scss new file mode 100644 index 0000000000..23f0eda5a9 --- /dev/null +++ b/packages/app/src/components/SearchGroupBy/SearchGroupBySelect.module.scss @@ -0,0 +1,41 @@ +// The label sits in a filled addon so it reads as the frame's name rather than +// something to press; only the value side is the button. +.control { + display: flex; + align-items: center; + flex-shrink: 0; + overflow: hidden; + background: var(--color-bg-surface); + border: 1px solid var(--color-border); + border-radius: var(--mantine-radius-default); +} + +.label { + align-self: stretch; + display: flex; + align-items: center; + padding-inline: 8px; + border-right: 1px solid var(--color-border); + background: var(--color-bg-muted); + white-space: nowrap; +} + +.target { + display: flex; + align-items: center; + gap: 6px; + height: 28px; + padding-inline: 8px; + white-space: nowrap; + + &:hover { + background: var(--color-bg-hover); + } +} + +// Long expressions truncate rather than pushing the row's other controls off. +.value { + overflow: hidden; + max-width: 220px; + text-overflow: ellipsis; +} diff --git a/packages/app/src/components/SearchGroupBy/SearchGroupBySelect.tsx b/packages/app/src/components/SearchGroupBy/SearchGroupBySelect.tsx new file mode 100644 index 0000000000..6794ad77fb --- /dev/null +++ b/packages/app/src/components/SearchGroupBy/SearchGroupBySelect.tsx @@ -0,0 +1,214 @@ +import { useMemo, useState } from 'react'; +import { splitAndTrimWithBracket } from '@hyperdx/common-utils/dist/core/utils'; +import { TSource } from '@hyperdx/common-utils/dist/types'; +import { + Button, + Group, + Popover, + SegmentedControl, + Text, + TextInput, + UnstyledButton, +} from '@mantine/core'; +import { IconChevronDown } from '@tabler/icons-react'; + +import SQLInlineEditor from '@/components/SQLEditor/SQLInlineEditor'; + +import { GroupByFieldList } from './GroupByFieldList'; +import { + defaultSearchGroupBy, + formatGroupByFields, + resolveSearchGroupBy, + SearchGroupByParam, +} from './searchGroupBy'; +import { useGroupByKeyOptions } from './useGroupByKeyOptions'; + +import classes from './SearchGroupBySelect.module.scss'; + +type Mode = 'fields' | 'sql'; + +function triggerLabel(fields: string[]) { + if (fields.length === 0) return 'none'; + if (fields.length === 1) return fields[0]; + return `${fields.length} fields`; +} + +export function SearchGroupBySelect({ + source, + value, + onApply, + dateRange, +}: { + source?: TSource; + value: SearchGroupByParam; + onApply: (next: SearchGroupByParam) => void; + dateRange: [Date, Date]; +}) { + const [opened, setOpened] = useState(false); + const [mode, setMode] = useState('fields'); + const [search, setSearch] = useState(''); + // Fields and SQL tabs edit the same selection; switching tabs converts one + // draft into the other so edits carry across in both directions. + const [draft, setDraft] = useState([]); + const [sqlDraft, setSqlDraft] = useState(''); + // Captured on open: live tail moves `dateRange` every refresh, which would + // refetch the keys and collapse the open list while it reloads. + const [keysDateRange, setKeysDateRange] = useState(dateRange); + + const { options, tableConnection } = useGroupByKeyOptions({ + source, + dateRange: keysDateRange, + enabled: opened, + }); + + const appliedFields = useMemo( + () => splitAndTrimWithBracket(resolveSearchGroupBy(value, source) ?? ''), + [value, source], + ); + + const handleOpenChange = (next: boolean) => { + if (next) { + setDraft(appliedFields); + setSqlDraft(formatGroupByFields(appliedFields)); + setSearch(''); + setMode('fields'); + setKeysDateRange(dateRange); + } + setOpened(next); + }; + + const handleModeChange = (next: string) => { + if (next === 'sql') { + setSqlDraft(formatGroupByFields(draft)); + setMode('sql'); + } else { + setDraft(splitAndTrimWithBracket(sqlDraft)); + setMode('fields'); + } + }; + + const visibleOptions = useMemo(() => { + const q = search.trim().toLowerCase(); + // Selected entries first, and kept even when absent from the key list so + // an expression typed in the SQL tab still shows as checked. + const union = Array.from(new Set([...draft, ...options])); + return q ? union.filter(f => f.toLowerCase().includes(q)) : union; + }, [search, draft, options]); + + const commit = (next: SearchGroupByParam) => { + onApply(next); + setOpened(false); + }; + + const applyDraft = () => + commit( + formatGroupByFields( + mode === 'sql' ? splitAndTrimWithBracket(sqlDraft) : draft, + ), + ); + + const toggleField = (field: string) => + setDraft(prev => + prev.includes(field) ? prev.filter(f => f !== field) : [...prev, field], + ); + + const label = triggerLabel(appliedFields); + + return ( + +
+ + Group by + + + handleOpenChange(!opened)} + data-testid="search-group-by-button" + aria-label={`Group by ${label}`} + > + + {label} + + + + +
+ + + {mode === 'sql' ? ( + + ) : ( + <> + setSearch(e.currentTarget.value)} + mb="xs" + autoFocus + data-testid="search-group-by-search" + /> + + + )} + + + + + + + + +
+ ); +} diff --git a/packages/app/src/components/SearchGroupBy/__tests__/SearchGroupBySelect.test.tsx b/packages/app/src/components/SearchGroupBy/__tests__/SearchGroupBySelect.test.tsx new file mode 100644 index 0000000000..23863694b0 --- /dev/null +++ b/packages/app/src/components/SearchGroupBy/__tests__/SearchGroupBySelect.test.tsx @@ -0,0 +1,218 @@ +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; +import { fireEvent, screen } from '@testing-library/react'; + +import { SearchGroupBySelect } from '@/components/SearchGroupBy/SearchGroupBySelect'; +import { useAllFields } from '@/hooks/useMetadata'; + +jest.mock('@/hooks/useMetadata', () => ({ + useAllFields: jest.fn(() => ({ + data: [ + { path: ['ServiceName'], type: 'String', jsType: 'string' }, + { + path: ['SeverityText'], + type: 'LowCardinality(String)', + jsType: 'string', + }, + { path: ['LogAttributes', 'user'], type: 'String', jsType: 'string' }, + { path: ['dash-column'], type: 'String', jsType: 'string' }, + ], + })), + useJsonColumns: () => ({ data: [] }), + useMapColumns: () => ({ data: ['LogAttributes'] }), +})); + +// jsdom has no layout, so the real virtualizer renders no rows. Render every +// row as if visible. +jest.mock('@tanstack/react-virtual', () => ({ + useVirtualizer: ({ count }: { count: number }) => ({ + getVirtualItems: () => + Array.from({ length: count }, (_, index) => ({ + index, + key: index, + start: index * 26, + })), + getTotalSize: () => count * 26, + measureElement: () => {}, + }), +})); + +// CodeMirror doesn't run in jsdom. +jest.mock('@/components/SQLEditor/SQLInlineEditor', () => ({ + __esModule: true, + default: ({ + value, + onChange, + onSubmit, + }: { + value: string; + onChange: (v: string) => void; + onSubmit: () => void; + }) => ( +