Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/stream-filter-values.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@hyperdx/app': patch
'@hyperdx/common-utils': patch
---

perf: stream filter values into the UI as each query finishes instead of
waiting for all of them
81 changes: 53 additions & 28 deletions packages/app/src/components/DBSearchPageFilters.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ import {
} from './DBSearchPageFilters/PinShareMenu';
import { SharedFiltersSection } from './DBSearchPageFilters/SharedFilters';
import {
type FacetEntry,
getFilterStateEntry,
groupFacetsByBaseName,
toQuotedClickHouseKeyExpression,
Expand Down Expand Up @@ -1016,7 +1017,7 @@ export const FilterGroup = ({
chevron: 'm-0',
label: 'p-0',
}}
className={hasOptions ? '' : 'opacity-50'}
className={hasOptions || optionsLoading ? '' : 'opacity-50'}
>
<Tooltip
openDelay={displayName.length > 26 ? 0 : 1500}
Expand All @@ -1028,12 +1029,21 @@ export const FilterGroup = ({
>
<Text size="xs" fw="500" truncate="end">
{displayName}
{showFilterCounts && (
<Text
component="span"
size="xs"
c="dimmed"
>{` (${totalAppliedFiltersSize > 0 ? totalAppliedFiltersSize : options.length})`}</Text>
{totalAppliedFiltersSize === 0 && optionsLoading ? (
<Loader
size={8}
color="var(--color-text-muted)"
ml={6}
data-testid={`filter-values-loading-${name}`}
/>
) : (
showFilterCounts && (
<Text
component="span"
size="xs"
c="dimmed"
>{` (${totalAppliedFiltersSize > 0 ? totalAppliedFiltersSize : options.length})`}</Text>
)
)}
</Text>
</Tooltip>
Expand Down Expand Up @@ -1232,7 +1242,7 @@ const DBSearchPageFiltersComponent = ({
filterState,
showMoreFields,
});
const facets = fetchFacetsData.keyValues;
const { keyValues: facets, pendingKeys } = fetchFacetsData;

useEffect(() => {
if (error) {
Expand All @@ -1246,7 +1256,7 @@ const DBSearchPageFiltersComponent = ({
}, [error]);

// Merge pinned filter values into the queried facets, so that pinned values are always available
const facetsWithPinnedValues = useMemo(() => {
const facetsWithPinnedValues = useMemo((): FacetEntry[] => {
const facetsMap = new Map((facets ?? []).map(f => [f.key, f.value]));
const mergedKeys = new Set<string>([
...facetsMap.keys(),
Expand All @@ -1262,9 +1272,13 @@ const DBSearchPageFiltersComponent = ({
...(pinnedValues ?? []),
]);

return { key, value: Array.from(mergedValues) };
return {
key,
value: Array.from(mergedValues),
isLoading: pendingKeys.has(key),
};
});
}, [facets, pinnedFilters, getPinnedFields]);
}, [facets, pinnedFilters, getPinnedFields, pendingKeys]);

// Build the set of team-pinned fields for the Shared Filters section,
// so we can avoid duplicating them in the regular Filters list below.
Expand All @@ -1278,7 +1292,7 @@ const DBSearchPageFiltersComponent = ({

// Build the facet list for the Shared Filters section.
// For each team-pinned field: merge pinned values with dynamic facet values.
const sharedFacets = useMemo(() => {
const sharedFacets = useMemo((): FacetEntry[] => {
if (sharedFilterKeys.size === 0) return [];

const facetMap = new Map(
Expand All @@ -1288,6 +1302,7 @@ const DBSearchPageFiltersComponent = ({
return Array.from(sharedFilterKeys).map(key => {
const teamVals = pinnedFiltersApiData?.team?.filters[key] ?? [];
const dynamicValues = facetMap.get(key) ?? [];
const isLoading = pendingKeys.has(key);

let merged: (string | boolean)[];
if (teamVals.length > 0) {
Expand All @@ -1302,12 +1317,17 @@ const DBSearchPageFiltersComponent = ({
merged = [...dynamicValues];
}

return { key, value: merged };
return { key, value: merged, isLoading };
});
}, [sharedFilterKeys, facetsWithPinnedValues, pinnedFiltersApiData]);
}, [
sharedFilterKeys,
facetsWithPinnedValues,
pinnedFiltersApiData,
pendingKeys,
]);

const shownFacets = useMemo(() => {
const _facets: { key: string; value: (string | boolean)[] }[] = [];
const _facets: FacetEntry[] = [];
for (const _facet of facetsWithPinnedValues ?? []) {
const facet = structuredClone(_facet);
if (jsonColumns?.some(col => facet.key.startsWith(col))) {
Expand All @@ -1319,12 +1339,18 @@ const DBSearchPageFiltersComponent = ({
continue;
}

// don't include empty facets, unless they are already selected or pinned
// don't include empty facets, unless they are already selected, pinned,
// or still waiting on their values
const filter = filterState[facet.key];
const hasSelectedValues =
filter && (filter.included.size > 0 || filter.excluded.size > 0);
const isPinned = isFieldPinned(facet.key);
if (facet.value?.length > 0 || hasSelectedValues || isPinned) {
if (
facet.value?.length > 0 ||
hasSelectedValues ||
isPinned ||
facet.isLoading
Comment thread
knudtty marked this conversation as resolved.
Resolved
) {
_facets.push(facet);
}
}
Expand Down Expand Up @@ -1466,7 +1492,7 @@ const DBSearchPageFiltersComponent = ({
*/
const renderFacetList = useCallback(
(
facets: { key: string; value: (string | boolean)[] }[],
facets: FacetEntry[],
options?: { keyPrefix?: string; isDefaultExpanded?: boolean },
) => {
const { keyPrefix = '', isDefaultExpanded: forceExpanded } =
Expand Down Expand Up @@ -1582,7 +1608,7 @@ const DBSearchPageFiltersComponent = ({
value,
label: value.toString(),
}))}
optionsLoading={isFacetsLoading}
optionsLoading={isFacetsLoading || !!facet.isLoading}
selectedValues={
getFilterStateEntry(filterState, facet.key) ?? {
included: new Set(),
Expand Down Expand Up @@ -1876,16 +1902,15 @@ const DBSearchPageFiltersComponent = ({
/>
)}

{isFacetsLoading ? (
<Flex align="center" justify="center">
<Loader size="xs" color="gray" />
</Flex>
) : (
shownFacets.length === 0 && (
{shownFacets.length === 0 &&
(isFacetsLoading ? (
<Flex align="center" justify="center">
<Loader size="xs" color="gray" />
</Flex>
) : (
<Text size="xxs">No filters available</Text>
)
)}
{/* Show facets even when loading to ensure pinned filters are visible while loading */}
))}
{/* Filters show their own loader until their values arrive */}
{renderFacetList(shownFacets)}

<Button
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,20 @@ import {
FILTER_SEARCH_MIN_ITEMS,
FilterSearchInput,
} from '@/components/DBSearchPageFilters/FilterSearchInput';
import type { FacetEntry } from '@/components/DBSearchPageFilters/utils';
import { isColumnInSelect } from '@/utils';

import classes from '@styles/SearchPage.module.scss';

type NestedFilterGroupProps = {
name: string;
childFilters: {
key: string;
value: (string | boolean)[];
childFilters: (FacetEntry & {
propertyPath: string;
// Canonical (quoted/bracket) SQL form of `key`, used wherever the key
// becomes a raw SQL expression (distribution query, "Add column" SELECT).
// Falls back to `key` when absent.
sqlKey?: string;
}[];
})[];
selectedValues?: FilterState;
onChange: (key: string, value: string | boolean) => void;
onClearClick: (key: string) => void;
Expand Down Expand Up @@ -258,7 +257,7 @@ export const NestedFilterGroup = ({
value: value,
label: value.toString(),
}))}
optionsLoading={false}
optionsLoading={child.isLoading ?? false}
selectedValues={childSelectedValues}
onChange={value => onChange(child.key, value)}
onClearClick={() => onClearClick(child.key)}
Expand Down
90 changes: 90 additions & 0 deletions packages/app/src/components/DBSearchPageFilters/hooks.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { act, renderHook, waitFor } from '@testing-library/react';

import api from '@/api';
import type { Facet } from '@/hooks/useMetadata';
import * as useMetadataModule from '@/hooks/useMetadata';
import * as searchFiltersModule from '@/searchFilters';
import * as sourceModule from '@/source';
Expand Down Expand Up @@ -68,6 +69,7 @@ jest.mock('@/hooks/useMetadata', () => ({
useMapColumns: jest.fn(),
useAllFields: jest.fn(),
useGetKeyValues: jest.fn(),
useFilterKeysFetchLimit: jest.fn(),
}));

const useMe = jest.mocked(api.useMe);
Expand All @@ -82,6 +84,9 @@ const useJsonColumns = jest.mocked(useMetadataModule.useJsonColumns);
const useMapColumns = jest.mocked(useMetadataModule.useMapColumns);
const useAllFields = jest.mocked(useMetadataModule.useAllFields);
const useGetKeyValues = jest.mocked(useMetadataModule.useGetKeyValues);
const useFilterKeysFetchLimit = jest.mocked(
useMetadataModule.useFilterKeysFetchLimit,
);

const CHART_CONFIG: BuilderChartConfigWithDateRange = {
connection: 'conn1',
Expand Down Expand Up @@ -121,6 +126,7 @@ const makeLogSource = (opts: { withMVs: boolean }) => ({

type SourceQueryResult = ReturnType<typeof sourceModule.useSource>;
type AllFieldsResult = ReturnType<typeof useMetadataModule.useAllFields>;
type KeyValuesResult = ReturnType<typeof useMetadataModule.useGetKeyValues>;
type MetadataWithSettings = ReturnType<
typeof useMetadataModule.useMetadataWithSettings
>;
Expand Down Expand Up @@ -192,6 +198,8 @@ function setupDefaultMocks({ withMVs }: { withMVs: boolean }) {
isFetching: false,
error: null,
} as any);

useFilterKeysFetchLimit.mockReturnValue(100);
}

describe('useFetchFacets', () => {
Expand Down Expand Up @@ -479,6 +487,88 @@ describe('useFetchFacets', () => {
});
});

describe('pending keys', () => {
const TWO_FIELDS = [
{
path: ['ServiceName'],
type: 'LowCardinality(String)',
jsType: 'string',
},
{
path: ['SeverityText'],
type: 'LowCardinality(String)',
jsType: 'string',
},
];

const mockKeyValues = (data: Facet[] | undefined, isFetching: boolean) =>
useGetKeyValues.mockReturnValue({
data,
isLoading: false,
isFetching,
error: null,
} as unknown as KeyValuesResult);

const renderAllMode = () =>
renderHook(
() =>
useFetchFacets({
chartConfig: CHART_CONFIG,
sourceId: 'source1',
dateRange: DATE_RANGE,
mode: 'all',
}),
{ wrapper: makeWrapper().wrapper },
);

it('lists keys whose values have not arrived yet as empty, pending facets', () => {
setupDefaultMocks({ withMVs: false });
useAllFields.mockReturnValue({
data: TWO_FIELDS,
} as unknown as AllFieldsResult);
mockKeyValues([{ key: 'SeverityText', value: ['info'] }], true);

const { result } = renderAllMode();

expect(result.current.data.keyValues).toEqual([
{ key: 'ServiceName', value: [] },
{ key: 'SeverityText', value: ['info'] },
]);
expect(result.current.data.pendingKeys).toEqual(new Set(['ServiceName']));
});

it('drops keys without values once the values query finishes', () => {
setupDefaultMocks({ withMVs: false });
useAllFields.mockReturnValue({
data: TWO_FIELDS,
} as unknown as AllFieldsResult);
mockKeyValues([{ key: 'SeverityText', value: ['info'] }], false);

const { result } = renderAllMode();

expect(result.current.data.keyValues).toEqual([
{ key: 'SeverityText', value: ['info'] },
]);
expect(result.current.data.pendingKeys.size).toBe(0);
});

it('does not list keys past the team key fetch limit', () => {
setupDefaultMocks({ withMVs: false });
useAllFields.mockReturnValue({
data: TWO_FIELDS,
} as unknown as AllFieldsResult);
useFilterKeysFetchLimit.mockReturnValue(1);
mockKeyValues(undefined, true);

const { result } = renderAllMode();

expect(result.current.data.keyValues).toEqual([
{ key: 'ServiceName', value: [] },
]);
expect(result.current.data.pendingKeys).toEqual(new Set(['ServiceName']));
});
});

describe('loadMoreFacetsForKey (raw-tables pipeline)', () => {
it('reports the key as loading while the fetch is in flight, then clears it', async () => {
setupDefaultMocks({ withMVs: false });
Expand Down
Loading
Loading