From d42dc1befe9a7a0b6a173fbc20ac2bf92e56c13c Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:27:01 -0400 Subject: [PATCH 01/10] feat: stream filter values to UI as they arrive --- .../DBSearchPageFilters/hooks.test.tsx | 109 ++++++------ .../components/DBSearchPageFilters/hooks.ts | 27 ++- .../__tests__/useAutoCompleteOptions.test.tsx | 15 +- .../src/hooks/__tests__/useMetadata.test.tsx | 123 ++++++++++++++ .../__tests__/useStreamingQuery.test.tsx | 132 +++++++++++++++ packages/app/src/hooks/useMetadata.tsx | 155 ++++++++++++------ packages/app/src/hooks/useStreamingQuery.tsx | 50 +++++- .../src/__tests__/metadata.test.ts | 95 +++++++++++ packages/common-utils/src/core/metadata.ts | 138 +++++++++++----- 9 files changed, 685 insertions(+), 159 deletions(-) diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx index a3c567233e..f5661230c5 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx +++ b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx @@ -10,6 +10,7 @@ import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { act, renderHook, waitFor } from '@testing-library/react'; import api from '@/api'; +import type { Facet } from '@/hooks/useMetadata'; import * as useMetadataModule from '@/hooks/useMetadata'; import * as searchFiltersModule from '@/searchFilters'; import * as sourceModule from '@/source'; @@ -22,17 +23,16 @@ enableMapSet(); * These tests focus on the two code paths inside `useFacets`: * * 1. Raw-tables pipeline: active when `mode === 'exact'`. Calls - * `useGetKeyValues({ mode: 'exact' })` and scopes "Load more" through + * `useGetKeyValues` and scopes "Load more" through * `metadata.getKeyValuesWithMVs`. - * 2. "All" pipeline: active when `mode === 'all'`. Calls - * `useGetKeyValues({ mode: 'all' })` — whose intelligent router picks + * 2. "All" pipeline: active when `mode === 'all'`. Calls `useAllKeyValues` + * — which streams from `metadata.streamAllKeyValues`, whose router picks * MV/text-index/raw internally — and delegates "Load more" to * `metadata.getAllKeyValues`. * - * Both paths share a single `useGetKeyValues` call whose behavior is driven - * entirely by the `mode` argument. Selection is mode-only; the presence or - * absence of metadata materialized views on the source does not affect which - * path runs. + * Both hooks are always called, and `mode` enables exactly one of them. + * Selection is mode-only; the presence or absence of metadata materialized + * views on the source does not affect which path runs. * * Plus the shared state layer that merges "load more" results into the * active path (union — primary values are preserved and never overridden by @@ -68,6 +68,7 @@ jest.mock('@/hooks/useMetadata', () => ({ useMapColumns: jest.fn(), useAllFields: jest.fn(), useGetKeyValues: jest.fn(), + useAllKeyValues: jest.fn(), })); const useMe = jest.mocked(api.useMe); @@ -82,6 +83,7 @@ const useJsonColumns = jest.mocked(useMetadataModule.useJsonColumns); const useMapColumns = jest.mocked(useMetadataModule.useMapColumns); const useAllFields = jest.mocked(useMetadataModule.useAllFields); const useGetKeyValues = jest.mocked(useMetadataModule.useGetKeyValues); +const useAllKeyValues = jest.mocked(useMetadataModule.useAllKeyValues); const CHART_CONFIG: BuilderChartConfigWithDateRange = { connection: 'conn1', @@ -191,6 +193,14 @@ function setupDefaultMocks({ withMVs }: { withMVs: boolean }) { isFetching: false, error: null, } as any); + + useAllKeyValues.mockReturnValue({ + data: undefined, + isLoading: false, + isFetching: false, + isError: false, + error: null, + }); } describe('useFetchFacets', () => { @@ -199,7 +209,7 @@ describe('useFetchFacets', () => { }); describe('pipeline selection', () => { - it('routes useGetKeyValues with mode="exact" when mode is exact', () => { + it('enables only useGetKeyValues when mode is exact', () => { setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); @@ -214,12 +224,11 @@ describe('useFetchFacets', () => { { wrapper }, ); - const call = useGetKeyValues.mock.calls.at(-1); - expect(call?.[0]?.mode).toBe('exact'); - expect(call?.[1]?.enabled).toBe(true); + expect(useGetKeyValues.mock.calls.at(-1)?.[1]?.enabled).toBe(true); + expect(useAllKeyValues.mock.calls.at(-1)?.[1]?.enabled).toBe(false); }); - it('routes useGetKeyValues with mode="all" when mode is all', () => { + it('enables only useAllKeyValues when mode is all', () => { setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); @@ -234,12 +243,11 @@ describe('useFetchFacets', () => { { wrapper }, ); - const call = useGetKeyValues.mock.calls.at(-1); - expect(call?.[0]?.mode).toBe('all'); - expect(call?.[1]?.enabled).toBe(true); + expect(useAllKeyValues.mock.calls.at(-1)?.[1]?.enabled).toBe(true); + expect(useGetKeyValues.mock.calls.at(-1)?.[1]?.enabled).toBe(false); }); - it('selection is mode-only: MV presence does not change which mode is passed', () => { + it('selection is mode-only: MV presence does not change which pipeline runs', () => { setupDefaultMocks({ withMVs: true }); const { wrapper } = makeWrapper(); @@ -254,8 +262,8 @@ describe('useFetchFacets', () => { { wrapper }, ); - const call = useGetKeyValues.mock.calls.at(-1); - expect(call?.[0]?.mode).toBe('exact'); + expect(useGetKeyValues.mock.calls.at(-1)?.[1]?.enabled).toBe(true); + expect(useAllKeyValues.mock.calls.at(-1)?.[1]?.enabled).toBe(false); }); }); @@ -264,7 +272,7 @@ describe('useFetchFacets', () => { // query — only firing that query once the user is actively searching on // a fully-formed key. Guard against a regression that couples the two. describe('disableValues', () => { - it('disables the useGetKeyValues query when disableValues is true', () => { + it('disables the values query when disableValues is true', () => { setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); @@ -280,11 +288,11 @@ describe('useFetchFacets', () => { { wrapper }, ); - const call = useGetKeyValues.mock.calls.at(-1); - expect(call?.[1]?.enabled).toBe(false); + expect(useAllKeyValues.mock.calls.at(-1)?.[1]?.enabled).toBe(false); + expect(useGetKeyValues.mock.calls.at(-1)?.[1]?.enabled).toBe(false); }); - it('enables the useGetKeyValues query when disableValues is false or omitted', () => { + it('enables the values query when disableValues is false or omitted', () => { setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); @@ -300,8 +308,7 @@ describe('useFetchFacets', () => { { wrapper }, ); - const call = useGetKeyValues.mock.calls.at(-1); - expect(call?.[1]?.enabled).toBe(true); + expect(useAllKeyValues.mock.calls.at(-1)?.[1]?.enabled).toBe(true); }); it('does not defer the field metadata query — useAllFields stays enabled', () => { @@ -352,29 +359,29 @@ describe('useFetchFacets', () => { }); describe('data selection', () => { - // Route mock responses by the `mode` arg so each pipeline sees a - // distinct fixture — that way an assertion against `data.keyValues` - // actually proves the active pipeline's response is returned. - function mockGetKeyValuesByMode(byMode: { exact: unknown; all: unknown }) { - useGetKeyValues.mockImplementation(((args: { mode?: 'all' | 'exact' }) => - args?.mode === 'all' ? byMode.all : byMode.exact) as any); + // Give each pipeline a distinct fixture, so an assertion against + // `data.keyValues` proves the active pipeline's response is returned. + function mockKeyValuesByMode(byMode: { exact: Facet[]; all: Facet[] }) { + useGetKeyValues.mockReturnValue({ + data: byMode.exact, + isLoading: false, + isFetching: false, + error: null, + } as any); + useAllKeyValues.mockReturnValue({ + data: byMode.all, + isLoading: false, + isFetching: false, + isError: false, + error: null, + }); } it('returns data from the raw-tables pipeline when mode is exact', () => { setupDefaultMocks({ withMVs: false }); - mockGetKeyValuesByMode({ - exact: { - data: [{ key: 'ServiceName', value: ['api', 'web'] }], - isLoading: false, - isFetching: false, - error: null, - }, - all: { - data: [{ key: 'ShouldNotBeUsed', value: ['x'] }], - isLoading: false, - isFetching: false, - error: null, - }, + mockKeyValuesByMode({ + exact: [{ key: 'ServiceName', value: ['api', 'web'] }], + all: [{ key: 'ShouldNotBeUsed', value: ['x'] }], }); const { wrapper } = makeWrapper(); @@ -397,19 +404,9 @@ describe('useFetchFacets', () => { it('returns data from the "all" pipeline when mode is all', () => { setupDefaultMocks({ withMVs: true }); - mockGetKeyValuesByMode({ - exact: { - data: [{ key: 'ShouldNotBeUsed', value: ['x'] }], - isLoading: false, - isFetching: false, - error: null, - }, - all: { - data: [{ key: 'ServiceName', value: ['api', 'web'] }], - isLoading: false, - isFetching: false, - error: null, - }, + mockKeyValuesByMode({ + exact: [{ key: 'ShouldNotBeUsed', value: ['x'] }], + all: [{ key: 'ServiceName', value: ['api', 'web'] }], }); const { wrapper } = makeWrapper(); diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index be73a26336..ffc18858ed 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -17,6 +17,7 @@ import { import { Facet, useAllFields, + useAllKeyValues, useColumns, useDateTimeColumns, useGetKeyValues, @@ -191,15 +192,29 @@ function useFacets({ [chartConfig, dateRange, mode], ); - const { data: rawFacets, ...rest } = useGetKeyValues( + const isValuesQueryEnabled = !!enabled && !disableValues; + const exactKeyValues = useGetKeyValues( + { + chartConfig: facetsChartConfig, + limit: INITIAL_LOAD_LIMIT, + keys: escapedKeysToFetch, + }, + { enabled: isValuesQueryEnabled && mode === 'exact' }, + ); + const allKeyValues = useAllKeyValues( { chartConfig: facetsChartConfig, limit: INITIAL_LOAD_LIMIT, keys: escapedKeysToFetch, - mode, }, - { enabled: enabled && !disableValues }, + { enabled: isValuesQueryEnabled && mode === 'all' }, ); + const { + data: rawFacets, + error: keyValuesError, + isLoading: isKeyValuesLoading, + isFetching, + } = mode === 'all' ? allKeyValues : exactKeyValues; // Map the (escaped) result keys back to the original UI keys. const facets = useMemo( @@ -288,10 +303,10 @@ function useFacets({ ); return { - ...rest, - error: allFieldsError ?? rest.error, + error: allFieldsError ?? keyValuesError, data: { keys: allFields, keyValues: facets }, - isLoading: isAllFieldsLoading || rest.isLoading, + isLoading: isAllFieldsLoading || isKeyValuesLoading, + isFetching, loadMoreFacetsForKey, }; } diff --git a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx index e54f4433f5..6d7e79d681 100644 --- a/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx +++ b/packages/app/src/hooks/__tests__/useAutoCompleteOptions.test.tsx @@ -11,7 +11,11 @@ import { tokenizeAtCursor, useAutoCompleteOptions, } from '@/hooks/useAutoCompleteOptions'; -import { useAllFields, useGetKeyValues } from '@/hooks/useMetadata'; +import { + useAllFields, + useAllKeyValues, + useGetKeyValues, +} from '@/hooks/useMetadata'; enableMapSet(); @@ -49,6 +53,7 @@ jest.mock('../useMetadata', () => { useAllFields: jest.fn(), useMultipleAllFields: jest.fn(), useGetKeyValues: jest.fn(), + useAllKeyValues: jest.fn(), useColumns: jest.fn().mockReturnValue({ data: [], isLoading: false }), useDateTimeColumns: jest.fn().mockReturnValue(new Map()), useJsonColumns: jest.fn().mockReturnValue({ data: [] }), @@ -114,6 +119,10 @@ describe('useAutoCompleteOptions', () => { data: null, isFetching: false, }); + (useAllKeyValues as jest.Mock).mockReturnValue({ + data: undefined, + isFetching: false, + }); wrapper = makeWrapper(); }); @@ -144,7 +153,7 @@ describe('useAutoCompleteOptions', () => { }); it('should return key value options with correct lucene formatting', () => { - (useGetKeyValues as jest.Mock).mockReturnValue({ + (useAllKeyValues as jest.Mock).mockReturnValue({ data: [ { key: 'ResourceAttributes.service.name', @@ -191,7 +200,7 @@ describe('useAutoCompleteOptions', () => { }); it('should handle nested key value options', () => { - (useGetKeyValues as jest.Mock).mockReturnValue({ + (useAllKeyValues as jest.Mock).mockReturnValue({ data: [ { key: 'ResourceAttributes', diff --git a/packages/app/src/hooks/__tests__/useMetadata.test.tsx b/packages/app/src/hooks/__tests__/useMetadata.test.tsx index d41a1ce2ed..cdfd794204 100644 --- a/packages/app/src/hooks/__tests__/useMetadata.test.tsx +++ b/packages/app/src/hooks/__tests__/useMetadata.test.tsx @@ -15,6 +15,7 @@ import api from '@/api'; import { deduplicate2dArray, intersect2dArray, + useAllKeyValues, useGetKeyValues, useMultipleAllFields, useMultipleGetKeyValues, @@ -276,6 +277,128 @@ describe('useGetKeyValues', () => { }); }); +describe('useAllKeyValues', () => { + let wrapper: React.ComponentType<{ children: any }>; + let mockMetadata: Metadata; + + const chartConfig = createMockChartConfig({ + source: 'source1', + timestampValueExpression: 'Timestamp', + dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], + }); + + beforeEach(() => { + jest.clearAllMocks(); + mockMetadata = new Metadata({} as ClickhouseClient, {} as MetadataCache); + jest.spyOn(metadataModule, 'getMetadata').mockReturnValue(mockMetadata); + jest.spyOn(api, 'useMe').mockReturnValue({ + data: { team: {} }, + isLoading: false, + } as any); + jest.mocked(useSources).mockReturnValue({ + data: [{ id: 'source1' }], + isLoading: false, + } as any); + + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + wrapper = ({ children }) => ( + {children} + ); + }); + + it("shows each query's values before the slowest finishes, in the order of keys", async () => { + let releaseSlowQuery: () => void = () => {}; + const slowQuery = new Promise(resolve => { + releaseSlowQuery = resolve; + }); + jest + .spyOn(mockMetadata, 'streamAllKeyValues') + .mockImplementation(async function* () { + yield [{ key: 'SeverityText', value: ['info'] }]; + await slowQuery; + yield [{ key: 'ServiceName', value: ['api'] }]; + }); + + const { result } = renderHook( + () => + useAllKeyValues({ + chartConfig, + keys: ['ServiceName', 'SeverityText'], + }), + { wrapper }, + ); + + await waitFor(() => + expect(result.current.data).toEqual([ + { key: 'SeverityText', value: ['info'] }, + ]), + ); + expect(result.current.isLoading).toBe(false); + expect(result.current.isFetching).toBe(true); + + releaseSlowQuery(); + + await waitFor(() => expect(result.current.isFetching).toBe(false)); + expect(result.current.data).toEqual([ + { key: 'ServiceName', value: ['api'] }, + { key: 'SeverityText', value: ['info'] }, + ]); + }); + + it('streams the keys, limit, and table from the chart config', async () => { + const streamAllKeyValues = jest + .spyOn(mockMetadata, 'streamAllKeyValues') + .mockImplementation(async function* () { + yield [{ key: 'StatusCode', value: [200, 500] }]; + }); + + const { result } = renderHook( + () => + useAllKeyValues({ + chartConfig, + keys: ['StatusCode'], + limit: 7, + }), + { wrapper }, + ); + + await waitFor(() => expect(result.current.isFetching).toBe(false)); + expect(streamAllKeyValues).toHaveBeenCalledWith( + expect.objectContaining({ + databaseName: 'telemetry', + tableName: 'traces', + connectionId: 'foo', + keyExpressions: ['StatusCode'], + maxValuesPerKey: 7, + dateRange: chartConfig.dateRange, + timestampValueExpression: 'Timestamp', + }), + ); + expect(result.current.data).toEqual([ + { key: 'StatusCode', value: ['200', '500'] }, + ]); + }); + + it('does not stream when disabled', () => { + const streamAllKeyValues = jest.spyOn(mockMetadata, 'streamAllKeyValues'); + + const { result } = renderHook( + () => + useAllKeyValues( + { chartConfig, keys: ['ServiceName'] }, + { enabled: false }, + ), + { wrapper }, + ); + + expect(streamAllKeyValues).not.toHaveBeenCalled(); + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + }); +}); + describe('useMultipleAllFields', () => { let queryClient: QueryClient; let wrapper: React.ComponentType<{ children: any }>; diff --git a/packages/app/src/hooks/__tests__/useStreamingQuery.test.tsx b/packages/app/src/hooks/__tests__/useStreamingQuery.test.tsx index b3ccfb9a8e..6c6c01f14d 100644 --- a/packages/app/src/hooks/__tests__/useStreamingQuery.test.tsx +++ b/packages/app/src/hooks/__tests__/useStreamingQuery.test.tsx @@ -111,6 +111,138 @@ describe('useStreamingQuery', () => { expect(result.current.data).toBeUndefined(); }); + it('does not write a superseded stream into the cache', async () => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + const wrapper = ({ children }: { children: React.ReactNode }) => ( + {children} + ); + let releaseLateChunk: () => void = () => {}; + const lateChunkGate = new Promise(resolve => { + releaseLateChunk = resolve; + }); + let isSupersededStreamDone = false; + const streams: Record AsyncGenerator> = { + a: async function* () { + try { + yield ['a1']; + await lateChunkGate; + yield ['a2']; + } finally { + isSupersededStreamDone = true; + } + }, + b: async function* () { + yield ['b1']; + }, + }; + + const { result, rerender } = renderHook( + ({ id }: { id: string }) => + useStreamingQuery({ + queryKey: [id], + streamFactory: streams[id], + flushIntervalMs: 0, + }), + { wrapper, initialProps: { id: 'a' } }, + ); + await waitFor(() => expect(result.current.data).toEqual(['a1'])); + + rerender({ id: 'b' }); + await waitFor(() => expect(result.current.data).toEqual(['b1'])); + releaseLateChunk(); + await waitFor(() => expect(isSupersededStreamDone).toBe(true)); + + // Reverted on cancel; a late write would be served as complete forever. + expect(queryClient.getQueryData(['a'])).toBeUndefined(); + }); + + describe('itemKey', () => { + type Item = { id: string; version: number }; + const itemKey = (item: Item) => item.id; + + it("shows a new key's stream over the last complete result", async () => { + let releaseLastChunk: () => void = () => {}; + const lastChunkGate = new Promise(resolve => { + releaseLastChunk = resolve; + }); + const streams: Record AsyncGenerator> = { + first: async function* () { + yield [ + { id: 'x', version: 1 }, + { id: 'y', version: 1 }, + ]; + }, + second: async function* () { + yield [{ id: 'y', version: 2 }]; + await lastChunkGate; + yield [{ id: 'z', version: 2 }]; + }, + }; + + const { result, rerender } = renderHook( + ({ id }: { id: string }) => + useStreamingQuery({ + queryKey: [id], + streamFactory: streams[id], + flushIntervalMs: 0, + itemKey, + }), + { wrapper: createWrapper(), initialProps: { id: 'first' } }, + ); + await waitFor(() => expect(result.current.isStreaming).toBe(false)); + + rerender({ id: 'second' }); + + // `x` keeps its old version until the stream completes; `y` is swapped + // in place rather than moved. + await waitFor(() => + expect(result.current.data).toEqual([ + { id: 'x', version: 1 }, + { id: 'y', version: 2 }, + ]), + ); + expect(result.current.isStreaming).toBe(true); + + releaseLastChunk(); + + await waitFor(() => expect(result.current.isStreaming).toBe(false)); + expect(result.current.data).toEqual([ + { id: 'y', version: 2 }, + { id: 'z', version: 2 }, + ]); + }); + + it('streams from empty when there is no complete result yet', async () => { + let releaseLastChunk: () => void = () => {}; + const lastChunkGate = new Promise(resolve => { + releaseLastChunk = resolve; + }); + const streamFactory = async function* () { + yield [{ id: 'x', version: 1 }]; + await lastChunkGate; + }; + + const { result } = renderHook( + () => + useStreamingQuery({ + queryKey: ['no-previous'], + streamFactory, + flushIntervalMs: 0, + itemKey, + }), + { wrapper: createWrapper() }, + ); + + await waitFor(() => + expect(result.current.data).toEqual([{ id: 'x', version: 1 }]), + ); + releaseLastChunk(); + await waitFor(() => expect(result.current.isStreaming).toBe(false)); + }); + }); + it('reports an empty stream as a completed empty result', async () => { const streamFactory = async function* (): AsyncGenerator { // no chunks diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index e9030abff3..a4b3e4a6f8 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -20,7 +20,6 @@ import { BuilderChartConfigWithDateRange, isLogSource, isTraceSource, - MetadataMaterializedViews, } from '@hyperdx/common-utils/dist/types'; import { keepPreviousData, @@ -33,6 +32,7 @@ import api from '@/api'; import { IS_LOCAL_MODE } from '@/config'; import { LOCAL_STORE_CONNECTIONS_KEY } from '@/connection'; import { DEFAULT_FILTER_KEYS_FETCH_LIMIT } from '@/defaults'; +import { useStreamingQuery } from '@/hooks/useStreamingQuery'; import { getMetadata } from '@/metadata'; import { useSource, useSources } from '@/source'; import { toArray } from '@/utils'; @@ -350,20 +350,15 @@ export function useMultipleGetKeyValues( keyConditions, limit, disableRowLimit, - mode = 'exact', - metadataMVs: metadataMVsOverride, }: { chartConfigs: | BuilderChartConfigWithDateRange | BuilderChartConfigWithDateRange[]; keys: string[]; - /** Per-key constraints for faceted ('exact' mode) value lookups. */ + /** Per-key constraints for faceted value lookups. */ keyConditions?: (FilterState | undefined)[]; limit?: number; disableRowLimit?: boolean; - mode?: 'all' | 'exact'; - /** Pass directly to skip source-based resolution (e.g. when chartConfig has no source) */ - metadataMVs?: MetadataMaterializedViews; }, options?: Omit, 'queryKey'>, ) { @@ -380,8 +375,6 @@ export function useMultipleGetKeyValues( const query = useQuery({ queryKey: [ 'useMetadata.useGetKeyValues', - mode, - metadataMVsOverride, ...chartConfigsArr.map(cc => ({ ...cc })), ...keys, // Serialized: react-query hashes keys with JSON.stringify, which would @@ -391,39 +384,6 @@ export function useMultipleGetKeyValues( maxKeys, ], queryFn: async ({ signal }) => { - if (mode === 'all') { - const firstConfig = chartConfigsArr[0]; - if (!firstConfig || keys.length === 0) return []; - - // Use explicit override, or resolve from source - const firstSource = firstConfig.source - ? sources?.find(s => s.id === firstConfig.source) - : undefined; - const metadataMVs = - metadataMVsOverride ?? - (firstSource && - (isLogSource(firstSource) || isTraceSource(firstSource)) - ? firstSource.metadataMaterializedViews - : undefined); - - const { databaseName, tableName } = firstConfig.from; - const connectionId = firstConfig.connection; - const dateRange = firstConfig.dateRange; - - return metadata.getAllKeyValues({ - databaseName, - tableName, - keyExpressions: keys.slice(0, maxKeys), - maxValuesPerKey: 20, - connectionId, - metadataMVs, - dateRange, - timestampValueExpression: firstConfig.timestampValueExpression, - signal, - }); - } - - // 'exact' mode return ( await Promise.all( chartConfigsArr.map(chartConfig => { @@ -497,8 +457,6 @@ export function useGetKeyValues( keyConditions, limit, disableRowLimit, - mode, - metadataMVs, }: { chartConfig?: BuilderChartConfigWithDateRange; keys: string[]; @@ -506,8 +464,6 @@ export function useGetKeyValues( keyConditions?: (FilterState | undefined)[]; limit?: number; disableRowLimit?: boolean; - mode?: 'all' | 'exact'; - metadataMVs?: MetadataMaterializedViews; }, options?: Omit, 'queryKey'>, ) { @@ -518,13 +474,116 @@ export function useGetKeyValues( keyConditions, limit, disableRowLimit, - mode, - metadataMVs, }, options, ); } +const facetKey = (facet: Facet) => facet.key; + +/** + * Values for each key across the whole date range, ignoring the search + * filters. `Metadata.getAllKeyValues` routes each key to a text index, the + * metadata materialized views, or a raw table scan; each of those queries' + * values show up as soon as it lands, rather than after the slowest one. + * + * Facets are kept in the order of `keys`, so each holds its place while the + * queries finish in any order, and while a new date range replaces the + * previous one's values. + */ +export function useAllKeyValues( + { + chartConfig, + keys, + limit = 20, + }: { + chartConfig: BuilderChartConfigWithDateRange; + keys: string[]; + /** Values per key. */ + limit?: number; + }, + { enabled = true }: { enabled?: boolean } = {}, +) { + const metadata = useMetadataWithSettings(); + const { data: me, isLoading: isLoadingMe } = api.useMe(); + const { data: sources, isLoading: isLoadingSources } = useSources(); + + const maxKeys = + me?.team?.filterKeysFetchLimit ?? DEFAULT_FILTER_KEYS_FETCH_LIMIT; + const keysToFetch = useMemo(() => keys.slice(0, maxKeys), [keys, maxKeys]); + + const source = chartConfig.source + ? sources?.find(s => s.id === chartConfig.source) + : undefined; + const metadataMVs = + source && (isLogSource(source) || isTraceSource(source)) + ? source.metadataMaterializedViews + : undefined; + + const { + connection: connectionId, + from: { databaseName, tableName }, + dateRange, + timestampValueExpression, + } = chartConfig; + + const streamed = useStreamingQuery({ + queryKey: [ + 'useMetadata.useAllKeyValues', + connectionId, + databaseName, + tableName, + timestampValueExpression, + dateRange[0].getTime(), + dateRange[1].getTime(), + metadataMVs, + limit, + keysToFetch, + ], + // Not memoized: read when the query runs, not hashed into its key. + streamFactory: async function* ({ signal }) { + for await (const keyValues of metadata.streamAllKeyValues({ + databaseName, + tableName, + connectionId, + keyExpressions: keysToFetch, + maxValuesPerKey: limit, + metadataMVs, + dateRange, + timestampValueExpression, + signal, + })) { + yield keyValues.map(({ key, value }) => ({ + key, + value: value.map(val => val.toString()), + })); + } + }, + enabled: + enabled && keysToFetch.length > 0 && !isLoadingSources && !isLoadingMe, + itemKey: facetKey, + }); + + const data = useMemo(() => { + if (!streamed.data) return undefined; + // Keys the result renders differently from the request (re-quoted + // identifiers) sort last, in arrival order. + const position = new Map(keysToFetch.map((key, i) => [key, i] as const)); + const rank = (facet: Facet) => + position.get(facet.key) ?? keysToFetch.length; + return [...streamed.data].sort((a, b) => rank(a) - rank(b)); + }, [streamed.data, keysToFetch]); + + return { + data, + error: streamed.error, + isError: streamed.isError, + isFetching: streamed.isStreaming, + // Only until the first values land; the stream then fills in. + isLoading: (!data && streamed.isStreaming) || isLoadingSources, + }; +} + /** * List metric names for one metrics table, ordered and matched server-side. * diff --git a/packages/app/src/hooks/useStreamingQuery.tsx b/packages/app/src/hooks/useStreamingQuery.tsx index 4bf0e13b95..33a86cbb89 100644 --- a/packages/app/src/hooks/useStreamingQuery.tsx +++ b/packages/app/src/hooks/useStreamingQuery.tsx @@ -1,3 +1,4 @@ +import { useMemo, useState } from 'react'; import { QueryKey, useQuery, useQueryClient } from '@tanstack/react-query'; // Chunks can arrive many times a second across several concurrent hooks, so @@ -8,6 +9,23 @@ type StreamFactory = (args: { signal?: AbortSignal; }) => AsyncIterable; +/** + * Swaps the items that have arrived into the last complete result, in place, + * and appends the ones it lacks. + */ +function overlayItems( + settled: TItem[], + partial: TItem[], + itemKey: (item: TItem) => string, +): TItem[] { + const arrived = new Map(partial.map(item => [itemKey(item), item] as const)); + const settledKeys = new Set(settled.map(itemKey)); + return [ + ...settled.map(item => arrived.get(itemKey(item)) ?? item), + ...partial.filter(item => !settledKeys.has(itemKey(item))), + ]; +} + /** * Runs an async-iterable query and exposes its results as they arrive. * @@ -21,11 +39,19 @@ export function useStreamingQuery({ streamFactory, enabled = true, flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS, + itemKey, }: { queryKey: QueryKey; streamFactory: StreamFactory; enabled?: boolean; flushIntervalMs?: number; + /** + * Identifies an item across results. When set, a stream that hasn't + * finished is shown over the last complete result rather than from empty: + * arrived items replace their old versions in place, and old items the new + * result lacks drop out once it completes. Must be referentially stable. + */ + itemKey?: (item: TItem) => string; }) { const queryClient = useQueryClient(); @@ -36,6 +62,10 @@ export function useStreamingQuery({ let lastFlushedAt = 0; for await (const chunk of streamFactory({ signal })) { + // A superseded query is reverted to its pre-fetch state on cancel. A + // write after that would land a partial result in the cache as if it + // were complete, and `staleTime: Infinity` would serve it forever. + if (signal.aborted) break; accumulated.push(...chunk); // Monotonic clock: this only ever measures an elapsed interval. if (performance.now() - lastFlushedAt >= flushIntervalMs) { @@ -56,11 +86,27 @@ export function useStreamingQuery({ retry: false, }); - return { + const [settled, setSettled] = useState(); + if ( + itemKey && + query.isSuccess && + !query.isFetching && + query.data !== settled + ) { + setSettled(query.data); + } + + const data = useMemo(() => { // Partial while streaming. Suppressed on error rather than handing back a // truncated list as though it were complete — the arrived chunks are still // in the cache, and `setQueryData(key, undefined)` is a no-op. - data: query.isError ? undefined : query.data, + if (query.isError) return undefined; + if (!itemKey || !settled || !query.isFetching) return query.data; + return overlayItems(settled, query.data ?? [], itemKey); + }, [query.isError, query.isFetching, query.data, settled, itemKey]); + + return { + data, isStreaming: query.isFetching, isError: query.isError, error: query.error, diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 207cc3abcb..d36144f949 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1709,6 +1709,101 @@ describe('Metadata', () => { consoleWarnSpy.mockRestore(); }); }); + + describe('streamAllKeyValues', () => { + const respond = (data: unknown[]) => + Promise.resolve({ json: () => Promise.resolve({ data }) }); + + const mvRow = { + ColumnIdentifier: 'NativeColumn', + Key: 'ServiceName', + Values: ['api'], + }; + const mapTextIndexRow = { + column: 'LogAttributes', + key: 'requestId', + value: ['r1'], + }; + + it('yields each query as it finishes, without waiting for the slowest', async () => { + setupDefaultLogsSchema(); + let releaseMapTextIndex: () => void = () => {}; + const mapTextIndexGate = new Promise(resolve => { + releaseMapTextIndex = resolve; + }); + (mockClickhouseClient.query as jest.Mock).mockImplementation( + ({ query }: { query: string }) => { + if (query.includes('startsWith(token,')) { + return mapTextIndexGate.then(() => respond([mapTextIndexRow])); + } + if (query.includes('ColumnIdentifier =')) { + return respond([mvRow]); + } + return respond([]); + }, + ); + + const stream = metadata.streamAllKeyValues({ + ...baseArgs, + keyExpressions: ["LogAttributes['requestId']", 'ServiceName'], + }); + + await expect(stream.next()).resolves.toEqual({ + done: false, + value: [{ key: 'ServiceName', value: ['api'] }], + }); + releaseMapTextIndex(); + await expect(stream.next()).resolves.toEqual({ + done: false, + value: [{ key: "LogAttributes['requestId']", value: ['r1'] }], + }); + await expect(stream.next()).resolves.toEqual({ + done: true, + value: undefined, + }); + }); + + it('skips a failed query and keeps yielding the rest', async () => { + setupDefaultLogsSchema(); + jest.spyOn(console, 'warn').mockImplementation(() => undefined); + (mockClickhouseClient.query as jest.Mock).mockImplementation( + ({ query }: { query: string }) => { + if (query.includes('startsWith(token,')) { + return Promise.reject(new Error('map text index unavailable')); + } + if (query.includes('ColumnIdentifier =')) { + return respond([mvRow]); + } + return respond([]); + }, + ); + + const yielded: unknown[] = []; + for await (const keyValues of metadata.streamAllKeyValues({ + ...baseArgs, + keyExpressions: ["LogAttributes['requestId']", 'ServiceName'], + })) { + yielded.push(keyValues); + } + + expect(yielded).toEqual([[{ key: 'ServiceName', value: ['api'] }]]); + }); + + it('yields nothing for empty keyExpressions', async () => { + setupDefaultLogsSchema(); + + const stream = metadata.streamAllKeyValues({ + ...baseArgs, + keyExpressions: [], + }); + + await expect(stream.next()).resolves.toEqual({ + done: true, + value: undefined, + }); + expect(mockClickhouseClient.query).not.toHaveBeenCalled(); + }); + }); }); describe('getValuesDistribution', () => { diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 7fcd3cf89f..605ef75f69 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -92,6 +92,34 @@ export type KeyValues = { value: string[] | number[]; }; +type AllKeyValuesQuery = { + databaseName: string; + tableName: string; + keyExpressions: string[]; + maxValuesPerKey?: number; + connectionId: string; + metadataMVs?: MetadataMaterializedViews; + dateRange: [Date, Date]; + timestampValueExpression: string; + signal?: AbortSignal; +}; + +/** Yields each promise's value as it resolves, fastest first. */ +async function* inCompletionOrder( + promises: Promise[], +): AsyncGenerator { + const pending = new Map( + promises.map( + (promise, i) => [i, promise.then(value => ({ i, value }))] as const, + ), + ); + while (pending.size > 0) { + const { i, value } = await Promise.race(pending.values()); + pending.delete(i); + yield value; + } +} + export type MetricNames = { names: string[]; /** True when more names matched than `limit`, so the page is incomplete. */ @@ -2603,50 +2631,74 @@ export class Metadata { * Fetches top values for one or more keys from the text index, metadataMV, or the raw table in a * single batched query. Falls back to getMapValues when no rollup is available. */ - async getAllKeyValues({ + async getAllKeyValues(query: AllKeyValuesQuery): Promise { + const results = await Promise.all( + await this.dispatchAllKeyValueQueries(query), + ); + return results.filter(v => v !== undefined).flat(); + } + + /** + * `getAllKeyValues`, yielding each query's values as soon as it finishes + * instead of waiting for the slowest. Every key is served by exactly one + * query, so its values arrive in a single yield. + */ + async *streamAllKeyValues( + query: AllKeyValuesQuery, + ): AsyncGenerator { + const queries = await this.dispatchAllKeyValueQueries(query); + for await (const keyValues of inCompletionOrder(queries)) { + if (keyValues?.length) yield keyValues; + } + } + + /** Starts every query the keys need, without waiting for any to finish. */ + private async dispatchAllKeyValueQueries({ + keyExpressions, + maxValuesPerKey = 20, + ...query + }: AllKeyValuesQuery): Promise>> { + if (keyExpressions.length === 0) return []; + + if (keyExpressions.length <= GET_ALL_KEY_VALUES_CHUNK_SIZE) { + return this.dispatchKeyValueQueries({ + ...query, + keyExpressions, + maxValuesPerKey, + }); + } + + // A batch that fails to plan is dropped; the others still answer. + const batches = await Promise.allSettled( + chunk(keyExpressions, GET_ALL_KEY_VALUES_CHUNK_SIZE).map(batch => + this.dispatchKeyValueQueries({ + ...query, + keyExpressions: batch, + maxValuesPerKey, + }), + ), + ); + return batches.filter(v => v.status === 'fulfilled').flatMap(v => v.value); + } + + /** + * Routes up to `GET_ALL_KEY_VALUES_CHUNK_SIZE` keys to their cheapest + * strategy and starts one query per strategy in use. + */ + private async dispatchKeyValueQueries({ databaseName, tableName, keyExpressions, - maxValuesPerKey = 20, + maxValuesPerKey, connectionId, metadataMVs, dateRange, timestampValueExpression, signal, - }: { - databaseName: string; - tableName: string; - keyExpressions: string[]; - maxValuesPerKey?: number; - connectionId: string; - metadataMVs?: MetadataMaterializedViews; - dateRange: [Date, Date]; - timestampValueExpression: string; - signal?: AbortSignal; - }): Promise { - if (keyExpressions.length === 0) return []; - - if (keyExpressions.length > GET_ALL_KEY_VALUES_CHUNK_SIZE) { - const batched = await Promise.allSettled( - chunk(keyExpressions, GET_ALL_KEY_VALUES_CHUNK_SIZE).map(batch => - this.getAllKeyValues({ - databaseName, - tableName, - keyExpressions: batch, - maxValuesPerKey, - connectionId, - metadataMVs, - dateRange, - timestampValueExpression, - signal, - }), - ), - ); - return batched - .filter(v => v.status === 'fulfilled') - .flatMap(v => v.value); - } - + }: AllKeyValuesQuery & + Required>): Promise< + Array> + > { // Parse all keys into (rollupColumn, rollupKey) pairs const parsed = keyExpressions.map(keyExpr => { const path = parseKeyPath(keyExpr); @@ -2795,9 +2847,8 @@ export class Metadata { if (rawQueryOptions.length > 0) { promises.push( // Isolate raw-table failures (timeout, abort, network) the same way - // the three sibling strategies do internally, so a single rejection - // here doesn't discard already-successful text-index / MV results - // when they're aggregated through `Promise.all` below. + // the three sibling strategies do internally, so the failure is + // logged rather than silently dropped below. this.getKeyValues({ chartConfig: { from: { @@ -2823,11 +2874,10 @@ export class Metadata { }), ); } - return (await Promise.allSettled(promises)) - .filter(res => res.status === 'fulfilled') - .map(v => v.value) - .filter(v => v !== undefined) - .flat(); + // The strategies isolate their own failures; this catches anything that + // slips past them, so one rejection can neither discard the others' + // results nor go unhandled while sibling batches are still being planned. + return promises.map(promise => promise.catch(() => undefined)); } async getKeyValues({ From 8889690db7c068cb92988847593b5710db2a5fbe Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:10:41 -0400 Subject: [PATCH 02/10] feat: show keys in filters bar as values are loading --- .../src/components/DBSearchPageFilters.tsx | 80 +++++++++++------ .../DBSearchPageFilters/NestedFilterGroup.tsx | 9 +- .../DBSearchPageFilters/hooks.test.tsx | 83 ++++++++++++++++++ .../components/DBSearchPageFilters/hooks.ts | 21 +++-- .../DBSearchPageFilters/utils.test.ts | 85 +++++++++++++++++++ .../components/DBSearchPageFilters/utils.ts | 70 +++++++++++++-- .../__tests__/DBSearchPageFilters.test.tsx | 44 ++++++++++ .../src/hooks/__tests__/useMetadata.test.tsx | 4 +- packages/app/src/hooks/useMetadata.tsx | 37 +++----- 9 files changed, 359 insertions(+), 74 deletions(-) diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 08c35d32da..e49b6846d8 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -74,6 +74,7 @@ import { } from './DBSearchPageFilters/PinShareMenu'; import { SharedFiltersSection } from './DBSearchPageFilters/SharedFilters'; import { + type FacetEntry, getFilterStateEntry, groupFacetsByBaseName, toQuotedClickHouseKeyExpression, @@ -1000,7 +1001,7 @@ export const FilterGroup = ({ chevron: 'm-0', label: 'p-0', }} - className={hasOptions ? '' : 'opacity-50'} + className={hasOptions || optionsLoading ? '' : 'opacity-50'} > 26 ? 0 : 1500} @@ -1012,12 +1013,21 @@ export const FilterGroup = ({ > {displayName} - {showFilterCounts && ( - {` (${totalAppliedFiltersSize > 0 ? totalAppliedFiltersSize : options.length})`} + {totalAppliedFiltersSize === 0 && optionsLoading ? ( + + ) : ( + showFilterCounts && ( + {` (${totalAppliedFiltersSize > 0 ? totalAppliedFiltersSize : options.length})`} + ) )} @@ -1205,6 +1215,7 @@ const DBSearchPageFiltersComponent = ({ isLoading: isFacetsLoading, isFetching: isFacetsFetching, error, + pendingKeys, loadMoreFacetsForKey, loadMoreLoadingKeys, extraFacetKeys, @@ -1230,7 +1241,7 @@ const DBSearchPageFiltersComponent = ({ }, [error]); // Merge pinned filter values into the queried facets, so that pinned values are always available - const facetsWithPinnedValues = useMemo(() => { + const facetsWithPinnedValues = useMemo((): FacetEntry[] => { const facetsMap = new Map((facets ?? []).map(f => [f.key, f.value])); const mergedKeys = new Set([ ...facetsMap.keys(), @@ -1246,9 +1257,13 @@ const DBSearchPageFiltersComponent = ({ ...(pinnedValues ?? []), ]); - return { key, value: Array.from(mergedValues) }; + return { + key, + value: Array.from(mergedValues), + isLoading: pendingKeys.has(key), + }; }); - }, [facets, pinnedFilters, getPinnedFields]); + }, [facets, pinnedFilters, getPinnedFields, pendingKeys]); // Build the set of team-pinned fields for the Shared Filters section, // so we can avoid duplicating them in the regular Filters list below. @@ -1262,7 +1277,7 @@ const DBSearchPageFiltersComponent = ({ // Build the facet list for the Shared Filters section. // For each team-pinned field: merge pinned values with dynamic facet values. - const sharedFacets = useMemo(() => { + const sharedFacets = useMemo((): FacetEntry[] => { if (sharedFilterKeys.size === 0) return []; const facetMap = new Map( @@ -1272,6 +1287,7 @@ const DBSearchPageFiltersComponent = ({ return Array.from(sharedFilterKeys).map(key => { const teamVals = pinnedFiltersApiData?.team?.filters[key] ?? []; const dynamicValues = facetMap.get(key) ?? []; + const isLoading = pendingKeys.has(key); let merged: (string | boolean)[]; if (teamVals.length > 0) { @@ -1286,12 +1302,17 @@ const DBSearchPageFiltersComponent = ({ merged = [...dynamicValues]; } - return { key, value: merged }; + return { key, value: merged, isLoading }; }); - }, [sharedFilterKeys, facetsWithPinnedValues, pinnedFiltersApiData]); + }, [ + sharedFilterKeys, + facetsWithPinnedValues, + pinnedFiltersApiData, + pendingKeys, + ]); const shownFacets = useMemo(() => { - const _facets: { key: string; value: (string | boolean)[] }[] = []; + const _facets: FacetEntry[] = []; for (const _facet of facetsWithPinnedValues ?? []) { const facet = structuredClone(_facet); if (jsonColumns?.some(col => facet.key.startsWith(col))) { @@ -1303,12 +1324,18 @@ const DBSearchPageFiltersComponent = ({ continue; } - // don't include empty facets, unless they are already selected or pinned + // don't include empty facets, unless they are already selected, pinned, + // or still waiting on their values const filter = filterState[facet.key]; const hasSelectedValues = filter && (filter.included.size > 0 || filter.excluded.size > 0); const isPinned = isFieldPinned(facet.key); - if (facet.value?.length > 0 || hasSelectedValues || isPinned) { + if ( + facet.value?.length > 0 || + hasSelectedValues || + isPinned || + facet.isLoading + ) { _facets.push(facet); } } @@ -1450,7 +1477,7 @@ const DBSearchPageFiltersComponent = ({ */ const renderFacetList = useCallback( ( - facets: { key: string; value: (string | boolean)[] }[], + facets: FacetEntry[], options?: { keyPrefix?: string; isDefaultExpanded?: boolean }, ) => { const { keyPrefix = '', isDefaultExpanded: forceExpanded } = @@ -1566,7 +1593,7 @@ const DBSearchPageFiltersComponent = ({ value, label: value.toString(), }))} - optionsLoading={isFacetsLoading} + optionsLoading={isFacetsLoading || !!facet.isLoading} selectedValues={ getFilterStateEntry(filterState, facet.key) ?? { included: new Set(), @@ -1860,16 +1887,15 @@ const DBSearchPageFiltersComponent = ({ /> )} - {isFacetsLoading ? ( - - - - ) : ( - shownFacets.length === 0 && ( + {shownFacets.length === 0 && + (isFacetsLoading ? ( + + + + ) : ( No filters available - ) - )} - {/* Show facets even when loading to ensure pinned filters are visible while loading */} + ))} + {/* Filters show their own loader until their values arrive */} {renderFacetList(shownFacets)}