diff --git a/.changeset/per-connection-all-fields.md b/.changeset/per-connection-all-fields.md new file mode 100644 index 0000000000..874e4d627e --- /dev/null +++ b/.changeset/per-connection-all-fields.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +refactor: useMultipleAllFields returns each table connection's fields and columns. diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx index a3c567233e..ddf2dd8bbc 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx +++ b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx @@ -120,6 +120,7 @@ const makeLogSource = (opts: { withMVs: boolean }) => ({ }); type SourceQueryResult = ReturnType; +type AllFieldsResult = ReturnType; type MetadataWithSettings = ReturnType< typeof useMetadataModule.useMetadataWithSettings >; @@ -452,6 +453,32 @@ describe('useFetchFacets', () => { }); }); + describe('errors', () => { + it('surfaces a field lookup error so the filter panel can report it', () => { + setupDefaultMocks({ withMVs: false }); + const fieldsError = new Error('DESCRIBE failed'); + useAllFields.mockReturnValue({ + data: [], + error: fieldsError, + isLoading: false, + } as unknown as AllFieldsResult); + const { wrapper } = makeWrapper(); + + const { result } = renderHook( + () => + useFetchFacets({ + chartConfig: CHART_CONFIG, + sourceId: 'source1', + dateRange: DATE_RANGE, + mode: 'exact', + }), + { wrapper }, + ); + + expect(result.current.error).toBe(fieldsError); + }); + }); + describe('loadMoreFacetsForKey (raw-tables pipeline)', () => { it('reports the key as loading while the fetch is in flight, then clears it', async () => { setupDefaultMocks({ withMVs: false }); diff --git a/packages/app/src/hooks/__tests__/useMetadata.test.tsx b/packages/app/src/hooks/__tests__/useMetadata.test.tsx index d41a1ce2ed..aa13cea0ce 100644 --- a/packages/app/src/hooks/__tests__/useMetadata.test.tsx +++ b/packages/app/src/hooks/__tests__/useMetadata.test.tsx @@ -1,6 +1,6 @@ import React from 'react'; import * as metadataModule from '@hyperdx/app/src/metadata'; -import { JSDataType } from '@hyperdx/common-utils/dist/clickhouse'; +import { ColumnMeta, JSDataType } from '@hyperdx/common-utils/dist/clickhouse'; import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/browser'; import { Field, @@ -303,14 +303,41 @@ describe('useMultipleAllFields', () => { connectionId: 'conn1', }; + const column = (name: string, type: string): ColumnMeta => ({ + name, + type, + codec_expression: '', + comment: '', + default_expression: '', + default_type: '', + ttl_expression: '', + }); + const columnsA = [ + column('col_a', 'String'), + column('attrs', 'Map(String, String)'), + column('json', 'JSON'), + ]; + const columnsB = [column('col_b', 'String')]; + + let team: { fieldMetadataDisabled?: boolean }; + beforeEach(() => { jest.clearAllMocks(); + team = {}; mockMetadata = new Metadata({} as ClickhouseClient, {} as MetadataCache); jest.spyOn(metadataModule, 'getMetadata').mockReturnValue(mockMetadata); - jest.spyOn(api, 'useMe').mockReturnValue({ - data: { team: {} }, - isFetched: true, - } as any); + jest.spyOn(api, 'useMe').mockImplementation( + () => + ({ + data: { team }, + isFetched: true, + }) as any, + ); + jest + .spyOn(mockMetadata, 'getColumns') + .mockImplementation(async ({ tableName }) => + tableName === 'table_a' ? columnsA : columnsB, + ); queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } }, @@ -320,7 +347,8 @@ describe('useMultipleAllFields', () => { ); }); - it('should return fields from successful connections and empty array for failed ones', async () => { + it('returns the other table connections and no error when only some fail', async () => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); jest .spyOn(mockMetadata, 'getAllFields') .mockResolvedValueOnce(fieldsA) @@ -332,8 +360,14 @@ describe('useMultipleAllFields', () => { await waitFor(() => expect(result.current.isSuccess).toBe(true)); - // Should contain only fieldsA since fieldsB failed expect(result.current.data).toEqual(fieldsA); + expect(result.current.errorsByTableConnection).toEqual( + new Map([[1, new Error('connection refused')]]), + ); + expect(result.current.isError).toBe(false); + expect(result.current.error).toBeNull(); + // Columns loaded before the field lookup failed, so they're kept. + expect(result.current.columnsByTableConnection.get(1)).toEqual(columnsB); }); it('falls back to each connection timestampValueExpression when none is passed', async () => { @@ -392,9 +426,12 @@ describe('useMultipleAllFields', () => { wrapper, }); - await waitFor(() => expect(result.current.isSuccess).toBe(true)); + await waitFor(() => expect(result.current.isError).toBe(true)); expect(result.current.data).toEqual([]); + expect([...result.current.errorsByTableConnection.keys()]).toEqual([0, 1]); + expect(result.current.isSuccess).toBe(false); + expect(result.current.error).toEqual(new Error('fail 1')); }); it('should log a warning for each failed connection', async () => { @@ -479,6 +516,89 @@ describe('useMultipleAllFields', () => { expect(getAllFields).not.toHaveBeenCalled(); }); + it('returns the fields and columns of each table connection in input order', async () => { + jest + .spyOn(mockMetadata, 'getAllFields') + .mockImplementation(async ({ tableName }) => + tableName === 'table_a' ? fieldsA : fieldsB, + ); + + const { result } = renderHook(() => useMultipleAllFields([tcA, tcB]), { + wrapper, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.fieldsByTableConnection).toEqual( + new Map([ + [0, fieldsA], + [1, fieldsB], + ]), + ); + expect(result.current.columnsByTableConnection).toEqual( + new Map([ + [0, columnsA], + [1, columnsB], + ]), + ); + expect(result.current.errorsByTableConnection.size).toBe(0); + expect(result.current.isError).toBe(false); + }); + + it('fails closed when intersecting and one table connection fails', async () => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + jest + .spyOn(mockMetadata, 'getAllFields') + .mockImplementation(async ({ tableName }) => { + if (tableName === 'table_b') throw new Error('timeout'); + return fieldsA; + }); + + const { result } = renderHook( + () => useMultipleAllFields([tcA, tcB], { intersect: true }), + { wrapper }, + ); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.data).toEqual([]); + }); + + it('returns only columns when the team disables field metadata', async () => { + team = { fieldMetadataDisabled: true }; + const getAllFields = jest.spyOn(mockMetadata, 'getAllFields'); + + const { result } = renderHook(() => useMultipleAllFields([tcA]), { + wrapper, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.data).toEqual([]); + expect(result.current.isFieldMetadataDisabled).toBe(true); + expect(result.current.columnsByTableConnection.get(0)).toEqual(columnsA); + expect(getAllFields).not.toHaveBeenCalled(); + }); + + it('keeps the same data reference across re-renders', async () => { + jest + .spyOn(mockMetadata, 'getAllFields') + .mockImplementation(async ({ tableName }) => + tableName === 'table_a' ? fieldsA : fieldsB, + ); + const tableConnections = [tcA, tcB]; + + const { result, rerender } = renderHook( + () => useMultipleAllFields(tableConnections), + { wrapper }, + ); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + const first = result.current; + rerender(); + expect(result.current.data).toBe(first.data); + expect(result.current.fieldsByTableConnection).toBe( + first.fieldsByTableConnection, + ); + }); + it('should fetch a populated table connection when the caller passes enabled: true', async () => { jest.spyOn(mockMetadata, 'getAllFields').mockResolvedValueOnce(fieldsA); diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index e9030abff3..81867d4859 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -224,87 +224,155 @@ export function useMapColumns( }); } +type TableConnectionFields = { + fields: Field[]; + columns: ColumnMeta[]; + /** The lookup failed; `columns` is kept if it loaded first. */ + error?: Error; +}; + +type MultipleAllFieldsOptions = { + dateRange?: [Date, Date]; + timestampValueExpression?: string; + // Return only fields present in EVERY table connection instead of the + // union. Use for a shared expression (e.g. a chart-level Group By over + // multiple series) that must be valid against all of them — the union + // would offer fields that exist in one table but not another. + intersect?: boolean; + enabled?: boolean; +}; + +/** + * Fields (columns plus Map / JSON keys) across one or more tables, merged into + * `data`, with each table connection's fields and raw columns also returned by index. + * + * A table connection whose lookup fails counts as no fields, so `data` still holds + * the others'. With `intersect`, a failed table connection makes `data` empty, since + * no field can be confirmed for it. Each failed table connection's error is in + * `errorsByTableConnection`; `isError` and `error` are set only when every + * table connection failed. + * + * When the team disables field metadata, `data` is `[]` but + * `columnsByTableConnection` is still filled. + */ export function useMultipleAllFields( tableConnections: TableConnection[], - options?: Partial> & { - dateRange?: [Date, Date]; - timestampValueExpression?: string; - // Return only fields present in EVERY table connection instead of the - // union. Use for a shared expression (e.g. a chart-level Group By over - // multiple series) that must be valid against all of them — the union - // would offer fields that exist in one table but not another. - intersect?: boolean; - }, -) { - const metadata = useMetadataWithSettings(); - const { data: me, isFetched } = api.useMe(); - const { + { dateRange, timestampValueExpression, intersect, - enabled: enabledOption = true, - ...queryOptions - } = options ?? {}; - return useQuery({ + enabled = true, + }: MultipleAllFieldsOptions = {}, +) { + const metadata = useMetadataWithSettings(); + const { data: me, isFetched } = api.useMe(); + const isFieldMetadataDisabled = !!me?.team?.fieldMetadataDisabled; + const allPopulated = tableConnections.every( + tc => !!tc.databaseName && !!tc.tableName && !!tc.connectionId, + ); + + const query = useQuery({ queryKey: [ 'useMetadata.useMultipleAllFields', ...tableConnections.map(tc => ({ ...tc })), dateRange ? [dateRange[0].getTime(), dateRange[1].getTime()] : undefined, timestampValueExpression, - intersect ?? false, + isFieldMetadataDisabled, ], - queryFn: async () => { - const team = me?.team; - if (team?.fieldMetadataDisabled) { - return []; - } - - const promiseResults = await Promise.allSettled( - tableConnections.map(tc => - metadata.getAllFields({ - ...tc, - dateRange, - timestampValueExpression: - timestampValueExpression ?? tc.timestampValueExpression, - }), - ), - ); - - const fields2d: Field[][] = promiseResults.map(result => { - if (result.status === 'rejected') { - console.warn( - 'Failed to fetch fields for table connection', - result.reason, - ); - return []; - } - return result.value; - }); + queryFn: () => + Promise.all( + tableConnections.map(async (tc): Promise => { + let columns: ColumnMeta[] = []; + try { + columns = await metadata.getColumns({ + databaseName: tc.databaseName, + tableName: tc.tableName, + connectionId: tc.connectionId, + }); + const fields = isFieldMetadataDisabled + ? [] + : await metadata.getAllFields({ + ...tc, + dateRange, + timestampValueExpression: + timestampValueExpression ?? tc.timestampValueExpression, + }); + return { fields, columns }; + } catch (error) { + console.warn('Failed to fetch fields for table connection', error); + return { + fields: [], + columns, + error: error instanceof Error ? error : new Error(String(error)), + }; + } + }), + ), + enabled: + enabled && tableConnections.length > 0 && allPopulated && isFetched, + }); + const tableConnectionFields = query.data; + const merged = useMemo(() => { + const fieldsByTableConnection = new Map(); + const columnsByTableConnection = new Map(); + const errorsByTableConnection = new Map(); + tableConnectionFields?.forEach(({ fields, columns, error }, index) => { + fieldsByTableConnection.set(index, fields); + columnsByTableConnection.set(index, columns); + if (error) errorsByTableConnection.set(index, error); + }); + const isError = + !!tableConnectionFields?.length && + errorsByTableConnection.size === tableConnectionFields.length; + + let data: Field[] | undefined; + if (tableConnectionFields) { + const fields2d = tableConnectionFields.map(({ fields }) => fields); // skip set logic if not needed - if (fields2d.length === 1) return fields2d[0]; + if (fields2d.length === 1) data = fields2d[0]; + else + data = intersect + ? intersect2dArray(fields2d) + : deduplicate2dArray(fields2d); + } - return intersect - ? intersect2dArray(fields2d) - : deduplicate2dArray(fields2d); - }, - ...queryOptions, - enabled: - enabledOption && - tableConnections.length > 0 && - tableConnections.every( - tc => !!tc.databaseName && !!tc.tableName && !!tc.connectionId, - ) && - isFetched, - }); + return { + data, + error: isError ? (tableConnectionFields?.[0].error ?? null) : null, + isError, + fieldsByTableConnection: fieldsByTableConnection as ReadonlyMap< + number, + Field[] + >, + columnsByTableConnection: columnsByTableConnection as ReadonlyMap< + number, + ColumnMeta[] + >, + errorsByTableConnection: errorsByTableConnection as ReadonlyMap< + number, + Error + >, + }; + }, [tableConnectionFields, intersect]); + + // The status fields come from `merged` rather than spreading `query`: + // `queryFn` never throws, so the query's own `status` and `isError` would + // report success when every table connection failed. + return { + ...merged, + isPending: query.isPending, + isSuccess: query.isSuccess && !merged.isError, + isLoading: query.isLoading, + isFetching: query.isFetching, + refetch: query.refetch, + isFieldMetadataDisabled, + }; } export function useAllFields( tableConnection: TableConnection | undefined, - options?: Partial> & { - dateRange?: [Date, Date]; - timestampValueExpression?: string; - }, + options?: Omit, ) { return useMultipleAllFields( tableConnection ? [tableConnection] : [],