From 761f2145e47988dd26657b10e2189a49742bba6c Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:06:58 -0400 Subject: [PATCH] refactor: remove TableConnection; pass sources instead --- .changeset/remove-table-connections.md | 18 + packages/api/src/controllers/ai.ts | 9 +- packages/app/src/DBDashboardPage.tsx | 21 +- packages/app/src/DBSearchPage.tsx | 18 +- packages/app/src/DBSearchPageAlertModal.tsx | 4 +- packages/app/src/DBServiceMapPage.tsx | 6 +- packages/app/src/KubernetesDashboardPage.tsx | 8 +- .../app/src/NamespaceDetailsSidePanel.tsx | 6 +- packages/app/src/NodeDetailsSidePanel.tsx | 6 +- packages/app/src/PodDetailsSidePanel.tsx | 6 +- .../ServicesDashboardPage.tsx | 4 +- packages/app/src/SessionSubpanel.tsx | 4 +- packages/app/src/SessionsPage.tsx | 4 +- packages/app/src/__tests__/utils.test.ts | 80 ++--- .../ChartEditor/RawSqlChartEditor.tsx | 20 +- .../app/src/components/ContextSidePanel.tsx | 3 +- packages/app/src/components/DBDeltaChart.tsx | 10 +- .../DBEditTimeChartForm/ChartActionBar.tsx | 12 +- .../ChartEditorControls.tsx | 35 +- .../DBEditTimeChartForm/ChartSeriesEditor.tsx | 38 +- .../DBEditTimeChartForm/EditTimeChartForm.tsx | 12 +- .../HeatmapSeriesEditor.tsx | 15 +- .../__tests__/ChartActionBar.test.tsx | 7 - .../__tests__/utils.test.ts | 49 +-- .../components/DBEditTimeChartForm/utils.ts | 33 +- .../components/DBRowSidePanelErrorState.tsx | 4 +- .../src/components/DBSearchPageFilters.tsx | 9 +- .../DBSearchPageFilters/hooks.test.tsx | 331 +++++++++--------- .../components/DBSearchPageFilters/hooks.ts | 126 +++---- packages/app/src/components/DBTracePanel.tsx | 4 +- .../src/components/DBTraceWaterfallChart.tsx | 7 +- .../QueryExpressionFilterEditForm.tsx | 18 +- .../src/components/HeatmapSettingsDrawer.tsx | 14 +- .../app/src/components/KubernetesFilters.tsx | 18 +- .../Patterns/PatternColumnSelector.tsx | 5 +- .../src/components/SQLEditor/SQLEditor.tsx | 31 +- .../components/SQLEditor/SQLInlineEditor.tsx | 32 +- .../__tests__/SQLInlineEditor.test.tsx | 12 +- .../Search/DBSearchHeatmapChart.tsx | 4 +- .../__tests__/DBSearchHeatmapChart.test.tsx | 4 - .../components/SearchInput/SearchInputV2.tsx | 26 +- .../SearchInput/SearchWhereInput.stories.tsx | 18 +- .../SearchInput/SearchWhereInput.tsx | 27 +- .../__tests__/SearchWhereInput.test.tsx | 24 +- ...rviceDashboardEndpointPerformanceChart.tsx | 6 +- .../Sources/SourceForm/ExpressionFormRow.tsx | 21 +- .../SourceForm/ExpressionValidationStatus.tsx | 12 +- .../SourceForm/HighlightedAttributes.tsx | 31 +- .../Sources/SourceForm/LogTableModelForm.tsx | 75 +--- .../Sources/SourceForm/MaterializedViews.tsx | 86 ++--- .../Sources/SourceForm/OrderByFormRow.tsx | 18 +- .../SourceForm/SessionTableModelForm.tsx | 14 +- .../SourceForm/TraceTableModelForm.tsx | 120 ++----- .../Sources/SourceForm/useFormSource.ts | 25 ++ .../src/components/alerts/EditAlertModal.tsx | 4 +- .../__tests__/useAutoCompleteOptions.test.tsx | 93 +++-- .../useFieldExpressionGenerator.test.tsx | 8 +- .../src/hooks/__tests__/useMetadata.test.tsx | 150 +++++--- .../app/src/hooks/useAutoCompleteOptions.tsx | 72 ++-- .../app/src/hooks/useExpressionValidation.ts | 14 +- .../src/hooks/useFieldExpressionGenerator.tsx | 3 +- packages/app/src/hooks/useMetadata.tsx | 95 ++--- .../src/llm/dashboard/LLMDashboardPage.tsx | 4 +- packages/app/src/llm/dashboard/LatencyTab.tsx | 3 +- .../llm/hooks/useLLMDashboardExpressions.ts | 15 +- packages/app/src/serviceDashboard.ts | 14 +- packages/app/src/source.ts | 20 ++ packages/app/src/utils.ts | 54 +-- .../utils/__tests__/materializedViews.test.ts | 88 +---- packages/app/src/utils/materializedViews.ts | 31 +- packages/cli/src/shared/tileConfig.ts | 2 +- .../src/__tests__/metadata.test.ts | 30 +- .../src/__tests__/queryParser.test.ts | 58 ++- .../src/core/materializedViews.ts | 33 +- packages/common-utils/src/core/metadata.ts | 156 ++++++--- packages/common-utils/src/queryParser.ts | 16 +- packages/common-utils/src/types.ts | 16 + 77 files changed, 1143 insertions(+), 1386 deletions(-) create mode 100644 .changeset/remove-table-connections.md create mode 100644 packages/app/src/components/Sources/SourceForm/useFormSource.ts diff --git a/.changeset/remove-table-connections.md b/.changeset/remove-table-connections.md new file mode 100644 index 0000000000..d94bddb348 --- /dev/null +++ b/.changeset/remove-table-connections.md @@ -0,0 +1,18 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/app': patch +'@hyperdx/api': patch +--- + +refactor: replace `TableConnection` with sources for field discovery + +`TableConnection`, `TableConnectionChoice`, `tcFromSource` and +`tcFromChartConfig` are removed from `@hyperdx/common-utils`. +`Metadata.getAllFields` and `genEnglishExplanation` now take a source (plus a +metric type and metric name for metric sources) and read the table, timestamp +expression and metadata materialized views from it. Search inputs, SQL editors +and facet hooks take `source` / `sourceTables` props in place of +`tableConnection` / `tableConnections` and `sourceId`. + +The raw SQL editor now autocompletes columns from metric sources' tables, which +it previously skipped. diff --git a/packages/api/src/controllers/ai.ts b/packages/api/src/controllers/ai.ts index eeaeae13bc..85bf7ed3db 100644 --- a/packages/api/src/controllers/ai.ts +++ b/packages/api/src/controllers/ai.ts @@ -3,7 +3,6 @@ import { createOpenAI } from '@ai-sdk/openai'; import { getMetadata, TableMetadata, - tcFromSource, } from '@hyperdx/common-utils/dist/core/metadata'; import { AILineTableResponse, @@ -103,13 +102,7 @@ export async function getAIMetadata(source: ISource) { connectionId, }); - const allFields = await metadata.getAllFields({ - databaseName, - tableName, - connectionId, - metadataMVs: tcFromSource(source).metadataMVs, - timestampValueExpression: source.timestampValueExpression, - }); + const allFields = await metadata.getAllFields({ source }); // TODO: Dedup with DBSearchPageFilters.tsx logic allFields.sort((a, b) => { diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index 40520a3e07..067172f7d6 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -22,10 +22,7 @@ import { import { ErrorBoundary } from 'react-error-boundary'; import RGL from 'react-grid-layout'; import { useForm, useWatch } from 'react-hook-form'; -import { - TableConnection, - tcFromSource, -} from '@hyperdx/common-utils/dist/core/metadata'; +import { SourceTable } from '@hyperdx/common-utils/dist/core/metadata'; import { convertToDashboardTemplate, displayTypeSupportsBuilderAlerts, @@ -1925,9 +1922,9 @@ function DBDashboardPage({ const { data: allDashboards } = useDashboards(); const [highlightedTileId] = useQueryState('highlightedTileId'); - const tableConnections = useMemo(() => { + const sourceTables = useMemo(() => { if (!dashboard) return []; - const tc: TableConnection[] = []; + const tables: SourceTable[] = []; for (const { config } of dashboard.tiles) { if (!isBuilderSavedChartConfig(config)) continue; @@ -1937,15 +1934,11 @@ function DBDashboardPage({ const firstSelect = config.select[0]; const metricType = typeof firstSelect !== 'string' ? firstSelect?.metricType : undefined; - const tableName = getMetricTableName(source, metricType); - if (!tableName) continue; - tc.push({ - ...tcFromSource(source), - tableName, - }); + if (!getMetricTableName(source, metricType)) continue; + tables.push({ source, metricType }); } - return tc; + return tables; }, [dashboard, sources]); const [granularity, setGranularity] = useQueryState( @@ -3160,7 +3153,7 @@ function DBDashboardPage({ }} > { @@ -2082,11 +2082,6 @@ function DBSearchPageContent() { ], ); - const inputSourceTableConnection = useMemo( - () => tcFromSource(inputSourceObj), - [inputSourceObj], - ); - const [isSourceSchemaPreviewOpen, setIsSourceSchemaPreviewOpen] = useState(false); @@ -2370,7 +2365,7 @@ function DBSearchPageContent() { /> <> @@ -2448,7 +2441,7 @@ function DBSearchPageContent() { /> tcFromSource(source), [source]); - const serviceNameKey = source?.serviceNameExpression ?? 'ServiceName'; const serviceNamesChartConfig = useMemo( () => @@ -370,7 +367,7 @@ function DBServiceMapPage() { data-testid="service-map-service-filter" /> (null); const { data: logSourceJsonColumns, isLoading: isLoadingJsonColumns } = - useJsonColumns( - logSource && { - databaseName: logSource.from.databaseName, - tableName: logSource.from.tableName, - connectionId: logSource.connection, - }, - ); + useJsonColumns({ source: logSource }); const eventAttributeExpressions = useMemo(() => { if (isLoadingJsonColumns || !logSource) { diff --git a/packages/app/src/NamespaceDetailsSidePanel.tsx b/packages/app/src/NamespaceDetailsSidePanel.tsx index 946bdd8abd..b88203adef 100644 --- a/packages/app/src/NamespaceDetailsSidePanel.tsx +++ b/packages/app/src/NamespaceDetailsSidePanel.tsx @@ -1,6 +1,6 @@ import * as React from 'react'; import { parseAsString, useQueryState } from 'nuqs'; -import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { getSourceTable } from '@hyperdx/common-utils/dist/core/metadata'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; import { TLogSource, TMetricSource } from '@hyperdx/common-utils/dist/types'; import { @@ -239,7 +239,9 @@ function NamespaceDetailsSidePanelInner({ return `${metricSource?.resourceAttributesExpression}.k8s.namespace.name:"${namespaceName}"`; }, [namespaceName, metricSource]); - const { data: logsTableMetadata } = useTableMetadata(tcFromSource(logSource)); + const { data: logsTableMetadata } = useTableMetadata( + getSourceTable({ source: logSource }), + ); let doesPrimaryOrSortingKeysContainServiceExpression = false; diff --git a/packages/app/src/NodeDetailsSidePanel.tsx b/packages/app/src/NodeDetailsSidePanel.tsx index c5a6d4ca81..2f5d3deb16 100644 --- a/packages/app/src/NodeDetailsSidePanel.tsx +++ b/packages/app/src/NodeDetailsSidePanel.tsx @@ -1,6 +1,6 @@ import * as React from 'react'; import { parseAsString, useQueryState } from 'nuqs'; -import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { getSourceTable } from '@hyperdx/common-utils/dist/core/metadata'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; import { TLogSource, TMetricSource } from '@hyperdx/common-utils/dist/types'; import { @@ -252,7 +252,9 @@ function NodeDetailsSidePanelInner({ return `${metricSource?.resourceAttributesExpression}.k8s.node.name:"${nodeName}"`; }, [nodeName, metricSource]); - const { data: logsTableMetadata } = useTableMetadata(tcFromSource(logSource)); + const { data: logsTableMetadata } = useTableMetadata( + getSourceTable({ source: logSource }), + ); let doesPrimaryOrSortingKeysContainServiceExpression = false; diff --git a/packages/app/src/PodDetailsSidePanel.tsx b/packages/app/src/PodDetailsSidePanel.tsx index f0ec35707c..0a5b109aa8 100644 --- a/packages/app/src/PodDetailsSidePanel.tsx +++ b/packages/app/src/PodDetailsSidePanel.tsx @@ -1,6 +1,6 @@ import * as React from 'react'; import { parseAsString, useQueryState } from 'nuqs'; -import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { getSourceTable } from '@hyperdx/common-utils/dist/core/metadata'; import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; import { TLogSource, TMetricSource } from '@hyperdx/common-utils/dist/types'; import { @@ -242,7 +242,9 @@ function PodDetailsSidePanelInner({ return `${metricSource?.resourceAttributesExpression}.k8s.pod.name:"${podName}"`; }, [podName, metricSource]); - const { data: logsTableMetadata } = useTableMetadata(tcFromSource(logSource)); + const { data: logsTableMetadata } = useTableMetadata( + getSourceTable({ source: logSource }), + ); let doesPrimaryOrSortingKeysContainServiceExpression = false; diff --git a/packages/app/src/ServicesDashboardPage/ServicesDashboardPage.tsx b/packages/app/src/ServicesDashboardPage/ServicesDashboardPage.tsx index d703020653..2c0b76d164 100644 --- a/packages/app/src/ServicesDashboardPage/ServicesDashboardPage.tsx +++ b/packages/app/src/ServicesDashboardPage/ServicesDashboardPage.tsx @@ -15,7 +15,6 @@ import { useQueryStates, } from 'nuqs'; import { UseControllerProps, useForm, useWatch } from 'react-hook-form'; -import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { PresetDashboard, SourceKind, @@ -361,8 +360,7 @@ function ServicesDashboardPage() { dateRange={searchedTimeRange} /> { ...createBaseSource(), metricTables: { gauge: 'gauge_table', - counter: 'counter_table', + sum: 'sum_table', }, }); @@ -152,8 +152,8 @@ describe('getMetricTableName', () => { expect(getMetricTableName(source, 'gauge' as MetricsDataType)).toBe( 'gauge_table', ); - expect(getMetricTableName(source, 'counter' as MetricsDataType)).toBe( - 'counter_table', + expect(getMetricTableName(source, 'sum' as MetricsDataType)).toBe( + 'sum_table', ); }); @@ -163,8 +163,8 @@ describe('getMetricTableName', () => { expect(getMetricTableName(source, 'GAUGE' as MetricsDataType)).toBe( 'gauge_table', ); - expect(getMetricTableName(source, 'Counter' as MetricsDataType)).toBe( - 'counter_table', + expect(getMetricTableName(source, 'Sum' as MetricsDataType)).toBe( + 'sum_table', ); }); @@ -229,25 +229,17 @@ describe('getAllMetricTables', () => { expect(getAllMetricTables(source)).toEqual([]); }); - it('returns TableConnection for each populated metric table', () => { + it('returns one entry per populated metric table', () => { const source = createMetricSource({ - Gauge: 'gauge_table', - Sum: 'sum_table', + gauge: 'gauge_table', + sum: 'sum_table', }); const result = getAllMetricTables(source); expect(result).toEqual( expect.arrayContaining([ - { - tableName: 'gauge_table', - databaseName: 'test_db', - connectionId: 'test-conn', - }, - { - tableName: 'sum_table', - databaseName: 'test_db', - connectionId: 'test-conn', - }, + { source, metricType: 'gauge' }, + { source, metricType: 'sum' }, ]), ); expect(result).toHaveLength(2); @@ -255,47 +247,43 @@ describe('getAllMetricTables', () => { it('filters out metric types with no table name', () => { const source = createMetricSource({ - Gauge: 'gauge_table', - Histogram: '', + gauge: 'gauge_table', + histogram: '', }); - const result = getAllMetricTables(source); - expect(result).toEqual([ - { - tableName: 'gauge_table', - databaseName: 'test_db', - connectionId: 'test-conn', - }, + expect(getAllMetricTables(source)).toEqual([ + { source, metricType: 'gauge' }, ]); }); - it('returns all four metric types when all are populated', () => { + it('returns every metric type when all are populated', () => { const source = createMetricSource({ - Gauge: 'gauge_t', - Histogram: 'histogram_t', - Sum: 'sum_t', - Summary: 'summary_t', + gauge: 'gauge_t', + histogram: 'histogram_t', + sum: 'sum_t', + summary: 'summary_t', + 'exponential histogram': 'exp_histogram_t', }); const result = getAllMetricTables(source); - expect(result).toHaveLength(4); - expect(result.map(t => t.tableName).sort()).toEqual([ - 'gauge_t', - 'histogram_t', - 'sum_t', - 'summary_t', - ]); - // All should share the same database and connection - for (const tc of result) { - expect(tc.databaseName).toBe('test_db'); - expect(tc.connectionId).toBe('test-conn'); - } + expect(result).toHaveLength(5); + expect( + result.map(({ metricType }) => getMetricTableName(source, metricType)), + ).toEqual( + expect.arrayContaining([ + 'gauge_t', + 'histogram_t', + 'sum_t', + 'summary_t', + 'exp_histogram_t', + ]), + ); }); it('returns empty array when all metric table values are falsy', () => { const source = createMetricSource({ - Gauge: '', - Histogram: '', + gauge: '', + histogram: '', }); expect(getAllMetricTables(source)).toEqual([]); diff --git a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx index a9d1949642..9fe7fcee80 100644 --- a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx @@ -1,9 +1,6 @@ import { useEffect, useMemo, useState } from 'react'; import { Control, UseFormSetValue, useWatch } from 'react-hook-form'; -import { - TableConnection, - tcFromSource, -} from '@hyperdx/common-utils/dist/core/metadata'; +import { SourceTable } from '@hyperdx/common-utils/dist/core/metadata'; import { displayTypeSupportsRawSqlAlerts, validateRawSqlChartConfig, @@ -13,6 +10,7 @@ import { ChartVariable, RawSqlChartConfig, SourceKind, + SourceLike, } from '@hyperdx/common-utils/dist/types'; import { DisplayType, @@ -44,7 +42,7 @@ import { SQLEditorControlled } from '@/components/SQLEditor/SQLEditor'; import { type SQLCompletion } from '@/components/SQLEditor/utils'; import { IS_LOCAL_MODE } from '@/config'; import useResizable from '@/hooks/useResizable'; -import { useSources } from '@/source'; +import { getMaterializedViewSource, useSources } from '@/source'; import { getAllMetricTables, usePrevious } from '@/utils'; import { DEFAULT_TILE_ALERT } from '@/utils/alerts'; @@ -223,15 +221,15 @@ export default function RawSqlChartEditor({ const [isSourceSchemaPreviewOpen, setIsSourceSchemaPreviewOpen] = useState(false); - const tableConnections: TableConnection[] = useMemo(() => { + const sourceTables = useMemo(() => { if (!sources) return []; return sources .filter(s => s.connection === connection && !s.disabled) .flatMap(source => { - const tables: TableConnection[] = getAllMetricTables(source); + const tables: SourceTable[] = getAllMetricTables(source); if (!isMetricSource(source)) { - tables.push(tcFromSource(source)); + tables.push({ source }); } if ( @@ -240,9 +238,7 @@ export default function RawSqlChartEditor({ ) { tables.push( ...source.materializedViews.map(mv => ({ - databaseName: mv.databaseName, - tableName: mv.tableName, - connectionId: source.connection, + source: getMaterializedViewSource(source, mv), })), ); } @@ -351,7 +347,7 @@ export default function RawSqlChartEditor({ height={`${size}vh`} enableLineWrapping placeholder={placeholderSQl} - tableConnections={tableConnections} + sourceTables={sourceTables} additionalCompletions={additionalCompletions} onSubmit={onSubmit} /> diff --git a/packages/app/src/components/ContextSidePanel.tsx b/packages/app/src/components/ContextSidePanel.tsx index fe2db7ae00..10118b1c68 100644 --- a/packages/app/src/components/ContextSidePanel.tsx +++ b/packages/app/src/components/ContextSidePanel.tsx @@ -2,7 +2,6 @@ import { use, useCallback, useEffect, useMemo, useState } from 'react'; import ms from 'ms'; import { ErrorBoundary } from 'react-error-boundary'; import { useForm, useWatch } from 'react-hook-form'; -import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { BuilderChartConfigWithDateRange, isLogSource, @@ -254,7 +253,7 @@ export default function ContextSubpanel({ {showCustomSearch && ( ; handleSubmit: UseFormHandleSubmit; - tableConnection: TableConnection; - sourceId?: string; + source?: TSource; dateRange?: [Date, Date]; activeTab: string; isRawSqlInput: boolean; @@ -99,8 +97,7 @@ type ChartActionBarProps = { export function ChartActionBar({ control, handleSubmit, - tableConnection, - sourceId, + source, dateRange, activeTab, isRawSqlInput, @@ -193,8 +190,7 @@ export function ChartActionBar({
; @@ -63,10 +62,8 @@ type ChartEditorControlsProps = { swapSeries: (from: number, to: number) => void; duplicateSeries: (index: number) => void; tableSource?: TSource; - tableConnection: TableConnection; allowedSourceKinds: SourceKind[]; databaseName?: string; - tableName?: string; dateRange: [Date, Date]; select: ChartEditorFormState['select']; displayType: DisplayType; @@ -100,10 +97,8 @@ export function ChartEditorControls({ swapSeries, duplicateSeries, tableSource, - tableConnection, allowedSourceKinds, databaseName, - tableName, dateRange, select, displayType, @@ -176,11 +171,11 @@ export function ChartEditorControls({ // The chart-level Group By must be valid against every series query. For // metric sources (which fan out to per-type tables) this means offering the - // intersection of each series' fields; see buildGroupByConnectionProps. + // intersection of each series' fields; see buildGroupBySourceProps. const series = useWatch({ control, name: 'series' }); - const groupByConnectionProps = useMemo( - () => buildGroupByConnectionProps({ tableSource, series, tableConnection }), - [tableSource, series, tableConnection], + const groupBySourceProps = useMemo( + () => buildGroupBySourceProps({ tableSource, series }), + [tableSource, series], ); // Grouped ratios can divide two ways (see RatioModeSchema); the mode toggle @@ -244,8 +239,7 @@ export function ChartEditorControls({ ) : displayType === DisplayType.EventPatterns ? ( )}
; databaseName: string; dateRange?: DateRange['dateRange']; - connectionId?: string; index: number; namePrefix: `series.${number}.`; parentRef?: HTMLElement | null; @@ -71,7 +70,6 @@ type ChartSeriesEditorProps = { showDuplicate: boolean; showColor: boolean; showSeriesNumberFormat?: boolean; - tableName: string; length: number; tableSource?: TSource; errors?: FieldErrors; @@ -82,7 +80,6 @@ export function ChartSeriesEditor({ control, databaseName, dateRange, - connectionId, index, namePrefix, onRemoveSeries, @@ -95,7 +92,6 @@ export function ChartSeriesEditor({ showDuplicate, showColor, showSeriesNumberFormat = true, - tableName: _tableName, parentRef, length, tableSource, @@ -151,11 +147,6 @@ export function ChartSeriesEditor({ } }, [tableSource?.kind, metricType, aggFn, namePrefix, setValue]); - const tableName = - tableSource?.kind === SourceKind.Metric - ? getMetricTableName(tableSource, metricType) - : _tableName; - const metricName = useWatch({ control, name: `${namePrefix}metricName` }); const aggCondition = useWatch({ control, @@ -257,15 +248,12 @@ export function ChartSeriesEditor({ const showWhere = aggFn !== 'none'; - const tableConnection = useMemo( - () => ({ - databaseName, - tableName: tableName ?? '', - connectionId: connectionId ?? '', - metricName: - tableSource?.kind === SourceKind.Metric ? metricName : undefined, - }), - [databaseName, tableName, connectionId, metricName, tableSource], + const sourceProps = useMemo( + () => + tableSource?.kind === SourceKind.Metric + ? { source: tableSource, metricType, metricName } + : { source: tableSource }, + [tableSource, metricType, metricName], ); const seriesNumberFormat = useWatch({ @@ -422,8 +410,7 @@ export function ChartSeriesEditor({ }} >
tcFromSource(tableSource), - [tableSource], - ); - const configTypeOptions = useMemo( () => [ { label: 'Builder', value: 'builder' }, @@ -1187,10 +1180,8 @@ export default function EditTimeChartForm({ swapSeries={swapSeries} duplicateSeries={duplicateSeries} tableSource={tableSource} - tableConnection={tableConnection} allowedSourceKinds={allowedSourceKinds} databaseName={databaseName} - tableName={tableName} dateRange={dateRange} select={select} displayType={displayType} @@ -1214,8 +1205,7 @@ export default function EditTimeChartForm({ tcFromSource(tableSource), [tableSource]); - return (
({ ), })); -const defaultTableConnection = { - databaseName: 'default', - tableName: 'logs', - connectionId: 'default', -}; - type WrapperProps = { children: (props: { control: any; handleSubmit: any }) => React.ReactNode; defaultValues?: Partial; @@ -76,7 +70,6 @@ const renderActionBar = ( { }); // --------------------------------------------------------------------------- -// buildGroupByConnectionProps +// buildGroupBySourceProps // --------------------------------------------------------------------------- -describe('buildGroupByConnectionProps', () => { - const tableConnection = { - databaseName: 'default', - tableName: 'logs', - connectionId: 'clickhouse', - }; - - it('falls back to the source tableConnection for non-metric sources', () => { - const result = buildGroupByConnectionProps({ +describe('buildGroupBySourceProps', () => { + it('falls back to the source for non-metric sources', () => { + const result = buildGroupBySourceProps({ tableSource: logSource, series: [{ metricType: 'gauge', metricName: 'cpu' }], - tableConnection, }); - expect(result).toEqual({ tableConnection }); + expect(result).toEqual({ source: logSource }); }); - it('builds one intersected connection per distinct metric table + name for mixed-type series', () => { - const result = buildGroupByConnectionProps({ + it('builds one intersected source table per distinct metric table + name for mixed-type series', () => { + const result = buildGroupBySourceProps({ tableSource: metricSource, series: [ { metricType: 'gauge', metricName: 'cpu' }, @@ -1154,34 +1147,22 @@ describe('buildGroupByConnectionProps', () => { // incomplete series — skipped { metricType: 'gauge' }, ], - tableConnection, }); - expect(result.tableConnection).toBeUndefined(); + expect(result.source).toBeUndefined(); expect(result.intersectFields).toBe(true); - expect(result.tableConnections).toEqual([ - { - databaseName: 'default', - tableName: 'metrics.gauge', - connectionId: 'clickhouse', - metricName: 'cpu', - }, - { - databaseName: 'default', - tableName: 'metrics.sum', - connectionId: 'clickhouse', - metricName: 'requests', - }, + expect(result.sourceTables).toEqual([ + { source: metricSource, metricType: 'gauge', metricName: 'cpu' }, + { source: metricSource, metricType: 'sum', metricName: 'requests' }, ]); }); - it('falls back to the tableConnection when a metric source has no resolvable series', () => { - const result = buildGroupByConnectionProps({ + it('falls back to the source when a metric source has no resolvable series', () => { + const result = buildGroupBySourceProps({ tableSource: metricSource, series: [{ metricName: 'cpu' }], // no metricType - tableConnection, }); - expect(result).toEqual({ tableConnection }); + expect(result).toEqual({ source: metricSource }); }); }); diff --git a/packages/app/src/components/DBEditTimeChartForm/utils.ts b/packages/app/src/components/DBEditTimeChartForm/utils.ts index 785ad572f9..7c807512e7 100644 --- a/packages/app/src/components/DBEditTimeChartForm/utils.ts +++ b/packages/app/src/components/DBEditTimeChartForm/utils.ts @@ -1,7 +1,7 @@ import z from 'zod'; import { - TableConnection, - TableConnectionChoice, + SourceTable, + SourceTableChoice, } from '@hyperdx/common-utils/dist/core/metadata'; import { displayTypeSupportsReducer, @@ -467,11 +467,11 @@ export function buildChartConfigForExplanations({ } /** - * Picks the table connection(s) that drive attribute autocomplete for the + * Picks the source table(s) that drive attribute autocomplete for the * chart-level Group By. * * Metric sources have no single `from.tableName` (they fan out to per-type - * metric tables), so we build one connection per series' metric table + name + * metric tables), so we pick one table per series' metric type + name * (deduped) and ask the editor to offer only fields present in ALL of them * (`intersectFields`). A ratio can mix metric types (e.g. gauge / sum) whose * tables have different native columns — a union would suggest a column that @@ -479,23 +479,21 @@ export function buildChartConfigForExplanations({ * By must be restricted to fields valid for every series. * * Non-metric sources (and metric sources with no resolvable series) fall back - * to the source's single `tableConnection`. + * to the source itself. */ -export function buildGroupByConnectionProps({ +export function buildGroupBySourceProps({ tableSource, series, - tableConnection, }: { tableSource: TSource | undefined; series: { metricType?: string; metricName?: string }[] | undefined; - tableConnection: TableConnection; -}): TableConnectionChoice & { intersectFields?: boolean } { +}): SourceTableChoice & { intersectFields?: boolean } { if (tableSource?.kind !== SourceKind.Metric || !Array.isArray(series)) { - return { tableConnection }; + return { source: tableSource }; } const seen = new Set(); - const connections: TableConnection[] = []; + const sourceTables: SourceTable[] = []; for (const s of series) { if (!s?.metricType || !s?.metricName) continue; const metricTable = getMetricTableName(tableSource, s.metricType); @@ -503,15 +501,14 @@ export function buildGroupByConnectionProps({ const key = `${metricTable}::${s.metricName}`; if (seen.has(key)) continue; seen.add(key); - connections.push({ - databaseName: tableSource.from.databaseName, - tableName: metricTable, - connectionId: tableSource.connection, + sourceTables.push({ + source: tableSource, + metricType: s.metricType, metricName: s.metricName, }); } - return connections.length > 0 - ? { tableConnections: connections, intersectFields: true } - : { tableConnection }; + return sourceTables.length > 0 + ? { sourceTables, intersectFields: true } + : { source: tableSource }; } diff --git a/packages/app/src/components/DBRowSidePanelErrorState.tsx b/packages/app/src/components/DBRowSidePanelErrorState.tsx index 673df7879e..07fb5ec3c4 100644 --- a/packages/app/src/components/DBRowSidePanelErrorState.tsx +++ b/packages/app/src/components/DBRowSidePanelErrorState.tsx @@ -3,7 +3,7 @@ import { ClickHouseQueryError, isMissingColumnError, } from '@hyperdx/common-utils/dist/clickhouse'; -import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; +import { getSourceTable } from '@hyperdx/common-utils/dist/core/metadata'; import { isLogSource, isTraceSource, @@ -118,7 +118,7 @@ export function DBRowSidePanelErrorState({ source: TSource; }) { const [editOpened, editModal] = useDisclosure(false); - const { data: tableMetadata } = useTableMetadata(tcFromSource(source)); + const { data: tableMetadata } = useTableMetadata(getSourceTable({ source })); const showHint = isMissingColumnError(error) && !!tableMetadata?.isPointerTable; diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 08c35d32da..16ceb5db94 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -1,8 +1,8 @@ import { memo, useCallback, useEffect, useMemo, useState } from 'react'; import cx from 'classnames'; import { + getSourceTable, TableMetadata, - tcFromSource, } from '@hyperdx/common-utils/dist/core/metadata'; import { FilterState } from '@hyperdx/common-utils/dist/filters'; import { @@ -1166,8 +1166,7 @@ const DBSearchPageFiltersComponent = ({ const { size, startResize } = useResizable(16, 'left'); const { data: source } = useSource({ id: sourceId }); - const sourceTableConnection = tcFromSource(source); - const { data: jsonColumns } = useJsonColumns(sourceTableConnection); + const { data: jsonColumns } = useJsonColumns({ source }); // Special case for live tail const [dateRange, setDateRange] = useState<[Date, Date]>( @@ -1180,7 +1179,7 @@ const DBSearchPageFiltersComponent = ({ connectionId: chartConfig.connection, }); - const { data: tableMetadata } = useTableMetadata(sourceTableConnection); + const { data: tableMetadata } = useTableMetadata(getSourceTable({ source })); useEffect(() => { if (!isLive) { @@ -1210,7 +1209,7 @@ const DBSearchPageFiltersComponent = ({ extraFacetKeys, } = useFetchFacets({ chartConfig, - sourceId: sourceId ?? null, + source, dateRange, mode: showAllValues ? 'all' : 'exact', filterState, diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx index a3c567233e..7859c2f070 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx +++ b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx @@ -5,6 +5,8 @@ import { FilterState } from '@hyperdx/common-utils/dist/filters'; import { BuilderChartConfigWithDateRange, SourceKind, + TLogSource, + TMetricSource, } from '@hyperdx/common-utils/dist/types'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { act, renderHook, waitFor } from '@testing-library/react'; @@ -12,7 +14,6 @@ import { act, renderHook, waitFor } from '@testing-library/react'; import api from '@/api'; import * as useMetadataModule from '@/hooks/useMetadata'; import * as searchFiltersModule from '@/searchFilters'; -import * as sourceModule from '@/source'; import { useFetchFacets } from './hooks'; @@ -48,11 +49,6 @@ jest.mock('@/api', () => ({ }, })); -jest.mock('@/source', () => ({ - __esModule: true, - useSource: jest.fn(), -})); - jest.mock('@/searchFilters', () => ({ __esModule: true, usePinnedFilters: jest.fn(), @@ -71,7 +67,6 @@ jest.mock('@/hooks/useMetadata', () => ({ })); const useMe = jest.mocked(api.useMe); -const useSource = jest.mocked(sourceModule.useSource); const usePinnedFilters = jest.mocked(searchFiltersModule.usePinnedFilters); const useMetadataWithSettings = jest.mocked( useMetadataModule.useMetadataWithSettings, @@ -98,9 +93,9 @@ const DATE_RANGE: [Date, Date] = [ new Date('2024-01-02'), ]; -const makeLogSource = (opts: { withMVs: boolean }) => ({ +const makeLogSource = (opts: { withMVs: boolean }): TLogSource => ({ id: 'source1', - kind: 'log', + kind: SourceKind.Log, name: 'logs', connection: 'conn1', from: { databaseName: 'db', tableName: 'logs' }, @@ -109,24 +104,34 @@ const makeLogSource = (opts: { withMVs: boolean }) => ({ ...(opts.withMVs ? { metadataMaterializedViews: { - granularity: 'PT1H', - keysAndValues: { - databaseName: 'db', - tableName: 'logs_mv', - }, + kvRollupTable: 'logs_kv_rollup', + granularity: '1 hour', }, } : {}), }); -type SourceQueryResult = ReturnType; +const METRIC_SOURCE: TMetricSource = { + id: 'metrics', + kind: SourceKind.Metric, + name: 'metrics', + connection: 'conn1', + from: { databaseName: 'db', tableName: '' }, + timestampValueExpression: 'TimeUnix', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + resourceAttributesExpression: 'ResourceAttributes', +}; + type MetadataWithSettings = ReturnType< typeof useMetadataModule.useMetadataWithSettings >; -const mockSourceQuery = (data: SourceQueryResult['data']) => - useSource.mockReturnValue({ data, isLoading: false } as SourceQueryResult); - const mockMetadata = (metadata: Partial) => useMetadataWithSettings.mockReturnValue(metadata as MetadataWithSettings); @@ -146,11 +151,6 @@ function setupDefaultMocks({ withMVs }: { withMVs: boolean }) { isFetched: true, } as any); - useSource.mockReturnValue({ - data: makeLogSource({ withMVs }), - isLoading: false, - } as any); - useColumns.mockReturnValue({ data: [ { name: 'Timestamp', type: 'DateTime' }, @@ -191,6 +191,8 @@ function setupDefaultMocks({ withMVs }: { withMVs: boolean }) { isFetching: false, error: null, } as any); + + return makeLogSource({ withMVs }); } describe('useFetchFacets', () => { @@ -200,14 +202,14 @@ describe('useFetchFacets', () => { describe('pipeline selection', () => { it('routes useGetKeyValues with mode="exact" when mode is exact', () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'exact', }), @@ -220,14 +222,14 @@ describe('useFetchFacets', () => { }); it('routes useGetKeyValues with mode="all" when mode is all', () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'all', }), @@ -240,14 +242,14 @@ describe('useFetchFacets', () => { }); it('selection is mode-only: MV presence does not change which mode is passed', () => { - setupDefaultMocks({ withMVs: true }); + const source = setupDefaultMocks({ withMVs: true }); const { wrapper } = makeWrapper(); renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'exact', }), @@ -265,14 +267,14 @@ describe('useFetchFacets', () => { // a fully-formed key. Guard against a regression that couples the two. describe('disableValues', () => { it('disables the useGetKeyValues query when disableValues is true', () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'all', disableValues: true, @@ -285,14 +287,14 @@ describe('useFetchFacets', () => { }); it('enables the useGetKeyValues query when disableValues is false or omitted', () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'all', disableValues: false, @@ -305,14 +307,14 @@ describe('useFetchFacets', () => { }); it('does not defer the field metadata query — useAllFields stays enabled', () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'all', disableValues: true, @@ -325,14 +327,14 @@ describe('useFetchFacets', () => { }); it('still surfaces field metadata via data.keys even while deferring values', () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); const { result } = renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'all', disableValues: true, @@ -361,7 +363,7 @@ describe('useFetchFacets', () => { } it('returns data from the raw-tables pipeline when mode is exact', () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); mockGetKeyValuesByMode({ exact: { data: [{ key: 'ServiceName', value: ['api', 'web'] }], @@ -383,7 +385,7 @@ describe('useFetchFacets', () => { () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'exact', }), @@ -396,7 +398,7 @@ describe('useFetchFacets', () => { }); it('returns data from the "all" pipeline when mode is all', () => { - setupDefaultMocks({ withMVs: true }); + const source = setupDefaultMocks({ withMVs: true }); mockGetKeyValuesByMode({ exact: { data: [{ key: 'ShouldNotBeUsed', value: ['x'] }], @@ -418,7 +420,7 @@ describe('useFetchFacets', () => { () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'all', }), @@ -431,14 +433,14 @@ describe('useFetchFacets', () => { }); it('returns undefined keyValues when the active pipeline has no data yet', () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); const { result } = renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'all', }), @@ -454,7 +456,7 @@ describe('useFetchFacets', () => { describe('loadMoreFacetsForKey (raw-tables pipeline)', () => { it('reports the key as loading while the fetch is in flight, then clears it', async () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); useGetKeyValues.mockReturnValue({ data: [{ key: 'ServiceName', value: ['api'] }], isLoading: false, @@ -475,7 +477,7 @@ describe('useFetchFacets', () => { () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'exact', }), @@ -504,7 +506,7 @@ describe('useFetchFacets', () => { }); it('adds the fetched key to extraFacetKeys after a successful load-more', async () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); useGetKeyValues.mockReturnValue({ data: [{ key: 'ServiceName', value: ['api'] }], isLoading: false, @@ -524,7 +526,7 @@ describe('useFetchFacets', () => { () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'exact', }), @@ -539,7 +541,7 @@ describe('useFetchFacets', () => { }); it('unions extra facet values with primary values when keys match, preserving primary values and primary order', async () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); useGetKeyValues.mockReturnValue({ data: [ { key: 'ServiceName', value: ['api', 'primary-only'] }, @@ -563,7 +565,7 @@ describe('useFetchFacets', () => { () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'exact', }), @@ -584,7 +586,7 @@ describe('useFetchFacets', () => { }); it('appends extra facets that were not in the primary list', async () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); useGetKeyValues.mockReturnValue({ data: [{ key: 'ServiceName', value: ['api'] }], isLoading: false, @@ -602,7 +604,7 @@ describe('useFetchFacets', () => { () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'exact', }), @@ -620,7 +622,7 @@ describe('useFetchFacets', () => { }); it('does not mutate state when the load-more strategy returns undefined (e.g. on error)', async () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); useGetKeyValues.mockReturnValue({ data: [{ key: 'ServiceName', value: ['api'] }], isLoading: false, @@ -640,7 +642,7 @@ describe('useFetchFacets', () => { () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'exact', }), @@ -663,7 +665,7 @@ describe('useFetchFacets', () => { describe('loadMoreFacetsForKey (MV pipeline)', () => { it('delegates to getAllKeyValues and merges the result', async () => { - setupDefaultMocks({ withMVs: true }); + const source = setupDefaultMocks({ withMVs: true }); const getAllKeyValues = jest .fn() .mockResolvedValue([{ key: 'ServiceName', value: ['api', 'web'] }]); @@ -678,7 +680,7 @@ describe('useFetchFacets', () => { () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'all', }), @@ -698,8 +700,8 @@ describe('useFetchFacets', () => { }); describe('extraFacets reset on prop change', () => { - it('clears extraFacets and extraFacetKeys when sourceId changes', async () => { - setupDefaultMocks({ withMVs: false }); + it('clears extraFacets and extraFacetKeys when the source changes', async () => { + const source = setupDefaultMocks({ withMVs: false }); useGetKeyValues.mockReturnValue({ data: [{ key: 'ServiceName', value: ['api'] }], isLoading: false, @@ -714,14 +716,14 @@ describe('useFetchFacets', () => { const { wrapper } = makeWrapper(); const { result, rerender } = renderHook( - (props: { sourceId: string }) => + (props: { source: TLogSource }) => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: props.sourceId, + source: props.source, dateRange: DATE_RANGE, mode: 'exact', }), - { wrapper, initialProps: { sourceId: 'source1' } }, + { wrapper, initialProps: { source } }, ); await act(async () => { @@ -734,7 +736,7 @@ describe('useFetchFacets', () => { { key: 'NewKey', value: ['n1'] }, ]); - rerender({ sourceId: 'source2' }); + rerender({ source: { ...source, id: 'source2' } }); expect(result.current.extraFacetKeys.size).toBe(0); expect(result.current.data.keyValues).toEqual([ @@ -743,7 +745,7 @@ describe('useFetchFacets', () => { }); it('clears extraFacets when dateRange changes', async () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); useGetKeyValues.mockReturnValue({ data: [{ key: 'ServiceName', value: ['api'] }], isLoading: false, @@ -761,7 +763,7 @@ describe('useFetchFacets', () => { (props: { dateRange: [Date, Date] }) => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: props.dateRange, mode: 'exact', }), @@ -788,7 +790,7 @@ describe('useFetchFacets', () => { }); it('clears extraFacets when filterState changes', async () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); useGetKeyValues.mockReturnValue({ data: [{ key: 'ServiceName', value: ['api'] }], isLoading: false, @@ -806,7 +808,7 @@ describe('useFetchFacets', () => { (props: { filterState: FilterState }) => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'exact', filterState: props.filterState, @@ -843,7 +845,7 @@ describe('useFetchFacets', () => { }); it('clears extraFacets when chartConfig.where changes', async () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); useGetKeyValues.mockReturnValue({ data: [{ key: 'ServiceName', value: ['api'] }], isLoading: false, @@ -861,7 +863,7 @@ describe('useFetchFacets', () => { (props: { chartConfig: BuilderChartConfigWithDateRange }) => useFetchFacets({ chartConfig: props.chartConfig, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: 'exact', }), @@ -889,7 +891,7 @@ describe('useFetchFacets', () => { }); it('clears extraFacets when mode changes', async () => { - setupDefaultMocks({ withMVs: false }); + const source = setupDefaultMocks({ withMVs: false }); useGetKeyValues.mockReturnValue({ data: [{ key: 'ServiceName', value: ['api'] }], isLoading: false, @@ -907,7 +909,7 @@ describe('useFetchFacets', () => { (props: { mode: 'all' | 'exact' }) => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', + source, dateRange: DATE_RANGE, mode: props.mode, }), @@ -929,149 +931,138 @@ describe('useFetchFacets', () => { }); }); - /** - * `tableConnection` is a fallback, not an override: whenever a source is - * available it wins, so its metadata materialized views keep serving key and - * value discovery. Only two cases reach the fallback. - * - * 1. No source id. The dashboard-wide WHERE spans every tile, so no single - * source names its table. Deriving discovery from the source id alone - * left `tcFromSource(undefined)` returning an all-empty connection and - * `useAllFields`'s enabled guard rejecting it, so that input offered zero - * suggestions. - * 2. A metric source, whose rows live in per-type tables its `from` doesn't - * name — KubernetesFilters always, and the tile editor and dashboard - * filters whenever a metric source is selected. - * - * Every other input passes a source id and never consults the fallback, even - * though it still passes a connection. - */ - describe('tableConnection fallback', () => { - const FALLBACK_TC = { - databaseName: 'other_db', - tableName: 'other_table', - connectionId: 'conn2', - }; - - const SOURCE_TC = expect.objectContaining({ - databaseName: 'db', - tableName: 'logs', - connectionId: 'conn1', - }); - - it('discovers fields from the connection when there is no sourceId', () => { - setupDefaultMocks({ withMVs: false }); - mockSourceQuery(undefined); + describe('source tables', () => { + it('discovers fields from the source, keeping its materialized views', () => { + // Losing the source here would drop `metadataMaterializedViews`, sending + // Map-key discovery to a raw table scan. + const source = setupDefaultMocks({ withMVs: true }); const { wrapper } = makeWrapper(); renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: null, - tableConnection: FALLBACK_TC, + source, dateRange: DATE_RANGE, mode: 'all', - disableValues: true, }), { wrapper }, ); - expect(useAllFields.mock.calls.at(-1)?.[0]).toEqual(FALLBACK_TC); - expect(useColumns.mock.calls.at(-1)?.[0]).toEqual(FALLBACK_TC); + expect(useAllFields.mock.calls.at(-1)?.[0]).toEqual({ + source, + metricType: undefined, + metricName: undefined, + }); + expect( + useAllFields.mock.calls.at(-1)?.[0]?.source?.metadataMaterializedViews, + ).toBeDefined(); + expect(useColumns.mock.calls.at(-1)?.[0]).toEqual({ + databaseName: 'db', + tableName: 'logs', + connectionId: 'conn1', + }); }); - it('prefers the source over the connection, keeping its materialized views', () => { - // Losing the source here would drop `metadataMVs`, sending Map-key - // discovery to a raw table scan. - setupDefaultMocks({ withMVs: true }); + it("narrows a metric source to the metric type's table", () => { + // Metric rows live in per-type tables (gauge/sum/...). Only the caller + // knows which one — and which metric — is in play. + setupDefaultMocks({ withMVs: false }); const { wrapper } = makeWrapper(); renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', - tableConnection: FALLBACK_TC, + source: METRIC_SOURCE, + metricType: 'gauge', + metricName: 'k8s.pod.cpu', dateRange: DATE_RANGE, mode: 'all', }), { wrapper }, ); - expect(useAllFields.mock.calls.at(-1)?.[0]).toEqual(SOURCE_TC); - expect(useAllFields.mock.calls.at(-1)?.[0]?.metadataMVs).toBeDefined(); + expect(useAllFields.mock.calls.at(-1)?.[0]).toEqual({ + source: METRIC_SOURCE, + metricType: 'gauge', + metricName: 'k8s.pod.cpu', + }); + expect(useColumns.mock.calls.at(-1)?.[0]).toEqual({ + databaseName: 'db', + tableName: 'otel_metrics_gauge', + connectionId: 'conn1', + }); }); - it('uses the connection for a metric source, whose from does not name a table', () => { - // Metric rows live in per-type tables (gauge/sum/...). Only the caller - // knows which one — and which metric — is in play. + it('resolves no table while the source is loading', () => { setupDefaultMocks({ withMVs: false }); - mockSourceQuery({ - id: 'source1', - kind: SourceKind.Metric, - name: 'metrics', - connection: 'conn1', - from: { databaseName: 'db', tableName: '' }, - timestampValueExpression: 'TimeUnix', - metricTables: { - gauge: 'otel_metrics_gauge', - histogram: 'otel_metrics_histogram', - sum: 'otel_metrics_sum', - summary: 'otel_metrics_summary', - 'exponential histogram': 'otel_metrics_exponential_histogram', - }, - resourceAttributesExpression: 'ResourceAttributes', - }); const { wrapper } = makeWrapper(); - const metricTc = { ...FALLBACK_TC, metricName: 'k8s.pod.cpu' }; renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', - tableConnection: metricTc, + source: undefined, dateRange: DATE_RANGE, mode: 'all', }), { wrapper }, ); - expect(useAllFields.mock.calls.at(-1)?.[0]).toEqual(metricTc); + expect(useColumns.mock.calls.at(-1)?.[0]).toEqual({ + databaseName: '', + tableName: '', + connectionId: '', + }); }); - it('ignores an incomplete connection', () => { - // What a `tcFromSource` of a not-yet-loaded source looks like — using it - // would disable the queries outright. + it("loads more values from a metric type's table", async () => { setupDefaultMocks({ withMVs: false }); - const { wrapper } = makeWrapper(); + const getAllKeyValues = jest + .fn() + .mockResolvedValue([{ key: 'ServiceName', value: ['api'] }]); + mockMetadata({ + getKeyValuesWithMVs: jest.fn(), + getAllKeyValues, + }); - renderHook( + const { wrapper } = makeWrapper(); + const { result } = renderHook( () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: 'source1', - tableConnection: { - databaseName: '', - tableName: '', - connectionId: '', - }, + source: METRIC_SOURCE, + metricType: 'gauge', dateRange: DATE_RANGE, mode: 'all', + disableValues: true, }), { wrapper }, ); - expect(useAllFields.mock.calls.at(-1)?.[0]).toEqual(SOURCE_TC); + await act(async () => { + await result.current.loadMoreFacetsForKey('ServiceName'); + }); + + expect(getAllKeyValues).toHaveBeenCalledWith( + expect.objectContaining({ + databaseName: 'db', + tableName: 'otel_metrics_gauge', + connectionId: 'conn1', + metadataMVs: undefined, + timestampValueExpression: 'TimeUnix', + keyExpressions: ['ServiceName'], + }), + ); + expect(result.current.data.keyValues).toEqual([ + { key: 'ServiceName', value: ['api'] }, + ]); }); - it('loads more values from the connection when there is no source', async () => { - setupDefaultMocks({ withMVs: false }); - mockSourceQuery(undefined); - const getAllKeyValues = jest - .fn() - .mockResolvedValue([{ key: 'ServiceName', value: ['api'] }]); + it("loads more values through the source's materialized views", async () => { + const source = setupDefaultMocks({ withMVs: true }); + const getAllKeyValues = jest.fn().mockResolvedValue([]); mockMetadata({ getKeyValuesWithMVs: jest.fn(), getAllKeyValues, @@ -1082,11 +1073,9 @@ describe('useFetchFacets', () => { () => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: null, - tableConnection: FALLBACK_TC, + source, dateRange: DATE_RANGE, mode: 'all', - disableValues: true, }), { wrapper }, ); @@ -1097,20 +1086,14 @@ describe('useFetchFacets', () => { expect(getAllKeyValues).toHaveBeenCalledWith( expect.objectContaining({ - databaseName: 'other_db', - tableName: 'other_table', - connectionId: 'conn2', - keyExpressions: ['ServiceName'], + tableName: 'logs', + metadataMVs: source.metadataMaterializedViews, }), ); - expect(result.current.data.keyValues).toEqual([ - { key: 'ServiceName', value: ['api'] }, - ]); }); - it('clears extraFacets when the connection changes', async () => { + it('clears extraFacets when the metric type changes', async () => { setupDefaultMocks({ withMVs: false }); - mockSourceQuery(undefined); mockMetadata({ getKeyValuesWithMVs: jest.fn(), getAllKeyValues: jest @@ -1120,16 +1103,16 @@ describe('useFetchFacets', () => { const { wrapper } = makeWrapper(); const { result, rerender } = renderHook( - (props: { tableConnection: typeof FALLBACK_TC }) => + (props: { metricType: string }) => useFetchFacets({ chartConfig: CHART_CONFIG, - sourceId: null, - tableConnection: props.tableConnection, + source: METRIC_SOURCE, + metricType: props.metricType, dateRange: DATE_RANGE, mode: 'all', disableValues: true, }), - { wrapper, initialProps: { tableConnection: FALLBACK_TC } }, + { wrapper, initialProps: { metricType: 'gauge' } }, ); await act(async () => { @@ -1138,9 +1121,7 @@ describe('useFetchFacets', () => { expect(result.current.extraFacetKeys.has('NewKey')).toBe(true); - rerender({ - tableConnection: { ...FALLBACK_TC, tableName: 'yet_another_table' }, - }); + rerender({ metricType: 'sum' }); expect(result.current.extraFacetKeys.size).toBe(0); }); diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index be73a26336..95f644b724 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -1,16 +1,14 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; import produce from 'immer'; -import { - TableConnection, - tcFromSource, -} from '@hyperdx/common-utils/dist/core/metadata'; +import { getSourceTable } from '@hyperdx/common-utils/dist/core/metadata'; import { FilterState, filtersToQuery, } from '@hyperdx/common-utils/dist/filters'; import { BuilderChartConfigWithDateRange, - isMetricSource, + isLogSource, + isTraceSource, TSource, } from '@hyperdx/common-utils/dist/types'; @@ -25,7 +23,6 @@ import { useMetadataWithSettings, } from '@/hooks/useMetadata'; import { escapeFilterStateKeys, usePinnedFilters } from '@/searchFilters'; -import { useSource } from '@/source'; import { mergePath } from '@/utils'; import { toQuotedClickHouseKeyExpression } from './utils'; @@ -35,49 +32,28 @@ const INITIAL_LOAD_LIMIT = 20; /* The maximum number of values per filter to load when "Load More" is clicked */ const LOAD_MORE_LOAD_LIMIT = 10000; -/** - * Decide which table key-value discovery reads from. - * - * The source stays authoritative whenever we have one, so its metadata - * materialized views keep serving discovery. `fallback` — the table connection - * a caller passes — is consulted in the two cases the source can't answer: - * - * - No source id is provided - * - A metric source, in which case the fallback provides the metric type's table name - */ -export function resolveTableConnection( - source: TSource | undefined, - fallback: TableConnection | undefined, -): TableConnection { - const isFallbackUsable = - !!fallback?.databaseName && !!fallback.tableName && !!fallback.connectionId; - if (!source) { - return isFallbackUsable ? fallback : tcFromSource(undefined); - } - if (isMetricSource(source) && isFallbackUsable) { - return fallback; - } - return tcFromSource(source); -} +type FacetsSource = { + /** Where keys and values are discovered, and whose pinned filters apply. */ + source: TSource | undefined; + /** Picks the table of a metric source, which keeps one per metric type. */ + metricType?: string; + /** Scopes key discovery to one metric's rows. */ + metricName?: string; +}; function useFacets({ chartConfig, - sourceId, - tableConnection: tableConnectionFallback, + source, + metricType, + metricName, mode, dateRange, filterState, showMoreFields, enabled, disableValues, -}: { +}: FacetsSource & { chartConfig: BuilderChartConfigWithDateRange; - sourceId: string | null; - /** - * A table where keys and values are discovered. Used when sourceId - * is not provided or references a metrics source. - */ - tableConnection?: TableConnection; mode: 'all' | 'exact'; dateRange: [Date, Date]; filterState?: FilterState; @@ -85,35 +61,27 @@ function useFacets({ enabled?: boolean; disableValues?: boolean; }) { - const { data: source } = useSource({ - id: sourceId, - }); - const tableConnection = useMemo( - () => resolveTableConnection(source, tableConnectionFallback), - [source, tableConnectionFallback], + const table = useMemo( + () => getSourceTable({ source, metricType }), + [source, metricType], ); - const { data: columns, isLoading: isColumnsLoading } = - useColumns(tableConnection); + const { data: columns, isLoading: isColumnsLoading } = useColumns(table); const dateTimeColumns = useDateTimeColumns(columns); const knownColumns = useMemo( () => (columns ? new Set(columns.map(c => c.name)) : new Set()), [columns], ); - const { data: jsonColumns } = useJsonColumns(tableConnection); - const { data: mapColumns } = useMapColumns(tableConnection); + const { data: jsonColumns } = useJsonColumns({ source, metricType }); + const { data: mapColumns } = useMapColumns({ source, metricType }); const { data: allFields, error: allFieldsError, isLoading: isAllFieldsLoading, - } = useAllFields(tableConnection, { - dateRange, - timestampValueExpression: source?.timestampValueExpression, - enabled, - }); + } = useAllFields({ source, metricType, metricName }, { dateRange, enabled }); const { isFieldPinned, isSharedFieldPinned } = usePinnedFilters( - sourceId ?? null, + source?.id ?? null, ); const keysToFetch = useMemo(() => { @@ -240,20 +208,15 @@ function useFacets({ }; } - if ( - !tableConnection.databaseName || - !tableConnection.tableName || - !tableConnection.connectionId - ) { - throw new Error( - 'loadMoreFacetsForKey: a source or table connection must be defined', - ); + if (!table.databaseName || !table.tableName || !table.connectionId) { + throw new Error('loadMoreFacetsForKey: a source must be defined'); } const newKeyVals = await metadata.getAllKeyValues({ - databaseName: tableConnection.databaseName, - tableName: tableConnection.tableName, - connectionId: tableConnection.connectionId, - metadataMVs: tableConnection.metadataMVs, + ...table, + metadataMVs: + source && (isLogSource(source) || isTraceSource(source)) + ? source.metadataMaterializedViews + : undefined, keyExpressions: [sqlKey], maxValuesPerKey: LOAD_MORE_LOAD_LIMIT, dateRange, @@ -276,7 +239,7 @@ function useFacets({ }, [ mode, - tableConnection, + table, metadata, chartConfig, dateRange, @@ -298,21 +261,16 @@ function useFacets({ export function useFetchFacets({ chartConfig, - sourceId, - tableConnection, + source, + metricType, + metricName, dateRange, mode, filterState, showMoreFields, disableValues, -}: { +}: FacetsSource & { chartConfig: BuilderChartConfigWithDateRange; - sourceId: string | null; - /** - * A table where keys and values are discovered. Used when sourceId - * is not provided or references a metrics source. - */ - tableConnection?: TableConnection; dateRange: [Date, Date]; mode: 'all' | 'exact'; filterState?: FilterState; @@ -321,8 +279,9 @@ export function useFetchFacets({ }) { const facetsQuery = useFacets({ chartConfig, - sourceId, - tableConnection, + source, + metricType, + metricName, mode, dateRange, filterState, @@ -331,6 +290,7 @@ export function useFetchFacets({ disableValues, }); + const table = getSourceTable({ source, metricType }); const [extraFacets, setExtraFacets] = useState(null); const facets = useMemo(() => { const base = facetsQuery.data.keyValues; @@ -409,10 +369,10 @@ export function useFetchFacets({ setExtraFacets(null); setExtraFacetKeys(new Set()); }, [ - sourceId, - tableConnection?.databaseName, - tableConnection?.tableName, - tableConnection?.connectionId, + source?.id, + table.databaseName, + table.tableName, + table.connectionId, dateRange, mode, filterState, diff --git a/packages/app/src/components/DBTracePanel.tsx b/packages/app/src/components/DBTracePanel.tsx index 822eb208d4..6fa72068b6 100644 --- a/packages/app/src/components/DBTracePanel.tsx +++ b/packages/app/src/components/DBTracePanel.tsx @@ -11,7 +11,6 @@ import { atomWithStorage } from 'jotai/utils'; import { useQueryState } from 'nuqs'; import { useForm, useWatch } from 'react-hook-form'; import { z } from 'zod'; -import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata'; import { isLogSource, isTraceSource, @@ -399,14 +398,13 @@ export default function DBTracePanel({ Trace ID Expression