Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .changeset/remove-table-connections.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
---
'@hyperdx/common-utils': minor
'@hyperdx/app': patch
'@hyperdx/api': patch
---

refactor: replace `TableConnection` with sources for field discovery

`TableConnection`, `TableConnectionChoice`, `tcFromSource` and
`tcFromChartConfig` are removed from `@hyperdx/common-utils`.
`Metadata.getAllFields` and `genEnglishExplanation` now take a source (plus a
metric type and metric name for metric sources) and read the table, timestamp
expression and metadata materialized views from it. Search inputs, SQL editors
and facet hooks take `source` / `sourceTables` props in place of
`tableConnection` / `tableConnections` and `sourceId`.

The raw SQL editor now autocompletes columns from metric sources' tables, which
it previously skipped.
9 changes: 1 addition & 8 deletions packages/api/src/controllers/ai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { createOpenAI } from '@ai-sdk/openai';
import {
getMetadata,
TableMetadata,
tcFromSource,
} from '@hyperdx/common-utils/dist/core/metadata';
import {
AILineTableResponse,
Expand Down Expand Up @@ -103,13 +102,7 @@ export async function getAIMetadata(source: ISource) {
connectionId,
});

const allFields = await metadata.getAllFields({
databaseName,
tableName,
connectionId,
metadataMVs: tcFromSource(source).metadataMVs,
timestampValueExpression: source.timestampValueExpression,
});
const allFields = await metadata.getAllFields({ source });

// TODO: Dedup with DBSearchPageFilters.tsx logic
allFields.sort((a, b) => {
Expand Down
21 changes: 7 additions & 14 deletions packages/app/src/DBDashboardPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,7 @@ import {
import { ErrorBoundary } from 'react-error-boundary';
import RGL from 'react-grid-layout';
import { useForm, useWatch } from 'react-hook-form';
import {
TableConnection,
tcFromSource,
} from '@hyperdx/common-utils/dist/core/metadata';
import { SourceTable } from '@hyperdx/common-utils/dist/core/metadata';
import {
convertToDashboardTemplate,
displayTypeSupportsBuilderAlerts,
Expand Down Expand Up @@ -1925,9 +1922,9 @@ function DBDashboardPage({
const { data: allDashboards } = useDashboards();

const [highlightedTileId] = useQueryState('highlightedTileId');
const tableConnections = useMemo(() => {
const sourceTables = useMemo(() => {
if (!dashboard) return [];
const tc: TableConnection[] = [];
const tables: SourceTable[] = [];

for (const { config } of dashboard.tiles) {
if (!isBuilderSavedChartConfig(config)) continue;
Expand All @@ -1937,15 +1934,11 @@ function DBDashboardPage({
const firstSelect = config.select[0];
const metricType =
typeof firstSelect !== 'string' ? firstSelect?.metricType : undefined;
const tableName = getMetricTableName(source, metricType);
if (!tableName) continue;
tc.push({
...tcFromSource(source),
tableName,
});
if (!getMetricTableName(source, metricType)) continue;
tables.push({ source, metricType });
}

return tc;
return tables;
}, [dashboard, sources]);

const [granularity, setGranularity] = useQueryState(
Expand Down Expand Up @@ -3160,7 +3153,7 @@ function DBDashboardPage({
}}
>
<SearchWhereInput
tableConnections={tableConnections}
sourceTables={sourceTables}
dateRange={searchedTimeRange}
control={control}
name="where"
Expand Down
18 changes: 5 additions & 13 deletions packages/app/src/DBSearchPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import {
ClickHouseQueryError,
ColumnMeta,
} from '@hyperdx/common-utils/dist/clickhouse';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import { getSourceTable } from '@hyperdx/common-utils/dist/core/metadata';
import { buildSearchChartConfig } from '@hyperdx/common-utils/dist/core/searchChartConfig';
import {
aliasMapToWithClauses,
Expand Down Expand Up @@ -874,7 +874,7 @@ export function useDefaultOrderBy(sourceID: string | undefined | null) {
id: sourceID,
kinds: [SourceKind.Log, SourceKind.Trace],
});
const { data: tableMetadata } = useTableMetadata(tcFromSource(source));
const { data: tableMetadata } = useTableMetadata(getSourceTable({ source }));

// When source changes, make sure select and orderby fields are set to default
return useMemo(() => {
Expand Down Expand Up @@ -2082,11 +2082,6 @@ function DBSearchPageContent() {
],
);

const inputSourceTableConnection = useMemo(
() => tcFromSource(inputSourceObj),
[inputSourceObj],
);

const [isSourceSchemaPreviewOpen, setIsSourceSchemaPreviewOpen] =
useState(false);

Expand Down Expand Up @@ -2370,7 +2365,7 @@ function DBSearchPageContent() {
/>
<Box style={{ flex: '1 1 0%', minWidth: 100 }}>
<SQLInlineEditorControlled
tableConnection={inputSourceTableConnection}
source={inputSourceObj}
control={control}
name="select"
defaultValue={defaultSearchConfig.select}
Expand All @@ -2380,20 +2375,18 @@ function DBSearchPageContent() {
size="xs"
allowMultiline
dateRange={searchedTimeRange}
sourceId={inputSource}
/>
</Box>
<Box style={{ maxWidth: 400, width: '20%' }}>
<SQLInlineEditorControlled
tableConnection={inputSourceTableConnection}
source={inputSourceObj}
control={control}
name="orderBy"
defaultValue={defaultSearchConfig.orderBy}
onSubmit={onSubmit}
label="ORDER BY"
size="xs"
dateRange={searchedTimeRange}
sourceId={inputSource}
/>
</Box>
<>
Expand Down Expand Up @@ -2448,7 +2441,7 @@ function DBSearchPageContent() {
/>
<Flex gap="sm" mt="sm" px="sm" wrap="wrap">
<SearchWhereInput
tableConnection={inputSourceTableConnection}
source={inputSourceObj}
control={control}
name="where"
onSubmit={onSubmit}
Expand All @@ -2458,7 +2451,6 @@ function DBSearchPageContent() {
data-testid="search-input"
minWidth="min(600px, 100%)"
dateRange={searchedTimeRange}
sourceId={inputSource}
size="xs"
/>
<Flex
Expand Down
4 changes: 1 addition & 3 deletions packages/app/src/DBSearchPageAlertModal.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import {
clampAlertDisplayName,
clampAlertTags,
} from '@hyperdx/common-utils/dist/alerts';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import {
type Alert,
alertDisplayNameSchema,
Expand Down Expand Up @@ -313,8 +312,7 @@ const AlertForm = ({
grouped by
</Text>
<SQLInlineEditorControlled
tableConnection={tcFromSource(source)}
sourceId={sourceId ?? undefined}
source={source}
control={control}
name={`groupBy`}
placeholder="SQL Columns"
Expand Down
6 changes: 1 addition & 5 deletions packages/app/src/DBServiceMapPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import {
useQueryStates,
} from 'nuqs';
import { useForm, useWatch } from 'react-hook-form';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import { SourceKind, TTraceSource } from '@hyperdx/common-utils/dist/types';
import {
Button,
Expand Down Expand Up @@ -143,8 +142,6 @@ function DBServiceMapPage() {
syncSourceParam(watchedSource);
}, [watchedSource]);

const sourceTableConnection = useMemo(() => tcFromSource(source), [source]);

const serviceNameKey = source?.serviceNameExpression ?? 'ServiceName';
const serviceNamesChartConfig = useMemo(
() =>
Expand Down Expand Up @@ -370,7 +367,7 @@ function DBServiceMapPage() {
data-testid="service-map-service-filter"
/>
<SearchWhereInput
tableConnection={sourceTableConnection}
source={source}
control={control}
name="where"
onSubmit={onSubmit}
Expand All @@ -381,7 +378,6 @@ function DBServiceMapPage() {
size="xs"
data-testid="service-map-search-input"
dateRange={searchedTimeRange}
sourceId={source?.id}
lucenePlaceholder="Filter spans w/ Lucene (ex. http.method:GET)"
sqlPlaceholder="SQL WHERE to filter spans (ex. Duration > 1000000)"
minWidth="min(500px, 100%)"
Expand Down
8 changes: 1 addition & 7 deletions packages/app/src/KubernetesDashboardPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -1258,13 +1258,7 @@ function KubernetesDashboardPage() {
const [_searchQuery, _setSearchQuery] = React.useState<string | null>(null);

const { data: logSourceJsonColumns, isLoading: isLoadingJsonColumns } =
useJsonColumns(
logSource && {
databaseName: logSource.from.databaseName,
tableName: logSource.from.tableName,
connectionId: logSource.connection,
},
);
useJsonColumns({ source: logSource });

const eventAttributeExpressions = useMemo(() => {
if (isLoadingJsonColumns || !logSource) {
Expand Down
6 changes: 4 additions & 2 deletions packages/app/src/NamespaceDetailsSidePanel.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import * as React from 'react';
import { parseAsString, useQueryState } from 'nuqs';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import { getSourceTable } from '@hyperdx/common-utils/dist/core/metadata';
import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils';
import { TLogSource, TMetricSource } from '@hyperdx/common-utils/dist/types';
import {
Expand Down Expand Up @@ -239,7 +239,9 @@ function NamespaceDetailsSidePanelInner({
return `${metricSource?.resourceAttributesExpression}.k8s.namespace.name:"${namespaceName}"`;
}, [namespaceName, metricSource]);

const { data: logsTableMetadata } = useTableMetadata(tcFromSource(logSource));
const { data: logsTableMetadata } = useTableMetadata(
getSourceTable({ source: logSource }),
);

let doesPrimaryOrSortingKeysContainServiceExpression = false;

Expand Down
6 changes: 4 additions & 2 deletions packages/app/src/NodeDetailsSidePanel.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import * as React from 'react';
import { parseAsString, useQueryState } from 'nuqs';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import { getSourceTable } from '@hyperdx/common-utils/dist/core/metadata';
import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils';
import { TLogSource, TMetricSource } from '@hyperdx/common-utils/dist/types';
import {
Expand Down Expand Up @@ -252,7 +252,9 @@ function NodeDetailsSidePanelInner({
return `${metricSource?.resourceAttributesExpression}.k8s.node.name:"${nodeName}"`;
}, [nodeName, metricSource]);

const { data: logsTableMetadata } = useTableMetadata(tcFromSource(logSource));
const { data: logsTableMetadata } = useTableMetadata(
getSourceTable({ source: logSource }),
);

let doesPrimaryOrSortingKeysContainServiceExpression = false;

Expand Down
6 changes: 4 additions & 2 deletions packages/app/src/PodDetailsSidePanel.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import * as React from 'react';
import { parseAsString, useQueryState } from 'nuqs';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import { getSourceTable } from '@hyperdx/common-utils/dist/core/metadata';
import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils';
import { TLogSource, TMetricSource } from '@hyperdx/common-utils/dist/types';
import {
Expand Down Expand Up @@ -242,7 +242,9 @@ function PodDetailsSidePanelInner({
return `${metricSource?.resourceAttributesExpression}.k8s.pod.name:"${podName}"`;
}, [podName, metricSource]);

const { data: logsTableMetadata } = useTableMetadata(tcFromSource(logSource));
const { data: logsTableMetadata } = useTableMetadata(
getSourceTable({ source: logSource }),
);

let doesPrimaryOrSortingKeysContainServiceExpression = false;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import {
useQueryStates,
} from 'nuqs';
import { UseControllerProps, useForm, useWatch } from 'react-hook-form';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import {
PresetDashboard,
SourceKind,
Expand Down Expand Up @@ -361,8 +360,7 @@ function ServicesDashboardPage() {
dateRange={searchedTimeRange}
/>
<SearchWhereInput
tableConnection={tcFromSource(source)}
sourceId={sourceId}
source={source}
dateRange={searchedTimeRange}
control={control}
name="where"
Expand Down
4 changes: 1 addition & 3 deletions packages/app/src/SessionSubpanel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import cx from 'classnames';
import throttle from 'lodash/throttle';
import { parseAsInteger, useQueryState } from 'nuqs';
import { useForm } from 'react-hook-form';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import {
ChartConfigWithOptDateRange,
DateRange,
Expand Down Expand Up @@ -457,8 +456,7 @@ export default function SessionSubpanel({
onSubmit={handleSubmit(handleWhereSubmit)}
>
<SearchWhereInput
tableConnection={tcFromSource(traceSource)}
sourceId={traceSource.id}
source={traceSource}
dateRange={eventsDateRange}
control={control}
name="where"
Expand Down
4 changes: 1 addition & 3 deletions packages/app/src/SessionsPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import {
useQueryStates,
} from 'nuqs';
import { useForm, useWatch } from 'react-hook-form';
import { tcFromSource } from '@hyperdx/common-utils/dist/core/metadata';
import {
SearchCondition,
SearchConditionLanguage,
Expand Down Expand Up @@ -449,9 +448,8 @@ function SessionsPage() {
allowedSourceKinds={[SourceKind.Session]}
/>
<SearchWhereInput
tableConnection={tcFromSource(traceTrace)}
// The WHERE runs against the trace source
sourceId={traceTrace?.id}
source={traceTrace}
dateRange={searchedTimeRange}
control={control}
name="where"
Expand Down
Loading
Loading