Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .changeset/promql-auto-detect-source.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
'@hyperdx/common-utils': patch
'@hyperdx/app': patch
---

feat: auto-detect a PromQL source during onboarding

When PromQL is enabled (`NEXT_PUBLIC_ENABLE_PROMQL=true`), onboarding source auto-detection now also creates a PromQL source for a ClickHouse TimeSeries engine table. It prefers `prometheus.metrics` (the table the ClickHouse Prometheus docs suggest), then a table named `metrics_ts`, and otherwise picks the only TimeSeries table on the connection. A service with only a TimeSeries table is no longer sent to manual source setup.
55 changes: 55 additions & 0 deletions packages/app/src/__tests__/source.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
getEventBody,
getSourceValidationNotificationId,
getTraceDurationNumberFormat,
pickTimeSeriesTable,
useChartNumberFormats,
useSingleSeriesNumberFormat,
useSources,
Expand Down Expand Up @@ -54,6 +55,60 @@ const TRACE_SOURCE: TTraceSource = {
defaultTableSelectExpression: 'Timestamp, ServiceName',
} as TTraceSource;

describe('pickTimeSeriesTable', () => {
const table = (databaseName: string, tableName: string) => ({
databaseName,
tableName,
});

it('prefers metrics_ts in the preferred database', () => {
const tables = [
table('default', 'metrics_ts'),
table('otel', 'metrics_ts'),
];
expect(pickTimeSeriesTable(tables, 'otel')).toEqual(
table('otel', 'metrics_ts'),
);
});

it('prefers prometheus.metrics over metrics_ts', () => {
const tables = [
table('otel', 'metrics_ts'),
table('prometheus', 'metrics'),
];
expect(pickTimeSeriesTable(tables, 'otel')).toEqual(
table('prometheus', 'metrics'),
);
});

it('only treats metrics in the prometheus database as the docs table', () => {
const tables = [table('default', 'metrics'), table('otel', 'metrics_ts')];
expect(pickTimeSeriesTable(tables, 'otel')).toEqual(
table('otel', 'metrics_ts'),
);
});

it('falls back to metrics_ts in any database', () => {
const tables = [table('a_db', 'custom_ts'), table('b_db', 'metrics_ts')];
expect(pickTimeSeriesTable(tables)).toEqual(table('b_db', 'metrics_ts'));
});

it('picks a sole TimeSeries table without the default name', () => {
expect(pickTimeSeriesTable([table('otel', 'prom')])).toEqual(
table('otel', 'prom'),
);
});

it('picks nothing from several tables without the default name', () => {
const tables = [table('otel', 'prom_a'), table('otel', 'prom_b')];
expect(pickTimeSeriesTable(tables, 'otel')).toBeUndefined();
});

it('picks nothing from no tables', () => {
expect(pickTimeSeriesTable([])).toBeUndefined();
});
});

describe('getEventBody', () => {
it('returns spanNameExpression for trace kind source when both bodyExpression and spanNameExpression are present', () => {
const result = getEventBody(TRACE_SOURCE);
Expand Down
59 changes: 49 additions & 10 deletions packages/app/src/components/OnboardingModal.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,17 @@ import { notifications } from '@mantine/notifications';
import { IconArrowLeft } from '@tabler/icons-react';

import { ConnectionForm } from '@/components/ConnectionForm';
import { IS_CLICKHOUSE_BUILD, IS_LOCAL_MODE } from '@/config';
import {
IS_CLICKHOUSE_BUILD,
IS_LOCAL_MODE,
IS_PROMQL_ENABLED,
} from '@/config';
import { useConnections, useCreateConnection } from '@/connection';
import { useMetadataWithSettings } from '@/hooks/useMetadata';
import {
inferTableSourceConfig,
pickTimeSeriesTable,
PROMQL_TIMESTAMP_EXPRESSION,
useCreateSource,
useDeleteSource,
useSources,
Expand Down Expand Up @@ -309,10 +315,25 @@ function OnboardingModalComponent({
setIsAutoDetecting(true);
setHasAutodetected(true);

// Try to detect OTEL tables
const otelTables = await metadata.getOtelTables({ connectionId });
const getTimeSeriesTables = IS_PROMQL_ENABLED
? metadata.getTimeSeriesTables({ connectionId }).catch(err => {
console.error('Error detecting TimeSeries tables:', err);
return [];
})
: [];

// Try to detect OTEL tables, and a TimeSeries table for PromQL
const [otelTables, timeSeriesTables] = await Promise.all([
metadata.getOtelTables({ connectionId }),
getTimeSeriesTables,
]);

const timeSeriesTable = pickTimeSeriesTable(
timeSeriesTables,
otelTables?.database,
);

if (!otelTables) {
if (!otelTables && !timeSeriesTable) {
Comment thread
karl-power marked this conversation as resolved.
Resolved
Comment thread
karl-power marked this conversation as resolved.
Resolved
// No tables detected, go to manual source setup
setStep('source');
return;
Expand All @@ -321,7 +342,7 @@ function OnboardingModalComponent({
const createdSources: TSource[] = [];

// Create Log Source if available
if (otelTables.tables.logs) {
if (otelTables?.tables.logs) {
const inferredConfig = await inferTableSourceConfig({
kind: SourceKind.Log,
databaseName: otelTables.database,
Expand Down Expand Up @@ -359,7 +380,7 @@ function OnboardingModalComponent({
}

// Create Trace Source if available
if (otelTables.tables.traces) {
if (otelTables?.tables.traces) {
const inferredConfig = await inferTableSourceConfig({
kind: SourceKind.Trace,
databaseName: otelTables.database,
Expand Down Expand Up @@ -406,9 +427,9 @@ function OnboardingModalComponent({
}

// Create Metrics Source if any metrics tables are available
const hasMetrics = Object.values(otelTables.tables.metrics).some(
t => t != null,
);
const hasMetrics =
otelTables != null &&
Object.values(otelTables.tables.metrics).some(t => t != null);
if (hasMetrics) {
const metricTables: MetricTable = {
[MetricsDataType.Gauge]: '',
Expand Down Expand Up @@ -455,7 +476,7 @@ function OnboardingModalComponent({
}

// Create Session Source if available
if (otelTables.tables.sessions) {
if (otelTables?.tables.sessions) {
const inferredConfig = await inferTableSourceConfig({
kind: SourceKind.Session,
databaseName: otelTables.database,
Expand Down Expand Up @@ -495,6 +516,24 @@ function OnboardingModalComponent({
}
}

// Create PromQL Source if a TimeSeries table is available
if (timeSeriesTable) {
try {
const promqlSource = await createSourceMutation.mutateAsync({
source: {
kind: SourceKind.Promql,
name: 'PromQL',
connection: connectionId,
from: timeSeriesTable,
timestampValueExpression: PROMQL_TIMESTAMP_EXPRESSION,
},
});
createdSources.push(promqlSource);
Comment thread
karl-power marked this conversation as resolved.
Resolved
} catch (err) {
console.error('Error creating PromQL source:', err);
}
}

if (createdSources.length === 0) {
console.error('No sources created due to missing required fields');
// No sources created, go to manual source setup
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { useEffect } from 'react';

import { PROMQL_TIMESTAMP_EXPRESSION } from '@/source';

import { MinAutoGranularityFormRow } from './MinAutoGranularityFormRow';
import { TableModelProps } from './types';

Expand All @@ -8,7 +10,7 @@ const PROMQL_MIN_AUTO_GRANULARITY_HELP_TEXT =

export function PromqlTableModelForm({ control, setValue }: TableModelProps) {
useEffect(() => {
setValue('timestampValueExpression' as any, 'timestamp');
setValue('timestampValueExpression' as any, PROMQL_TIMESTAMP_EXPRESSION);
}, [setValue]);

// PromQL sources use the standard database + table fields from BaseSourceSchema.
Expand Down
176 changes: 176 additions & 0 deletions packages/app/src/components/__tests__/OnboardingModal.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
import { SourceKind, TSourceNoId } from '@hyperdx/common-utils/dist/types';
import { notifications } from '@mantine/notifications';
import { screen } from '@testing-library/react';

import OnboardingModal from '@/components/OnboardingModal';

const mockConfig = { IS_PROMQL_ENABLED: true };
jest.mock('@/config', () => ({
IS_CLICKHOUSE_BUILD: false,
IS_LOCAL_MODE: false,
get IS_PROMQL_ENABLED() {
return mockConfig.IS_PROMQL_ENABLED;
},
}));

const mockCreateSource = jest.fn();
const mockUpdateSource = jest.fn();
// Stable references: new objects on every render would re-run the modal's
// effects, which depend on them.
const mockSourcesResult = { data: [] };
const mockCreateSourceMutation = { mutateAsync: mockCreateSource };
const mockUpdateSourceMutation = { mutateAsync: mockUpdateSource };
jest.mock('@/source', () => ({
pickTimeSeriesTable: jest.requireActual('@/source').pickTimeSeriesTable,
PROMQL_TIMESTAMP_EXPRESSION:
jest.requireActual('@/source').PROMQL_TIMESTAMP_EXPRESSION,
inferTableSourceConfig: async ({ kind }: { kind: SourceKind }) => ({
kind,
timestampValueExpression: 'Timestamp',
defaultTableSelectExpression: 'Timestamp, Body',
}),
useSources: () => mockSourcesResult,
useCreateSource: () => mockCreateSourceMutation,
useUpdateSource: () => mockUpdateSourceMutation,
useDeleteSource: () => ({ mutateAsync: jest.fn() }),
}));

const mockConnectionsResult = {
data: [{ id: 'conn-1', name: 'Default', host: 'http://localhost:8123' }],
};
jest.mock('@/connection', () => ({
useConnections: () => mockConnectionsResult,
useCreateConnection: () => ({ mutate: jest.fn() }),
}));

const mockMetadata = {
getOtelTables: jest.fn(),
getTimeSeriesTables: jest.fn(),
};
jest.mock('@/hooks/useMetadata', () => ({
useMetadataWithSettings: () => mockMetadata,
}));

jest.mock('@/theme/ThemeProvider', () => ({
useBrandDisplayName: () => 'HyperDX',
}));
jest.mock('@/components/ConnectionForm', () => ({
ConnectionForm: () => null,
}));
jest.mock('@/components/Sources/SourceForm', () => ({
TableSourceForm: () => null,
}));
jest.mock('@/components/Sources/SourcesList', () => ({
SourcesList: () => null,
}));

const OTEL_LOGS_ONLY = {
database: 'otel',
tables: { logs: 'otel_logs', metrics: {} },
};

const createdKinds = () =>
mockCreateSource.mock.calls.map(
([{ source }]: [{ source: TSourceNoId }]) => source.kind,
);

describe('OnboardingModal auto-detection', () => {
afterEach(() => {
jest.restoreAllMocks();
});

beforeEach(() => {
// The notifications store is global, so earlier tests' toasts linger.
notifications.clean();
jest.clearAllMocks();
mockConfig.IS_PROMQL_ENABLED = true;
mockCreateSource.mockImplementation(
async ({ source }: { source: TSourceNoId }) => ({
...source,
id: `${source.kind}-id`,
}),
);
mockUpdateSource.mockResolvedValue(undefined);
mockMetadata.getOtelTables.mockResolvedValue(OTEL_LOGS_ONLY);
mockMetadata.getTimeSeriesTables.mockResolvedValue([
{ databaseName: 'default', tableName: 'metrics_ts' },
]);
});

it('creates a PromQL source for a detected TimeSeries table', async () => {
renderWithMantine(<OnboardingModal />);

await screen.findByText('Automatically detected and created 2 sources.');
expect(mockMetadata.getTimeSeriesTables).toHaveBeenCalledWith({
connectionId: 'conn-1',
});
expect(createdKinds()).toEqual([SourceKind.Log, SourceKind.Promql]);
expect(mockCreateSource).toHaveBeenLastCalledWith({
source: {
kind: SourceKind.Promql,
name: 'PromQL',
connection: 'conn-1',
from: { databaseName: 'default', tableName: 'metrics_ts' },
timestampValueExpression: 'timestamp',
},
});
});

it('skips PromQL detection when PromQL is disabled', async () => {
mockConfig.IS_PROMQL_ENABLED = false;

renderWithMantine(<OnboardingModal />);

await screen.findByText('Automatically detected and created 1 source.');
expect(mockMetadata.getTimeSeriesTables).not.toHaveBeenCalled();
expect(createdKinds()).toEqual([SourceKind.Log]);
});

it('creates a PromQL source when no OTel tables exist', async () => {
mockMetadata.getOtelTables.mockResolvedValue(null);

renderWithMantine(<OnboardingModal />);

await screen.findByText('Automatically detected and created 1 source.');
expect(createdKinds()).toEqual([SourceKind.Promql]);
});

it('keeps the other sources when the PromQL source fails', async () => {
jest.spyOn(console, 'error').mockImplementation(() => {});
mockCreateSource.mockImplementation(
async ({ source }: { source: TSourceNoId }) => {
if (source.kind === SourceKind.Promql) {
throw new Error('boom');
}
return { ...source, id: `${source.kind}-id` };
},
);

renderWithMantine(<OnboardingModal />);

await screen.findByText('Automatically detected and created 1 source.');
expect(createdKinds()).toEqual([SourceKind.Log, SourceKind.Promql]);
});

it('keeps OTel detection when TimeSeries detection fails', async () => {
jest.spyOn(console, 'error').mockImplementation(() => {});
mockMetadata.getTimeSeriesTables.mockRejectedValue(new Error('boom'));

renderWithMantine(<OnboardingModal />);

await screen.findByText('Automatically detected and created 1 source.');
expect(createdKinds()).toEqual([SourceKind.Log]);
});

it('falls back to manual setup when nothing is detected', async () => {
mockMetadata.getOtelTables.mockResolvedValue(null);
mockMetadata.getTimeSeriesTables.mockResolvedValue([]);

renderWithMantine(<OnboardingModal />);

await screen.findByText(
'Lets set up a source table to query telemetry from.',
);
expect(mockCreateSource).not.toHaveBeenCalled();
});
});
Loading
Loading