diff --git a/.changeset/promql-auto-detect-source.md b/.changeset/promql-auto-detect-source.md
new file mode 100644
index 0000000000..4dc304ffc4
--- /dev/null
+++ b/.changeset/promql-auto-detect-source.md
@@ -0,0 +1,8 @@
+---
+'@hyperdx/common-utils': patch
+'@hyperdx/app': patch
+---
+
+feat: auto-detect a PromQL source during onboarding
+
+When PromQL is enabled (`NEXT_PUBLIC_ENABLE_PROMQL=true`), onboarding source auto-detection now also creates a PromQL source for a ClickHouse TimeSeries engine table. It prefers `prometheus.metrics` (the table the ClickHouse Prometheus docs suggest), then a table named `metrics_ts`, and otherwise picks the only TimeSeries table on the connection. A service with only a TimeSeries table is no longer sent to manual source setup.
diff --git a/packages/app/src/__tests__/source.test.ts b/packages/app/src/__tests__/source.test.ts
index 420cbdc59c..7f6fd08cb2 100644
--- a/packages/app/src/__tests__/source.test.ts
+++ b/packages/app/src/__tests__/source.test.ts
@@ -14,6 +14,7 @@ import {
getEventBody,
getSourceValidationNotificationId,
getTraceDurationNumberFormat,
+ pickTimeSeriesTable,
useChartNumberFormats,
useSingleSeriesNumberFormat,
useSources,
@@ -54,6 +55,60 @@ const TRACE_SOURCE: TTraceSource = {
defaultTableSelectExpression: 'Timestamp, ServiceName',
} as TTraceSource;
+describe('pickTimeSeriesTable', () => {
+ const table = (databaseName: string, tableName: string) => ({
+ databaseName,
+ tableName,
+ });
+
+ it('prefers metrics_ts in the preferred database', () => {
+ const tables = [
+ table('default', 'metrics_ts'),
+ table('otel', 'metrics_ts'),
+ ];
+ expect(pickTimeSeriesTable(tables, 'otel')).toEqual(
+ table('otel', 'metrics_ts'),
+ );
+ });
+
+ it('prefers prometheus.metrics over metrics_ts', () => {
+ const tables = [
+ table('otel', 'metrics_ts'),
+ table('prometheus', 'metrics'),
+ ];
+ expect(pickTimeSeriesTable(tables, 'otel')).toEqual(
+ table('prometheus', 'metrics'),
+ );
+ });
+
+ it('only treats metrics in the prometheus database as the docs table', () => {
+ const tables = [table('default', 'metrics'), table('otel', 'metrics_ts')];
+ expect(pickTimeSeriesTable(tables, 'otel')).toEqual(
+ table('otel', 'metrics_ts'),
+ );
+ });
+
+ it('falls back to metrics_ts in any database', () => {
+ const tables = [table('a_db', 'custom_ts'), table('b_db', 'metrics_ts')];
+ expect(pickTimeSeriesTable(tables)).toEqual(table('b_db', 'metrics_ts'));
+ });
+
+ it('picks a sole TimeSeries table without the default name', () => {
+ expect(pickTimeSeriesTable([table('otel', 'prom')])).toEqual(
+ table('otel', 'prom'),
+ );
+ });
+
+ it('picks nothing from several tables without the default name', () => {
+ const tables = [table('otel', 'prom_a'), table('otel', 'prom_b')];
+ expect(pickTimeSeriesTable(tables, 'otel')).toBeUndefined();
+ });
+
+ it('picks nothing from no tables', () => {
+ expect(pickTimeSeriesTable([])).toBeUndefined();
+ });
+});
+
describe('getEventBody', () => {
it('returns spanNameExpression for trace kind source when both bodyExpression and spanNameExpression are present', () => {
const result = getEventBody(TRACE_SOURCE);
diff --git a/packages/app/src/components/OnboardingModal.tsx b/packages/app/src/components/OnboardingModal.tsx
index de7648fde3..dfccade434 100644
--- a/packages/app/src/components/OnboardingModal.tsx
+++ b/packages/app/src/components/OnboardingModal.tsx
@@ -15,11 +15,17 @@ import { notifications } from '@mantine/notifications';
import { IconArrowLeft } from '@tabler/icons-react';
import { ConnectionForm } from '@/components/ConnectionForm';
-import { IS_CLICKHOUSE_BUILD, IS_LOCAL_MODE } from '@/config';
+import {
+ IS_CLICKHOUSE_BUILD,
+ IS_LOCAL_MODE,
+ IS_PROMQL_ENABLED,
+} from '@/config';
import { useConnections, useCreateConnection } from '@/connection';
import { useMetadataWithSettings } from '@/hooks/useMetadata';
import {
inferTableSourceConfig,
+ pickTimeSeriesTable,
+ PROMQL_TIMESTAMP_EXPRESSION,
useCreateSource,
useDeleteSource,
useSources,
@@ -309,10 +315,25 @@ function OnboardingModalComponent({
setIsAutoDetecting(true);
setHasAutodetected(true);
- // Try to detect OTEL tables
- const otelTables = await metadata.getOtelTables({ connectionId });
+ const getTimeSeriesTables = IS_PROMQL_ENABLED
+ ? metadata.getTimeSeriesTables({ connectionId }).catch(err => {
+ console.error('Error detecting TimeSeries tables:', err);
+ return [];
+ })
+ : [];
+
+ // Try to detect OTEL tables, and a TimeSeries table for PromQL
+ const [otelTables, timeSeriesTables] = await Promise.all([
+ metadata.getOtelTables({ connectionId }),
+ getTimeSeriesTables,
+ ]);
+
+ const timeSeriesTable = pickTimeSeriesTable(
+ timeSeriesTables,
+ otelTables?.database,
+ );
- if (!otelTables) {
+ if (!otelTables && !timeSeriesTable) {
// No tables detected, go to manual source setup
setStep('source');
return;
@@ -321,7 +342,7 @@ function OnboardingModalComponent({
const createdSources: TSource[] = [];
// Create Log Source if available
- if (otelTables.tables.logs) {
+ if (otelTables?.tables.logs) {
const inferredConfig = await inferTableSourceConfig({
kind: SourceKind.Log,
databaseName: otelTables.database,
@@ -359,7 +380,7 @@ function OnboardingModalComponent({
}
// Create Trace Source if available
- if (otelTables.tables.traces) {
+ if (otelTables?.tables.traces) {
const inferredConfig = await inferTableSourceConfig({
kind: SourceKind.Trace,
databaseName: otelTables.database,
@@ -406,9 +427,9 @@ function OnboardingModalComponent({
}
// Create Metrics Source if any metrics tables are available
- const hasMetrics = Object.values(otelTables.tables.metrics).some(
- t => t != null,
- );
+ const hasMetrics =
+ otelTables != null &&
+ Object.values(otelTables.tables.metrics).some(t => t != null);
if (hasMetrics) {
const metricTables: MetricTable = {
[MetricsDataType.Gauge]: '',
@@ -455,7 +476,7 @@ function OnboardingModalComponent({
}
// Create Session Source if available
- if (otelTables.tables.sessions) {
+ if (otelTables?.tables.sessions) {
const inferredConfig = await inferTableSourceConfig({
kind: SourceKind.Session,
databaseName: otelTables.database,
@@ -495,6 +516,24 @@ function OnboardingModalComponent({
}
}
+ // Create PromQL Source if a TimeSeries table is available
+ if (timeSeriesTable) {
+ try {
+ const promqlSource = await createSourceMutation.mutateAsync({
+ source: {
+ kind: SourceKind.Promql,
+ name: 'PromQL',
+ connection: connectionId,
+ from: timeSeriesTable,
+ timestampValueExpression: PROMQL_TIMESTAMP_EXPRESSION,
+ },
+ });
+ createdSources.push(promqlSource);
+ } catch (err) {
+ console.error('Error creating PromQL source:', err);
+ }
+ }
+
if (createdSources.length === 0) {
console.error('No sources created due to missing required fields');
// No sources created, go to manual source setup
diff --git a/packages/app/src/components/Sources/SourceForm/PromqlTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/PromqlTableModelForm.tsx
index ff3df91ae5..2f228c4583 100644
--- a/packages/app/src/components/Sources/SourceForm/PromqlTableModelForm.tsx
+++ b/packages/app/src/components/Sources/SourceForm/PromqlTableModelForm.tsx
@@ -1,5 +1,7 @@
import { useEffect } from 'react';
+import { PROMQL_TIMESTAMP_EXPRESSION } from '@/source';
+
import { MinAutoGranularityFormRow } from './MinAutoGranularityFormRow';
import { TableModelProps } from './types';
@@ -8,7 +10,7 @@ const PROMQL_MIN_AUTO_GRANULARITY_HELP_TEXT =
export function PromqlTableModelForm({ control, setValue }: TableModelProps) {
useEffect(() => {
- setValue('timestampValueExpression' as any, 'timestamp');
+ setValue('timestampValueExpression' as any, PROMQL_TIMESTAMP_EXPRESSION);
}, [setValue]);
// PromQL sources use the standard database + table fields from BaseSourceSchema.
diff --git a/packages/app/src/components/__tests__/OnboardingModal.test.tsx b/packages/app/src/components/__tests__/OnboardingModal.test.tsx
new file mode 100644
index 0000000000..3a7407eb26
--- /dev/null
+++ b/packages/app/src/components/__tests__/OnboardingModal.test.tsx
@@ -0,0 +1,176 @@
+import { SourceKind, TSourceNoId } from '@hyperdx/common-utils/dist/types';
+import { notifications } from '@mantine/notifications';
+import { screen } from '@testing-library/react';
+
+import OnboardingModal from '@/components/OnboardingModal';
+
+const mockConfig = { IS_PROMQL_ENABLED: true };
+jest.mock('@/config', () => ({
+ IS_CLICKHOUSE_BUILD: false,
+ IS_LOCAL_MODE: false,
+ get IS_PROMQL_ENABLED() {
+ return mockConfig.IS_PROMQL_ENABLED;
+ },
+}));
+
+const mockCreateSource = jest.fn();
+const mockUpdateSource = jest.fn();
+// Stable references: new objects on every render would re-run the modal's
+// effects, which depend on them.
+const mockSourcesResult = { data: [] };
+const mockCreateSourceMutation = { mutateAsync: mockCreateSource };
+const mockUpdateSourceMutation = { mutateAsync: mockUpdateSource };
+jest.mock('@/source', () => ({
+ pickTimeSeriesTable: jest.requireActual('@/source').pickTimeSeriesTable,
+ PROMQL_TIMESTAMP_EXPRESSION:
+ jest.requireActual('@/source').PROMQL_TIMESTAMP_EXPRESSION,
+ inferTableSourceConfig: async ({ kind }: { kind: SourceKind }) => ({
+ kind,
+ timestampValueExpression: 'Timestamp',
+ defaultTableSelectExpression: 'Timestamp, Body',
+ }),
+ useSources: () => mockSourcesResult,
+ useCreateSource: () => mockCreateSourceMutation,
+ useUpdateSource: () => mockUpdateSourceMutation,
+ useDeleteSource: () => ({ mutateAsync: jest.fn() }),
+}));
+
+const mockConnectionsResult = {
+ data: [{ id: 'conn-1', name: 'Default', host: 'http://localhost:8123' }],
+};
+jest.mock('@/connection', () => ({
+ useConnections: () => mockConnectionsResult,
+ useCreateConnection: () => ({ mutate: jest.fn() }),
+}));
+
+const mockMetadata = {
+ getOtelTables: jest.fn(),
+ getTimeSeriesTables: jest.fn(),
+};
+jest.mock('@/hooks/useMetadata', () => ({
+ useMetadataWithSettings: () => mockMetadata,
+}));
+
+jest.mock('@/theme/ThemeProvider', () => ({
+ useBrandDisplayName: () => 'HyperDX',
+}));
+jest.mock('@/components/ConnectionForm', () => ({
+ ConnectionForm: () => null,
+}));
+jest.mock('@/components/Sources/SourceForm', () => ({
+ TableSourceForm: () => null,
+}));
+jest.mock('@/components/Sources/SourcesList', () => ({
+ SourcesList: () => null,
+}));
+
+const OTEL_LOGS_ONLY = {
+ database: 'otel',
+ tables: { logs: 'otel_logs', metrics: {} },
+};
+
+const createdKinds = () =>
+ mockCreateSource.mock.calls.map(
+ ([{ source }]: [{ source: TSourceNoId }]) => source.kind,
+ );
+
+describe('OnboardingModal auto-detection', () => {
+ afterEach(() => {
+ jest.restoreAllMocks();
+ });
+
+ beforeEach(() => {
+ // The notifications store is global, so earlier tests' toasts linger.
+ notifications.clean();
+ jest.clearAllMocks();
+ mockConfig.IS_PROMQL_ENABLED = true;
+ mockCreateSource.mockImplementation(
+ async ({ source }: { source: TSourceNoId }) => ({
+ ...source,
+ id: `${source.kind}-id`,
+ }),
+ );
+ mockUpdateSource.mockResolvedValue(undefined);
+ mockMetadata.getOtelTables.mockResolvedValue(OTEL_LOGS_ONLY);
+ mockMetadata.getTimeSeriesTables.mockResolvedValue([
+ { databaseName: 'default', tableName: 'metrics_ts' },
+ ]);
+ });
+
+ it('creates a PromQL source for a detected TimeSeries table', async () => {
+ renderWithMantine();
+
+ await screen.findByText('Automatically detected and created 2 sources.');
+ expect(mockMetadata.getTimeSeriesTables).toHaveBeenCalledWith({
+ connectionId: 'conn-1',
+ });
+ expect(createdKinds()).toEqual([SourceKind.Log, SourceKind.Promql]);
+ expect(mockCreateSource).toHaveBeenLastCalledWith({
+ source: {
+ kind: SourceKind.Promql,
+ name: 'PromQL',
+ connection: 'conn-1',
+ from: { databaseName: 'default', tableName: 'metrics_ts' },
+ timestampValueExpression: 'timestamp',
+ },
+ });
+ });
+
+ it('skips PromQL detection when PromQL is disabled', async () => {
+ mockConfig.IS_PROMQL_ENABLED = false;
+
+ renderWithMantine();
+
+ await screen.findByText('Automatically detected and created 1 source.');
+ expect(mockMetadata.getTimeSeriesTables).not.toHaveBeenCalled();
+ expect(createdKinds()).toEqual([SourceKind.Log]);
+ });
+
+ it('creates a PromQL source when no OTel tables exist', async () => {
+ mockMetadata.getOtelTables.mockResolvedValue(null);
+
+ renderWithMantine();
+
+ await screen.findByText('Automatically detected and created 1 source.');
+ expect(createdKinds()).toEqual([SourceKind.Promql]);
+ });
+
+ it('keeps the other sources when the PromQL source fails', async () => {
+ jest.spyOn(console, 'error').mockImplementation(() => {});
+ mockCreateSource.mockImplementation(
+ async ({ source }: { source: TSourceNoId }) => {
+ if (source.kind === SourceKind.Promql) {
+ throw new Error('boom');
+ }
+ return { ...source, id: `${source.kind}-id` };
+ },
+ );
+
+ renderWithMantine();
+
+ await screen.findByText('Automatically detected and created 1 source.');
+ expect(createdKinds()).toEqual([SourceKind.Log, SourceKind.Promql]);
+ });
+
+ it('keeps OTel detection when TimeSeries detection fails', async () => {
+ jest.spyOn(console, 'error').mockImplementation(() => {});
+ mockMetadata.getTimeSeriesTables.mockRejectedValue(new Error('boom'));
+
+ renderWithMantine();
+
+ await screen.findByText('Automatically detected and created 1 source.');
+ expect(createdKinds()).toEqual([SourceKind.Log]);
+ });
+
+ it('falls back to manual setup when nothing is detected', async () => {
+ mockMetadata.getOtelTables.mockResolvedValue(null);
+ mockMetadata.getTimeSeriesTables.mockResolvedValue([]);
+
+ renderWithMantine();
+
+ await screen.findByText(
+ 'Lets set up a source table to query telemetry from.',
+ );
+ expect(mockCreateSource).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts
index e16093f067..2479582e63 100644
--- a/packages/app/src/source.ts
+++ b/packages/app/src/source.ts
@@ -480,6 +480,47 @@ export async function inferTableSourceConfig({
};
}
+// The TimeSeries table the ClickHouse Prometheus docs tell users to create,
+// which is what ClickHouse Cloud PromQL preview services will likely have.
+// https://clickhouse.com/docs/concepts/features/interfaces/prometheus#create-a-timeseries-table
+const DOCS_TIME_SERIES_TABLE = {
+ databaseName: 'prometheus',
+ tableName: 'metrics',
+};
+
+// The TimeSeries table name used by the collector's PromQL schema migration
+// and the dev ClickHouse remote-write handler.
+const COLLECTOR_TIME_SERIES_TABLE_NAME = 'metrics_ts';
+
+// The time column of a TimeSeries table's data inner table, which every PromQL
+// source uses.
+export const PROMQL_TIMESTAMP_EXPRESSION = 'timestamp';
+
+/**
+ * Picks the TimeSeries table to back an auto-provisioned PromQL source. Prefers
+ * the docs' `prometheus.metrics`, then the collector's `metrics_ts` in
+ * `preferredDatabase` (the OTel tables' database), then `metrics_ts` anywhere.
+ * Without either name, a sole TimeSeries table is still picked.
+ */
+export function pickTimeSeriesTable(
+ tables: TPromqlSource['from'][],
+ preferredDatabase?: string,
+): TPromqlSource['from'] | undefined {
+ const collectorNamed = tables.filter(
+ t => t.tableName === COLLECTOR_TIME_SERIES_TABLE_NAME,
+ );
+ return (
+ tables.find(
+ t =>
+ t.databaseName === DOCS_TIME_SERIES_TABLE.databaseName &&
+ t.tableName === DOCS_TIME_SERIES_TABLE.tableName,
+ ) ??
+ collectorNamed.find(t => t.databaseName === preferredDatabase) ??
+ collectorNamed[0] ??
+ (tables.length === 1 ? tables[0] : undefined)
+ );
+}
+
export function getDurationMsExpression(source: TTraceSource) {
return `(${source.durationExpression})/1e${(source.durationPrecision ?? 9) - 3}`;
}
diff --git a/packages/common-utils/src/__tests__/metadata.int.test.ts b/packages/common-utils/src/__tests__/metadata.int.test.ts
index a780102bd1..e27a2ccc0e 100644
--- a/packages/common-utils/src/__tests__/metadata.int.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.int.test.ts
@@ -1307,6 +1307,47 @@ describe('Metadata Integration Tests', () => {
});
});
+ describe('getTimeSeriesTables', () => {
+ const database = 'test_ts_discovery';
+
+ const timeSeriesCommand = (query: string) =>
+ client.command({
+ query,
+ clickhouse_settings: { allow_experimental_time_series_table: 1 },
+ });
+
+ beforeAll(async () => {
+ await timeSeriesCommand(`DROP DATABASE IF EXISTS ${database}`);
+ await timeSeriesCommand(`CREATE DATABASE ${database}`);
+ await timeSeriesCommand(
+ `CREATE TABLE ${database}.metrics_ts ENGINE = TimeSeries`,
+ );
+ await timeSeriesCommand(
+ `CREATE TABLE ${database}.custom_ts ENGINE = TimeSeries`,
+ );
+ await timeSeriesCommand(
+ `CREATE TABLE ${database}.not_ts (ts DateTime) ENGINE = MergeTree ORDER BY ts`,
+ );
+ });
+
+ afterAll(async () => {
+ await timeSeriesCommand(`DROP DATABASE IF EXISTS ${database}`);
+ });
+
+ it('lists only TimeSeries tables, without their inner tables', async () => {
+ const metadata = new Metadata(hdxClient, new MetadataCache());
+
+ const tables = await metadata.getTimeSeriesTables({
+ connectionId: 'test_connection',
+ });
+
+ expect(tables.filter(t => t.databaseName === database)).toEqual([
+ { databaseName: database, tableName: 'custom_ts' },
+ { databaseName: database, tableName: 'metrics_ts' },
+ ]);
+ });
+ });
+
describe('getMapKeys - unbounded scan guard (#3037)', () => {
const tableName = 'test_map_keys_scan';
let metadata: Metadata;
diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts
index d00818f0e9..207cc3abcb 100644
--- a/packages/common-utils/src/__tests__/metadata.test.ts
+++ b/packages/common-utils/src/__tests__/metadata.test.ts
@@ -3449,6 +3449,57 @@ describe('Metadata', () => {
});
});
+ describe('getTimeSeriesTables', () => {
+ beforeEach(() => {
+ mockCache.getOrFetch.mockImplementation((key, queryFn) => queryFn());
+ });
+
+ it('returns the TimeSeries tables from system.tables', async () => {
+ const tables = [
+ { databaseName: 'default', tableName: 'metrics_ts' },
+ { databaseName: 'otel', tableName: 'prom' },
+ ];
+ (mockClickhouseClient.query as jest.Mock).mockResolvedValue({
+ json: jest.fn().mockResolvedValue({ data: tables }),
+ });
+
+ const result = await metadata.getTimeSeriesTables({
+ connectionId: 'test_connection',
+ });
+
+ expect(result).toEqual(tables);
+ const { query } = (mockClickhouseClient.query as jest.Mock).mock
+ .calls[0][0];
+ expect(query).toContain("engine = 'TimeSeries'");
+ expect(mockCache.getOrFetch).toHaveBeenCalledWith(
+ 'test_connection.timeSeriesTables',
+ expect.any(Function),
+ );
+ });
+
+ it('returns an empty list when permissions error occurs', async () => {
+ (mockClickhouseClient.query as jest.Mock).mockRejectedValue(
+ new Error('Not enough privileges'),
+ );
+
+ const result = await metadata.getTimeSeriesTables({
+ connectionId: 'test_connection',
+ });
+
+ expect(result).toEqual([]);
+ });
+
+ it('rethrows other errors', async () => {
+ (mockClickhouseClient.query as jest.Mock).mockRejectedValue(
+ new Error('Connection refused'),
+ );
+
+ await expect(
+ metadata.getTimeSeriesTables({ connectionId: 'test_connection' }),
+ ).rejects.toThrow('Connection refused');
+ });
+ });
+
describe('doMetadataMVsAggregateColumn (word-boundary token match)', () => {
const tableConn = {
databaseName: 'default',
diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts
index 662fa5cba7..52429d6e4f 100644
--- a/packages/common-utils/src/core/metadata.ts
+++ b/packages/common-utils/src/core/metadata.ts
@@ -653,6 +653,50 @@ export class Metadata {
);
}
+ async getTimeSeriesTables({
+ connectionId,
+ }: {
+ connectionId: string;
+ }): Promise[]> {
+ return this.cache.getOrFetch(
+ `${connectionId}.timeSeriesTables`,
+ async () => {
+ const sql = chSql`
+ SELECT database AS databaseName, name AS tableName
+ FROM system.tables
+ WHERE engine = 'TimeSeries'
+ ORDER BY database, name
+ `;
+ try {
+ const json = await this.clickhouseClient
+ .query<'JSON'>({
+ connectionId,
+ query: sql.sql,
+ query_params: sql.params,
+ clickhouse_settings: this.getClickHouseSettings(),
+ })
+ .then(res =>
+ res.json>(),
+ );
+ return json.data;
+ } catch (e) {
+ if (
+ e instanceof Error &&
+ e.message.includes('Not enough privileges')
+ ) {
+ console.warn(
+ 'Not enough privileges to fetch TimeSeries tables:',
+ e,
+ );
+ return [];
+ }
+
+ throw e;
+ }
+ },
+ );
+ }
+
async getMaterializedColumnsLookupTable({
databaseName,
tableName,