diff --git a/.changeset/promql-default-source-gate.md b/.changeset/promql-default-source-gate.md new file mode 100644 index 0000000000..98d1a010ed --- /dev/null +++ b/.changeset/promql-default-source-gate.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/api': patch +--- + +feat: only provision a default PromQL source when PromQL is enabled + +`DEFAULT_SOURCES` entries of kind `promql` are now skipped unless +`ENABLE_PROMQL=true`, so a single sources list can include a PromQL source +without creating it on deployments that have PromQL turned off. diff --git a/packages/api/.env.development b/packages/api/.env.development index 9e15745242..d4995195fd 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -22,7 +22,8 @@ USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}] -DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] +DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"},{"kind":"promql","name":"PromQL","connection":"Local ClickHouse","from":{"databaseName":"default","tableName":"metrics_ts"},"timestampValueExpression":"timestamp"}] + INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY ANTHROPIC_API_KEY="your-anthropic-api-key-here" diff --git a/packages/api/src/__tests__/setupDefaults.test.ts b/packages/api/src/__tests__/setupDefaults.test.ts new file mode 100644 index 0000000000..f7b42bc64f --- /dev/null +++ b/packages/api/src/__tests__/setupDefaults.test.ts @@ -0,0 +1,87 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; + +const configState = { IS_PROMQL_ENABLED: false }; + +jest.mock('@/config', () => ({ + DEFAULT_CONNECTIONS: undefined, + DEFAULT_SOURCES: JSON.stringify([ + { + kind: 'log', + name: 'Logs', + connection: 'Local ClickHouse', + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: 'Timestamp, Body', + }, + { + kind: 'promql', + name: 'PromQL', + connection: 'Local ClickHouse', + from: { databaseName: 'default', tableName: 'metrics_ts' }, + timestampValueExpression: 'timestamp', + }, + ]), + get IS_PROMQL_ENABLED() { + return configState.IS_PROMQL_ENABLED; + }, +})); + +jest.mock('@/controllers/team', () => ({ + getTeam: jest.fn().mockResolvedValue({ _id: 'team-1' }), +})); +jest.mock('@/controllers/connection', () => ({ + createConnection: jest.fn(), + getConnections: jest + .fn() + .mockResolvedValue([ + { _id: 'conn-1', team: 'team-1', name: 'Local ClickHouse' }, + ]), +})); +jest.mock('@/controllers/sources', () => ({ + getSources: jest.fn().mockResolvedValue([]), + createSource: jest.fn(async (_teamId: string, source: { name: string }) => ({ + _id: `${source.name}-id`, + toObject: () => source, + })), + updateSource: jest.fn(), +})); +jest.mock('@/utils/logger', () => ({ + __esModule: true, + default: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})); + +import { createSource } from '@/controllers/sources'; +import { setupTeamDefaults } from '@/setupDefaults'; + +const createdKinds = () => + jest.mocked(createSource).mock.calls.map(([, source]) => source.kind); + +describe('setupTeamDefaults', () => { + beforeEach(() => { + jest.mocked(createSource).mockClear(); + }); + + it('creates a default PromQL source when PromQL is enabled', async () => { + configState.IS_PROMQL_ENABLED = true; + + await setupTeamDefaults('team-1'); + + expect(createdKinds()).toEqual([SourceKind.Log, SourceKind.Promql]); + expect(createSource).toHaveBeenLastCalledWith( + 'team-1', + expect.objectContaining({ + kind: SourceKind.Promql, + connection: 'conn-1', + from: { databaseName: 'default', tableName: 'metrics_ts' }, + }), + ); + }); + + it('skips a default PromQL source when PromQL is disabled', async () => { + configState.IS_PROMQL_ENABLED = false; + + await setupTeamDefaults('team-1'); + + expect(createdKinds()).toEqual([SourceKind.Log]); + }); +}); diff --git a/packages/api/src/setupDefaults.ts b/packages/api/src/setupDefaults.ts index f4584c92ff..afab8f99a6 100644 --- a/packages/api/src/setupDefaults.ts +++ b/packages/api/src/setupDefaults.ts @@ -1,4 +1,10 @@ -import { DEFAULT_CONNECTIONS, DEFAULT_SOURCES } from '@/config'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { + DEFAULT_CONNECTIONS, + DEFAULT_SOURCES, + IS_PROMQL_ENABLED, +} from '@/config'; import { createConnection, getConnections } from '@/controllers/connection'; import { createSource, getSources, updateSource } from '@/controllers/sources'; import { getTeam } from '@/controllers/team'; @@ -122,6 +128,13 @@ export async function setupTeamDefaults(teamId: string) { continue; } + if (sourceConfig.kind === SourceKind.Promql && !IS_PROMQL_ENABLED) { + logger.info( + `Skipping default source ${sourceConfig.name}: PromQL is disabled`, + ); + continue; + } + // Store the config by name for later reference sourceConfigsByName[sourceConfig.name] = sourceConfig;