Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changeset/promql-default-source-gate.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
'@hyperdx/api': patch
---

feat: only provision a default PromQL source when PromQL is enabled

`DEFAULT_SOURCES` entries of kind `promql` are now skipped unless
`ENABLE_PROMQL=true`, so a single sources list can include a PromQL source
without creating it on deployments that have PromQL turned off.
3 changes: 2 additions & 1 deletion packages/api/.env.development
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ USAGE_STATS_ENABLED=false
NODE_OPTIONS="--max-http-header-size=131072"
ENABLE_SWAGGER=true
DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}]
DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}]
DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"},{"kind":"promql","name":"PromQL","connection":"Local ClickHouse","from":{"databaseName":"default","tableName":"metrics_ts"},"timestampValueExpression":"timestamp"}]

INGESTION_API_KEY="super-secure-ingestion-api-key"
HYPERDX_API_KEY=$INGESTION_API_KEY
ANTHROPIC_API_KEY="your-anthropic-api-key-here"
Expand Down
87 changes: 87 additions & 0 deletions packages/api/src/__tests__/setupDefaults.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import { SourceKind } from '@hyperdx/common-utils/dist/types';

const configState = { IS_PROMQL_ENABLED: false };

jest.mock('@/config', () => ({
DEFAULT_CONNECTIONS: undefined,
DEFAULT_SOURCES: JSON.stringify([
{
kind: 'log',
name: 'Logs',
connection: 'Local ClickHouse',
from: { databaseName: 'default', tableName: 'otel_logs' },
timestampValueExpression: 'Timestamp',
defaultTableSelectExpression: 'Timestamp, Body',
},
{
kind: 'promql',
name: 'PromQL',
connection: 'Local ClickHouse',
from: { databaseName: 'default', tableName: 'metrics_ts' },
timestampValueExpression: 'timestamp',
},
]),
get IS_PROMQL_ENABLED() {
return configState.IS_PROMQL_ENABLED;
},
}));

jest.mock('@/controllers/team', () => ({
getTeam: jest.fn().mockResolvedValue({ _id: 'team-1' }),
}));
jest.mock('@/controllers/connection', () => ({
createConnection: jest.fn(),
getConnections: jest
.fn()
.mockResolvedValue([
{ _id: 'conn-1', team: 'team-1', name: 'Local ClickHouse' },
]),
}));
jest.mock('@/controllers/sources', () => ({
getSources: jest.fn().mockResolvedValue([]),
createSource: jest.fn(async (_teamId: string, source: { name: string }) => ({
_id: `${source.name}-id`,
toObject: () => source,
})),
updateSource: jest.fn(),
}));
jest.mock('@/utils/logger', () => ({
__esModule: true,
default: { info: jest.fn(), warn: jest.fn(), error: jest.fn() },
}));

import { createSource } from '@/controllers/sources';
import { setupTeamDefaults } from '@/setupDefaults';

const createdKinds = () =>
jest.mocked(createSource).mock.calls.map(([, source]) => source.kind);

describe('setupTeamDefaults', () => {
beforeEach(() => {
jest.mocked(createSource).mockClear();
});

it('creates a default PromQL source when PromQL is enabled', async () => {
configState.IS_PROMQL_ENABLED = true;

await setupTeamDefaults('team-1');

expect(createdKinds()).toEqual([SourceKind.Log, SourceKind.Promql]);
expect(createSource).toHaveBeenLastCalledWith(
'team-1',
expect.objectContaining({
kind: SourceKind.Promql,
connection: 'conn-1',
from: { databaseName: 'default', tableName: 'metrics_ts' },
}),
);
});

it('skips a default PromQL source when PromQL is disabled', async () => {
configState.IS_PROMQL_ENABLED = false;

await setupTeamDefaults('team-1');

expect(createdKinds()).toEqual([SourceKind.Log]);
});
});
15 changes: 14 additions & 1 deletion packages/api/src/setupDefaults.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
import { DEFAULT_CONNECTIONS, DEFAULT_SOURCES } from '@/config';
import { SourceKind } from '@hyperdx/common-utils/dist/types';

import {
DEFAULT_CONNECTIONS,
DEFAULT_SOURCES,
IS_PROMQL_ENABLED,
} from '@/config';
import { createConnection, getConnections } from '@/controllers/connection';
import { createSource, getSources, updateSource } from '@/controllers/sources';
import { getTeam } from '@/controllers/team';
Expand Down Expand Up @@ -122,6 +128,13 @@ export async function setupTeamDefaults(teamId: string) {
continue;
}

if (sourceConfig.kind === SourceKind.Promql && !IS_PROMQL_ENABLED) {
logger.info(
`Skipping default source ${sourceConfig.name}: PromQL is disabled`,
);
Comment on lines +132 to +134

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Skipped sources lack metrics

The new PromQL skip writes a countable log event but emits no metric or span attribute. The repository guide requires both for logged skips, so developers can count them and see the reason on the active span. Add these alongside the log before merging to satisfy that requirement.

Context Used: AGENTS.md (source)

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Fix in Claude Code Fix in Conductor Fix in Cursor Fix in Codex

continue;
}

// Store the config by name for later reference
sourceConfigsByName[sourceConfig.name] = sourceConfig;

Expand Down
Loading