Skip to content

Commit d5fe3ef

Browse files
committed
fix: support https for the PromQL remote-write endpoint
CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT now accepts an http:// or https:// URL as well as a bare host:port. entrypoint.sh splits off the scheme and passes it to the prometheusremotewrite exporter through HYPERDX_PROMETHEUS_SCHEME and HYPERDX_PROMETHEUS_TLS_INSECURE. Both the standalone and OpAMP-managed configs read these, with defaults that keep plain HTTP for older collector images. A bare host:port behaves as before. A path or an unsupported scheme now fails at startup with a clear error.
1 parent 1be9de8 commit d5fe3ef

10 files changed

Lines changed: 208 additions & 7 deletions

File tree

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
'@hyperdx/otel-collector': patch
3+
'@hyperdx/api': patch
4+
---
5+
6+
fix: support https for the PromQL remote-write endpoint
7+
8+
`CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT` now accepts an `http://` or `https://` URL as well as a bare `host:port`. With `https://`, the collector's `prometheusremotewrite` exporter sends metrics over TLS and verifies the server certificate against the system CAs. Previously the exporter always used plain HTTP, so an HTTPS-only endpoint could not receive metrics. This applies to both standalone and OpAMP-managed collectors.
9+
10+
A bare `host:port` keeps plain HTTP, so existing setups are unchanged. The value must not include a path (`/write` is still appended automatically); the collector now exits at startup with a clear error if it does, or if the scheme is anything other than `http` or `https`.

‎docker-compose.dev.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ services:
3131
OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION}
3232
environment:
3333
CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
34+
# host:port (plain HTTP), http://host:port or https://host:port
3435
CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
3536
HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
3637
HYPERDX_API_KEY: ${HYPERDX_API_KEY}

‎docker/otel-collector/config.standalone.promql.yaml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,21 @@
11
# Enables remote-write of OTLP metrics to a Prometheus-compatible endpoint.
22
# Only included when CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT is set.
33
#
4+
# entrypoint.sh strips any http:// or https:// scheme from
5+
# CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT and exports it as
6+
# HYPERDX_PROMETHEUS_SCHEME / HYPERDX_PROMETHEUS_TLS_INSECURE. The defaults
7+
# keep plain HTTP if those are unset.
8+
#
49
# Mirrors the OpAMP-managed behavior in
510
# packages/api/src/opamp/controllers/opampController.ts (gated on
611
# IS_PROMQL_ENABLED). When updating this file, keep it in sync with
712
# buildOtelCollectorConfig().
813

914
exporters:
1015
prometheusremotewrite:
11-
endpoint: http://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write
16+
endpoint: ${env:HYPERDX_PROMETHEUS_SCHEME:-http}://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write
1217
tls:
13-
insecure: true
18+
insecure: ${env:HYPERDX_PROMETHEUS_TLS_INSECURE:-true}
1419
resource_to_telemetry_conversion:
1520
enabled: true
1621

‎docker/otel-collector/entrypoint.sh‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,43 @@ if [ "$HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE" = "true" ]; then
2424
export HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA=true
2525
fi
2626

27+
# CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT accepts a bare host:port (plain HTTP,
28+
# the original format) or an http:// or https:// URL. Both the standalone
29+
# config and the OpAMP-managed config (opampController.ts) need it as
30+
# host:port, because it is also a Prometheus scrape target, so the scheme is
31+
# split off here and handed to the prometheusremotewrite exporter through
32+
# HYPERDX_PROMETHEUS_SCHEME and HYPERDX_PROMETHEUS_TLS_INSECURE. Runs before
33+
# migrations so a bad value fails fast without touching ClickHouse.
34+
prometheus_endpoint_error() {
35+
echo "ERROR: CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT $1. Use host:port, http://host:port or https://host:port." >&2
36+
exit 1
37+
}
38+
39+
if [ -n "$CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT" ]; then
40+
HYPERDX_PROMETHEUS_SCHEME=http
41+
HYPERDX_PROMETHEUS_TLS_INSECURE=true
42+
case "$CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT" in
43+
https://*)
44+
HYPERDX_PROMETHEUS_SCHEME=https
45+
HYPERDX_PROMETHEUS_TLS_INSECURE=false
46+
;;
47+
http://*) ;;
48+
# Only the scheme is echoed back: the rest of the URL may carry credentials.
49+
*://*) prometheus_endpoint_error "has an unsupported scheme '${CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT%%://*}'" ;;
50+
esac
51+
52+
PROMETHEUS_HOST_PORT="${CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT#*://}"
53+
PROMETHEUS_HOST_PORT="${PROMETHEUS_HOST_PORT%/}"
54+
case "$PROMETHEUS_HOST_PORT" in
55+
"") prometheus_endpoint_error "has no host" ;;
56+
*/*) prometheus_endpoint_error "must not include a path (/write is appended automatically)" ;;
57+
esac
58+
59+
export CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT="$PROMETHEUS_HOST_PORT"
60+
export HYPERDX_PROMETHEUS_SCHEME HYPERDX_PROMETHEUS_TLS_INSECURE
61+
echo "CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT scheme: $HYPERDX_PROMETHEUS_SCHEME"
62+
fi
63+
2764
# Run ClickHouse schema migrations if not using legacy schema creation
2865
if [ "$HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA" != "true" ]; then
2966
# Run Go-based migrate tool with TLS support

‎packages/api/src/opamp/controllers/__tests__/opampController.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,24 @@ describe('opampController', () => {
6464
});
6565
});
6666

67+
describe('buildOtelCollectorConfig prometheusremotewrite exporter', () => {
68+
it('takes the scheme and TLS mode from the collector environment, defaulting to plain HTTP', () => {
69+
configState.IS_PROMQL_ENABLED = true;
70+
71+
const cfg = buildOtelCollectorConfig([
72+
{ apiKey: 'k1', collectorAuthenticationEnforced: false },
73+
]);
74+
75+
// The defaults matter: collector images older than this change don't
76+
// export HYPERDX_PROMETHEUS_*, and must keep today's plain-HTTP config.
77+
expect(cfg.exporters?.prometheusremotewrite).toMatchObject({
78+
endpoint:
79+
'${env:HYPERDX_PROMETHEUS_SCHEME:-http}://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write',
80+
tls: { insecure: '${env:HYPERDX_PROMETHEUS_TLS_INSECURE:-true}' },
81+
});
82+
});
83+
});
84+
6785
describe('buildOtelCollectorConfig otlp/hyperdx authentication', () => {
6886
it('accepts bare and bearer-prefixed authorization values for each API key', () => {
6987
const cfg = buildOtelCollectorConfig([

‎packages/api/src/opamp/controllers/opampController.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ type CollectorConfig = {
149149
prometheusremotewrite?: {
150150
endpoint: string;
151151
tls: {
152-
insecure: boolean;
152+
insecure: string;
153153
};
154154
resource_to_telemetry_conversion: {
155155
enabled: boolean;
@@ -336,10 +336,14 @@ export const buildOtelCollectorConfig = (
336336
);
337337

338338
if (config.IS_PROMQL_ENABLED && otelCollectorConfig.exporters) {
339+
// The collector's entrypoint.sh derives the scheme and TLS mode from
340+
// CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT. The defaults keep plain HTTP
341+
// for collector images that predate it.
339342
otelCollectorConfig.exporters.prometheusremotewrite = {
340-
endpoint: 'http://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write',
343+
endpoint:
344+
'${env:HYPERDX_PROMETHEUS_SCHEME:-http}://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write',
341345
tls: {
342-
insecure: true,
346+
insecure: '${env:HYPERDX_PROMETHEUS_TLS_INSECURE:-true}',
343347
},
344348
resource_to_telemetry_conversion: {
345349
enabled: true,
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"resourceMetrics": [
3+
{
4+
"resource": {
5+
"attributes": [
6+
{
7+
"key": "suite-id",
8+
"value": {
9+
"stringValue": "promql-remote-write"
10+
}
11+
},
12+
{
13+
"key": "service.name",
14+
"value": {
15+
"stringValue": "promql-remote-write-test-service"
16+
}
17+
}
18+
]
19+
},
20+
"scopeMetrics": [
21+
{
22+
"scope": {},
23+
"metrics": [
24+
{
25+
"name": "smoke.promql.url.gauge",
26+
"gauge": {
27+
"dataPoints": [
28+
{
29+
"timeUnixNano": "1901999590000000000",
30+
"asDouble": 42
31+
}
32+
]
33+
}
34+
}
35+
]
36+
}
37+
]
38+
}
39+
]
40+
}

‎smoke-tests/otel-collector/docker-compose.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,56 @@ services:
108108
ch-server:
109109
condition: service_healthy
110110

111+
# HDX-4633: CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT given as a URL rather than
112+
# a bare host:port. entrypoint.sh splits the scheme off for the
113+
# prometheusremotewrite exporter; see promql-remote-write.bats.
114+
otel-collector-promql-url:
115+
build:
116+
context: ../..
117+
dockerfile: docker/otel-collector/Dockerfile
118+
target: dev
119+
args:
120+
OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-0.155.0}
121+
OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-1.61.0}
122+
environment:
123+
- CLICKHOUSE_ENDPOINT=tcp://ch-server:9000?dial_timeout=10s
124+
- CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=http://ch-server:9363/
125+
- CLICKHOUSE_USER=default
126+
- CLICKHOUSE_PASSWORD=
127+
- HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=promql_url
128+
- HYPERDX_LOG_LEVEL=info
129+
- HYPERDX_OTEL_BATCH_TIMEOUT=100ms
130+
# OPAMP_SERVER_URL is intentionally not set to run in standalone mode
131+
ports:
132+
- 64318:4318 # OTLP http receiver
133+
networks:
134+
- internal
135+
depends_on:
136+
ch-server:
137+
condition: service_healthy
138+
139+
# HDX-4633: a path in CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT can't be used as
140+
# a scrape target, so entrypoint.sh rejects it before migrations. Expected to
141+
# exit immediately; see promql-remote-write.bats. CLICKHOUSE_ENDPOINT is
142+
# unreachable on purpose, as in otel-collector-oidc-missing-audience.
143+
otel-collector-promql-bad-endpoint:
144+
build:
145+
context: ../..
146+
dockerfile: docker/otel-collector/Dockerfile
147+
target: dev
148+
args:
149+
OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-0.155.0}
150+
OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-1.61.0}
151+
environment:
152+
- CLICKHOUSE_ENDPOINT=tcp://127.0.0.1:1/?dial_timeout=1s
153+
- CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=https://ch-server:9363/write
154+
- CLICKHOUSE_USER=default
155+
- CLICKHOUSE_PASSWORD=
156+
- HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=promql_bad_endpoint
157+
- HYPERDX_LOG_LEVEL=info
158+
networks:
159+
- internal
160+
111161
# Older ClickHouse (< 26.2) to test the compatibility schema (bloom filters
112162
# instead of full text indexes)
113163
ch-server-compat:
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/usr/bin/env bats
2+
3+
# HDX-4633: CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT may be a URL. entrypoint.sh
4+
# splits the scheme off for the prometheusremotewrite exporter
5+
# (config.standalone.promql.yaml) and keeps host:port for everything else.
6+
7+
load 'test_helpers/utilities.bash'
8+
load 'test_helpers/assertions.bash'
9+
10+
@test "PromQL: an http:// endpoint URL still remote-writes metrics into the TimeSeries table" {
11+
run docker compose logs otel-collector-promql-url
12+
[ "$status" -eq 0 ]
13+
[[ "$output" == *"CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT scheme: http"* ]]
14+
15+
# ClickHouse's remote_write handler (docker/clickhouse/local/config.xml)
16+
# always writes to default.metrics_ts, but the collector only migrates that
17+
# table with ENABLE_PROMQL=true and into its own database. Creating it here
18+
# avoids racing the otel-collector service's migrations on `default`.
19+
run clickhouse-client --port=9000 --query="CREATE TABLE IF NOT EXISTS default.metrics_ts ENGINE = TimeSeries SETTINGS allow_experimental_time_series_table = 1"
20+
[ "$status" -eq 0 ]
21+
22+
emit_otel_data "http://localhost:64318" "data/metrics/promql-remote-write" "metrics"
23+
wait_for_rows 9000 "SELECT count() FROM timeSeriesTags(default, metrics_ts) WHERE metric_name = 'smoke_promql_url_gauge' SETTINGS allow_experimental_time_series_table = 1" 1
24+
}
25+
26+
@test "PromQL: collector fails fast with a clear error when the endpoint includes a path" {
27+
run docker compose logs otel-collector-promql-bad-endpoint
28+
[ "$status" -eq 0 ]
29+
[[ "$output" == *"CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT must not include a path"* ]]
30+
31+
run docker compose ps --status running --services
32+
[ "$status" -eq 0 ]
33+
[[ "$output" != *"otel-collector-promql-bad-endpoint"* ]]
34+
}

‎smoke-tests/otel-collector/setup_suite.bash‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@ setup_suite() {
1717
wait_for_ready "otel-collector-custom"
1818
wait_for_ready "otel-collector-oidc"
1919
wait_for_ready "otel-collector-bearer-auth"
20-
# otel-collector-oidc-missing-audience is expected to exit immediately,
21-
# not become ready -- checked directly in oidc-auth.bats instead.
20+
wait_for_ready "otel-collector-promql-url"
21+
# otel-collector-oidc-missing-audience and otel-collector-promql-bad-endpoint
22+
# are expected to exit immediately, not become ready -- checked directly in
23+
# oidc-auth.bats and promql-remote-write.bats instead.
2224
}
2325

2426
teardown_suite() {

0 commit comments

Comments
 (0)