From 1cae7b0eeb6647abeb2ca997e40c613246179fed Mon Sep 17 00:00:00 2001 From: Karl Power Date: Thu, 8 Oct 2026 18:12:31 +0200 Subject: [PATCH 1/2] fix: support https for the PromQL remote-write endpoint CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT now accepts an http:// or https:// URL as well as a bare host:port. entrypoint.sh splits off the scheme and passes it to the prometheusremotewrite exporter through HYPERDX_PROMETHEUS_SCHEME and HYPERDX_PROMETHEUS_TLS_INSECURE. Both the standalone and OpAMP-managed configs read these, with defaults that keep plain HTTP for older collector images. A bare host:port behaves as before. A path or an unsupported scheme now fails at startup with a clear error. --- .changeset/promql-remote-write-https.md | 10 ++++ docker-compose.dev.yml | 1 + .../config.standalone.promql.yaml | 9 +++- docker/otel-collector/entrypoint.sh | 37 ++++++++++++++ .../__tests__/opampController.test.ts | 18 +++++++ .../src/opamp/controllers/opampController.ts | 10 ++-- .../metrics/promql-remote-write/input.json | 40 +++++++++++++++ .../otel-collector/docker-compose.yaml | 50 +++++++++++++++++++ .../otel-collector/promql-remote-write.bats | 34 +++++++++++++ smoke-tests/otel-collector/setup_suite.bash | 6 ++- 10 files changed, 208 insertions(+), 7 deletions(-) create mode 100644 .changeset/promql-remote-write-https.md create mode 100644 smoke-tests/otel-collector/data/metrics/promql-remote-write/input.json create mode 100644 smoke-tests/otel-collector/promql-remote-write.bats diff --git a/.changeset/promql-remote-write-https.md b/.changeset/promql-remote-write-https.md new file mode 100644 index 0000000000..5de82e9e19 --- /dev/null +++ b/.changeset/promql-remote-write-https.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/otel-collector': patch +'@hyperdx/api': patch +--- + +fix: support https for the PromQL remote-write endpoint + +`CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT` now accepts an `http://` or `https://` URL as well as a bare `host:port`. With `https://`, the collector's `prometheusremotewrite` exporter sends metrics over TLS and verifies the server certificate against the system CAs. Previously the exporter always used plain HTTP, so an HTTPS-only endpoint could not receive metrics. This applies to both standalone and OpAMP-managed collectors. + +A bare `host:port` keeps plain HTTP, so existing setups are unchanged. The value must not include a path (`/write` is still appended automatically); the collector now exits at startup with a clear error if it does, or if the scheme is anything other than `http` or `https`. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 9c3c0491ec..7f35762648 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -31,6 +31,7 @@ services: OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION} environment: CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s' + # host:port (plain HTTP), http://host:port or https://host:port CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363' HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE} HYPERDX_API_KEY: ${HYPERDX_API_KEY} diff --git a/docker/otel-collector/config.standalone.promql.yaml b/docker/otel-collector/config.standalone.promql.yaml index d2af395930..f6f10a5125 100644 --- a/docker/otel-collector/config.standalone.promql.yaml +++ b/docker/otel-collector/config.standalone.promql.yaml @@ -1,6 +1,11 @@ # Enables remote-write of OTLP metrics to a Prometheus-compatible endpoint. # Only included when CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT is set. # +# entrypoint.sh strips any http:// or https:// scheme from +# CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT and exports it as +# HYPERDX_PROMETHEUS_SCHEME / HYPERDX_PROMETHEUS_TLS_INSECURE. The defaults +# keep plain HTTP if those are unset. +# # Mirrors the OpAMP-managed behavior in # packages/api/src/opamp/controllers/opampController.ts (gated on # IS_PROMQL_ENABLED). When updating this file, keep it in sync with @@ -8,9 +13,9 @@ exporters: prometheusremotewrite: - endpoint: http://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write + endpoint: ${env:HYPERDX_PROMETHEUS_SCHEME:-http}://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write tls: - insecure: true + insecure: ${env:HYPERDX_PROMETHEUS_TLS_INSECURE:-true} resource_to_telemetry_conversion: enabled: true diff --git a/docker/otel-collector/entrypoint.sh b/docker/otel-collector/entrypoint.sh index 8995f71c06..058b94ec41 100644 --- a/docker/otel-collector/entrypoint.sh +++ b/docker/otel-collector/entrypoint.sh @@ -24,6 +24,43 @@ if [ "$HYPERDX_OTEL_EXPORTER_CLICKHOUSE_JSON_ENABLE" = "true" ]; then export HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA=true fi +# CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT accepts a bare host:port (plain HTTP, +# the original format) or an http:// or https:// URL. Both the standalone +# config and the OpAMP-managed config (opampController.ts) need it as +# host:port, because it is also a Prometheus scrape target, so the scheme is +# split off here and handed to the prometheusremotewrite exporter through +# HYPERDX_PROMETHEUS_SCHEME and HYPERDX_PROMETHEUS_TLS_INSECURE. Runs before +# migrations so a bad value fails fast without touching ClickHouse. +prometheus_endpoint_error() { + echo "ERROR: CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT $1. Use host:port, http://host:port or https://host:port." >&2 + exit 1 +} + +if [ -n "$CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT" ]; then + HYPERDX_PROMETHEUS_SCHEME=http + HYPERDX_PROMETHEUS_TLS_INSECURE=true + case "$CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT" in + https://*) + HYPERDX_PROMETHEUS_SCHEME=https + HYPERDX_PROMETHEUS_TLS_INSECURE=false + ;; + http://*) ;; + # Only the scheme is echoed back: the rest of the URL may carry credentials. + *://*) prometheus_endpoint_error "has an unsupported scheme '${CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT%%://*}'" ;; + esac + + PROMETHEUS_HOST_PORT="${CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT#*://}" + PROMETHEUS_HOST_PORT="${PROMETHEUS_HOST_PORT%/}" + case "$PROMETHEUS_HOST_PORT" in + "") prometheus_endpoint_error "has no host" ;; + */*) prometheus_endpoint_error "must not include a path (/write is appended automatically)" ;; + esac + + export CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT="$PROMETHEUS_HOST_PORT" + export HYPERDX_PROMETHEUS_SCHEME HYPERDX_PROMETHEUS_TLS_INSECURE + echo "CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT scheme: $HYPERDX_PROMETHEUS_SCHEME" +fi + # Run ClickHouse schema migrations if not using legacy schema creation if [ "$HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA" != "true" ]; then # Run Go-based migrate tool with TLS support diff --git a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts index 42d26941d3..9fc4996fb4 100644 --- a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts +++ b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts @@ -64,6 +64,24 @@ describe('opampController', () => { }); }); + describe('buildOtelCollectorConfig prometheusremotewrite exporter', () => { + it('takes the scheme and TLS mode from the collector environment, defaulting to plain HTTP', () => { + configState.IS_PROMQL_ENABLED = true; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + // The defaults matter: collector images older than this change don't + // export HYPERDX_PROMETHEUS_*, and must keep today's plain-HTTP config. + expect(cfg.exporters?.prometheusremotewrite).toMatchObject({ + endpoint: + '${env:HYPERDX_PROMETHEUS_SCHEME:-http}://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write', + tls: { insecure: '${env:HYPERDX_PROMETHEUS_TLS_INSECURE:-true}' }, + }); + }); + }); + describe('buildOtelCollectorConfig otlp/hyperdx authentication', () => { it('accepts bare and bearer-prefixed authorization values for each API key', () => { const cfg = buildOtelCollectorConfig([ diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index c4e13d3f6c..13a1e42ccb 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -149,7 +149,7 @@ type CollectorConfig = { prometheusremotewrite?: { endpoint: string; tls: { - insecure: boolean; + insecure: string; }; resource_to_telemetry_conversion: { enabled: boolean; @@ -336,10 +336,14 @@ export const buildOtelCollectorConfig = ( ); if (config.IS_PROMQL_ENABLED && otelCollectorConfig.exporters) { + // The collector's entrypoint.sh derives the scheme and TLS mode from + // CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT. The defaults keep plain HTTP + // for collector images that predate it. otelCollectorConfig.exporters.prometheusremotewrite = { - endpoint: 'http://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write', + endpoint: + '${env:HYPERDX_PROMETHEUS_SCHEME:-http}://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write', tls: { - insecure: true, + insecure: '${env:HYPERDX_PROMETHEUS_TLS_INSECURE:-true}', }, resource_to_telemetry_conversion: { enabled: true, diff --git a/smoke-tests/otel-collector/data/metrics/promql-remote-write/input.json b/smoke-tests/otel-collector/data/metrics/promql-remote-write/input.json new file mode 100644 index 0000000000..ca54e13d9c --- /dev/null +++ b/smoke-tests/otel-collector/data/metrics/promql-remote-write/input.json @@ -0,0 +1,40 @@ +{ + "resourceMetrics": [ + { + "resource": { + "attributes": [ + { + "key": "suite-id", + "value": { + "stringValue": "promql-remote-write" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "promql-remote-write-test-service" + } + } + ] + }, + "scopeMetrics": [ + { + "scope": {}, + "metrics": [ + { + "name": "smoke.promql.url.gauge", + "gauge": { + "dataPoints": [ + { + "timeUnixNano": "1901999590000000000", + "asDouble": 42 + } + ] + } + } + ] + } + ] + } + ] +} diff --git a/smoke-tests/otel-collector/docker-compose.yaml b/smoke-tests/otel-collector/docker-compose.yaml index 604ea99534..0c0201d9d7 100644 --- a/smoke-tests/otel-collector/docker-compose.yaml +++ b/smoke-tests/otel-collector/docker-compose.yaml @@ -108,6 +108,56 @@ services: ch-server: condition: service_healthy + # HDX-4633: CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT given as a URL rather than + # a bare host:port. entrypoint.sh splits the scheme off for the + # prometheusremotewrite exporter; see promql-remote-write.bats. + otel-collector-promql-url: + build: + context: ../.. + dockerfile: docker/otel-collector/Dockerfile + target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-0.155.0} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-1.61.0} + environment: + - CLICKHOUSE_ENDPOINT=tcp://ch-server:9000?dial_timeout=10s + - CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=http://ch-server:9363/ + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD= + - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=promql_url + - HYPERDX_LOG_LEVEL=info + - HYPERDX_OTEL_BATCH_TIMEOUT=100ms + # OPAMP_SERVER_URL is intentionally not set to run in standalone mode + ports: + - 64318:4318 # OTLP http receiver + networks: + - internal + depends_on: + ch-server: + condition: service_healthy + + # HDX-4633: a path in CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT can't be used as + # a scrape target, so entrypoint.sh rejects it before migrations. Expected to + # exit immediately; see promql-remote-write.bats. CLICKHOUSE_ENDPOINT is + # unreachable on purpose, as in otel-collector-oidc-missing-audience. + otel-collector-promql-bad-endpoint: + build: + context: ../.. + dockerfile: docker/otel-collector/Dockerfile + target: dev + args: + OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-0.155.0} + OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-1.61.0} + environment: + - CLICKHOUSE_ENDPOINT=tcp://127.0.0.1:1/?dial_timeout=1s + - CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT=https://ch-server:9363/write + - CLICKHOUSE_USER=default + - CLICKHOUSE_PASSWORD= + - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=promql_bad_endpoint + - HYPERDX_LOG_LEVEL=info + networks: + - internal + # Older ClickHouse (< 26.2) to test the compatibility schema (bloom filters # instead of full text indexes) ch-server-compat: diff --git a/smoke-tests/otel-collector/promql-remote-write.bats b/smoke-tests/otel-collector/promql-remote-write.bats new file mode 100644 index 0000000000..b7662a1764 --- /dev/null +++ b/smoke-tests/otel-collector/promql-remote-write.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +# HDX-4633: CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT may be a URL. entrypoint.sh +# splits the scheme off for the prometheusremotewrite exporter +# (config.standalone.promql.yaml) and keeps host:port for everything else. + +load 'test_helpers/utilities.bash' +load 'test_helpers/assertions.bash' + +@test "PromQL: an http:// endpoint URL still remote-writes metrics into the TimeSeries table" { + run docker compose logs otel-collector-promql-url + [ "$status" -eq 0 ] + [[ "$output" == *"CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT scheme: http"* ]] + + # ClickHouse's remote_write handler (docker/clickhouse/local/config.xml) + # always writes to default.metrics_ts, but the collector only migrates that + # table with ENABLE_PROMQL=true and into its own database. Creating it here + # avoids racing the otel-collector service's migrations on `default`. + run clickhouse-client --port=9000 --query="CREATE TABLE IF NOT EXISTS default.metrics_ts ENGINE = TimeSeries SETTINGS allow_experimental_time_series_table = 1" + [ "$status" -eq 0 ] + + emit_otel_data "http://localhost:64318" "data/metrics/promql-remote-write" "metrics" + wait_for_rows 9000 "SELECT count() FROM timeSeriesTags(default, metrics_ts) WHERE metric_name = 'smoke_promql_url_gauge' SETTINGS allow_experimental_time_series_table = 1" 1 +} + +@test "PromQL: collector fails fast with a clear error when the endpoint includes a path" { + run docker compose logs otel-collector-promql-bad-endpoint + [ "$status" -eq 0 ] + [[ "$output" == *"CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT must not include a path"* ]] + + run docker compose ps --status running --services + [ "$status" -eq 0 ] + [[ "$output" != *"otel-collector-promql-bad-endpoint"* ]] +} diff --git a/smoke-tests/otel-collector/setup_suite.bash b/smoke-tests/otel-collector/setup_suite.bash index 63afed6af1..82bc5cb6b3 100644 --- a/smoke-tests/otel-collector/setup_suite.bash +++ b/smoke-tests/otel-collector/setup_suite.bash @@ -17,8 +17,10 @@ setup_suite() { wait_for_ready "otel-collector-custom" wait_for_ready "otel-collector-oidc" wait_for_ready "otel-collector-bearer-auth" - # otel-collector-oidc-missing-audience is expected to exit immediately, - # not become ready -- checked directly in oidc-auth.bats instead. + wait_for_ready "otel-collector-promql-url" + # otel-collector-oidc-missing-audience and otel-collector-promql-bad-endpoint + # are expected to exit immediately, not become ready -- checked directly in + # oidc-auth.bats and promql-remote-write.bats instead. } teardown_suite() { From c22f6dcaf8c9ea7a47ef73e18c84979470e08edb Mon Sep 17 00:00:00 2001 From: Karl Power Date: Mon, 12 Oct 2026 11:29:55 +0200 Subject: [PATCH 2/2] fix: scrape ClickHouse metrics over https in the OpAMP config --- .changeset/promql-remote-write-https.md | 4 ++- .../__tests__/opampController.test.ts | 28 +++++++++++++++++++ .../src/opamp/controllers/opampController.ts | 16 ++++++++--- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/.changeset/promql-remote-write-https.md b/.changeset/promql-remote-write-https.md index 5de82e9e19..57e6300f18 100644 --- a/.changeset/promql-remote-write-https.md +++ b/.changeset/promql-remote-write-https.md @@ -3,8 +3,10 @@ '@hyperdx/api': patch --- -fix: support https for the PromQL remote-write endpoint +fix: support https for the PromQL remote-write endpoint and ClickHouse scraping `CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT` now accepts an `http://` or `https://` URL as well as a bare `host:port`. With `https://`, the collector's `prometheusremotewrite` exporter sends metrics over TLS and verifies the server certificate against the system CAs. Previously the exporter always used plain HTTP, so an HTTPS-only endpoint could not receive metrics. This applies to both standalone and OpAMP-managed collectors. A bare `host:port` keeps plain HTTP, so existing setups are unchanged. The value must not include a path (`/write` is still appended automatically); the collector now exits at startup with a clear error if it does, or if the scheme is anything other than `http` or `https`. + +The OpAMP-managed collector scraped the endpoint over plain HTTP regardless of its scheme. ClickHouse is now scraped in its own `clickhouse` job (previously the `otelcol` job) using the endpoint's scheme, with TLS verification on. The collector's own metrics endpoint stays on plain HTTP. A bare `host:port` or `http://` endpoint behaves as before. Scraped ClickHouse metrics now carry `job="clickhouse"` instead of `job="otelcol"`. diff --git a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts index 9fc4996fb4..b6d274e970 100644 --- a/packages/api/src/opamp/controllers/__tests__/opampController.test.ts +++ b/packages/api/src/opamp/controllers/__tests__/opampController.test.ts @@ -82,6 +82,34 @@ describe('opampController', () => { }); }); + describe('buildOtelCollectorConfig prometheus receiver', () => { + it('scrapes ClickHouse in its own job so the scheme follows the endpoint', () => { + configState.IS_PROMQL_ENABLED = true; + + const cfg = buildOtelCollectorConfig([ + { apiKey: 'k1', collectorAuthenticationEnforced: false }, + ]); + + const scrapeConfigs = cfg.receivers.prometheus?.config.scrape_configs; + // The collector's own endpoint is always plain HTTP: no scheme here. + expect(scrapeConfigs?.[0]).toEqual({ + job_name: 'otelcol', + scrape_interval: '30s', + static_configs: [{ targets: ['0.0.0.0:8888'] }], + }); + // The default keeps collector images that predate + // HYPERDX_PROMETHEUS_SCHEME on plain HTTP. + expect(scrapeConfigs?.[1]).toEqual({ + job_name: 'clickhouse', + scrape_interval: '30s', + scheme: '${env:HYPERDX_PROMETHEUS_SCHEME:-http}', + static_configs: [ + { targets: ['${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}'] }, + ], + }); + }); + }); + describe('buildOtelCollectorConfig otlp/hyperdx authentication', () => { it('accepts bare and bearer-prefixed authorization values for each API key', () => { const cfg = buildOtelCollectorConfig([ diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index 13a1e42ccb..22325ad2bc 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -76,6 +76,7 @@ type CollectorConfig = { scrape_configs: Array<{ job_name: string; scrape_interval: string; + scheme?: string; static_configs: Array<{ targets: string[]; }>; @@ -216,12 +217,19 @@ export const buildOtelCollectorConfig = ( { job_name: 'otelcol', scrape_interval: '30s', + static_configs: [{ targets: ['0.0.0.0:8888'] }], + }, + // Separate job because the collector's own endpoint is always + // plain HTTP, while ClickHouse may be served over https. The + // default keeps collector images that predate + // HYPERDX_PROMETHEUS_SCHEME on http. + { + job_name: 'clickhouse', + scrape_interval: '30s', + scheme: '${env:HYPERDX_PROMETHEUS_SCHEME:-http}', static_configs: [ { - targets: [ - '0.0.0.0:8888', - '${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}', - ], + targets: ['${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}'], }, ], },