Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .changeset/replicated-engine.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
---
'@hyperdx/otel-collector': minor
---

feat: opt-in Replicated database and table engines for the schema seed

When `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED=true`, the migrate tool
creates the target database with the `Replicated` engine and seeds
`ReplicatedMergeTree` / `ReplicatedSummingMergeTree` tables instead of
`MergeTree` / `SummingMergeTree`. If the database already exists with another
engine, it logs a warning and seeds the plain engines as before. Off by default.

The migrate tool also warns when it targets a database other than `default`
while `default` still holds ClickStack tables, since those are not migrated.

With `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE` set (off by default),
a target database that does not exist yet yields to the fallback database when
the fallback already has ClickStack tables. The collector then writes to the
fallback database too.
7 changes: 7 additions & 0 deletions docker-compose.dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,13 @@ services:
# HYPERDX_OTEL_EXPORTER_LOGS_TTL / _TRACES_TTL / _METRICS_TTL / _SESSIONS_TTL
# Apply TTL changes to already-existing tables too (default false):
# HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL: 'true'
# Create the database with the Replicated engine and seed Replicated* table
# engines (default false). The database must not exist yet (or already be
# Replicated) and the server needs {shard}/{replica} macros:
# HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: 'true'
# Keep using this database instead when HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE
# does not exist yet but this one already has ClickStack tables (default off):
# HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE: 'default'
ENABLE_PROMQL: 'true'
volumes:
- ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
Expand Down
10 changes: 9 additions & 1 deletion docker/otel-collector/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,15 @@ if [ "$HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA" != "true" ]; then
# - CLICKHOUSE_TLS_SERVER_NAME_OVERRIDE: Server name for TLS verification
# - CLICKHOUSE_TLS_INSECURE_SKIP_VERIFY: Skip TLS verification (set to "true")
echo "🚀 Using Go-based migrate tool with TLS support 🔐"
migrate /etc/otel/schema/seed
# With HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE set, migrate records
# the database it seeded here so the collector writes to the same one.
RESOLVED_DATABASE_PATH="${HYPERDX_OTEL_EXPORTER_RESOLVED_DATABASE_PATH:-/tmp/hyperdx-otel-exporter-database}"
rm -f "$RESOLVED_DATABASE_PATH"
HYPERDX_OTEL_EXPORTER_RESOLVED_DATABASE_PATH="$RESOLVED_DATABASE_PATH" migrate /etc/otel/schema/seed
if [ -s "$RESOLVED_DATABASE_PATH" ]; then
export HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE="$(cat "$RESOLVED_DATABASE_PATH")"
echo "Using ClickHouse database from migrate: $HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE"
fi
fi

# Check if OPAMP_SERVER_URL is defined to determine mode
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
-- +goose Up
-- +goose NO TRANSACTION
CREATE DATABASE IF NOT EXISTS ${DATABASE};
CREATE DATABASE IF NOT EXISTS ${DATABASE}${DATABASE_ENGINE};

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the possible value of DATABASE_ENGINE here?

2 changes: 1 addition & 1 deletion docker/otel-collector/schema/seed/00002_otel_logs.sql
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs
INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array'),
INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha')
)
ENGINE = MergeTree
ENGINE = ${ENGINE_PREFIX}MergeTree
PARTITION BY toDate(Timestamp)
ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp)
TTL toDateTime(Timestamp) + ${LOGS_TTL}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
)
ENGINE = MergeTree
ENGINE = ${ENGINE_PREFIX}MergeTree
PARTITION BY toDate(Timestamp)
ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp)
TTL toDateTime(Timestamp) + ${LOGS_TTL}
Expand Down
10 changes: 5 additions & 5 deletions docker/otel-collector/schema/seed/00003_otel_metrics.sql
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_gauge
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1
)
ENGINE = MergeTree
ENGINE = ${ENGINE_PREFIX}MergeTree
PARTITION BY toDate(TimeUnix)
ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix)
TTL toDateTime(TimeUnix) + ${METRICS_TTL}
Expand Down Expand Up @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_sum
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1
)
ENGINE = MergeTree
ENGINE = ${ENGINE_PREFIX}MergeTree
PARTITION BY toDate(TimeUnix)
ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix)
TTL toDateTime(TimeUnix) + ${METRICS_TTL}
Expand Down Expand Up @@ -115,7 +115,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_histogram
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1
)
ENGINE = MergeTree
ENGINE = ${ENGINE_PREFIX}MergeTree
PARTITION BY toDate(TimeUnix)
ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix)
TTL toDateTime(TimeUnix) + ${METRICS_TTL}
Expand Down Expand Up @@ -164,7 +164,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_exponential_histogram
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1
)
ENGINE = MergeTree
ENGINE = ${ENGINE_PREFIX}MergeTree
PARTITION BY toDate(TimeUnix)
ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix)
TTL toDateTime(TimeUnix) + ${METRICS_TTL}
Expand Down Expand Up @@ -201,7 +201,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_summary
INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1
)
ENGINE = MergeTree
ENGINE = ${ENGINE_PREFIX}MergeTree
PARTITION BY toDate(TimeUnix)
ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix)
TTL toDateTime(TimeUnix) + ${METRICS_TTL}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
)
ENGINE = MergeTree
ENGINE = ${ENGINE_PREFIX}MergeTree
PARTITION BY toDate(TimestampTime)
PRIMARY KEY (ServiceName, TimestampTime)
ORDER BY (ServiceName, TimestampTime, Timestamp)
Expand Down
2 changes: 1 addition & 1 deletion docker/otel-collector/schema/seed/00005_otel_traces.sql
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces
INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName))
)
ENGINE = MergeTree
ENGINE = ${ENGINE_PREFIX}MergeTree
PARTITION BY toDate(Timestamp)
ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
TTL toDate(Timestamp) + ${TRACES_TTL}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces
INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
)
ENGINE = MergeTree
ENGINE = ${ENGINE_PREFIX}MergeTree
PARTITION BY toDate(Timestamp)
ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
TTL toDate(Timestamp) + ${TRACES_TTL}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs_kv_rollup_15m
INDEX idx_count_minmax count TYPE minmax GRANULARITY 1,
INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1
)
ENGINE = SummingMergeTree
ENGINE = ${ENGINE_PREFIX}SummingMergeTree
PARTITION BY toDate(Timestamp)
ORDER BY (ColumnIdentifier, Key, Timestamp, Value)
TTL Timestamp + ${LOGS_TTL}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces_kv_rollup_15m
INDEX idx_count_minmax count TYPE minmax GRANULARITY 1,
INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1
)
ENGINE = SummingMergeTree
ENGINE = ${ENGINE_PREFIX}SummingMergeTree
PARTITION BY toDate(Timestamp)
ORDER BY (ColumnIdentifier, Key, Timestamp, Value)
TTL Timestamp + ${TRACES_TTL}
Expand Down
151 changes: 145 additions & 6 deletions packages/otel-collector/cmd/migrate/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"crypto/tls"
"crypto/x509"
"database/sql"
"errors"
"fmt"
"log"
"net/url"
Expand Down Expand Up @@ -42,6 +43,17 @@ type Config struct {
// Reconcile TTL on already-existing tables, not just newly created ones.
ReconcileTableTTL bool

// Create the database with the Replicated engine and seed Replicated*
// table engines, unless the database already exists with another engine.
Replicated bool

// Database to stay on when Database does not exist yet but this one already
// holds ClickStack tables. Empty disables the fallback.
FallbackDatabase string

// Where to record the seeded database for the entrypoint (fallback only).
ResolvedDatabasePath string

// TLS settings
TLSCAFile string
TLSCertFile string
Expand All @@ -68,6 +80,9 @@ func loadConfig() (*Config, error) {
MetricsTTL: getEnv("HYPERDX_OTEL_EXPORTER_METRICS_TTL", tablesTTL),
SessionsTTL: getEnv("HYPERDX_OTEL_EXPORTER_SESSIONS_TTL", tablesTTL),
ReconcileTableTTL: getEnv("HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL", "false") == "true",
Replicated: getEnv("HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED", "false") == "true",
FallbackDatabase: getEnv("HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE", ""),
ResolvedDatabasePath: getEnv("HYPERDX_OTEL_EXPORTER_RESOLVED_DATABASE_PATH", "/tmp/hyperdx-otel-exporter-database"),
TLSCAFile: getEnv("CLICKHOUSE_TLS_CA_FILE", ""),
TLSCertFile: getEnv("CLICKHOUSE_TLS_CERT_FILE", ""),
TLSKeyFile: getEnv("CLICKHOUSE_TLS_KEY_FILE", ""),
Expand Down Expand Up @@ -528,14 +543,23 @@ func reconcileTableTTLs(ctx context.Context, db *sql.DB, database string, ttlExp
}

// processSchemaDir creates a temporary directory with SQL files that have the
// ${DATABASE} and per-signal ${*_TTL} macros replaced with actual values.
// ${DATABASE}, ${DATABASE_ENGINE}, ${ENGINE_PREFIX} and per-signal ${*_TTL}
// macros replaced with actual values. When replicated is set, the database is
// created with the Replicated engine and MergeTree becomes ReplicatedMergeTree;
// otherwise both macros are empty.
// ttlExprs maps a macro name (e.g. "LOGS_TTL") to its ClickHouse interval expr.
func processSchemaDir(schemaDir, database string, ttlExprs map[string]string) (string, error) {
func processSchemaDir(schemaDir, database string, replicated bool, ttlExprs map[string]string) (string, error) {
tempDir, err := os.MkdirTemp("", "schema-*")
if err != nil {
return "", fmt.Errorf("failed to create temp directory: %w", err)
}

databaseEngine, enginePrefix := "", ""
if replicated {
databaseEngine = fmt.Sprintf(" ENGINE = Replicated('/clickhouse/databases/%s', '{shard}', '{replica}')", database)
enginePrefix = "Replicated"
}

// Walk through the schema directory and process SQL files
err = filepath.Walk(schemaDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
Expand All @@ -562,6 +586,8 @@ func processSchemaDir(schemaDir, database string, ttlExprs map[string]string) (s

// Replace macros with actual values
processedContent := strings.ReplaceAll(string(content), "${DATABASE}", database)
processedContent = strings.ReplaceAll(processedContent, "${DATABASE_ENGINE}", databaseEngine)
processedContent = strings.ReplaceAll(processedContent, "${ENGINE_PREFIX}", enginePrefix)
for macro, expr := range ttlExprs {
processedContent = strings.ReplaceAll(processedContent, "${"+macro+"}", expr)
}
Expand All @@ -582,6 +608,75 @@ func processSchemaDir(schemaDir, database string, ttlExprs map[string]string) (s
return tempDir, nil
}

// planReplicated decides whether to seed Replicated engines. Argument-less
// Replicated* table engines only work inside a Replicated database and an
// existing database cannot change engine, so only a missing or already
// Replicated database qualifies. Returns a warning when the flag cannot apply.
func planReplicated(enabled, exists bool, database, engine string) (bool, string) {
switch {
case !enabled:
return false, ""
case !exists, engine == "Replicated":
return true, ""
default:
return false, fmt.Sprintf("database %s already exists with engine %s, so tables use non-replicated engines; "+
"set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE to a new database name to seed a Replicated database", database, engine)
}
}

// getDatabaseEngine returns the engine of a database, or exists=false if it is missing.
func getDatabaseEngine(ctx context.Context, db *sql.DB, database string) (engine string, exists bool, err error) {
err = db.QueryRowContext(ctx, "SELECT engine FROM system.databases WHERE name = ?", database).Scan(&engine)
if errors.Is(err, sql.ErrNoRows) {
return "", false, nil
}
if err != nil {
return "", false, fmt.Errorf("failed to query engine of database %s: %w", database, err)
}
return engine, true, nil
}

// legacyDatabase is where an earlier install keeps its ClickStack tables: the
// fallback database when set, otherwise default.
func legacyDatabase(fallback string) string {
if fallback != "" {
return fallback
}
return "default"
}

// hasClickStackTables reports whether database already holds ClickStack tables.
func hasClickStackTables(ctx context.Context, db *sql.DB, database string) (bool, error) {
var n uint64
err := db.QueryRowContext(ctx,
"SELECT count() FROM system.tables WHERE database = ? AND name IN ('otel_logs', 'otel_traces')", database).Scan(&n)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we want to check only logs and traces here?

if err != nil {
return false, fmt.Errorf("failed to check database %s for ClickStack tables: %w", database, err)
}
return n > 0, nil
}

// resolveDatabase picks the database to seed and hand to the collector. With a
// fallback set, a target that does not exist yet yields to a fallback that
// already holds ClickStack tables, so an upgrade that switched databases keeps
// writing where HyperDX sources read. Otherwise legacy tables only warrant a
// warning, as they are not migrated.
func resolveDatabase(target, fallback string, targetExists, legacyHasTables bool) (string, string) {
legacy := legacyDatabase(fallback)
switch {
case legacy == target || !legacyHasTables:
return target, ""
case fallback != "" && !targetExists:
return fallback, fmt.Sprintf("database %s does not exist and %s already has ClickStack tables, so staying on %s. "+
"Set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=%s to silence this, or clear HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE to move to %s",
target, fallback, fallback, fallback, target)
default:
return target, fmt.Sprintf("database %s has existing ClickStack tables that are not migrated to %s; "+
"HyperDX sources created earlier still point at %s. Set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=%s to keep using the old database",
legacy, target, legacy, legacy)
}
}

// runMigrationWithRetry runs goose seed with exponential backoff retry
func runMigrationWithRetry(ctx context.Context, db *sql.DB, migrationsDir string, maxRetries int) error {
var lastErr error
Expand Down Expand Up @@ -788,9 +883,41 @@ func main() {
log.Printf("Table TTLs: logs=%s traces=%s metrics=%s sessions=%s (default %s)",
cfg.LogsTTL, cfg.TracesTTL, cfg.MetricsTTL, cfg.SessionsTTL, cfg.TablesTTL)

// Process schema directory (replace ${DATABASE} and ${*_TTL} macros)
log.Printf("Preparing SQL files with database: %s", cfg.Database)
tempDir, err := processSchemaDir(cfg.SchemaDir, cfg.Database, ttlExprs)
// The legacy tables check is best effort unless the fallback depends on it.
var legacyHasTables bool
if legacy := legacyDatabase(cfg.FallbackDatabase); legacy != cfg.Database {
if legacyHasTables, err = hasClickStackTables(ctx, db, legacy); err != nil && cfg.FallbackDatabase != "" {
log.Fatalf("Failed to check fallback database: %v", err)
}
}

var dbEngine string
var dbExists bool
if cfg.Replicated || cfg.FallbackDatabase != "" {
if dbEngine, dbExists, err = getDatabaseEngine(ctx, db, cfg.Database); err != nil {
log.Fatalf("Failed to check database engine: %v", err)
}
}
database, warning := resolveDatabase(cfg.Database, cfg.FallbackDatabase, dbExists, legacyHasTables)
if warning != "" {
log.Printf("WARNING: %s", warning)
}

// A fallback database keeps the plain engines it was seeded with.
replicated := false
if database == cfg.Database {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔵 minor — Replicated mode is dropped without a warning when the fallback database is used

With HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED=true and a fallback that wins (the target is missing and the fallback has tables), planReplicated never runs. The fallback is seeded with plain engines and nothing says Replicated was not applied. Log a warning in that branch, for example: if cfg.Replicated && database != cfg.Database { log.Printf("WARNING: Replicated engines not applied; staying on fallback %s", database) }.

Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond

replicated, warning = planReplicated(cfg.Replicated, dbExists, database, dbEngine)
if warning != "" {
log.Printf("WARNING: %s", warning)
}
if replicated {
log.Printf("Using Replicated database and table engines for %s", database)
}
}

// Process schema directory (replace ${DATABASE}, ${DATABASE_ENGINE}, ${ENGINE_PREFIX} and ${*_TTL} macros)
log.Printf("Preparing SQL files with database: %s", database)
tempDir, err := processSchemaDir(cfg.SchemaDir, database, replicated, ttlExprs)
if err != nil {
log.Fatalf("Failed to process schema directory: %v", err)
}
Expand Down Expand Up @@ -841,11 +968,23 @@ func main() {
// CREATE TABLE IF NOT EXISTS won't update TTL on existing tables; reconcile
// (opt-in) applies it to them too. Non-fatal so it can't block startup.
if cfg.ReconcileTableTTL {
if err := reconcileTableTTLs(ctx, db, cfg.Database, ttlExprs); err != nil {
if err := reconcileTableTTLs(ctx, db, database, ttlExprs); err != nil {
log.Printf("WARNING: reconcile: %v", err)
}
}

// Record the seeded database so the entrypoint hands it to the collector.
if cfg.FallbackDatabase != "" {
if err := os.WriteFile(cfg.ResolvedDatabasePath, []byte(database+"\n"), 0644); err != nil {
// The collector would otherwise write to a database that was not seeded.
if database != cfg.Database {
log.Fatalf("Failed to write resolved database to %s: %v", cfg.ResolvedDatabasePath, err)
}

log.Printf("WARNING: failed to write resolved database to %s: %v", cfg.ResolvedDatabasePath, err)
}
}

log.Println("========================================")
log.Println("Schema seed completed successfully")
log.Println("========================================")
Expand Down
Loading
Loading