diff --git a/.changeset/replicated-engine.md b/.changeset/replicated-engine.md new file mode 100644 index 0000000000..fe38cf1591 --- /dev/null +++ b/.changeset/replicated-engine.md @@ -0,0 +1,19 @@ +--- +'@hyperdx/otel-collector': minor +--- + +feat: opt-in Replicated database and table engines for the schema seed + +When `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED=true`, the migrate tool +creates the target database with the `Replicated` engine and seeds +`ReplicatedMergeTree` / `ReplicatedSummingMergeTree` tables instead of +`MergeTree` / `SummingMergeTree`. If the database already exists with another +engine, it logs a warning and seeds the plain engines as before. Off by default. + +The migrate tool also warns when it targets a database other than `default` +while `default` still holds ClickStack tables, since those are not migrated. + +With `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE` set (off by default), +a target database that does not exist yet yields to the fallback database when +the fallback already has ClickStack tables. The collector then writes to the +fallback database too. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 9c3c0491ec..572c9cbcf3 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -44,6 +44,13 @@ services: # HYPERDX_OTEL_EXPORTER_LOGS_TTL / _TRACES_TTL / _METRICS_TTL / _SESSIONS_TTL # Apply TTL changes to already-existing tables too (default false): # HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL: 'true' + # Create the database with the Replicated engine and seed Replicated* table + # engines (default false). The database must not exist yet (or already be + # Replicated) and the server needs {shard}/{replica} macros: + # HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: 'true' + # Keep using this database instead when HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + # does not exist yet but this one already has ClickStack tables (default off): + # HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE: 'default' ENABLE_PROMQL: 'true' volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml diff --git a/docker/otel-collector/entrypoint.sh b/docker/otel-collector/entrypoint.sh index 8995f71c06..00a490f959 100644 --- a/docker/otel-collector/entrypoint.sh +++ b/docker/otel-collector/entrypoint.sh @@ -34,7 +34,15 @@ if [ "$HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA" != "true" ]; then # - CLICKHOUSE_TLS_SERVER_NAME_OVERRIDE: Server name for TLS verification # - CLICKHOUSE_TLS_INSECURE_SKIP_VERIFY: Skip TLS verification (set to "true") echo "🚀 Using Go-based migrate tool with TLS support 🔐" - migrate /etc/otel/schema/seed + # With HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE set, migrate records + # the database it seeded here so the collector writes to the same one. + RESOLVED_DATABASE_PATH="${HYPERDX_OTEL_EXPORTER_RESOLVED_DATABASE_PATH:-/tmp/hyperdx-otel-exporter-database}" + rm -f "$RESOLVED_DATABASE_PATH" + HYPERDX_OTEL_EXPORTER_RESOLVED_DATABASE_PATH="$RESOLVED_DATABASE_PATH" migrate /etc/otel/schema/seed + if [ -s "$RESOLVED_DATABASE_PATH" ]; then + export HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE="$(cat "$RESOLVED_DATABASE_PATH")" + echo "Using ClickHouse database from migrate: $HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE" + fi fi # Check if OPAMP_SERVER_URL is defined to determine mode diff --git a/docker/otel-collector/schema/seed/00001_create_database.sql b/docker/otel-collector/schema/seed/00001_create_database.sql index 2199bf8e40..cc4f5aa75e 100644 --- a/docker/otel-collector/schema/seed/00001_create_database.sql +++ b/docker/otel-collector/schema/seed/00001_create_database.sql @@ -1,3 +1,3 @@ -- +goose Up -- +goose NO TRANSACTION -CREATE DATABASE IF NOT EXISTS ${DATABASE}; +CREATE DATABASE IF NOT EXISTS ${DATABASE}${DATABASE_ENGINE}; diff --git a/docker/otel-collector/schema/seed/00002_otel_logs.sql b/docker/otel-collector/schema/seed/00002_otel_logs.sql index 2664576ec8..8a875afc82 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs.sql @@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) TTL toDateTime(Timestamp) + ${LOGS_TTL} diff --git a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql index 2aa6f1c82f..3c82c9a858 100644 --- a/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql +++ b/docker/otel-collector/schema/seed/00002_otel_logs_compat.sql @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) TTL toDateTime(Timestamp) + ${LOGS_TTL} diff --git a/docker/otel-collector/schema/seed/00003_otel_metrics.sql b/docker/otel-collector/schema/seed/00003_otel_metrics.sql index 03938e811a..e6e90923d0 100644 --- a/docker/otel-collector/schema/seed/00003_otel_metrics.sql +++ b/docker/otel-collector/schema/seed/00003_otel_metrics.sql @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_gauge INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${METRICS_TTL} @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_sum INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${METRICS_TTL} @@ -115,7 +115,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_histogram INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${METRICS_TTL} @@ -164,7 +164,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_exponential_histogram INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${METRICS_TTL} @@ -201,7 +201,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_summary INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimeUnix) ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) TTL toDateTime(TimeUnix) + ${METRICS_TTL} diff --git a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql index d82bccfd55..3e85bb8bf2 100644 --- a/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql +++ b/docker/otel-collector/schema/seed/00004_hyperdx_sessions.sql @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.hyperdx_sessions INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(TimestampTime) PRIMARY KEY (ServiceName, TimestampTime) ORDER BY (ServiceName, TimestampTime, Timestamp) diff --git a/docker/otel-collector/schema/seed/00005_otel_traces.sql b/docker/otel-collector/schema/seed/00005_otel_traces.sql index ce964efb37..58e6de69f1 100644 --- a/docker/otel-collector/schema/seed/00005_otel_traces.sql +++ b/docker/otel-collector/schema/seed/00005_otel_traces.sql @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces INDEX idx_duration Duration TYPE minmax GRANULARITY 1, INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName)) ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(Timestamp) ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) TTL toDate(Timestamp) + ${TRACES_TTL} diff --git a/docker/otel-collector/schema/seed/00005_otel_traces_compat.sql b/docker/otel-collector/schema/seed/00005_otel_traces_compat.sql index 480be6b093..3e3ebdfce4 100644 --- a/docker/otel-collector/schema/seed/00005_otel_traces_compat.sql +++ b/docker/otel-collector/schema/seed/00005_otel_traces_compat.sql @@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces INDEX idx_duration Duration TYPE minmax GRANULARITY 1, INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 ) -ENGINE = MergeTree +ENGINE = ${ENGINE_PREFIX}MergeTree PARTITION BY toDate(Timestamp) ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) TTL toDate(Timestamp) + ${TRACES_TTL} diff --git a/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql b/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql index 82db4d1be6..6509663795 100644 --- a/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql +++ b/docker/otel-collector/schema/seed/00006_otel_logs_rollups.sql @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs_kv_rollup_15m INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 ) -ENGINE = SummingMergeTree +ENGINE = ${ENGINE_PREFIX}SummingMergeTree PARTITION BY toDate(Timestamp) ORDER BY (ColumnIdentifier, Key, Timestamp, Value) TTL Timestamp + ${LOGS_TTL} diff --git a/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql b/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql index 80afc569cf..07ed18e573 100644 --- a/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql +++ b/docker/otel-collector/schema/seed/00007_otel_traces_rollups.sql @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces_kv_rollup_15m INDEX idx_count_minmax count TYPE minmax GRANULARITY 1, INDEX idx_timestamp_minmax Timestamp TYPE minmax GRANULARITY 1 ) -ENGINE = SummingMergeTree +ENGINE = ${ENGINE_PREFIX}SummingMergeTree PARTITION BY toDate(Timestamp) ORDER BY (ColumnIdentifier, Key, Timestamp, Value) TTL Timestamp + ${TRACES_TTL} diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index fc4e514ed0..c525c5e715 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -9,6 +9,7 @@ import ( "crypto/tls" "crypto/x509" "database/sql" + "errors" "fmt" "log" "net/url" @@ -42,6 +43,17 @@ type Config struct { // Reconcile TTL on already-existing tables, not just newly created ones. ReconcileTableTTL bool + // Create the database with the Replicated engine and seed Replicated* + // table engines, unless the database already exists with another engine. + Replicated bool + + // Database to stay on when Database does not exist yet but this one already + // holds ClickStack tables. Empty disables the fallback. + FallbackDatabase string + + // Where to record the seeded database for the entrypoint (fallback only). + ResolvedDatabasePath string + // TLS settings TLSCAFile string TLSCertFile string @@ -68,6 +80,9 @@ func loadConfig() (*Config, error) { MetricsTTL: getEnv("HYPERDX_OTEL_EXPORTER_METRICS_TTL", tablesTTL), SessionsTTL: getEnv("HYPERDX_OTEL_EXPORTER_SESSIONS_TTL", tablesTTL), ReconcileTableTTL: getEnv("HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL", "false") == "true", + Replicated: getEnv("HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED", "false") == "true", + FallbackDatabase: getEnv("HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE", ""), + ResolvedDatabasePath: getEnv("HYPERDX_OTEL_EXPORTER_RESOLVED_DATABASE_PATH", "/tmp/hyperdx-otel-exporter-database"), TLSCAFile: getEnv("CLICKHOUSE_TLS_CA_FILE", ""), TLSCertFile: getEnv("CLICKHOUSE_TLS_CERT_FILE", ""), TLSKeyFile: getEnv("CLICKHOUSE_TLS_KEY_FILE", ""), @@ -528,14 +543,23 @@ func reconcileTableTTLs(ctx context.Context, db *sql.DB, database string, ttlExp } // processSchemaDir creates a temporary directory with SQL files that have the -// ${DATABASE} and per-signal ${*_TTL} macros replaced with actual values. +// ${DATABASE}, ${DATABASE_ENGINE}, ${ENGINE_PREFIX} and per-signal ${*_TTL} +// macros replaced with actual values. When replicated is set, the database is +// created with the Replicated engine and MergeTree becomes ReplicatedMergeTree; +// otherwise both macros are empty. // ttlExprs maps a macro name (e.g. "LOGS_TTL") to its ClickHouse interval expr. -func processSchemaDir(schemaDir, database string, ttlExprs map[string]string) (string, error) { +func processSchemaDir(schemaDir, database string, replicated bool, ttlExprs map[string]string) (string, error) { tempDir, err := os.MkdirTemp("", "schema-*") if err != nil { return "", fmt.Errorf("failed to create temp directory: %w", err) } + databaseEngine, enginePrefix := "", "" + if replicated { + databaseEngine = fmt.Sprintf(" ENGINE = Replicated('/clickhouse/databases/%s', '{shard}', '{replica}')", database) + enginePrefix = "Replicated" + } + // Walk through the schema directory and process SQL files err = filepath.Walk(schemaDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -562,6 +586,8 @@ func processSchemaDir(schemaDir, database string, ttlExprs map[string]string) (s // Replace macros with actual values processedContent := strings.ReplaceAll(string(content), "${DATABASE}", database) + processedContent = strings.ReplaceAll(processedContent, "${DATABASE_ENGINE}", databaseEngine) + processedContent = strings.ReplaceAll(processedContent, "${ENGINE_PREFIX}", enginePrefix) for macro, expr := range ttlExprs { processedContent = strings.ReplaceAll(processedContent, "${"+macro+"}", expr) } @@ -582,6 +608,75 @@ func processSchemaDir(schemaDir, database string, ttlExprs map[string]string) (s return tempDir, nil } +// planReplicated decides whether to seed Replicated engines. Argument-less +// Replicated* table engines only work inside a Replicated database and an +// existing database cannot change engine, so only a missing or already +// Replicated database qualifies. Returns a warning when the flag cannot apply. +func planReplicated(enabled, exists bool, database, engine string) (bool, string) { + switch { + case !enabled: + return false, "" + case !exists, engine == "Replicated": + return true, "" + default: + return false, fmt.Sprintf("database %s already exists with engine %s, so tables use non-replicated engines; "+ + "set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE to a new database name to seed a Replicated database", database, engine) + } +} + +// getDatabaseEngine returns the engine of a database, or exists=false if it is missing. +func getDatabaseEngine(ctx context.Context, db *sql.DB, database string) (engine string, exists bool, err error) { + err = db.QueryRowContext(ctx, "SELECT engine FROM system.databases WHERE name = ?", database).Scan(&engine) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("failed to query engine of database %s: %w", database, err) + } + return engine, true, nil +} + +// legacyDatabase is where an earlier install keeps its ClickStack tables: the +// fallback database when set, otherwise default. +func legacyDatabase(fallback string) string { + if fallback != "" { + return fallback + } + return "default" +} + +// hasClickStackTables reports whether database already holds ClickStack tables. +func hasClickStackTables(ctx context.Context, db *sql.DB, database string) (bool, error) { + var n uint64 + err := db.QueryRowContext(ctx, + "SELECT count() FROM system.tables WHERE database = ? AND name IN ('otel_logs', 'otel_traces')", database).Scan(&n) + if err != nil { + return false, fmt.Errorf("failed to check database %s for ClickStack tables: %w", database, err) + } + return n > 0, nil +} + +// resolveDatabase picks the database to seed and hand to the collector. With a +// fallback set, a target that does not exist yet yields to a fallback that +// already holds ClickStack tables, so an upgrade that switched databases keeps +// writing where HyperDX sources read. Otherwise legacy tables only warrant a +// warning, as they are not migrated. +func resolveDatabase(target, fallback string, targetExists, legacyHasTables bool) (string, string) { + legacy := legacyDatabase(fallback) + switch { + case legacy == target || !legacyHasTables: + return target, "" + case fallback != "" && !targetExists: + return fallback, fmt.Sprintf("database %s does not exist and %s already has ClickStack tables, so staying on %s. "+ + "Set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=%s to silence this, or clear HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE to move to %s", + target, fallback, fallback, fallback, target) + default: + return target, fmt.Sprintf("database %s has existing ClickStack tables that are not migrated to %s; "+ + "HyperDX sources created earlier still point at %s. Set HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=%s to keep using the old database", + legacy, target, legacy, legacy) + } +} + // runMigrationWithRetry runs goose seed with exponential backoff retry func runMigrationWithRetry(ctx context.Context, db *sql.DB, migrationsDir string, maxRetries int) error { var lastErr error @@ -788,9 +883,41 @@ func main() { log.Printf("Table TTLs: logs=%s traces=%s metrics=%s sessions=%s (default %s)", cfg.LogsTTL, cfg.TracesTTL, cfg.MetricsTTL, cfg.SessionsTTL, cfg.TablesTTL) - // Process schema directory (replace ${DATABASE} and ${*_TTL} macros) - log.Printf("Preparing SQL files with database: %s", cfg.Database) - tempDir, err := processSchemaDir(cfg.SchemaDir, cfg.Database, ttlExprs) + // The legacy tables check is best effort unless the fallback depends on it. + var legacyHasTables bool + if legacy := legacyDatabase(cfg.FallbackDatabase); legacy != cfg.Database { + if legacyHasTables, err = hasClickStackTables(ctx, db, legacy); err != nil && cfg.FallbackDatabase != "" { + log.Fatalf("Failed to check fallback database: %v", err) + } + } + + var dbEngine string + var dbExists bool + if cfg.Replicated || cfg.FallbackDatabase != "" { + if dbEngine, dbExists, err = getDatabaseEngine(ctx, db, cfg.Database); err != nil { + log.Fatalf("Failed to check database engine: %v", err) + } + } + database, warning := resolveDatabase(cfg.Database, cfg.FallbackDatabase, dbExists, legacyHasTables) + if warning != "" { + log.Printf("WARNING: %s", warning) + } + + // A fallback database keeps the plain engines it was seeded with. + replicated := false + if database == cfg.Database { + replicated, warning = planReplicated(cfg.Replicated, dbExists, database, dbEngine) + if warning != "" { + log.Printf("WARNING: %s", warning) + } + if replicated { + log.Printf("Using Replicated database and table engines for %s", database) + } + } + + // Process schema directory (replace ${DATABASE}, ${DATABASE_ENGINE}, ${ENGINE_PREFIX} and ${*_TTL} macros) + log.Printf("Preparing SQL files with database: %s", database) + tempDir, err := processSchemaDir(cfg.SchemaDir, database, replicated, ttlExprs) if err != nil { log.Fatalf("Failed to process schema directory: %v", err) } @@ -841,11 +968,23 @@ func main() { // CREATE TABLE IF NOT EXISTS won't update TTL on existing tables; reconcile // (opt-in) applies it to them too. Non-fatal so it can't block startup. if cfg.ReconcileTableTTL { - if err := reconcileTableTTLs(ctx, db, cfg.Database, ttlExprs); err != nil { + if err := reconcileTableTTLs(ctx, db, database, ttlExprs); err != nil { log.Printf("WARNING: reconcile: %v", err) } } + // Record the seeded database so the entrypoint hands it to the collector. + if cfg.FallbackDatabase != "" { + if err := os.WriteFile(cfg.ResolvedDatabasePath, []byte(database+"\n"), 0644); err != nil { + // The collector would otherwise write to a database that was not seeded. + if database != cfg.Database { + log.Fatalf("Failed to write resolved database to %s: %v", cfg.ResolvedDatabasePath, err) + } + + log.Printf("WARNING: failed to write resolved database to %s: %v", cfg.ResolvedDatabasePath, err) + } + } + log.Println("========================================") log.Println("Schema seed completed successfully") log.Println("========================================") diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go index ba066ef6b8..a7ba097496 100644 --- a/packages/otel-collector/cmd/migrate/main_test.go +++ b/packages/otel-collector/cmd/migrate/main_test.go @@ -10,6 +10,7 @@ import ( "math/big" "os" "path/filepath" + "regexp" "strings" "testing" "time" @@ -557,7 +558,7 @@ func TestProcessSchemaDir_Subdirectories(t *testing.T) { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "testdb", map[string]string{"TABLES_TTL": "toIntervalHour(48)"}) + tempDir, err := processSchemaDir(schemaDir, "testdb", false, map[string]string{"TABLES_TTL": "toIntervalHour(48)"}) if err != nil { t.Fatal(err) } @@ -580,7 +581,7 @@ func TestProcessSchemaDir_MultipleReplacements(t *testing.T) { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "db", map[string]string{"TABLES_TTL": "toIntervalDay(7)"}) + tempDir, err := processSchemaDir(schemaDir, "db", false, map[string]string{"TABLES_TTL": "toIntervalDay(7)"}) if err != nil { t.Fatal(err) } @@ -603,7 +604,7 @@ func TestProcessSchemaDir_NoMacros(t *testing.T) { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "db", map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) + tempDir, err := processSchemaDir(schemaDir, "db", false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) if err != nil { t.Fatal(err) } @@ -619,7 +620,7 @@ func TestProcessSchemaDir_NoMacros(t *testing.T) { } func TestProcessSchemaDir_NonexistentDir(t *testing.T) { - _, err := processSchemaDir("/nonexistent/schema", "db", map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) + _, err := processSchemaDir("/nonexistent/schema", "db", false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) if err == nil { t.Fatal("expected error for nonexistent schema dir") } @@ -804,7 +805,7 @@ func TestProcessSchemaDir_PerSignalMacros(t *testing.T) { if err := os.WriteFile(filepath.Join(schemaDir, "001.sql"), []byte(content), 0644); err != nil { t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "db", map[string]string{ + tempDir, err := processSchemaDir(schemaDir, "db", false, map[string]string{ "LOGS_TTL": "toIntervalDay(180)", "TRACES_TTL": "toIntervalDay(180)", "METRICS_TTL": "toIntervalDay(30)", @@ -824,6 +825,227 @@ func TestProcessSchemaDir_PerSignalMacros(t *testing.T) { } } +// --------------------------------------------------------------------------- +// replicated engines +// --------------------------------------------------------------------------- + +func TestLoadConfig_Replicated(t *testing.T) { + origArgs := os.Args + t.Cleanup(func() { os.Args = origArgs }) + os.Args = []string{"migrate", t.TempDir()} + + for _, c := range []struct { + value string + want bool + }{{"", false}, {"1", false}, {"TRUE", false}, {"true", true}} { + t.Setenv("HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED", c.value) + cfg, err := loadConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.Replicated != c.want { + t.Errorf("REPLICATED=%q: got %v, want %v", c.value, cfg.Replicated, c.want) + } + } +} + +func TestProcessSchemaDir_ReplicatedMacros(t *testing.T) { + schemaDir := t.TempDir() + content := "CREATE DATABASE IF NOT EXISTS ${DATABASE}${DATABASE_ENGINE};\n" + + "ENGINE = ${ENGINE_PREFIX}MergeTree; ENGINE = ${ENGINE_PREFIX}SummingMergeTree; ENGINE = TimeSeries" + if err := os.WriteFile(filepath.Join(schemaDir, "001.sql"), []byte(content), 0644); err != nil { + t.Fatal(err) + } + for _, c := range []struct { + replicated bool + want string + }{ + {false, "CREATE DATABASE IF NOT EXISTS otel;\n" + + "ENGINE = MergeTree; ENGINE = SummingMergeTree; ENGINE = TimeSeries"}, + {true, "CREATE DATABASE IF NOT EXISTS otel ENGINE = Replicated('/clickhouse/databases/otel', '{shard}', '{replica}');\n" + + "ENGINE = ReplicatedMergeTree; ENGINE = ReplicatedSummingMergeTree; ENGINE = TimeSeries"}, + } { + tempDir, err := processSchemaDir(schemaDir, "otel", c.replicated, map[string]string{}) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + got, err := os.ReadFile(filepath.Join(tempDir, "001.sql")) + if err != nil { + t.Fatal(err) + } + if string(got) != c.want { + t.Errorf("replicated=%v: got %q, want %q", c.replicated, string(got), c.want) + } + } +} + +// Guards the real seed files: every MergeTree-family engine must use the +// ${ENGINE_PREFIX} macro and take no arguments (a Replicated database rejects +// explicit Keeper path/replica arguments), and the database must be created +// with the Replicated engine only when enabled. +func TestSeedSchemaEngines(t *testing.T) { + seedDir := filepath.Join("..", "..", "..", "..", "docker", "otel-collector", "schema", "seed") + ttlExprs := map[string]string{} + for _, m := range []string{"TABLES_TTL", "LOGS_TTL", "TRACES_TTL", "METRICS_TTL", "SESSIONS_TTL"} { + ttlExprs[m] = "toIntervalDay(30)" + } + engineRe := regexp.MustCompile(`(?im)^\s*ENGINE\s*=\s*(.*)$`) + for _, replicated := range []bool{false, true} { + tempDir, err := processSchemaDir(seedDir, "otel", replicated, ttlExprs) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + files, err := listSQLFiles(tempDir) + if err != nil { + t.Fatal(err) + } + mergeTrees := 0 + for _, f := range files { + content, err := os.ReadFile(filepath.Join(tempDir, f)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(content), "${") { + t.Errorf("replicated=%v: %s has an unreplaced macro", replicated, f) + } + for _, m := range engineRe.FindAllStringSubmatch(string(content), -1) { + engine := strings.TrimSpace(m[1]) + if engine == "TimeSeries" { + continue + } + mergeTrees++ + if strings.Contains(engine, "(") { + t.Errorf("%s: engine %q must not take arguments", f, engine) + } + if strings.HasPrefix(engine, "Replicated") != replicated || !strings.HasSuffix(engine, "MergeTree") { + t.Errorf("replicated=%v: %s has engine %q", replicated, f, engine) + } + } + } + if mergeTrees == 0 { + t.Errorf("replicated=%v: no MergeTree engines found in %s", replicated, seedDir) + } + + createDB, err := os.ReadFile(filepath.Join(tempDir, "00001_create_database.sql")) + if err != nil { + t.Fatal(err) + } + want := "CREATE DATABASE IF NOT EXISTS otel;\n" + if replicated { + want = "CREATE DATABASE IF NOT EXISTS otel ENGINE = Replicated('/clickhouse/databases/otel', '{shard}', '{replica}');\n" + } + if !strings.HasSuffix(string(createDB), want) { + t.Errorf("replicated=%v: 00001_create_database.sql = %q; want suffix %q", replicated, string(createDB), want) + } + } +} + +func TestPlanReplicated(t *testing.T) { + cases := []struct { + name string + enabled, exists bool + engine string + want, warns bool + }{ + {"flag off", false, true, "Atomic", false, false}, + {"flag off, database missing", false, false, "", false, false}, + {"flag off, database Replicated", false, true, "Replicated", false, false}, + {"database missing", true, false, "", true, false}, + {"database Replicated", true, true, "Replicated", true, false}, + {"database Atomic", true, true, "Atomic", false, true}, + {"database Shared", true, true, "Shared", false, true}, + } + for _, c := range cases { + got, warning := planReplicated(c.enabled, c.exists, "otel", c.engine) + if got != c.want { + t.Errorf("%s: replicated = %v; want %v", c.name, got, c.want) + } + if (warning != "") != c.warns { + t.Errorf("%s: warning = %q; want warning=%v", c.name, warning, c.warns) + } + if c.warns && (!strings.Contains(warning, c.engine) || !strings.Contains(warning, "otel")) { + t.Errorf("%s: warning should name the database and engine: %q", c.name, warning) + } + } +} + +// --------------------------------------------------------------------------- +// fallback database +// --------------------------------------------------------------------------- + +func TestLoadConfig_FallbackDatabase(t *testing.T) { + origArgs := os.Args + t.Cleanup(func() { os.Args = origArgs }) + os.Args = []string{"migrate", t.TempDir()} + + cfg, err := loadConfig() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.FallbackDatabase != "" { + t.Errorf("FallbackDatabase should default to empty, got %q", cfg.FallbackDatabase) + } + if cfg.ResolvedDatabasePath != "/tmp/hyperdx-otel-exporter-database" { + t.Errorf("ResolvedDatabasePath: got %q", cfg.ResolvedDatabasePath) + } + + t.Setenv("HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE", "default") + t.Setenv("HYPERDX_OTEL_EXPORTER_RESOLVED_DATABASE_PATH", "/run/hdx/database") + if cfg, err = loadConfig(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.FallbackDatabase != "default" || cfg.ResolvedDatabasePath != "/run/hdx/database" { + t.Errorf("overrides not applied: fallback=%q path=%q", cfg.FallbackDatabase, cfg.ResolvedDatabasePath) + } +} + +func TestResolveDatabase(t *testing.T) { + cases := []struct { + name string + target, fallback string + targetExists, legacyHasTables bool + want string + wantWarning []string // substrings; nil means no warning + }{ + {"no fallback, target default", "default", "", true, true, "default", nil}, + {"no fallback, legacy tables in default", "otel", "", false, true, "otel", + []string{"database default has existing ClickStack tables", "not migrated to otel", "HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default"}}, + {"no fallback, default empty", "otel", "", false, false, "otel", nil}, + {"fallback, target missing, fallback seeded", "otel", "default", false, true, "default", + []string{"database otel does not exist", "staying on default", "HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE=default", "clear HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE to move to otel"}}, + {"fallback, target exists, fallback seeded", "otel", "default", true, true, "otel", + []string{"database default has existing ClickStack tables", "not migrated to otel"}}, + {"fallback, target missing, fallback empty", "otel", "default", false, false, "otel", nil}, + {"fallback equals target", "default", "default", true, true, "default", nil}, + {"custom fallback", "otel", "legacy", false, true, "legacy", []string{"staying on legacy"}}, + } + for _, c := range cases { + got, warning := resolveDatabase(c.target, c.fallback, c.targetExists, c.legacyHasTables) + if got != c.want { + t.Errorf("%s: database = %q; want %q", c.name, got, c.want) + } + if c.wantWarning == nil && warning != "" { + t.Errorf("%s: unexpected warning %q", c.name, warning) + } + for _, sub := range c.wantWarning { + if !strings.Contains(warning, sub) { + t.Errorf("%s: warning %q should contain %q", c.name, warning, sub) + } + } + } +} + +func TestLegacyDatabase(t *testing.T) { + if got := legacyDatabase(""); got != "default" { + t.Errorf("legacyDatabase(\"\") = %q; want default", got) + } + if got := legacyDatabase("legacy"); got != "legacy" { + t.Errorf("legacyDatabase(\"legacy\") = %q; want legacy", got) + } +} + // --------------------------------------------------------------------------- // parseTTLDuration / ttlToClickHouseInterval (existing tests below) // --------------------------------------------------------------------------- @@ -1151,7 +1373,7 @@ SETTINGS ttl_only_drop_parts = 1;` t.Fatal(err) } - tempDir, err := processSchemaDir(schemaDir, "mydb", map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) + tempDir, err := processSchemaDir(schemaDir, "mydb", false, map[string]string{"TABLES_TTL": "toIntervalDay(30)"}) if err != nil { t.Fatal(err) }