Skip to content

Commit 46f0ead

Browse files
committed
feat: let operators opt out of the text index block size ALTER
1 parent 001b112 commit 46f0ead

4 files changed

Lines changed: 31 additions & 4 deletions

File tree

‎.changeset/text-index-dictionary-block-size.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,7 @@ made searches for a common value 10-30% faster.
1515

1616
Because it's an `ALTER`, existing installs get it too: new parts use 8,192-token
1717
blocks, and existing parts keep theirs until they're merged. A table that
18-
already sets `text_index_dictionary_block_size` is left alone. ClickHouse
19-
26.2–26.5 doesn't have the setting, so those servers keep 512-token blocks.
18+
already sets `text_index_dictionary_block_size` is left alone, and operators who
19+
manage these tables themselves can turn the change off with
20+
`HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE=false`. ClickHouse 26.2–26.5
21+
doesn't have the setting, so those servers keep 512-token blocks.

‎docker-compose.dev.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ services:
4545
# HYPERDX_OTEL_EXPORTER_LOGS_TTL / _TRACES_TTL / _METRICS_TTL / _SESSIONS_TTL
4646
# Apply TTL changes to already-existing tables too (default false):
4747
# HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL: 'true'
48+
# Leave text index dictionary block sizes alone (default: set 8192 on 26.6+):
49+
# HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE: 'false'
4850
ENABLE_PROMQL: 'true'
4951
volumes:
5052
- ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml

‎packages/otel-collector/cmd/migrate/main.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ type Config struct {
4343
// Reconcile TTL on already-existing tables, not just newly created ones.
4444
ReconcileTableTTL bool
4545

46+
// Set 8,192-token text index dictionary blocks on the seeded tables.
47+
SetTextIndexBlockSize bool
48+
4649
// TLS settings
4750
TLSCAFile string
4851
TLSCertFile string
@@ -69,6 +72,7 @@ func loadConfig() (*Config, error) {
6972
MetricsTTL: getEnv("HYPERDX_OTEL_EXPORTER_METRICS_TTL", tablesTTL),
7073
SessionsTTL: getEnv("HYPERDX_OTEL_EXPORTER_SESSIONS_TTL", tablesTTL),
7174
ReconcileTableTTL: getEnv("HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL", "false") == "true",
75+
SetTextIndexBlockSize: getEnv("HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE", "true") != "false",
7276
TLSCAFile: getEnv("CLICKHOUSE_TLS_CA_FILE", ""),
7377
TLSCertFile: getEnv("CLICKHOUSE_TLS_CERT_FILE", ""),
7478
TLSKeyFile: getEnv("CLICKHOUSE_TLS_KEY_FILE", ""),
@@ -919,8 +923,10 @@ func main() {
919923
os.Exit(1)
920924
}
921925

922-
// Non-fatal so it can't block startup; older servers keep 512-token blocks.
923-
if supportsTextIndexTableSettings(chMajor, chMinor) {
926+
// On by default; HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE=false lets
927+
// operators who manage these tables opt out. Non-fatal so it can't block
928+
// startup; older servers keep 512-token blocks.
929+
if cfg.SetTextIndexBlockSize && supportsTextIndexTableSettings(chMajor, chMinor) {
924930
if err := setTextIndexBlockSize(ctx, db, cfg.Database); err != nil {
925931
log.Printf("WARNING: text index block size: %v", err)
926932
}

‎packages/otel-collector/cmd/migrate/main_test.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,9 @@ func TestLoadConfig(t *testing.T) {
111111
if cfg.ReconcileTableTTL {
112112
t.Error("ReconcileTableTTL should default to false")
113113
}
114+
if !cfg.SetTextIndexBlockSize {
115+
t.Error("SetTextIndexBlockSize should default to true")
116+
}
114117
if cfg.SchemaDir != dir {
115118
t.Errorf("SchemaDir: got %q, want %q", cfg.SchemaDir, dir)
116119
}
@@ -629,6 +632,20 @@ func TestProcessSchemaDir_NonexistentDir(t *testing.T) {
629632
// per-signal TTL resolution + reconcile helpers
630633
// ---------------------------------------------------------------------------
631634

635+
func TestLoadConfig_TextIndexBlockSizeOptOut(t *testing.T) {
636+
dir := t.TempDir()
637+
os.Args = []string{"migrate", dir}
638+
t.Setenv("HYPERDX_OTEL_EXPORTER_SET_TEXT_INDEX_BLOCK_SIZE", "false")
639+
640+
cfg, err := loadConfig()
641+
if err != nil {
642+
t.Fatalf("unexpected error: %v", err)
643+
}
644+
if cfg.SetTextIndexBlockSize {
645+
t.Error("SetTextIndexBlockSize should be false")
646+
}
647+
}
648+
632649
func TestLoadConfig_PerSignalTTL(t *testing.T) {
633650
dir := t.TempDir()
634651
os.Args = []string{"migrate", dir}

0 commit comments

Comments
 (0)