diff --git a/docs/use-cases/observability/clickstack/ingesting-data/schemas.md b/docs/use-cases/observability/clickstack/ingesting-data/schemas.md index 572ef81a6d8..f504cdadb27 100644 --- a/docs/use-cases/observability/clickstack/ingesting-data/schemas.md +++ b/docs/use-cases/observability/clickstack/ingesting-data/schemas.md @@ -44,19 +44,22 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)), + `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))), + `ScopeAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ScopeAttributes::Array(Tuple(String, String))), + `LogAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), LogAttributes::Array(Tuple(String, String))), INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_scope_attr_items ScopeAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE text(tokenizer = 'array'), - INDEX idx_log_attr_value mapValues(LogAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_log_attr_items LogAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_lower_body lower(Body) TYPE text(tokenizer = 'splitByNonAlpha') ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) -TTL toDateTime(Timestamp) + ${TABLES_TTL} +TTL toDateTime(Timestamp) + ${LOGS_TTL} SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1, enable_block_number_column = 1, enable_block_offset_column = 1; ``` @@ -92,19 +95,21 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces `Links.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), `__hdx_materialized_rum.sessionId` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(1)), `SampleRate` UInt64 MATERIALIZED greatest(toUInt64OrZero(SpanAttributes['SampleRate']), 1) CODEC(T64, ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + `ResourceAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), ResourceAttributes::Array(Tuple(String, String))), + `SpanAttributeItems` Array(String) ALIAS arrayMap((arr) -> concat(arr.1, '=', arr.2), SpanAttributes::Array(Tuple(String, String))), + INDEX idx_trace_id TraceId TYPE text(tokenizer = 'array'), + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_res_attr_items ResourceAttributeItems TYPE text(tokenizer = 'array'), + INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE text(tokenizer = 'array'), + INDEX idx_span_attr_items SpanAttributeItems TYPE text(tokenizer = 'array'), INDEX idx_duration Duration TYPE minmax GRANULARITY 1, - INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 + INDEX idx_lower_span_name SpanName TYPE text(tokenizer = 'splitByNonAlpha', preprocessor=lower(SpanName)) ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) -TTL toDate(Timestamp) + ${TABLES_TTL} +TTL toDate(Timestamp) + ${TRACES_TTL} SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; ``` @@ -126,16 +131,16 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_gauge `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), `ScopeSchemaUrl` String CODEC(ZSTD(1)), `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), + `MetricName` LowCardinality(String) CODEC(ZSTD(1)), `MetricDescription` String CODEC(ZSTD(1)), `MetricUnit` String CODEC(ZSTD(1)), `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(1)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(1)), `Value` Float64 CODEC(ZSTD(1)), `Flags` UInt32 CODEC(ZSTD(1)), `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(1)), `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), @@ -144,12 +149,13 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_gauge INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) -TTL toDateTime(TimeUnix) + ${TABLES_TTL} +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) +TTL toDateTime(TimeUnix) + ${METRICS_TTL} SETTINGS ttl_only_drop_parts = 1; ``` @@ -169,16 +175,16 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_sum `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), `ScopeSchemaUrl` String CODEC(ZSTD(1)), `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), + `MetricName` LowCardinality(String) CODEC(ZSTD(1)), `MetricDescription` String CODEC(ZSTD(1)), `MetricUnit` String CODEC(ZSTD(1)), `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(1)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(1)), `Value` Float64 CODEC(ZSTD(1)), `Flags` UInt32 CODEC(ZSTD(1)), `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(1)), `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), @@ -189,12 +195,13 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_sum INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) -TTL toDateTime(TimeUnix) + ${TABLES_TTL} +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) +TTL toDateTime(TimeUnix) + ${METRICS_TTL} SETTINGS ttl_only_drop_parts = 1; ``` @@ -214,18 +221,18 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_histogram `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), `ScopeSchemaUrl` String CODEC(ZSTD(1)), `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), + `MetricName` LowCardinality(String) CODEC(ZSTD(1)), `MetricDescription` String CODEC(ZSTD(1)), `MetricUnit` String CODEC(ZSTD(1)), `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(1)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(1)), `Count` UInt64 CODEC(Delta(8), ZSTD(1)), `Sum` Float64 CODEC(ZSTD(1)), `BucketCounts` Array(UInt64) CODEC(ZSTD(1)), `ExplicitBounds` Array(Float64) CODEC(ZSTD(1)), `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(1)), `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), @@ -238,12 +245,13 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_histogram INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) -TTL toDateTime(TimeUnix) + ${TABLES_TTL} +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) +TTL toDateTime(TimeUnix) + ${METRICS_TTL} SETTINGS ttl_only_drop_parts = 1; ``` @@ -267,12 +275,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_exponential_histogram `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), `ScopeSchemaUrl` String CODEC(ZSTD(1)), `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), + `MetricName` LowCardinality(String) CODEC(ZSTD(1)), `MetricDescription` String CODEC(ZSTD(1)), `MetricUnit` String CODEC(ZSTD(1)), `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(1)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(1)), `Count` UInt64 CODEC(Delta(8), ZSTD(1)), `Sum` Float64 CODEC(ZSTD(1)), `Scale` Int32 CODEC(ZSTD(1)), @@ -282,7 +290,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_exponential_histogram `NegativeOffset` Int32 CODEC(ZSTD(1)), `NegativeBucketCounts` Array(UInt64) CODEC(ZSTD(1)), `Exemplars.FilteredAttributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), - `Exemplars.TimeUnix` Array(DateTime64(9)) CODEC(ZSTD(1)), + `Exemplars.TimeUnix` Array(DateTime) CODEC(ZSTD(1)), `Exemplars.Value` Array(Float64) CODEC(ZSTD(1)), `Exemplars.SpanId` Array(String) CODEC(ZSTD(1)), `Exemplars.TraceId` Array(String) CODEC(ZSTD(1)), @@ -295,12 +303,13 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_exponential_histogram INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) -TTL toDateTime(TimeUnix) + ${TABLES_TTL} +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) +TTL toDateTime(TimeUnix) + ${METRICS_TTL} SETTINGS ttl_only_drop_parts = 1; ``` @@ -320,12 +329,12 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_summary `ScopeDroppedAttrCount` UInt32 CODEC(ZSTD(1)), `ScopeSchemaUrl` String CODEC(ZSTD(1)), `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `MetricName` String CODEC(ZSTD(1)), + `MetricName` LowCardinality(String) CODEC(ZSTD(1)), `MetricDescription` String CODEC(ZSTD(1)), `MetricUnit` String CODEC(ZSTD(1)), `Attributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `StartTimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TimeUnix` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `StartTimeUnix` DateTime CODEC(Delta, ZSTD(1)), + `TimeUnix` DateTime CODEC(Delta, ZSTD(1)), `Count` UInt64 CODEC(Delta(8), ZSTD(1)), `Sum` Float64 CODEC(ZSTD(1)), `ValueAtQuantiles.Quantile` Array(Float64) CODEC(ZSTD(1)), @@ -336,12 +345,13 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_summary INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_attr_key mapKeys(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_attr_value mapValues(Attributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_time_minmax TimeUnix TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(TimeUnix) -ORDER BY (ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) -TTL toDateTime(TimeUnix) + ${TABLES_TTL} +ORDER BY (ServiceName, MetricName, toStartOfHour(TimeUnix), cityHash64(Attributes), TimeUnix) +TTL toDateTime(TimeUnix) + ${METRICS_TTL} SETTINGS ttl_only_drop_parts = 1; ``` @@ -382,7 +392,7 @@ ENGINE = MergeTree PARTITION BY toDate(TimestampTime) PRIMARY KEY (ServiceName, TimestampTime) ORDER BY (ServiceName, TimestampTime, Timestamp) -TTL TimestampTime + ${TABLES_TTL} +TTL TimestampTime + ${SESSIONS_TTL} SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; ```