diff --git a/docs/ngc-managed/cluster-management/configuration.md b/docs/ngc-managed/cluster-management/configuration.md index 0749820a8..ce8a9a62f 100644 --- a/docs/ngc-managed/cluster-management/configuration.md +++ b/docs/ngc-managed/cluster-management/configuration.md @@ -788,10 +788,13 @@ BYOO metric subset example: - 'metric.name != "BpsInstrument"' byooWorkloadMetrics: dropLabels: - - metric_subset_enabled - custom_label ``` +When `byooMetricSubset.enabled` is true, `dropLabels` extends the default +`metric_subset_enabled` label. The configured labels are removed from both the +primary metrics pipeline and the metric subset endpoint on port `19091`. + **Apply via Helm:** ```bash diff --git a/docs/user/cluster-management/configuration.md b/docs/user/cluster-management/configuration.md index be3f81e5d..1b309466a 100644 --- a/docs/user/cluster-management/configuration.md +++ b/docs/user/cluster-management/configuration.md @@ -918,10 +918,13 @@ agentConfig: - 'metric.name != "BpsInstrument"' byooWorkloadMetrics: dropLabels: - - metric_subset_enabled - custom_label ``` +When `byooMetricSubset.enabled` is true, `dropLabels` extends the default +`metric_subset_enabled` label. The configured labels are removed from both the +primary metrics pipeline and the metric subset endpoint on port `19091`. + Apply via Helm: ```bash diff --git a/src/compute-plane-services/byoo-otel-collector/README.md b/src/compute-plane-services/byoo-otel-collector/README.md index 749de5d73..89d50ec7f 100644 --- a/src/compute-plane-services/byoo-otel-collector/README.md +++ b/src/compute-plane-services/byoo-otel-collector/README.md @@ -141,7 +141,7 @@ Chunking is disabled by default. Configure it with: - `BYOO_OTEL_COLLECTOR_CONFIG_B64`: optional base64-encoded JSON for advanced collector rendering overrides, such as exporterhelper timeout, retry, sending queue, sending queue batch, memory limiter, batch, log batch, and separate log and trace sampler settings. Both samplers support sampling percentage, mode, hash seed, and fail closed. The log sampler also supports attribute source, source attribute, and sampling priority. - `BYOO_METRIC_SUBSET_ENABLED`: enables an additional OTLP-only metrics pipeline that exposes filtered user metrics through a Prometheus exporter on port `19091`. Disabled by default. - `BYOO_METRIC_SUBSET_FILTER_CONFIG`: optional YAML filter processor config for the metric subset pipeline. If unset, the default drops every metric except `BpsInstrument`, `FpsInstrument`, `RtdInstrument`, and `StageOpenDuration`, and drops datapoints/resources explicitly labeled `metric_subset_enabled=false`. -- `BYOO_WORKLOAD_METRICS_DROP_LABELS`: comma-separated resource attribute names removed from the generated workload `metrics` pipeline. If unset, defaults to `metric_subset_enabled` only when the metric subset pipeline is enabled. +- `BYOO_WORKLOAD_METRICS_DROP_LABELS`: comma-separated resource attribute names removed from the generated workload metrics pipelines. When the metric subset pipeline is enabled, configured labels extend the default `metric_subset_enabled` label. Labels are removed from both the primary and metric subset pipelines. When chunking is enabled, each emitted chunk preserves the original log metadata and adds these attributes so chunks can be grouped in the backend: diff --git a/src/compute-plane-services/byoo-otel-collector/examples/otelconfigs/k8s/config_function_container_metric_subset.yaml b/src/compute-plane-services/byoo-otel-collector/examples/otelconfigs/k8s/config_function_container_metric_subset.yaml index 705c77998..5642a5071 100644 --- a/src/compute-plane-services/byoo-otel-collector/examples/otelconfigs/k8s/config_function_container_metric_subset.yaml +++ b/src/compute-plane-services/byoo-otel-collector/examples/otelconfigs/k8s/config_function_container_metric_subset.yaml @@ -285,6 +285,7 @@ service: - memory_limiter - filter/metric_subset - resource + - resource/workload_metrics_drop_labels - metrics_transform - batch/metric_subset extensions: diff --git a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/otelconfig_test.go b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/otelconfig_test.go index 6e7ede922..771cec7d1 100644 --- a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/otelconfig_test.go +++ b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/otelconfig_test.go @@ -212,7 +212,7 @@ func TestGetTemplateConfig(t *testing.T) { "NVCF_ZONE_NAME": "zone-1", "BYOO_METRIC_SUBSET_ENABLED": "true", "BYOO_METRIC_SUBSET_FILTER_CONFIG": "error_mode: ignore\nmetric_conditions:\n - 'metric.name == \"drop\"'\n", - "BYOO_WORKLOAD_METRICS_DROP_LABELS": "metric_subset_enabled, custom_label, metric_subset_enabled", + "BYOO_WORKLOAD_METRICS_DROP_LABELS": "custom_label", }, expectErr: false, expect: func(t *testing.T, cfg TemplateConfig) { diff --git a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go index 60983fa7b..34501e3fc 100644 --- a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go +++ b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render.go @@ -248,6 +248,12 @@ func resolvedWorkloadMetricsDropLabels(configured string, metricSubsetEnabled bo seen := map[string]struct{}{} labels := []string{} + if metricSubsetEnabled { + for _, label := range defaultWorkloadMetricsDropLabels { + seen[label] = struct{}{} + labels = append(labels, label) + } + } for _, label := range strings.Split(configured, ",") { label = strings.TrimSpace(label) if label == "" { @@ -578,7 +584,7 @@ func cloneConfigValue(value interface{}) interface{} { } } -func addMetricSubsetPipeline(otelConfig *OpenTelemetryConfig, config MetricSubsetConfig) { +func addMetricSubsetPipeline(otelConfig *OpenTelemetryConfig, config MetricSubsetConfig, workloadMetricsDropLabelsProcessor string) { addMetricSubsetExporter(otelConfig) filterConfig := config.FilterConfig @@ -604,9 +610,14 @@ func addMetricSubsetPipeline(otelConfig *OpenTelemetryConfig, config MetricSubse "memory_limiter", metricSubsetFilterProcessorID, "resource", + } + if workloadMetricsDropLabelsProcessor != "" { + metricSubsetPipeline.Processors = append(metricSubsetPipeline.Processors, workloadMetricsDropLabelsProcessor) + } + metricSubsetPipeline.Processors = append(metricSubsetPipeline.Processors, "metrics_transform", metricSubsetBatchProcessorID, - } + ) otelConfig.Service.Pipelines["metrics/metric_subset"] = metricSubsetPipeline } @@ -888,14 +899,15 @@ func generateExportersAndService(config TelemetryConfig, otelConfig *OpenTelemet metricPipeline.Receivers = []string{"otlp", "prometheus"} metricPipeline.Exporters = []string{exporterId} metricPipeline.Processors = []string{"memory_limiter", "filter/metrics", "resource"} - if processorID := addWorkloadMetricsDropLabelsProcessor(otelConfig, tmplConfig.WorkloadMetrics.DropLabels); processorID != "" { - metricPipeline.Processors = append(metricPipeline.Processors, processorID) + workloadMetricsDropLabelsProcessor := addWorkloadMetricsDropLabelsProcessor(otelConfig, tmplConfig.WorkloadMetrics.DropLabels) + if workloadMetricsDropLabelsProcessor != "" { + metricPipeline.Processors = append(metricPipeline.Processors, workloadMetricsDropLabelsProcessor) } metricPipeline.Processors = append(metricPipeline.Processors, "metrics_transform", "batch") otelConfig.Service.Pipelines["metrics"] = metricPipeline if tmplConfig.MetricSubset.Enabled { - addMetricSubsetPipeline(otelConfig, tmplConfig.MetricSubset) + addMetricSubsetPipeline(otelConfig, tmplConfig.MetricSubset, workloadMetricsDropLabelsProcessor) } } diff --git a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go index 5ccb31755..401747522 100644 --- a/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go +++ b/src/compute-plane-services/byoo-otel-collector/internal/otelconfig/render_test.go @@ -159,6 +159,7 @@ func TestRenderOtelConfigWithMetricSubsetPipeline(t *testing.T) { "memory_limiter", metricSubsetFilterProcessorID, "resource", + workloadMetricsDropLabelsProcessorID, "metrics_transform", metricSubsetBatchProcessorID, }, otelConfig.Service.Pipelines["metrics/metric_subset"].Processors) @@ -856,7 +857,7 @@ func TestGenerateExportersAndServiceAddsMetricSubsetPipeline(t *testing.T) { FilterConfig: filterConfig, }, WorkloadMetrics: WorkloadMetricsConfig{ - DropLabels: []string{"metric_subset_enabled"}, + DropLabels: []string{"metric_subset_enabled", "custom_label"}, }, }) @@ -888,6 +889,10 @@ func TestGenerateExportersAndServiceAddsMetricSubsetPipeline(t *testing.T) { "key": "metric_subset_enabled", "action": "delete", }, + { + "key": "custom_label", + "action": "delete", + }, }, }, otelConfig.Processors[workloadMetricsDropLabelsProcessorID]) @@ -898,6 +903,7 @@ func TestGenerateExportersAndServiceAddsMetricSubsetPipeline(t *testing.T) { "memory_limiter", metricSubsetFilterProcessorID, "resource", + workloadMetricsDropLabelsProcessorID, "metrics_transform", metricSubsetBatchProcessorID, }, metricSubsetPipeline.Processors)