Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion docs/ngc-managed/cluster-management/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -788,10 +788,13 @@ BYOO metric subset example:
- 'metric.name != "BpsInstrument"'
byooWorkloadMetrics:
dropLabels:
- metric_subset_enabled
- custom_label
```

When `byooMetricSubset.enabled` is true, `dropLabels` extends the default
`metric_subset_enabled` label. The configured labels are removed from both the
primary metrics pipeline and the metric subset endpoint on port `19091`.

**Apply via Helm:**

```bash
Expand Down
5 changes: 4 additions & 1 deletion docs/user/cluster-management/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -918,10 +918,13 @@ agentConfig:
- 'metric.name != "BpsInstrument"'
byooWorkloadMetrics:
dropLabels:
- metric_subset_enabled
- custom_label
```

When `byooMetricSubset.enabled` is true, `dropLabels` extends the default
`metric_subset_enabled` label. The configured labels are removed from both the
primary metrics pipeline and the metric subset endpoint on port `19091`.

Apply via Helm:

```bash
Expand Down
2 changes: 1 addition & 1 deletion src/compute-plane-services/byoo-otel-collector/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ Chunking is disabled by default. Configure it with:
- `BYOO_OTEL_COLLECTOR_CONFIG_B64`: optional base64-encoded JSON for advanced collector rendering overrides, such as exporterhelper timeout, retry, sending queue, sending queue batch, memory limiter, batch, log batch, and separate log and trace sampler settings. Both samplers support sampling percentage, mode, hash seed, and fail closed. The log sampler also supports attribute source, source attribute, and sampling priority.
- `BYOO_METRIC_SUBSET_ENABLED`: enables an additional OTLP-only metrics pipeline that exposes filtered user metrics through a Prometheus exporter on port `19091`. Disabled by default.
- `BYOO_METRIC_SUBSET_FILTER_CONFIG`: optional YAML filter processor config for the metric subset pipeline. If unset, the default drops every metric except `BpsInstrument`, `FpsInstrument`, `RtdInstrument`, and `StageOpenDuration`, and drops datapoints/resources explicitly labeled `metric_subset_enabled=false`.
- `BYOO_WORKLOAD_METRICS_DROP_LABELS`: comma-separated resource attribute names removed from the generated workload `metrics` pipeline. If unset, defaults to `metric_subset_enabled` only when the metric subset pipeline is enabled.
- `BYOO_WORKLOAD_METRICS_DROP_LABELS`: comma-separated resource attribute names removed from the generated workload metrics pipelines. When the metric subset pipeline is enabled, configured labels extend the default `metric_subset_enabled` label. Labels are removed from both the primary and metric subset pipelines.

When chunking is enabled, each emitted chunk preserves the original log metadata and adds these attributes so chunks can be grouped in the backend:

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,7 @@ service:
- memory_limiter
- filter/metric_subset
- resource
- resource/workload_metrics_drop_labels
- metrics_transform
- batch/metric_subset
extensions:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ func TestGetTemplateConfig(t *testing.T) {
"NVCF_ZONE_NAME": "zone-1",
"BYOO_METRIC_SUBSET_ENABLED": "true",
"BYOO_METRIC_SUBSET_FILTER_CONFIG": "error_mode: ignore\nmetric_conditions:\n - 'metric.name == \"drop\"'\n",
"BYOO_WORKLOAD_METRICS_DROP_LABELS": "metric_subset_enabled, custom_label, metric_subset_enabled",
"BYOO_WORKLOAD_METRICS_DROP_LABELS": "custom_label",
},
expectErr: false,
expect: func(t *testing.T, cfg TemplateConfig) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,12 @@ func resolvedWorkloadMetricsDropLabels(configured string, metricSubsetEnabled bo

seen := map[string]struct{}{}
labels := []string{}
if metricSubsetEnabled {
for _, label := range defaultWorkloadMetricsDropLabels {
seen[label] = struct{}{}
labels = append(labels, label)
}
}
for _, label := range strings.Split(configured, ",") {
label = strings.TrimSpace(label)
if label == "" {
Expand Down Expand Up @@ -578,7 +584,7 @@ func cloneConfigValue(value interface{}) interface{} {
}
}

func addMetricSubsetPipeline(otelConfig *OpenTelemetryConfig, config MetricSubsetConfig) {
func addMetricSubsetPipeline(otelConfig *OpenTelemetryConfig, config MetricSubsetConfig, workloadMetricsDropLabelsProcessor string) {
addMetricSubsetExporter(otelConfig)

filterConfig := config.FilterConfig
Expand All @@ -604,9 +610,14 @@ func addMetricSubsetPipeline(otelConfig *OpenTelemetryConfig, config MetricSubse
"memory_limiter",
metricSubsetFilterProcessorID,
"resource",
}
if workloadMetricsDropLabelsProcessor != "" {
metricSubsetPipeline.Processors = append(metricSubsetPipeline.Processors, workloadMetricsDropLabelsProcessor)
}
metricSubsetPipeline.Processors = append(metricSubsetPipeline.Processors,
"metrics_transform",
metricSubsetBatchProcessorID,
}
)
otelConfig.Service.Pipelines["metrics/metric_subset"] = metricSubsetPipeline
}

Expand Down Expand Up @@ -888,14 +899,15 @@ func generateExportersAndService(config TelemetryConfig, otelConfig *OpenTelemet
metricPipeline.Receivers = []string{"otlp", "prometheus"}
metricPipeline.Exporters = []string{exporterId}
metricPipeline.Processors = []string{"memory_limiter", "filter/metrics", "resource"}
if processorID := addWorkloadMetricsDropLabelsProcessor(otelConfig, tmplConfig.WorkloadMetrics.DropLabels); processorID != "" {
metricPipeline.Processors = append(metricPipeline.Processors, processorID)
workloadMetricsDropLabelsProcessor := addWorkloadMetricsDropLabelsProcessor(otelConfig, tmplConfig.WorkloadMetrics.DropLabels)
if workloadMetricsDropLabelsProcessor != "" {
metricPipeline.Processors = append(metricPipeline.Processors, workloadMetricsDropLabelsProcessor)
}
metricPipeline.Processors = append(metricPipeline.Processors, "metrics_transform", "batch")
otelConfig.Service.Pipelines["metrics"] = metricPipeline

if tmplConfig.MetricSubset.Enabled {
addMetricSubsetPipeline(otelConfig, tmplConfig.MetricSubset)
addMetricSubsetPipeline(otelConfig, tmplConfig.MetricSubset, workloadMetricsDropLabelsProcessor)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@ func TestRenderOtelConfigWithMetricSubsetPipeline(t *testing.T) {
"memory_limiter",
metricSubsetFilterProcessorID,
"resource",
workloadMetricsDropLabelsProcessorID,
"metrics_transform",
metricSubsetBatchProcessorID,
}, otelConfig.Service.Pipelines["metrics/metric_subset"].Processors)
Expand Down Expand Up @@ -856,7 +857,7 @@ func TestGenerateExportersAndServiceAddsMetricSubsetPipeline(t *testing.T) {
FilterConfig: filterConfig,
},
WorkloadMetrics: WorkloadMetricsConfig{
DropLabels: []string{"metric_subset_enabled"},
DropLabels: []string{"metric_subset_enabled", "custom_label"},
},
})

Expand Down Expand Up @@ -888,6 +889,10 @@ func TestGenerateExportersAndServiceAddsMetricSubsetPipeline(t *testing.T) {
"key": "metric_subset_enabled",
"action": "delete",
},
{
"key": "custom_label",
"action": "delete",
},
},
}, otelConfig.Processors[workloadMetricsDropLabelsProcessorID])

Expand All @@ -898,6 +903,7 @@ func TestGenerateExportersAndServiceAddsMetricSubsetPipeline(t *testing.T) {
"memory_limiter",
metricSubsetFilterProcessorID,
"resource",
workloadMetricsDropLabelsProcessorID,
"metrics_transform",
metricSubsetBatchProcessorID,
}, metricSubsetPipeline.Processors)
Expand Down
Loading