From e25b75cd430b82cf834609d08d575d4503693236 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 06:43:50 +1100 Subject: [PATCH 1/2] fix(sink): set the producer compression codec once Vector writes sink.compression as compression.codec. The generator also wrote compression.type. librdkafka reads both as one setting. rdkafka applies a client's options in hash order. With compression.codec: gzip in sink.librdkafka_options, 30 starts of Vector 0.61.0 on the generated sink gave 12 gzip and 18 zstd. The generated sink alone gave 30 zstd, because both spellings carried the same value. The sink sets the codec once through Vector's compression field. It drops compression.codec and compression.type from the profile's librdkafka_options. Both spellings are derived options, so config validation rejects either in sink.librdkafka_options. --- docs/librdkafka.md | 4 +- src/config/generate.rs | 83 +++++++++++++++++++++---------- tests/integration/config_reach.rs | 3 +- 3 files changed, 60 insertions(+), 30 deletions(-) diff --git a/docs/librdkafka.md b/docs/librdkafka.md index ec4741b..39ef1fe 100644 --- a/docs/librdkafka.md +++ b/docs/librdkafka.md @@ -29,7 +29,7 @@ A few keys are derived from the big dials and written after the merge, so `librd | Key | Owned by | Written when | |---|---|---| -| `compression.type` | `sink.compression` | always, on the sink | +| `compression.type`, `compression.codec` | `sink.compression` | always, on the sink, as Vector's `compression` field | | `security.protocol` | `{source,sink}.sasl.enabled` + `.tls.enabled` | either is on | | `sasl.mechanism` | `{source,sink}.sasl.mechanism` | SASL is on | | `enable.ssl.certificate.verification` | `{source,sink}.tls.skip_verify` | TLS is on and verification is skipped | @@ -48,7 +48,7 @@ sink: ## Example -For a producer with no central file, `sink.compression: lz4` and `sink.librdkafka_options: {linger.ms: "20"}`, the supervisor writes `batch.timeout_secs: 0.02` on the sink and, in `librdkafka_options`, `compression.type=lz4`, `queue.buffering.max.kbytes=262144`, `enable.idempotence=true`, `socket.nagle.disable=true`, `statistics.interval.ms=1000` and `message.max.bytes=16777216`, the last because no layer set it. +For a producer with no central file, `sink.compression: lz4` and `sink.librdkafka_options: {linger.ms: "20"}`, the supervisor writes `batch.timeout_secs: 0.02` and `compression: lz4` on the sink and, in `librdkafka_options`, `queue.buffering.max.kbytes=262144`, `enable.idempotence=true`, `socket.nagle.disable=true`, `statistics.interval.ms=1000` and `message.max.bytes=16777216`, the last because no layer set it. Add a central `producer.production` naming only `linger.ms: "50"` and `socket.nagle.disable` and `statistics.interval.ms` drop out of `librdkafka_options`, because the central profile replaced the constants rather than merging over them. Vector still writes `statistics.interval.ms=1000` itself. The delay stays 20 ms: the per-instance option outranks the central file. Drop that option and it becomes 50 ms. diff --git a/src/config/generate.rs b/src/config/generate.rs index 698201d..eaaacbb 100644 --- a/src/config/generate.rs +++ b/src/config/generate.rs @@ -348,10 +348,10 @@ fn kafka_sink_component(sink: &SinkConfig, inputs: &[String], acknowledged: bool encoding.insert(key("codec"), val(&sink.encoding)); component.insert(key("encoding"), Value::Object(encoding)); - // Compression - if sink.compression != "none" { - component.insert(key("compression"), val(&sink.compression)); - } + // Compression. Vector writes this as `compression.codec`, the only + // statement of it: librdkafka reads `compression.type` as the same setting, + // so `build_sink_librdkafka` keeps that spelling out. + component.insert(key("compression"), val(&sink.compression)); // Acknowledgements — on the sink (not source) per Vector 0.53+ guidance component.insert(key("acknowledgements"), enabled_block(acknowledged)); @@ -509,11 +509,15 @@ pub fn derived_source_options(source: &SourceConfig) -> Vec<(&'static str, &'sta /// Sink counterpart of [`derived_source_options`]. /// -/// `compression.type` is unconditional: the sink always writes it from +/// `compression.type` and `compression.codec`, librdkafka's two spellings of +/// one setting, are unconditional: the sink always writes the codec from /// `sink.compression`. #[must_use] pub fn derived_sink_options(sink: &SinkConfig) -> Vec<(&'static str, &'static str)> { - let mut derived = vec![("compression.type", "sink.compression")]; + let mut derived = vec![ + ("compression.type", "sink.compression"), + ("compression.codec", "sink.compression"), + ]; if security_protocol(&sink.sasl, &sink.tls).is_some() { derived.push(( "security.protocol", @@ -606,15 +610,10 @@ fn build_sink_librdkafka(sink: &SinkConfig) -> Mapping { m.insert(key(k), val(v)); } - // Compression from big-dial config (overrides scalo default if different) - let compression = match sink.compression.as_str() { - "none" => "none", - "gzip" => "gzip", - "lz4" => "lz4", - "snappy" => "snappy", - _ => "zstd", - }; - m.insert(key("compression.type"), val(compression)); + // The codec comes from `sink.compression` through Vector's own field, so a + // profile's spelling of it is dropped. + m.remove("compression.type"); + m.remove("compression.codec"); insert_security_options(&mut m, &sink.sasl, &sink.tls); m } @@ -1386,7 +1385,7 @@ mod tests { assert!(text.contains("codec: json")); assert!(text.contains("compression: zstd")); // Production librdkafka defaults (from scalo kafka_config) - assert!(text.contains("compression.type: zstd")); + assert!(!text.contains("compression.type")); assert!(text.contains("socket.nagle.disable: \"true\"")); assert!(text.contains("statistics.interval.ms: \"1000\"")); // Service-specific override: producer queue cap @@ -1546,16 +1545,16 @@ mod tests { sink.get("compression").and_then(Value::as_str), Some("zstd") ); - assert_eq!( - sink.get("librdkafka_options") - .and_then(|o| o.get("compression.type")) - .and_then(Value::as_str), - Some("zstd") - ); + let options = sink + .get("librdkafka_options") + .and_then(Value::as_object) + .expect("the librdkafka options"); + assert!(!options.contains_key("compression.type")); + assert!(!options.contains_key("compression.codec")); } #[test] - fn sink_yaml_no_compression_sets_none_in_librdkafka() { + fn sink_yaml_no_compression_sets_none_on_the_vector_field() { let sink = SinkConfig { compression: "none".into(), ..Default::default() @@ -1563,10 +1562,40 @@ mod tests { let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true); let text = scalo::yaml::to_string(&yaml).unwrap(); - // Vector-level compression omitted - assert!(!text.contains("\n compression:")); - // librdkafka compression.type set to none - assert!(text.contains("compression.type: none")); + assert!(text.contains("\n compression: none")); + assert!(!text.contains("compression.type")); + } + + /// A sink states its compression codec once, as Vector's `compression` + /// field. Vector writes that as `compression.codec`, which librdkafka reads + /// as `compression.type`, so a second statement leaves the producer with + /// whichever one the client applied last. + #[test] + fn a_kafka_sink_states_its_compression_codec_once() { + for codec in ["none", "gzip", "lz4", "snappy", "zstd"] { + let sink = SinkConfig { + compression: codec.into(), + ..Default::default() + }; + let component = kafka_sink_component(&sink, &["src".into()], true); + let options = component["librdkafka_options"].as_object().unwrap(); + assert_eq!(component["compression"], codec); + assert!(!options.contains_key("compression.type"), "{codec}"); + assert!(!options.contains_key("compression.codec"), "{codec}"); + } + } + + /// Either spelling in `sink.librdkafka_options` is refused, with the dial + /// that owns the codec named. + #[test] + fn both_spellings_of_the_compression_codec_are_derived_options() { + let derived = derived_sink_options(&SinkConfig::default()); + for spelling in ["compression.type", "compression.codec"] { + assert!( + derived.contains(&(spelling, "sink.compression")), + "{spelling}" + ); + } } #[test] diff --git a/tests/integration/config_reach.rs b/tests/integration/config_reach.rs index 2bdca66..ffc66cf 100644 --- a/tests/integration/config_reach.rs +++ b/tests/integration/config_reach.rs @@ -219,8 +219,9 @@ fn a_misspelt_logging_value_is_named_as_a_config_error() { /// `sink.compression: zstd` produced zstd and logged nothing. #[test] fn a_librdkafka_option_the_generator_overwrites_is_refused() { - let cases: [(&str, &str, &str); 3] = [ + let cases: [(&str, &str, &str); 4] = [ ("sink", "compression.type", "sink.compression"), + ("sink", "compression.codec", "sink.compression"), ("sink", "security.protocol", "sink.sasl.enabled"), ("source", "sasl.mechanism", "source.sasl.mechanism"), ]; From e7a862f720e99b88eb6fb876fe0e288bbee7e427 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 06:51:13 +1100 Subject: [PATCH 2/2] docs(librdkafka): link the scalo-rs profile source The profile constants file is in scalo-rs. The hyperi-ci doc-paths check treats the path in docs/librdkafka.md as a missing local file. Replace the path with a link to that file. --- docs/librdkafka.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/librdkafka.md b/docs/librdkafka.md index 39ef1fe..6480acf 100644 --- a/docs/librdkafka.md +++ b/docs/librdkafka.md @@ -119,7 +119,7 @@ Only the `production` profile of each side is read. A profile in the file replac ## Source code references -- Profile constants: scalo-rs `src/kafka_config.rs` +- Profile constants: [scalo-rs src/kafka_config.rs](https://github.com/hyperi-io/scalo-rs/blob/main/src/kafka_config.rs) - Central file loader: `src/config/kafka_defaults.rs` (`consumer_profile()`, `producer_profile()`, the `OnceLock` cache, `merge_layers()`) - Layer merge and derived keys: `src/config/generate.rs` (`build_source_librdkafka`, `build_sink_librdkafka`, `SERVICE_CONSUMER_OVERRIDES`, `SERVICE_PRODUCER_OVERRIDES`) - Config fields: `src/config/loader.rs` (`SourceConfig.librdkafka_options`, `SinkConfig.librdkafka_options`)