Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions docs/librdkafka.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ A few keys are derived from the big dials and written after the merge, so `librd

| Key | Owned by | Written when |
|---|---|---|
| `compression.type` | `sink.compression` | always, on the sink |
| `compression.type`, `compression.codec` | `sink.compression` | always, on the sink, as Vector's `compression` field |
| `security.protocol` | `{source,sink}.sasl.enabled` + `.tls.enabled` | either is on |
| `sasl.mechanism` | `{source,sink}.sasl.mechanism` | SASL is on |
| `enable.ssl.certificate.verification` | `{source,sink}.tls.skip_verify` | TLS is on and verification is skipped |
Expand All @@ -48,7 +48,7 @@ sink:

## Example

For a producer with no central file, `sink.compression: lz4` and `sink.librdkafka_options: {linger.ms: "20"}`, the supervisor writes `batch.timeout_secs: 0.02` on the sink and, in `librdkafka_options`, `compression.type=lz4`, `queue.buffering.max.kbytes=262144`, `enable.idempotence=true`, `socket.nagle.disable=true`, `statistics.interval.ms=1000` and `message.max.bytes=16777216`, the last because no layer set it.
For a producer with no central file, `sink.compression: lz4` and `sink.librdkafka_options: {linger.ms: "20"}`, the supervisor writes `batch.timeout_secs: 0.02` and `compression: lz4` on the sink and, in `librdkafka_options`, `queue.buffering.max.kbytes=262144`, `enable.idempotence=true`, `socket.nagle.disable=true`, `statistics.interval.ms=1000` and `message.max.bytes=16777216`, the last because no layer set it.

Add a central `producer.production` naming only `linger.ms: "50"` and `socket.nagle.disable` and `statistics.interval.ms` drop out of `librdkafka_options`, because the central profile replaced the constants rather than merging over them. Vector still writes `statistics.interval.ms=1000` itself. The delay stays 20 ms: the per-instance option outranks the central file. Drop that option and it becomes 50 ms.

Expand Down Expand Up @@ -119,7 +119,7 @@ Only the `production` profile of each side is read. A profile in the file replac

## Source code references

- Profile constants: scalo-rs `src/kafka_config.rs`
- Profile constants: [scalo-rs src/kafka_config.rs](https://github.com/hyperi-io/scalo-rs/blob/main/src/kafka_config.rs)
- Central file loader: `src/config/kafka_defaults.rs` (`consumer_profile()`, `producer_profile()`, the `OnceLock` cache, `merge_layers()`)
- Layer merge and derived keys: `src/config/generate.rs` (`build_source_librdkafka`, `build_sink_librdkafka`, `SERVICE_CONSUMER_OVERRIDES`, `SERVICE_PRODUCER_OVERRIDES`)
- Config fields: `src/config/loader.rs` (`SourceConfig.librdkafka_options`, `SinkConfig.librdkafka_options`)
Expand Down
83 changes: 56 additions & 27 deletions src/config/generate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -348,10 +348,10 @@ fn kafka_sink_component(sink: &SinkConfig, inputs: &[String], acknowledged: bool
encoding.insert(key("codec"), val(&sink.encoding));
component.insert(key("encoding"), Value::Object(encoding));

// Compression
if sink.compression != "none" {
component.insert(key("compression"), val(&sink.compression));
}
// Compression. Vector writes this as `compression.codec`, the only
// statement of it: librdkafka reads `compression.type` as the same setting,
// so `build_sink_librdkafka` keeps that spelling out.
component.insert(key("compression"), val(&sink.compression));

// Acknowledgements — on the sink (not source) per Vector 0.53+ guidance
component.insert(key("acknowledgements"), enabled_block(acknowledged));
Expand Down Expand Up @@ -509,11 +509,15 @@ pub fn derived_source_options(source: &SourceConfig) -> Vec<(&'static str, &'sta

/// Sink counterpart of [`derived_source_options`].
///
/// `compression.type` is unconditional: the sink always writes it from
/// `compression.type` and `compression.codec`, librdkafka's two spellings of
/// one setting, are unconditional: the sink always writes the codec from
/// `sink.compression`.
#[must_use]
pub fn derived_sink_options(sink: &SinkConfig) -> Vec<(&'static str, &'static str)> {
let mut derived = vec![("compression.type", "sink.compression")];
let mut derived = vec![
("compression.type", "sink.compression"),
("compression.codec", "sink.compression"),
];
if security_protocol(&sink.sasl, &sink.tls).is_some() {
derived.push((
"security.protocol",
Expand Down Expand Up @@ -606,15 +610,10 @@ fn build_sink_librdkafka(sink: &SinkConfig) -> Mapping {
m.insert(key(k), val(v));
}

// Compression from big-dial config (overrides scalo default if different)
let compression = match sink.compression.as_str() {
"none" => "none",
"gzip" => "gzip",
"lz4" => "lz4",
"snappy" => "snappy",
_ => "zstd",
};
m.insert(key("compression.type"), val(compression));
// The codec comes from `sink.compression` through Vector's own field, so a
// profile's spelling of it is dropped.
m.remove("compression.type");
m.remove("compression.codec");
insert_security_options(&mut m, &sink.sasl, &sink.tls);
m
}
Expand Down Expand Up @@ -1386,7 +1385,7 @@ mod tests {
assert!(text.contains("codec: json"));
assert!(text.contains("compression: zstd"));
// Production librdkafka defaults (from scalo kafka_config)
assert!(text.contains("compression.type: zstd"));
assert!(!text.contains("compression.type"));
assert!(text.contains("socket.nagle.disable: \"true\""));
assert!(text.contains("statistics.interval.ms: \"1000\""));
// Service-specific override: producer queue cap
Expand Down Expand Up @@ -1546,27 +1545,57 @@ mod tests {
sink.get("compression").and_then(Value::as_str),
Some("zstd")
);
assert_eq!(
sink.get("librdkafka_options")
.and_then(|o| o.get("compression.type"))
.and_then(Value::as_str),
Some("zstd")
);
let options = sink
.get("librdkafka_options")
.and_then(Value::as_object)
.expect("the librdkafka options");
assert!(!options.contains_key("compression.type"));
assert!(!options.contains_key("compression.codec"));
}

#[test]
fn sink_yaml_no_compression_sets_none_in_librdkafka() {
fn sink_yaml_no_compression_sets_none_on_the_vector_field() {
let sink = SinkConfig {
compression: "none".into(),
..Default::default()
};
let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true);
let text = scalo::yaml::to_string(&yaml).unwrap();

// Vector-level compression omitted
assert!(!text.contains("\n compression:"));
// librdkafka compression.type set to none
assert!(text.contains("compression.type: none"));
assert!(text.contains("\n compression: none"));
assert!(!text.contains("compression.type"));
}

/// A sink states its compression codec once, as Vector's `compression`
/// field. Vector writes that as `compression.codec`, which librdkafka reads
/// as `compression.type`, so a second statement leaves the producer with
/// whichever one the client applied last.
#[test]
fn a_kafka_sink_states_its_compression_codec_once() {
for codec in ["none", "gzip", "lz4", "snappy", "zstd"] {
let sink = SinkConfig {
compression: codec.into(),
..Default::default()
};
let component = kafka_sink_component(&sink, &["src".into()], true);
let options = component["librdkafka_options"].as_object().unwrap();
assert_eq!(component["compression"], codec);
assert!(!options.contains_key("compression.type"), "{codec}");
assert!(!options.contains_key("compression.codec"), "{codec}");
}
}

/// Either spelling in `sink.librdkafka_options` is refused, with the dial
/// that owns the codec named.
#[test]
fn both_spellings_of_the_compression_codec_are_derived_options() {
let derived = derived_sink_options(&SinkConfig::default());
for spelling in ["compression.type", "compression.codec"] {
assert!(
derived.contains(&(spelling, "sink.compression")),
"{spelling}"
);
}
}

#[test]
Expand Down
3 changes: 2 additions & 1 deletion tests/integration/config_reach.rs
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,9 @@ fn a_misspelt_logging_value_is_named_as_a_config_error() {
/// `sink.compression: zstd` produced zstd and logged nothing.
#[test]
fn a_librdkafka_option_the_generator_overwrites_is_refused() {
let cases: [(&str, &str, &str); 3] = [
let cases: [(&str, &str, &str); 4] = [
("sink", "compression.type", "sink.compression"),
("sink", "compression.codec", "sink.compression"),
("sink", "security.protocol", "sink.sasl.enabled"),
("source", "sasl.mechanism", "source.sasl.mechanism"),
];
Expand Down
Loading