diff --git a/docs/config-schema.json b/docs/config-schema.json index c592cf5..be7a8ae 100644 --- a/docs/config-schema.json +++ b/docs/config-schema.json @@ -70,6 +70,7 @@ "localhost:9092" ], "topic": "", + "dlq_topic": "dfe_transform_dlq", "key_field": "", "encoding": "json", "compression": "zstd", @@ -468,6 +469,11 @@ "type": "string", "default": "" }, + "dlq_topic": { + "description": "Topic a record over the producer's size ceiling is dead-lettered to on\nthe bus transport, as a DLQ envelope the loader reads. Unused on direct,\nwhere the bridge's `dlq` holds it, and while `dlq.enabled` is false.", + "type": "string", + "default": "dfe_transform_dlq" + }, "key_field": { "description": "Event field path for Kafka partition key (e.g., \".org_id\").", "type": "string", @@ -795,7 +801,7 @@ } }, "DeadLetterConfig": { - "description": "The bridge's dead-letter queue: scalo's `DlqConfig` as written.\n\nThe default writes NDJSON files under `/var/spool/scalo/dlq`, which the\ndeployment contract declares writable. With `enabled: false` such a record\nis dropped and counted on `pipeline_dead_letters_dropped_total`. Only the\ndirect transport's bridge has one: on the bus Vector's own Kafka sink\ndelivers.", + "description": "The bridge's dead-letter queue: scalo's `DlqConfig` as written.\n\nThe default writes NDJSON files under `/var/spool/scalo/dlq`, which the\ndeployment contract declares writable. With `enabled: false` such a record\nis dropped and counted on `pipeline_dead_letters_dropped_total`. Only the\ndirect transport's bridge writes through it. On the bus a record over the\nsink's size ceiling goes to `sink.dlq_topic`, and only `enabled` and\n`max_payload_bytes` apply.", "$ref": "#/$defs/DlqConfig" }, "DlqConfig": { diff --git a/docs/config-schema.yaml b/docs/config-schema.yaml index 0ce8701..a5f620f 100644 --- a/docs/config-schema.yaml +++ b/docs/config-schema.yaml @@ -92,6 +92,7 @@ properties: brokers: - localhost:9092 topic: "" + dlq_topic: dfe_transform_dlq key_field: "" encoding: json compression: zstd @@ -458,6 +459,13 @@ $defs: the downstream stage picks its table by, so a source keeps its name. type: string default: "" + dlq_topic: + description: |- + Topic a record over the producer's size ceiling is dead-lettered to on + the bus transport, as a DLQ envelope the loader reads. Unused on direct, + where the bridge's `dlq` holds it, and while `dlq.enabled` is false. + type: string + default: dfe_transform_dlq key_field: description: Event field path for Kafka partition key (e.g., ".org_id"). type: string @@ -810,8 +818,9 @@ $defs: The default writes NDJSON files under `/var/spool/scalo/dlq`, which the deployment contract declares writable. With `enabled: false` such a record is dropped and counted on `pipeline_dead_letters_dropped_total`. Only the - direct transport's bridge has one: on the bus Vector's own Kafka sink - delivers. + direct transport's bridge writes through it. On the bus a record over the + sink's size ceiling goes to `sink.dlq_topic`, and only `enabled` and + `max_payload_bytes` apply. $ref: "#/$defs/DlqConfig" DlqConfig: description: Top-level DLQ configuration. diff --git a/docs/internals.md b/docs/internals.md index 71d2a67..dc21496 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -55,13 +55,15 @@ transforms: | `00_global.yaml` | `timezone: UTC`, `data_dir`, the API switch, one `directory` secret backend per credential source | | `00_source.yaml` | `dfe_source`: a `kafka` source on bus, carrying the librdkafka layers in [librdkafka.md](librdkafka.md), or a `vector` source on `bridge.to_vector` on direct | | `50_NNN_` | Each transform file, in filename order, re-serialised with its wired `inputs` | -| `90_sink.yaml` | `dfe_sink`: on bus a `kafka` sink behind the `dfe_size_cap` filter, on direct a `vector` sink dialling `bridge.from_vector` | +| `90_sink.yaml` | `dfe_sink`: on bus a `kafka` sink behind the `dfe_size_cap` route, plus `dfe_size_cap_dlq` and `dfe_dlq_sink` for the records it holds back, on direct a `vector` sink dialling `bridge.from_vector` | | `99_observability.yaml` | `internal_metrics` into a `prometheus_exporter` on `metrics.vector_metrics_address` | | `.secrets/` | Owner-only files for a SASL credential given as text | Credentials appear only as references: `SECRET[dfe_source_sasl.username]` for a `secret_dir`, `SECRET[dfe_credentials.source_sasl_username]` for a credential given as text. The assembler rejects a `secret_dir` that lacks its `username` or `password` file. -`dfe_size_cap` drops a record with an encoded length over the producer's `message.max.bytes` less 128 bytes for the key, headers and framing. `message.max.bytes` is scalo's pipeline-wide record ceiling of 16 MiB unless the central profile or `sink.librdkafka_options` sets it. librdkafka's default of 1,000,000 bytes would refuse records the rest of the chain accepts. +`dfe_size_cap` is a `route` that sends a record whose encoded length is over the producer's `message.max.bytes` less 128 bytes for the key, headers and framing out of its `_unmatched` output, and the rest out of `within_cap` to the sink. `message.max.bytes` is scalo's pipeline-wide record ceiling of 16 MiB unless the central profile or `sink.librdkafka_options` sets it. librdkafka's default of 1,000,000 bytes would refuse records the rest of the chain accepts. + +While `dlq.enabled` is true, `dfe_size_cap_dlq` remaps each record from `_unmatched` into scalo's DLQ entry and `dfe_dlq_sink` writes it to `sink.dlq_topic` (`dfe_transform_dlq`, the shared topic dfe-loader loads). The entry is `truncated`: `original_bytes` is the record's length, `reason` names it and the ceiling, `destination` is `sink.topic`, and `_raw` is the first bytes as base64 with `_raw_encoding: base64`. The `_org_id`, `_source`, `_timestamp_received` and `_tags` values that fit are copied beside it, and any that do not are named in `header_dropped`. The head and the copied values share `dlq.max_payload_bytes`, 512 KiB by default, held to half of what `message.max.bytes` leaves. `dfe_dlq_sink` uses the main sink's brokers, credentials and librdkafka options, writes JSON with no key, and holds in a 500-event memory buffer that blocks when full, so an unreachable broker back-pressures the source rather than dropping the entry. With `dlq.enabled: false` there is no remap or second sink, and `dfe_size_cap` is a `filter` that drops the record. The wiring follows these rules: @@ -153,9 +155,10 @@ Metric names are bare unless `metrics.namespace` is set, and `dfe-transform-vect |---|---| | `records_received_total` | `vector_component_received_events_total{component_id="dfe_source"}` | | `records_processed_total`, `records_delivered_total` | `vector_component_sent_events_total{component_id="dfe_sink"}` | -| `records_error_total` | `vector_component_discarded_events_total` on `dfe_sink` and on `dfe_size_cap` | +| `records_error_total` | `vector_component_discarded_events_total` on `dfe_sink`, `dfe_size_cap` and `dfe_dlq_sink` | | `transform_vector_sink_errors_total` | `vector_component_errors_total{component_id="dfe_sink"}` | -| `pipeline_dead_letters_dropped_total{reason="too_large"}` | the growth of `vector_component_discarded_events_total` on `dfe_size_cap` | +| `pipeline_dead_letters_dropped_total{reason="too_large"}` | the growth of `vector_component_discarded_events_total` on `dfe_size_cap` and on `dfe_dlq_sink` | +| Dead letters held by the DLQ topic, with no supervisor counter of their own | `vector_component_sent_events_total{component_id="dfe_dlq_sink"}`, merged with the other `vector_*` series | | `pipeline_dead_letters_dropped_total{reason="rejected"}` | the growth of `vector_component_discarded_events_total` on `dfe_sink` | The scrape reads Vector's exporter on scalo's metrics interval and re-registers every sample. A failed scrape is not fatal: it counts `transform_vector_scrape_failures_total`, warns at most once every five minutes, and leaves readiness as it was. A response over 4 MiB is a failed scrape, not a partial merge. A counter is set with `absolute` and carried across Vector restarts: a total that falls is a restarted Vector, and the old count becomes a base. A histogram arrives pre-aggregated, so each `_bucket` registers as a counter with its `le` label. A gauge Vector stops reporting is zeroed after `metrics.vector_metrics_expiry_ticks` scrapes, a counter is forgotten after 240, and at most 50,000 series merge. @@ -166,7 +169,7 @@ dfe-infra's values set a CPU target and, on its single, scale and mesh profiles, With `source.acknowledgements.enabled`, the default, delivery is at-least-once on both transports. On bus the sink's acknowledgements hold Vector's source offsets until delivery, so a crash redelivers from the last committed offset. On direct a push is answered once the next hop has it. With acknowledgements off, both answer at receipt. -On direct a record over the next stage's message-size ceiling goes to scalo's file DLQ, NDJSON under `/var/spool/scalo/dlq` on the contract's `dlq` writable path. With `dlq.enabled: false` it is dropped and counted in `pipeline_dead_letters_dropped_total{reason="too_large"}`. A record Vector rejects for good is dropped and counted under `reason="rejected"`, whatever `dlq` says. On bus no DLQ runs: the size cap's drops count under `too_large` and the Kafka sink's under `rejected`. When no enabled DLQ backend can start, such as a file DLQ with no writable directory off the image, a direct run logs `DLQ enabled but no backend started` and leaves dead letters unacknowledged at their source. Point `dlq.file.path` at a directory you can write. +On direct a record over the next stage's message-size ceiling goes to scalo's file DLQ, NDJSON under `/var/spool/scalo/dlq` on the contract's `dlq` writable path. With `dlq.enabled: false` it is dropped and counted in `pipeline_dead_letters_dropped_total{reason="too_large"}`. A record Vector rejects for good is dropped and counted under `reason="rejected"`, whatever `dlq` says. On bus a record over the producer's ceiling goes to `sink.dlq_topic` (see [The assembled directory](#the-assembled-directory)), counted in `vector_component_sent_events_total{component_id="dfe_dlq_sink"}` and in no drop counter. It counts under `too_large` only when the DLQ sink rejects it for good, or with `dlq.enabled: false`, and the Kafka sink's rejections count under `rejected`. When no enabled DLQ backend can start, such as a file DLQ with no writable directory off the image, a direct run logs `DLQ enabled but no backend started` and leaves dead letters unacknowledged at their source. Point `dlq.file.path` at a directory you can write. On bus Vector holds the records, bounded by its `buffer` settings and the container's memory limit. diff --git a/src/config/assembler.rs b/src/config/assembler.rs index 1b45adb..9bd5324 100644 --- a/src/config/assembler.rs +++ b/src/config/assembler.rs @@ -111,6 +111,7 @@ pub fn assemble_for(config: &Config, output_dir: &Path, published: &Path) -> Res &config.bridge, &wiring.sink_inputs, config.source.acknowledgements.enabled, + &config.dlq.0, ); write_yaml(output_dir, "90_sink.yaml", &sink_yaml)?; diff --git a/src/config/generate.rs b/src/config/generate.rs index eaaacbb..04092f7 100644 --- a/src/config/generate.rs +++ b/src/config/generate.rs @@ -26,6 +26,7 @@ use std::collections::{BTreeMap, HashMap}; +use scalo::dlq::{DEFAULT_MAX_PAYLOAD_BYTES, DlqConfig}; use scalo::transport::kafka::MESSAGE_MAX_BYTES; use serde_json::{Map, Number, Value}; use tracing::warn; @@ -46,10 +47,37 @@ pub const SOURCE_LABEL: &str = "dfe_source"; /// Canonical label for the generated sink component. pub const SINK_LABEL: &str = "dfe_sink"; -/// Label of the filter that keeps records over the producer's size ceiling off -/// the Kafka sink. +/// Label of the transform that keeps records over the producer's size ceiling +/// off the Kafka sink: a `route` whose unmatched output is dead-lettered, or a +/// `filter` that drops them while `dlq.enabled` is false. pub const SIZE_CAP_LABEL: &str = "dfe_size_cap"; +/// Label of the remap that turns a record over the ceiling into a DLQ entry. +pub const SIZE_CAP_DLQ_LABEL: &str = "dfe_size_cap_dlq"; + +/// Label of the Kafka sink that writes those entries to `sink.dlq_topic`. +pub const DLQ_SINK_LABEL: &str = "dfe_dlq_sink"; + +/// The size cap route's output for a record that fits. +const WITHIN_CAP_OUTPUT: &str = "within_cap"; + +/// Vector's output of a `route` for the events no route matched. +const UNMATCHED_OUTPUT: &str = "_unmatched"; + +/// `service` in a dead-letter entry, as scalo's DLQ names its producer. +const DLQ_SERVICE: &str = env!("CARGO_PKG_NAME"); + +/// The common-header fields scalo's DLQ copies from the original event beside +/// its payload. +const DLQ_HEADER_FIELDS: [&str; 4] = ["_org_id", "_source", "_timestamp_received", "_tags"]; + +/// Bytes kept back from a dead letter's Kafka ceiling for its envelope, the +/// key and the record framing. +const DLQ_ENVELOPE_BYTES: u64 = 8192; + +/// Events the dead letter sink's memory buffer holds before it blocks. +const DLQ_BUFFER_EVENTS: u64 = 500; + /// Vector's `type` for a Kafka source or sink. pub const KAFKA_COMPONENT_TYPE: &str = "kafka"; @@ -221,18 +249,22 @@ fn section(name: &str, label: &str, component: Mapping) -> Mapping { /// /// On the bus that is a Kafka producer with production-tuned librdkafka /// options, Vector-level batching, buffer config and acknowledgements, behind -/// a filter that holds back records over the producer's size ceiling. On -/// direct it is Vector's own `vector` sink, dialling the supervisor's bridge, -/// which pushes the records on to `sink.endpoint`. +/// a size cap. A record over the producer's ceiling goes through a remap into +/// a DLQ entry and a second Kafka sink on `sink.dlq_topic`, or is dropped while +/// `dlq.enabled` is false. On direct it is Vector's own `vector` sink, +/// dialling the supervisor's bridge, which pushes the records on to +/// `sink.endpoint`. /// /// `acknowledged` is `source.acknowledgements.enabled`: the sink's end-to-end -/// acknowledgement is what lets the source hold its own. +/// acknowledgement is what lets the source hold its own. `dlq` is the app's +/// `dlq` block, whose `enabled` and `max_payload_bytes` the size cap reads. #[must_use] pub fn generate_sink_yaml( sink: &SinkConfig, bridge: &BridgeConfig, inputs: &[String], acknowledged: bool, + dlq: &DlqConfig, ) -> Value { if sink.transport.is_direct() { return wrap( @@ -242,16 +274,42 @@ pub fn generate_sink_yaml( ); } - let mut root = section( - "sinks", - SINK_LABEL, - kafka_sink_component(sink, &[SIZE_CAP_LABEL.to_string()], acknowledged), + let dead_lettering = dlq.enabled; + let within_cap = if dead_lettering { + format!("{SIZE_CAP_LABEL}.{WITHIN_CAP_OUTPUT}") + } else { + SIZE_CAP_LABEL.to_string() + }; + + let mut sinks = Mapping::new(); + sinks.insert( + key(SINK_LABEL), + Value::Object(kafka_sink_component(sink, &[within_cap], acknowledged)), ); - root.extend(section( - "transforms", - SIZE_CAP_LABEL, - size_cap_component(sink, inputs), - )); + let mut transforms = Mapping::new(); + if dead_lettering { + transforms.insert( + key(SIZE_CAP_LABEL), + Value::Object(size_cap_route_component(sink, inputs)), + ); + transforms.insert( + key(SIZE_CAP_DLQ_LABEL), + Value::Object(dead_letter_remap_component(sink, dlq)), + ); + sinks.insert( + key(DLQ_SINK_LABEL), + Value::Object(dead_letter_sink_component(sink, acknowledged)), + ); + } else { + transforms.insert( + key(SIZE_CAP_LABEL), + Value::Object(size_cap_filter_component(sink, inputs)), + ); + } + + let mut root = Mapping::new(); + root.insert(key("sinks"), Value::Object(sinks)); + root.insert(key("transforms"), Value::Object(transforms)); Value::Object(root) } @@ -276,25 +334,157 @@ fn pipeline_message_max_bytes() -> u64 { u64::from(MESSAGE_MAX_BYTES.unsigned_abs()) } -/// The filter in front of the Kafka sink that drops a record over -/// [`max_record_bytes`], counted in `component_discarded_events_total`. -/// -/// Measures the record as the sink's codec encodes it: the JSON of the whole -/// event, or the `message` field for `raw_bytes`. -fn size_cap_component(sink: &SinkConfig, inputs: &[String]) -> Mapping { - let encoded = match sink.encoding.as_str() { +/// The VRL expression for a record as the sink's codec encodes it: the JSON of +/// the whole event, or the `message` field for `raw_bytes`. +fn encoded_record(sink: &SinkConfig) -> &'static str { + match sink.encoding.as_str() { "raw_bytes" => r#"string(.message) ?? """#, _ => "encode_json(.)", - }; - let condition = format!("length({encoded}) <= {}", max_record_bytes(sink)); + } +} + +/// The VRL condition that holds for a record within [`max_record_bytes`]. +fn within_cap_condition(sink: &SinkConfig) -> String { + format!( + "length({}) <= {}", + encoded_record(sink), + max_record_bytes(sink) + ) +} + +/// The route in front of the Kafka sink. A record over [`max_record_bytes`] +/// leaves on `_unmatched`, which [`dead_letter_remap_component`] reads. +fn size_cap_route_component(sink: &SinkConfig, inputs: &[String]) -> Mapping { + let mut routes = Mapping::new(); + routes.insert(key(WITHIN_CAP_OUTPUT), val(&within_cap_condition(sink))); + let mut component = Mapping::new(); + component.insert(key("type"), val("route")); + component.insert(key("inputs"), input_list(inputs)); + component.insert(key("route"), Value::Object(routes)); + component +} + +/// The filter in front of the Kafka sink while `dlq.enabled` is false: it drops +/// a record over [`max_record_bytes`], counted in +/// `component_discarded_events_total`. +fn size_cap_filter_component(sink: &SinkConfig, inputs: &[String]) -> Mapping { let mut component = Mapping::new(); component.insert(key("type"), val("filter")); component.insert(key("inputs"), input_list(inputs)); - component.insert(key("condition"), val(&condition)); + component.insert(key("condition"), val(&within_cap_condition(sink))); + component +} + +/// The most bytes of a dead letter's payload and copied header: `dlq`'s +/// `max_payload_bytes`, held to what the sink's Kafka ceiling takes once the +/// payload is base64. +#[must_use] +pub fn dead_letter_payload_cap(sink: &SinkConfig, dlq: &DlqConfig) -> u64 { + let configured = dlq + .max_payload_bytes + .map_or(DEFAULT_MAX_PAYLOAD_BYTES as u64, |n| n as u64); + let kafka_fit = max_record_bytes(sink).saturating_sub(DLQ_ENVELOPE_BYTES) / 2; + configured.min(kafka_fit) +} + +/// A VRL string literal for `text`. +fn vrl_string(text: &str) -> String { + let mut literal = String::with_capacity(text.len() + 2); + literal.push('"'); + for c in text.chars() { + match c { + '\\' => literal.push_str("\\\\"), + '"' => literal.push_str("\\\""), + '\n' => literal.push_str("\\n"), + '\r' => literal.push_str("\\r"), + '\t' => literal.push_str("\\t"), + other => literal.push(other), + } + } + literal.push('"'); + literal +} + +/// The VRL that builds a scalo DLQ entry from a record over the ceiling. +/// +/// The entry is `truncated`: the record's first bytes in `_raw` as base64, its +/// length in `original_bytes`, and the header fields scalo copies beside it. A +/// header value that does not fit in the cap with the others is left out and +/// named in `header_dropped`. +fn dead_letter_source(sink: &SinkConfig, dlq: &DlqConfig) -> String { + let cap = dead_letter_payload_cap(sink, dlq); + let mut program = format!( + "body = {}\noriginal = length(body)\nheader = {{}}\nheader_bytes = 0\nheader_dropped = []\n", + encoded_record(sink), + ); + for field in DLQ_HEADER_FIELDS { + program.push_str(&format!( + "if !is_null(.{field}) {{\n size = length(encode_json(.{field}))\n \ + if header_bytes + size <= {cap} {{\n header.{field} = .{field}\n \ + header_bytes = header_bytes + size\n }} else {{\n \ + header_dropped = push(header_dropped, \"{field}\")\n }}\n}}\n" + )); + } + program.push_str(&format!( + "head = slice!(body, 0, {cap} - header_bytes)\n\ + . = header\n\ + .timestamp = to_unix_timestamp(now(), unit: \"milliseconds\")\n\ + .service = {service}\n\ + .reason = \"record of \" + to_string(original) + \" bytes is over the sink's {ceiling}-byte ceiling\"\n\ + .attempts = 0\n\ + .truncated = true\n\ + .original_bytes = original\n\ + .destination = {destination}\n\ + ._raw = encode_base64(head)\n\ + ._raw_encoding = \"base64\"\n\ + if length(header_dropped) > 0 {{\n .header_dropped = header_dropped\n}}\n", + service = vrl_string(DLQ_SERVICE), + ceiling = max_record_bytes(sink), + destination = vrl_string(&sink.topic), + )); + program +} + +/// The remap that turns what the size cap route left unmatched into a dead +/// letter entry. +fn dead_letter_remap_component(sink: &SinkConfig, dlq: &DlqConfig) -> Mapping { + let mut component = Mapping::new(); + component.insert(key("type"), val("remap")); + component.insert( + key("inputs"), + input_list(&[format!("{SIZE_CAP_LABEL}.{UNMATCHED_OUTPUT}")]), + ); + component.insert(key("source"), val(&dead_letter_source(sink, dlq))); component } +/// The Kafka sink for dead letter entries. +/// +/// It takes the main sink's brokers, credentials, compression and librdkafka +/// options, and writes `sink.dlq_topic` as JSON with no partition key, as +/// scalo's DLQ does. Its buffer holds in memory and blocks when full: an entry +/// the broker cannot take yet back-pressures the source rather than drops. +fn dead_letter_sink_component(sink: &SinkConfig, acknowledged: bool) -> Mapping { + let dead_letters = SinkConfig { + topic: sink.dlq_topic.clone(), + key_field: String::new(), + encoding: "json".to_string(), + buffer: BufferConfig { + buffer_type: "memory".to_string(), + max_events: Some(DLQ_BUFFER_EVENTS), + max_size: None, + when_full: "block".to_string(), + }, + ..sink.clone() + }; + kafka_sink_component( + &dead_letters, + &[SIZE_CAP_DLQ_LABEL.to_string()], + acknowledged, + ) +} + /// The `vector` sink that hands the transformed records back to the supervisor. /// /// The buffer is the operator's dial on both transports, so it is carried here @@ -1010,6 +1200,7 @@ mod tests { &BridgeConfig::default(), &["src".into()], true, + &scalo::dlq::DlqConfig::default(), )) .unwrap(); @@ -1048,6 +1239,7 @@ mod tests { &BridgeConfig::default(), &["src".into()], true, + &scalo::dlq::DlqConfig::default(), )) .unwrap(); assert!( @@ -1066,6 +1258,7 @@ mod tests { &BridgeConfig::default(), &["src".into()], true, + &scalo::dlq::DlqConfig::default(), )) .unwrap(); assert!(text.contains("enable.idempotence: \"false\"")); @@ -1086,41 +1279,186 @@ mod tests { .and_then(Value::as_str) } - /// The size cap's filter condition. + /// The size cap's condition, on its route or its filter. fn size_cap_condition(yaml: &Value) -> Option<&str> { - yaml.get("transforms") - .and_then(|t| t.get(SIZE_CAP_LABEL)) - .and_then(|c| c.get("condition")) + let cap = yaml.get("transforms").and_then(|t| t.get(SIZE_CAP_LABEL))?; + cap.get("route") + .and_then(|r| r.get(WITHIN_CAP_OUTPUT)) + .or_else(|| cap.get("condition")) .and_then(Value::as_str) } - /// The Kafka sink reads from the size cap, and the cap from the wired - /// inputs. + /// A generated sink's `inputs`, as the labels they name. + fn inputs_of<'a>(yaml: &'a Value, section: &str, label: &str) -> Vec<&'a str> { + yaml.get(section) + .and_then(|s| s.get(label)) + .and_then(|c| c.get("inputs")) + .and_then(Value::as_array) + .map(|inputs| inputs.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default() + } + + fn bus_yaml(sink: &SinkConfig, dlq: &DlqConfig) -> Value { + generate_sink_yaml(sink, &BridgeConfig::default(), &["last".into()], true, dlq) + } + + /// The Kafka sink reads the size cap's `within_cap` output, and the cap + /// reads the wired inputs. #[test] fn the_kafka_sink_sits_behind_the_size_cap() { - let yaml = generate_sink_yaml( - &SinkConfig::default(), - &BridgeConfig::default(), - &["last".into()], - true, + let yaml = bus_yaml(&SinkConfig::default(), &DlqConfig::default()); + assert_eq!( + inputs_of(&yaml, "sinks", SINK_LABEL), + vec!["dfe_size_cap.within_cap"] ); - let sink_inputs = yaml - .get("sinks") - .and_then(|s| s.get(SINK_LABEL)) - .and_then(|s| s.get("inputs")) - .and_then(Value::as_array) - .expect("sink inputs"); - assert_eq!(sink_inputs, &vec![val(SIZE_CAP_LABEL)]); - + assert_eq!(inputs_of(&yaml, "transforms", SIZE_CAP_LABEL), vec!["last"]); let cap = yaml .get("transforms") .and_then(|t| t.get(SIZE_CAP_LABEL)) .expect("the size cap transform"); - assert_eq!(cap.get("type").and_then(Value::as_str), Some("filter")); + assert_eq!(cap.get("type").and_then(Value::as_str), Some("route")); + } + + /// A record over the ceiling leaves the route on `_unmatched`, becomes an + /// entry in the remap and is written to `sink.dlq_topic` as JSON with no + /// key, whichever encoding the main sink uses. + #[test] + fn an_oversize_record_is_dead_lettered_whatever_the_encoding() { + for (encoding, encoded) in [ + ("json", "body = encode_json(.)"), + ("raw_bytes", r#"body = string(.message) ?? """#), + ] { + let sink = SinkConfig { + encoding: encoding.into(), + key_field: ".org_id".into(), + topic: "syslog_load".into(), + ..Default::default() + }; + let yaml = bus_yaml(&sink, &DlqConfig::default()); + + assert_eq!( + inputs_of(&yaml, "transforms", SIZE_CAP_DLQ_LABEL), + vec!["dfe_size_cap._unmatched"], + "{encoding}" + ); + let source = yaml["transforms"][SIZE_CAP_DLQ_LABEL]["source"] + .as_str() + .expect("the remap source"); + assert!(source.starts_with(encoded), "{encoding}: {source}"); + assert!( + source.contains(r#"._raw_encoding = "base64""#), + "{encoding}: {source}" + ); + assert!( + source.contains(r#".destination = "syslog_load""#), + "{encoding}: {source}" + ); + + let dlq = &yaml["sinks"][DLQ_SINK_LABEL]; + assert_eq!( + inputs_of(&yaml, "sinks", DLQ_SINK_LABEL), + vec![SIZE_CAP_DLQ_LABEL] + ); + assert_eq!(dlq["type"], "kafka", "{encoding}"); + assert_eq!(dlq["topic"], "dfe_transform_dlq", "{encoding}"); + assert_eq!(dlq["encoding"]["codec"], "json", "{encoding}"); + assert!(dlq.get("key_field").is_none(), "{encoding}: {dlq:?}"); + assert_eq!(dlq["buffer"]["type"], "memory", "{encoding}"); + assert_eq!(dlq["buffer"]["when_full"], "block", "{encoding}"); + assert_eq!( + yaml["sinks"][SINK_LABEL]["key_field"], ".org_id", + "the main sink keeps its key" + ); + } + } + + /// The dead letter sink reaches the broker as the main sink does: the same + /// credentials by reference, TLS, and `message.max.bytes`. + #[test] + fn the_dead_letter_sink_shares_the_main_sinks_broker_settings() { + let mut sink = SinkConfig { + sasl: SaslConfig { + enabled: true, + username: "kafka-user".into(), + password: "kafka-pass".into(), + ..SaslConfig::default() + }, + tls: TlsConfig { + enabled: true, + ..Default::default() + }, + brokers: vec!["kafka-1:9093".into()], + ..Default::default() + }; + sink.librdkafka_options + .insert("message.max.bytes".into(), "2000000".into()); + let yaml = bus_yaml(&sink, &DlqConfig::default()); + let (main, dead) = (&yaml["sinks"][SINK_LABEL], &yaml["sinks"][DLQ_SINK_LABEL]); + for field in [ + "bootstrap_servers", + "sasl", + "tls", + "compression", + "librdkafka_options", + ] { + assert_eq!(main[field], dead[field], "{field}"); + } assert_eq!( - cap.get("inputs").and_then(Value::as_array), - Some(&vec![val("last")]) + dead["sasl"]["username"], + "SECRET[dfe_credentials.sink_sasl_username]" ); + assert!(!yaml.to_string().contains("kafka-pass")); + } + + /// With `dlq.enabled` false the cap is the filter that drops, and no remap + /// or second sink is written. + #[test] + fn a_disabled_dlq_leaves_the_size_cap_dropping() { + let dlq = DlqConfig { + enabled: false, + ..DlqConfig::default() + }; + let yaml = bus_yaml(&SinkConfig::default(), &dlq); + assert_eq!(inputs_of(&yaml, "sinks", SINK_LABEL), vec![SIZE_CAP_LABEL]); + assert_eq!(yaml["transforms"][SIZE_CAP_LABEL]["type"], "filter"); + assert!(yaml["transforms"].get(SIZE_CAP_DLQ_LABEL).is_none()); + assert!(yaml["sinks"].get(DLQ_SINK_LABEL).is_none()); + } + + /// The dead letter keeps `dlq.max_payload_bytes`, held to what the sink's + /// Kafka ceiling takes once the payload is base64. + #[test] + fn the_dead_letter_payload_cap_follows_the_dlq_and_the_ceiling() { + // (message.max.bytes, dlq.max_payload_bytes, cap) + for (ceiling, configured, cap) in [ + (None, Some(DEFAULT_MAX_PAYLOAD_BYTES), 524_288_u64), + (None, Some(1024), 1024), + (None, None, 524_288), + (Some("200000"), Some(DEFAULT_MAX_PAYLOAD_BYTES), 95_840), + (Some("8192"), Some(DEFAULT_MAX_PAYLOAD_BYTES), 0), + ] { + let mut sink = SinkConfig::default(); + if let Some(ceiling) = ceiling { + sink.librdkafka_options + .insert("message.max.bytes".into(), ceiling.into()); + } + let dlq = DlqConfig { + max_payload_bytes: configured, + ..DlqConfig::default() + }; + assert_eq!( + dead_letter_payload_cap(&sink, &dlq), + cap, + "{ceiling:?} {configured:?}" + ); + } + } + + /// A topic name is written as a VRL string, so a quote or backslash in it + /// cannot end the literal. + #[test] + fn vrl_strings_escape_quotes_and_backslashes() { + assert_eq!(vrl_string("a\"b\\c"), "\"a\\\"b\\\\c\""); } /// A sink no deployment has tuned carries a 2 MiB record. librdkafka's own @@ -1130,7 +1468,13 @@ mod tests { fn a_default_sink_carries_a_2_mib_record() { const TWO_MIB: u64 = 2 * 1024 * 1024; let sink = SinkConfig::default(); - let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true); + let yaml = generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ); assert_eq!( written_message_max_bytes(&yaml), @@ -1161,7 +1505,13 @@ mod tests { sink.librdkafka_options .insert("message.max.bytes".into(), configured.into()); assert_eq!(max_record_bytes(&sink), cap, "{configured}"); - let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true); + let yaml = generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ); assert_eq!( written_message_max_bytes(&yaml), Some(configured), @@ -1180,7 +1530,13 @@ mod tests { transport: crate::config::Transport::Direct, ..Default::default() }; - let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true); + let yaml = generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ); assert!(yaml.get("transforms").is_none()); } @@ -1195,11 +1551,17 @@ mod tests { batch, ..Default::default() }; - generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true) - .get("sinks") - .and_then(|s| s.get(SINK_LABEL)) - .and_then(|s| s.get("batch")) - .cloned() + generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ) + .get("sinks") + .and_then(|s| s.get(SINK_LABEL)) + .and_then(|s| s.get("batch")) + .cloned() }; assert_eq!( batch_of(BatchConfig { @@ -1229,12 +1591,18 @@ mod tests { transport, ..Default::default() }; - generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true) - .get("sinks") - .and_then(|s| s.get(SINK_LABEL)) - .and_then(|s| s.get("healthcheck")) - .and_then(|h| h.get("enabled")) - .and_then(Value::as_bool) + generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ) + .get("sinks") + .and_then(|s| s.get(SINK_LABEL)) + .and_then(|s| s.get("healthcheck")) + .and_then(|h| h.get("enabled")) + .and_then(Value::as_bool) }; assert_eq!(healthcheck(crate::config::Transport::Direct), Some(false)); assert_eq!(healthcheck(crate::config::Transport::Bus), Some(true)); @@ -1263,6 +1631,7 @@ mod tests { &BridgeConfig::default(), &["src".into()], false, + &scalo::dlq::DlqConfig::default(), ); assert_eq!( yaml.get("sinks") @@ -1349,7 +1718,13 @@ mod tests { }, ..Default::default() }; - let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true); + let yaml = generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ); let text = scalo::yaml::to_string(&yaml).unwrap(); // Vector TLS block @@ -1374,6 +1749,7 @@ mod tests { &BridgeConfig::default(), &["last_transform".into()], true, + &scalo::dlq::DlqConfig::default(), ); let text = scalo::yaml::to_string(&yaml).unwrap(); @@ -1423,7 +1799,13 @@ mod tests { }, ..Default::default() }; - let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true); + let yaml = generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ); let text = scalo::yaml::to_string(&yaml).unwrap(); assert!(text.contains("type: memory")); @@ -1442,7 +1824,13 @@ mod tests { }, ..Default::default() }; - let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true); + let yaml = generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ); let text = scalo::yaml::to_string(&yaml).unwrap(); assert!(text.contains("type: disk")); @@ -1460,7 +1848,13 @@ mod tests { }, ..Default::default() }; - let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true); + let yaml = generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ); let text = scalo::yaml::to_string(&yaml).unwrap(); assert!(text.contains("max_events: 5000")); @@ -1536,6 +1930,7 @@ mod tests { &BridgeConfig::default(), &["src".into()], true, + &scalo::dlq::DlqConfig::default(), ); let sink = yaml .get("sinks") @@ -1559,7 +1954,13 @@ mod tests { compression: "none".into(), ..Default::default() }; - let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true); + let yaml = generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ); let text = scalo::yaml::to_string(&yaml).unwrap(); assert!(text.contains("\n compression: none")); @@ -1674,7 +2075,13 @@ mod tests { }, ..Default::default() }; - let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true); + let yaml = generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ); let written = yaml .get("sinks") .and_then(|s| s.get(SINK_LABEL)) @@ -1826,7 +2233,13 @@ librdkafka_options: }, ..Default::default() }; - let yaml = generate_sink_yaml(&sink, &BridgeConfig::default(), &["src".into()], true); + let yaml = generate_sink_yaml( + &sink, + &BridgeConfig::default(), + &["src".into()], + true, + &scalo::dlq::DlqConfig::default(), + ); let component = yaml .get("sinks") .and_then(|s| s.get(SINK_LABEL)) diff --git a/src/config/loader.rs b/src/config/loader.rs index d8f81e0..ef592be 100644 --- a/src/config/loader.rs +++ b/src/config/loader.rs @@ -227,8 +227,9 @@ pub struct Config { /// The default writes NDJSON files under `/var/spool/scalo/dlq`, which the /// deployment contract declares writable. With `enabled: false` such a record /// is dropped and counted on `pipeline_dead_letters_dropped_total`. Only the -/// direct transport's bridge has one: on the bus Vector's own Kafka sink -/// delivers. +/// direct transport's bridge writes through it. On the bus a record over the +/// sink's size ceiling goes to `sink.dlq_topic`, and only `enabled` and +/// `max_payload_bytes` apply. #[derive(Debug, Clone, Default, Serialize, Deserialize, schemars::JsonSchema)] #[serde(transparent)] pub struct DeadLetterConfig(pub scalo::dlq::DlqConfig); @@ -435,6 +436,10 @@ impl Default for SourceConfig { } } +/// The fleet's shared dead-letter topic for the transforms, which a DFE +/// bootstrap creates and the loader loads. +pub const DEFAULT_DLQ_TOPIC: &str = "dfe_transform_dlq"; + /// Sink configuration: the bus topic to produce to, or the Push listener to /// send the transformed records on to. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] @@ -449,6 +454,10 @@ pub struct SinkConfig { /// Output topic name. On the direct transport it is still the routing key /// the downstream stage picks its table by, so a source keeps its name. pub topic: String, + /// Topic a record over the producer's size ceiling is dead-lettered to on + /// the bus transport, as a DLQ envelope the loader reads. Unused on direct, + /// where the bridge's `dlq` holds it, and while `dlq.enabled` is false. + pub dlq_topic: String, /// Event field path for Kafka partition key (e.g., ".org_id"). pub key_field: String, /// Encoding codec: json, raw_bytes. @@ -482,6 +491,7 @@ impl Default for SinkConfig { endpoint: "http://dfe-loader:6000".to_string(), brokers: vec!["localhost:9092".to_string()], topic: String::new(), + dlq_topic: DEFAULT_DLQ_TOPIC.to_string(), key_field: String::new(), encoding: "json".to_string(), compression: "zstd".to_string(), @@ -892,6 +902,9 @@ impl ApplyFlatEnv for Config { if let Some(v) = flat_env::flat_env_string(prefix, "SINK_TOPIC") { self.sink.topic = v; } + if let Some(v) = flat_env::flat_env_string(prefix, "SINK_DLQ_TOPIC") { + self.sink.dlq_topic = v; + } if let Some(v) = flat_env::flat_env_string(prefix, "SINK_KEY_FIELD") { self.sink.key_field = v; } @@ -1230,6 +1243,11 @@ impl Config { "sink.topic must not be empty".into(), )); } + if self.dlq.0.enabled && self.sink.dlq_topic.is_empty() { + return Err(crate::Error::Validation( + "sink.dlq_topic must not be empty while dlq.enabled is true".into(), + )); + } } Self::validate_sasl("sink.sasl", &self.sink.sasl)?; Self::validate_buffer(&self.sink.buffer)?; diff --git a/src/metrics/scrape.rs b/src/metrics/scrape.rs index 5ea1a62..659c5f5 100644 --- a/src/metrics/scrape.rs +++ b/src/metrics/scrape.rs @@ -39,7 +39,7 @@ use tokio::net::TcpStream; use tracing::{debug, info, warn}; use super::{TOO_LARGE, WrapperMetrics, count_dropped_dead_letters}; -use crate::config::generate::{SINK_LABEL, SIZE_CAP_LABEL, SOURCE_LABEL}; +use crate::config::generate::{DLQ_SINK_LABEL, SINK_LABEL, SIZE_CAP_LABEL, SOURCE_LABEL}; /// Vector's per-component counter of events read in. const VECTOR_RECEIVED: &str = "vector_component_received_events_total"; @@ -115,16 +115,29 @@ pub struct Derived { /// `vector_component_errors_total` on the wrapper's sink. pub sink_errors: f64, /// `vector_component_discarded_events_total` on the size cap: records over - /// the producer's ceiling, held back from the sink. + /// the producer's ceiling, dropped there while `dlq.enabled` is false. pub oversize: f64, + /// `vector_component_sent_events_total` on the dead letter sink: records + /// over the producer's ceiling that the DLQ topic holds. + pub dead_lettered: f64, + /// `vector_component_discarded_events_total` on the dead letter sink: + /// dead letters the broker refused for good. + pub dlq_discarded: f64, } impl Derived { + /// Records over the producer's ceiling that were lost: dropped by the size + /// cap, or refused by the dead letter sink. + #[must_use] + pub fn too_large_lost(&self) -> f64 { + self.oversize + self.dlq_discarded + } + /// Records the pipeline lost after taking them in: rejected by the sink, - /// or held back by the size cap. + /// or over the ceiling and not dead-lettered. #[must_use] pub fn rejected(&self) -> f64 { - self.sink_discarded + self.oversize + self.sink_discarded + self.too_large_lost() } /// Records the sink has taken on and neither delivered nor dropped. @@ -238,6 +251,8 @@ pub struct LifetimeTotals { sink_discarded: Rebased, sink_errors: Rebased, oversize: Rebased, + dead_lettered: Rebased, + dlq_discarded: Rebased, } impl LifetimeTotals { @@ -250,6 +265,8 @@ impl LifetimeTotals { &mut self.sink_discarded, &mut self.sink_errors, &mut self.oversize, + &mut self.dead_lettered, + &mut self.dlq_discarded, ] { total.restart(); } @@ -264,6 +281,8 @@ impl LifetimeTotals { sink_discarded: self.sink_discarded.lifetime(derived.sink_discarded), sink_errors: self.sink_errors.lifetime(derived.sink_errors), oversize: self.oversize.lifetime(derived.oversize), + dead_lettered: self.dead_lettered.lifetime(derived.dead_lettered), + dlq_discarded: self.dlq_discarded.lifetime(derived.dlq_discarded), } } } @@ -367,6 +386,8 @@ impl ExpositionMerger { (VECTOR_DISCARDED, Some(SINK_LABEL)) => derived.sink_discarded += v, (VECTOR_ERRORS, Some(SINK_LABEL)) => derived.sink_errors += v, (VECTOR_DISCARDED, Some(SIZE_CAP_LABEL)) => derived.oversize += v, + (VECTOR_SENT, Some(DLQ_SINK_LABEL)) => derived.dead_lettered += v, + (VECTOR_DISCARDED, Some(DLQ_SINK_LABEL)) => derived.dlq_discarded += v, _ => {} } } @@ -583,8 +604,8 @@ pub fn apply_derived(metrics: &WrapperMetrics, derived: &Derived) { // The platform's sink-side counter is the same number by another name and // reads 0 without this. ServiceMetrics only exposes an increment. metrics::counter!("records_delivered_total").absolute(to_u64(derived.sent)); - // A record Vector rejected or the size cap held back is gone, so it counts - // where every other app counts a lost record. + // A record Vector rejected, or that went over the ceiling and no DLQ holds, + // is gone, so it counts where every other app counts a lost record. metrics .app .records_error @@ -611,13 +632,14 @@ pub struct DeadLetterDrops { } impl DeadLetterDrops { - /// The size cap's drops, records over the producer's ceiling, under - /// `reason="too_large"`. + /// Records over the producer's ceiling that were lost, under + /// `reason="too_large"`: dropped by the size cap while `dlq.enabled` is + /// false, or refused by the dead letter sink. #[must_use] pub fn too_large() -> Self { Self { reason: TOO_LARGE, - total: |d| d.oversize, + total: Derived::too_large_lost, seen: 0.0, } } @@ -976,6 +998,30 @@ vector_component_errors_total{component_id=\"dfe_sink\",error_type=\"request_fai assert_eq!(derived.outstanding(), 6.0); } + /// A record over the ceiling that the DLQ topic holds is a dead letter, not + /// a loss. Only the dead letter sink's own refusals, and the size cap's + /// drops while `dlq.enabled` is false, count as lost. + #[test] + fn dead_letters_are_counted_apart_from_the_records_lost_over_the_ceiling() { + let text = "\ +# TYPE vector_component_sent_events_total counter +vector_component_sent_events_total{component_id=\"dfe_dlq_sink\",component_kind=\"sink\"} 5 +# TYPE vector_component_discarded_events_total counter +vector_component_discarded_events_total{component_id=\"dfe_dlq_sink\",intentional=\"false\"} 2 +vector_component_discarded_events_total{component_id=\"dfe_sink\",intentional=\"false\"} 4 +vector_component_discarded_events_total{component_id=\"dfe_size_cap\",intentional=\"true\"} 1 +"; + let derived = ExpositionMerger::default().merge(text); + assert_eq!(derived.dead_lettered, 5.0); + assert_eq!(derived.dlq_discarded, 2.0); + assert_eq!(derived.too_large_lost(), 3.0); + assert_eq!( + derived.rejected(), + 7.0, + "the five dead letters are held, not lost" + ); + } + /// The size cap's running total reaches scalo's drop counter once: each /// exposition adds what grew, and a restarted Vector's fresh total counts /// whole. diff --git a/tests/e2e/kafka.rs b/tests/e2e/kafka.rs index 3edb055..e4e3531 100644 --- a/tests/e2e/kafka.rs +++ b/tests/e2e/kafka.rs @@ -34,6 +34,7 @@ use std::process::Stdio; use std::time::{Duration, Instant}; use dfe_transform_vector::config::assembler; +use dfe_transform_vector::config::generate::dead_letter_payload_cap; use dfe_transform_vector::config::loader::{ Config, DecodingConfig, MetricsConfig, PipelineConfig, SaslConfig, SinkConfig, SourceConfig, TlsConfig, TransformConfig, VectorConfig, @@ -368,6 +369,115 @@ async fn e2e_a_scram_broker_passes_startup_validation_and_delivers() { /// `message.max.bytes` default of 1,000,000 bytes. const TWO_MIB: usize = 2 * 1024 * 1024; +/// The dead letter on `dlq_topic` whose copied `_org_id` is `org`, waiting up to +/// `wait` for it, and every entry's `_org_id` read on the way. +async fn dead_letter_for( + kf_base: &KafkaConfig, + dlq_topic: &str, + group_id: &str, + org: &str, + wait: Duration, +) -> Result { + let mut consumer_cfg = kf_base.clone(); + consumer_cfg.group = format!("{group_id}-dlq-consumer"); + consumer_cfg.topics = vec![dlq_topic.to_string()]; + consumer_cfg.auto_offset_reset = "earliest".into(); + let consumer = KafkaTransport::new(&consumer_cfg) + .await + .map_err(|e| format!("dlq consumer transport: {e}"))?; + let deadline = Instant::now() + wait; + let mut seen: Vec = Vec::new(); + while Instant::now() < deadline { + match consumer.recv(10).await { + Ok(batch) => { + for record in &batch.records { + let Ok(entry) = serde_json::from_slice::(&record.payload) + else { + return Err(format!( + "a dead letter that is not JSON: {} bytes", + record.payload.len() + )); + }; + let found = entry["_org_id"].as_str().unwrap_or_default().to_string(); + if found == org { + let _ = consumer.close().await; + return Ok(entry); + } + seen.push(found); + } + } + Err(_) => sleep(Duration::from_millis(500)).await, + } + } + let _ = consumer.close().await; + Err(format!( + "no dead letter for {org} on {dlq_topic} within {wait:?}; saw {seen:?}" + )) +} + +/// What a dead letter for a record whose size is in `original` must carry: +/// scalo's DLQ envelope, truncated to `cap` bytes of payload and copied header. +fn check_dead_letter( + entry: &serde_json::Value, + original: std::ops::RangeInclusive, + org: &str, + destination: &str, + cap: u64, +) -> Result<(), String> { + let original = entry["original_bytes"] + .as_u64() + .and_then(|n| usize::try_from(n).ok()) + .filter(|n| original.contains(n)) + .ok_or_else(|| { + format!( + "original_bytes should be in {original:?}, got {}", + entry["original_bytes"] + ) + })?; + let expect = |field: &str, want: serde_json::Value| { + if entry[field] == want { + Ok(()) + } else { + Err(format!("{field}: wanted {want}, got {}", entry[field])) + } + }; + expect("service", "dfe-transform-vector".into())?; + expect("attempts", 0.into())?; + expect("truncated", true.into())?; + expect("original_bytes", original.into())?; + expect("destination", destination.into())?; + expect("_raw_encoding", "base64".into())?; + let reason = entry["reason"].as_str().unwrap_or_default(); + if !reason.starts_with(&format!("record of {original} bytes is over the sink's ")) { + return Err(format!("reason does not name the record's size: {reason}")); + } + if entry["timestamp"].as_i64().unwrap_or_default() <= 0 { + return Err(format!( + "timestamp is not epoch milliseconds: {}", + entry["timestamp"] + )); + } + // The copied header takes its bytes from the cap, then the payload's head + // takes the rest, base64 encoded. + let header_bytes = serde_json::to_string(org).map(|t| t.len()).unwrap_or(0) as u64; + let head_bytes = cap - header_bytes; + let raw = entry["_raw"].as_str().unwrap_or_default(); + if raw.len() as u64 != head_bytes.div_ceil(3) * 4 { + return Err(format!( + "_raw holds {} base64 characters, a {head_bytes}-byte head is {}", + raw.len(), + head_bytes.div_ceil(3) * 4 + )); + } + if !raw.starts_with("eyJ") { + return Err(format!( + "_raw does not start with a JSON object: {}", + &raw[..8.min(raw.len())] + )); + } + Ok(()) +} + /// The record the transform triples past the pipeline's 16 MiB ceiling. Under /// that ceiling itself, so the topic and the test producer both take it. const SIX_MIB: usize = 6 * 1024 * 1024; @@ -396,7 +506,7 @@ fn write_growth_transform(dir: &std::path::Path) { /// A sink no deployment has tuned carries a 2 MiB record to the next topic, /// and a record the transform grows past the pipeline's ceiling is held back -/// and counted once as too large. +/// from it and dead-lettered once, to the DLQ topic, as scalo's envelope. /// /// Owns its broker, so it runs by default rather than opt-in. #[tokio::test] @@ -419,6 +529,7 @@ async fn e2e_a_default_sink_carries_2_mib_and_counts_a_record_grown_past_the_cei let suffix = unique_suffix(); let source_topic = format!("dfe_e2e_ceiling_src_{suffix}"); let sink_topic = format!("dfe_e2e_ceiling_sink_{suffix}"); + let dlq_topic = format!("dfe_e2e_ceiling_dlq_{suffix}"); let group_id = format!("dfe-e2e-ceiling-{suffix}"); // scalo creates each topic with the pipeline's record ceiling, as a DFE @@ -427,7 +538,11 @@ async fn e2e_a_default_sink_carries_2_mib_and_counts_a_record_grown_past_the_cei admin_cfg.group = group_id.clone(); let admin = KafkaAdmin::new(&admin_cfg).expect("admin"); admin - .create_topics(&[(&source_topic, 1, 1), (&sink_topic, 1, 1)]) + .create_topics(&[ + (&source_topic, 1, 1), + (&sink_topic, 1, 1), + (&dlq_topic, 1, 1), + ]) .await .expect("create topics"); @@ -448,10 +563,12 @@ async fn e2e_a_default_sink_carries_2_mib_and_counts_a_record_grown_past_the_cei &exporter_addr, ); dfe_config.vector.binary = vector_bin.to_string_lossy().into_owned(); + dfe_config.sink.dlq_topic = dlq_topic.clone(); assert!( dfe_config.sink.librdkafka_options.is_empty(), "the case is a sink no deployment has tuned" ); + let cap = dead_letter_payload_cap(&dfe_config.sink, &dfe_config.dlq.0); dfe_config.validate().expect("config validates"); assembler::assemble(&dfe_config, &config_dir).expect("assemble config"); @@ -468,18 +585,26 @@ async fn e2e_a_default_sink_carries_2_mib_and_counts_a_record_grown_past_the_cei let outcome = run_ceiling_assertions( &kf_base, - &source_topic, - &sink_topic, + &CeilingTopics { + source: &source_topic, + sink: &sink_topic, + dlq: &dlq_topic, + }, &group_id, &suffix, &exporter_addr, + cap, ) .await; let _ = vector_child.start_kill(); let _ = tokio::time::timeout(Duration::from_secs(3), vector_child.wait()).await; let _ = admin - .delete_topics(&[source_topic.as_str(), sink_topic.as_str()]) + .delete_topics(&[ + source_topic.as_str(), + sink_topic.as_str(), + dlq_topic.as_str(), + ]) .await; if let Err(msg) = outcome { @@ -487,17 +612,25 @@ async fn e2e_a_default_sink_carries_2_mib_and_counts_a_record_grown_past_the_cei } } +/// The topics the record ceiling test reads and writes. +struct CeilingTopics<'a> { + source: &'a str, + sink: &'a str, + dlq: &'a str, +} + /// Send a record the transform grows past the ceiling, then a 2 MiB one; wait -/// for the 2 MiB record downstream, then check the grown one was held back and -/// reached scalo's too-large count exactly once. +/// for the 2 MiB record downstream, then check the grown one was kept off the +/// sink topic and written once to the DLQ topic as scalo's envelope. async fn run_ceiling_assertions( kf_base: &KafkaConfig, - source_topic: &str, - sink_topic: &str, + topics: &CeilingTopics<'_>, group_id: &str, suffix: &str, exporter_addr: &str, + cap: u64, ) -> Result<(), String> { + let (source_topic, sink_topic) = (topics.source, topics.sink); sleep(Duration::from_secs(3)).await; let mut producer_cfg = kf_base.clone(); @@ -506,14 +639,18 @@ async fn run_ceiling_assertions( let producer = KafkaTransport::new(&producer_cfg) .await .map_err(|e| format!("producer transport: {e}"))?; + let org = format!("{suffix}-grow"); let grown = format!( - r#"{{"id":"{suffix}-grow","grow":true,"blob":"{}"}}"#, + r#"{{"_org_id":"{org}","id":"{suffix}-grow","grow":true,"blob":"{}"}}"#, "g".repeat(SIX_MIB) ); let two_mib = format!( r#"{{"id":"{suffix}-2mib","blob":"{}"}}"#, "y".repeat(TWO_MIB) ); + // The transform adds `enriched` and two copies of `blob`. + let grown_after_transform = + grown.len() + r#","enriched":true"#.len() + 2 * (SIX_MIB + r#","copy_a":"""#.len()); for payload in [grown, two_mib] { let sent = producer .send(source_topic, bytes::Bytes::from(payload.into_bytes())) @@ -571,24 +708,33 @@ async fn run_ceiling_assertions( return Err("the record grown past the ceiling reached the sink topic".into()); } + let entry = + dead_letter_for(kf_base, topics.dlq, group_id, &org, Duration::from_secs(60)).await?; + check_dead_letter( + &entry, + grown_after_transform..=grown_after_transform + KAFKA_SOURCE_FIELDS_MAX, + &org, + sink_topic, + cap, + )?; + // Vector publishes internal metrics on its own flush interval. let mut merger = dfe_transform_vector::metrics::scrape::ExpositionMerger::default(); let mut derived = dfe_transform_vector::metrics::scrape::Derived::default(); let deadline = Instant::now() + Duration::from_secs(20); - while derived.oversize < 1.0 && Instant::now() < deadline { + while derived.dead_lettered < 1.0 && Instant::now() < deadline { if let Ok(body) = dfe_transform_vector::metrics::scrape::fetch_exposition(exporter_addr).await { derived = merger.merge(&body); } - if derived.oversize < 1.0 { + if derived.dead_lettered < 1.0 { sleep(Duration::from_millis(500)).await; } } - if derived.oversize != 1.0 || derived.sink_discarded != 0.0 { + if derived.dead_lettered != 1.0 || derived.rejected() != 0.0 { return Err(format!( - "the size cap should hold back exactly the grown record and the sink \ - refuse nothing: {derived:?}" + "the DLQ sink should hold exactly the grown record and nothing be lost: {derived:?}" )); } @@ -597,9 +743,9 @@ async fn run_ceiling_assertions( let mut drops = DeadLetterDrops::too_large(); metrics::with_local_recorder(&recorder, || drops.record(&derived)); let counted = too_large_since(&snapshotter); - if counted != 1 { + if counted != 0 { return Err(format!( - "the grown record must count once as too large, counted {counted}" + "a dead-lettered record is not a drop, counted {counted} as too large" )); } Ok(()) @@ -609,15 +755,15 @@ async fn run_ceiling_assertions( /// oversize record is far under the broker's and the test producer's limits. const SMALL_MESSAGE_MAX_BYTES: &str = "200000"; -/// A record on the bus over the sink's ceiling is dropped by the size cap and -/// counts once in `pipeline_dead_letters_dropped_total{reason="too_large"}`, -/// however many times the exporter is scraped afterwards. +/// A record on the bus over the sink's ceiling is dead-lettered once, to the +/// DLQ topic, with the reason and its first bytes, however many times the +/// exporter is scraped afterwards, and counts as no drop. #[tokio::test] #[ignore] // requires Vector binary + Kafka (live or Docker) -async fn e2e_an_oversize_record_on_the_bus_counts_one_too_large_drop() { +async fn e2e_an_oversize_record_on_the_bus_is_dead_lettered_once() { crate::common::skip_if_no_vector!(); - let Some(fixture) = KafkaFixture::acquire("kafka-oversize-counts-once").await else { + let Some(fixture) = KafkaFixture::acquire("kafka-oversize-dead-letters").await else { crate::common::require_service_in_ci("Kafka", "live unreachable and no Docker"); eprintln!("Skipping: no Kafka backend available (live unreachable AND no Docker)"); return; @@ -628,6 +774,7 @@ async fn e2e_an_oversize_record_on_the_bus_counts_one_too_large_drop() { let suffix = unique_suffix(); let source_topic = format!("dfe_e2e_big_src_{suffix}"); let sink_topic = format!("dfe_e2e_big_sink_{suffix}"); + let dlq_topic = format!("dfe_e2e_big_dlq_{suffix}"); let group_id = format!("dfe-e2e-big-{suffix}"); let mut admin_cfg = kf_base.clone(); @@ -635,7 +782,11 @@ async fn e2e_an_oversize_record_on_the_bus_counts_one_too_large_drop() { admin_cfg.topics = vec![source_topic.clone()]; let admin = KafkaAdmin::new(&admin_cfg).expect("admin"); admin - .create_topics(&[(&source_topic, 1, 1), (&sink_topic, 1, 1)]) + .create_topics(&[ + (&source_topic, 1, 1), + (&sink_topic, 1, 1), + (&dlq_topic, 1, 1), + ]) .await .expect("create topics"); @@ -655,10 +806,12 @@ async fn e2e_an_oversize_record_on_the_bus_counts_one_too_large_drop() { &data_dir.to_string_lossy(), &exporter_addr, ); + dfe_config.sink.dlq_topic = dlq_topic.clone(); dfe_config .sink .librdkafka_options .insert("message.max.bytes".into(), SMALL_MESSAGE_MAX_BYTES.into()); + let cap = dead_letter_payload_cap(&dfe_config.sink, &dfe_config.dlq.0); dfe_config.validate().expect("config validates"); assembler::assemble(&dfe_config, &config_dir).expect("assemble config"); @@ -675,18 +828,26 @@ async fn e2e_an_oversize_record_on_the_bus_counts_one_too_large_drop() { let outcome = run_oversize_assertions( &kf_base, - &source_topic, - &sink_topic, + &CeilingTopics { + source: &source_topic, + sink: &sink_topic, + dlq: &dlq_topic, + }, &group_id, &suffix, &exporter_addr, + cap, ) .await; let _ = vector_child.start_kill(); let _ = tokio::time::timeout(Duration::from_secs(3), vector_child.wait()).await; let _ = admin - .delete_topics(&[source_topic.as_str(), sink_topic.as_str()]) + .delete_topics(&[ + source_topic.as_str(), + sink_topic.as_str(), + dlq_topic.as_str(), + ]) .await; if let Err(msg) = outcome { @@ -715,16 +876,28 @@ fn too_large_since(snapshotter: &metrics_util::debugging::Snapshotter) -> u64 { .sum() } +/// Bytes of the fields Vector's Kafka source adds to an event (`topic`, +/// `partition`, `offset`, `timestamp`, `source_type`, `message_key`), which +/// the transforms and the sink then carry, and an upper bound for the +/// transform's own additions. +const KAFKA_SOURCE_FIELDS_MAX: usize = 512; + +/// The `blob` of the record the oversize test sends, which puts it over +/// `SMALL_MESSAGE_MAX_BYTES`. +const OVERSIZE_BLOB_BYTES: usize = 300_000; + /// Send one oversize record then one marker, wait for the marker downstream, -/// then check the size cap's drop reaches scalo's counter exactly once. +/// then check the oversize record is on the DLQ topic as scalo's envelope, and +/// that the exporter counts it once as a dead letter and never as a drop. async fn run_oversize_assertions( kf_base: &KafkaConfig, - source_topic: &str, - sink_topic: &str, + topics: &CeilingTopics<'_>, group_id: &str, suffix: &str, exporter_addr: &str, + cap: u64, ) -> Result<(), String> { + let (source_topic, sink_topic) = (topics.source, topics.sink); sleep(Duration::from_secs(3)).await; let mut producer_cfg = kf_base.clone(); @@ -734,10 +907,12 @@ async fn run_oversize_assertions( .await .map_err(|e| format!("producer transport: {e}"))?; // One partition, so the marker reaches the size cap after the oversize one. + let org = format!("{suffix}-big"); let oversize = format!( - r#"{{"id":"{suffix}-big","blob":"{}"}}"#, - "x".repeat(300_000) + r#"{{"_org_id":"{org}","id":"{org}","blob":"{}"}}"#, + "x".repeat(OVERSIZE_BLOB_BYTES) ); + let sent_bytes = oversize.len(); let marker = format!(r#"{{"id":"{suffix}-marker"}}"#); for payload in [oversize, marker] { let sent = producer @@ -782,23 +957,35 @@ async fn run_oversize_assertions( return Err("the oversize record reached the sink topic past the size cap".into()); } + let entry = + dead_letter_for(kf_base, topics.dlq, group_id, &org, Duration::from_secs(30)).await?; + // The enrich transform adds `enriched` and a timestamp, and the Kafka + // source its own fields. + check_dead_letter( + &entry, + sent_bytes + 64..=sent_bytes + KAFKA_SOURCE_FIELDS_MAX, + &org, + sink_topic, + cap, + )?; + // Vector publishes internal metrics on its own flush interval. let mut merger = dfe_transform_vector::metrics::scrape::ExpositionMerger::default(); let mut derived = dfe_transform_vector::metrics::scrape::Derived::default(); let deadline = Instant::now() + Duration::from_secs(20); - while derived.oversize < 1.0 && Instant::now() < deadline { + while derived.dead_lettered < 1.0 && Instant::now() < deadline { if let Ok(body) = dfe_transform_vector::metrics::scrape::fetch_exposition(exporter_addr).await { derived = merger.merge(&body); } - if derived.oversize < 1.0 { + if derived.dead_lettered < 1.0 { sleep(Duration::from_millis(500)).await; } } - if derived.oversize != 1.0 { + if derived.dead_lettered != 1.0 || derived.rejected() != 0.0 { return Err(format!( - "the size cap should have dropped exactly one record: {derived:?}" + "the DLQ sink should hold exactly one dead letter and nothing be lost: {derived:?}" )); } @@ -806,20 +993,18 @@ async fn run_oversize_assertions( let snapshotter = recorder.snapshotter(); let mut drops = DeadLetterDrops::too_large(); metrics::with_local_recorder(&recorder, || drops.record(&derived)); - let first = too_large_since(&snapshotter); - - // A second scrape of the same total must add nothing. + // A second scrape of the same totals must change nothing. let body = dfe_transform_vector::metrics::scrape::fetch_exposition(exporter_addr) .await .map_err(|e| format!("second scrape: {e}"))?; let again = merger.merge(&body); metrics::with_local_recorder(&recorder, || drops.record(&again)); - let second = too_large_since(&snapshotter); - - if (first, second) != (1, 0) { + let counted = too_large_since(&snapshotter); + if counted != 0 || again.dead_lettered != 1.0 { return Err(format!( - "too_large must rise by exactly one and not again on a rescrape: \ - first scrape {first}, second {second}" + "a dead letter is not a drop and a rescrape adds none: counted {counted} as too \ + large, {} dead-lettered after the second scrape", + again.dead_lettered )); } Ok(()) diff --git a/tests/integration/config.rs b/tests/integration/config.rs index 92d0e08..cdb07d2 100644 --- a/tests/integration/config.rs +++ b/tests/integration/config.rs @@ -148,8 +148,8 @@ fn end_to_end_assembly_with_transforms() { assert!(source.contains("SECRET[dfe_credentials.source_sasl_username]")); assert!(source.contains("cooperative-sticky")); - // Verify sink YAML content — the size cap reads the last transform, and - // the sink reads the size cap + // Verify sink YAML content — the size cap reads the last transform, the + // sink reads what fits, and the dead letter sink reads what does not let sink = fs::read_to_string(output_dir.path().join("90_sink.yaml")).unwrap(); let sink_yaml: serde_json::Value = scalo::yaml::from_str(&sink, "config.yaml", scalo::yaml::Profile::Config).unwrap(); @@ -160,7 +160,15 @@ fn end_to_end_assembly_with_transforms() { ); assert_eq!( sink_yaml["sinks"]["dfe_sink"]["inputs"][0].as_str(), - Some("dfe_size_cap") + Some("dfe_size_cap.within_cap") + ); + assert_eq!( + sink_yaml["transforms"]["dfe_size_cap_dlq"]["inputs"][0].as_str(), + Some("dfe_size_cap._unmatched") + ); + assert_eq!( + sink_yaml["sinks"]["dfe_dlq_sink"]["topic"].as_str(), + Some("dfe_transform_dlq") ); assert!(sink.contains("enriched_syslog_land")); @@ -317,6 +325,33 @@ fn config_validation_catches_missing_sink_topic() { assert!(err.to_string().contains("sink.topic")); } +/// A record over the ceiling needs a topic to go to while the DLQ is on, and +/// needs none once it is off or the sink is direct. +#[test] +fn config_validation_needs_a_dlq_topic_only_where_one_is_written() { + for (dlq_enabled, direct, empty_topic_refused) in [ + (true, false, true), + (false, false, false), + (true, true, false), + ] { + let mut config = full_config(None); + config.sink.dlq_topic = String::new(); + config.dlq.0.enabled = dlq_enabled; + if direct { + config.sink.transport = Transport::Direct; + } + let verdict = config.validate(); + assert_eq!( + verdict.is_err(), + empty_topic_refused, + "dlq.enabled {dlq_enabled}, direct {direct}: {verdict:?}" + ); + if let Err(e) = verdict { + assert!(e.to_string().contains("sink.dlq_topic"), "{e}"); + } + } +} + /// With the commit timer armed, handing offset-storing back to librdkafka /// turns an unclean pod death into skipped records rather than replayed ones. /// @@ -692,7 +727,13 @@ fn written_kafka_components(config: &Config) -> (String, String) { use dfe_transform_vector::config::generate; let source = generate::generate_source_yaml(&config.source, &config.bridge); - let sink = generate::generate_sink_yaml(&config.sink, &config.bridge, &["t".into()], true); + let sink = generate::generate_sink_yaml( + &config.sink, + &config.bridge, + &["t".into()], + true, + &scalo::dlq::DlqConfig::default(), + ); ( scalo::yaml::to_string(&source).unwrap(), scalo::yaml::to_string(&sink).unwrap(), @@ -1388,6 +1429,7 @@ fn config_with_all_fields_populated_validates() { endpoint: "http://dfe-loader:6000".into(), brokers: vec!["kafka-1:9092".into()], topic: "enriched_load".into(), + dlq_topic: "enriched_dlq".into(), key_field: ".org_id".into(), encoding: "raw_bytes".into(), compression: "lz4".into(), diff --git a/tests/integration/dead_letter_vrl.rs b/tests/integration/dead_letter_vrl.rs new file mode 100644 index 0000000..45bfe5a --- /dev/null +++ b/tests/integration/dead_letter_vrl.rs @@ -0,0 +1,191 @@ +// Project: dfe-transform-vector +// File: tests/integration/dead_letter_vrl.rs +// Purpose: The size cap's dead letter remap, run through Vector's own VRL +// Language: Rust +// +// License: BUSL-1.1 +// Copyright: (c) 2026 HYPERI PTY LIMITED + +//! The remap that turns a record over the producer's ceiling into scalo's DLQ +//! entry, run through `vector vrl` so the program is the one the assembler +//! writes and the VRL is Vector's. +//! +//! Skips where no Vector binary is available. + +use std::fs; +use std::process::Command; + +use dfe_transform_vector::config::generate::{ + SIZE_CAP_DLQ_LABEL, generate_sink_yaml, max_record_bytes, +}; +use dfe_transform_vector::config::loader::{BridgeConfig, SinkConfig}; +use scalo::dlq::DlqConfig; +use serde_json::{Value, json}; +use tempfile::TempDir; + +use crate::common; + +/// The payload and copied header budget the tests give the DLQ. +const CAP: usize = 40; + +/// Base64 characters for `bytes` bytes. +fn base64_len(bytes: usize) -> usize { + bytes.div_ceil(3) * 4 +} + +/// Run `events` through the remap `sink` generates, one entry per event. +fn run_remap(sink: &SinkConfig, events: &[Value]) -> Vec { + let dlq = DlqConfig { + max_payload_bytes: Some(CAP), + ..DlqConfig::default() + }; + let yaml = generate_sink_yaml(sink, &BridgeConfig::default(), &["last".into()], true, &dlq); + let program = yaml["transforms"][SIZE_CAP_DLQ_LABEL]["source"] + .as_str() + .expect("the remap source"); + + let dir = TempDir::new().expect("work dir"); + let (program_file, input_file) = (dir.path().join("dlq.vrl"), dir.path().join("in.json")); + fs::write(&program_file, program).expect("write the program"); + let lines: Vec = events.iter().map(Value::to_string).collect(); + fs::write(&input_file, lines.join("\n")).expect("write the events"); + + let vector = common::vector_binary_path().expect("a Vector binary"); + let output = Command::new(vector) + .args(["vrl", "--quiet", "--print-object", "--input"]) + .arg(&input_file) + .arg("--program") + .arg(&program_file) + .output() + .expect("run vector vrl"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "vector vrl failed: {}\n{stdout}\n--- program ---\n{program}", + String::from_utf8_lossy(&output.stderr) + ); + stdout + .lines() + .filter(|line| line.starts_with('{')) + .map(|line| serde_json::from_str(line).expect("an entry that is JSON")) + .collect() +} + +fn json_sink() -> SinkConfig { + SinkConfig { + topic: "syslog_load".into(), + ..SinkConfig::default() + } +} + +/// The envelope scalo's DLQ writes for a record cut to its first bytes. +fn assert_envelope(entry: &Value, sink: &SinkConfig, original: usize) { + assert_eq!(entry["service"], "dfe-transform-vector", "{entry}"); + assert_eq!(entry["attempts"], 0, "{entry}"); + assert_eq!(entry["truncated"], true, "{entry}"); + assert_eq!(entry["original_bytes"], original, "{entry}"); + assert_eq!(entry["destination"], sink.topic.as_str(), "{entry}"); + assert_eq!(entry["_raw_encoding"], "base64", "{entry}"); + assert_eq!( + entry["reason"], + format!( + "record of {original} bytes is over the sink's {}-byte ceiling", + max_record_bytes(sink) + ), + "{entry}" + ); + assert!( + entry["timestamp"] + .as_i64() + .is_some_and(|t| t > 1_700_000_000_000), + "timestamp should be epoch milliseconds: {entry}" + ); +} + +/// The header fields that fit are copied beside the payload and take their +/// bytes from the cap, so the payload's head is what they leave. +#[test] +fn a_dead_letter_copies_its_header_and_cuts_the_rest_to_the_cap() { + common::skip_if_no_vector!(); + let sink = json_sink(); + let event = json!({"_org_id": "acme", "_source": "syslog", "id": 7, "blob": "x".repeat(100)}); + let original = event.to_string().len(); + + let entries = run_remap(&sink, &[event]); + + assert_eq!(entries.len(), 1); + let entry = &entries[0]; + assert_envelope(entry, &sink, original); + assert_eq!(entry["_org_id"], "acme"); + assert_eq!(entry["_source"], "syslog"); + // `"acme"` and `"syslog"` are 6 and 8 bytes of the 40. + assert_eq!( + entry["_raw"].as_str().map(str::len), + Some(base64_len(CAP - 14)) + ); + assert!(entry["_raw"].as_str().is_some_and(|r| r.starts_with("eyJ"))); + assert!(entry.get("header_dropped").is_none(), "{entry}"); +} + +/// A header value that does not fit is left out and named, as scalo's DLQ does. +#[test] +fn a_header_value_over_the_cap_is_named_in_header_dropped() { + common::skip_if_no_vector!(); + let sink = json_sink(); + let event = json!({"_tags": ["t".repeat(60)], "id": 1}); + let original = event.to_string().len(); + + let entries = run_remap(&sink, &[event]); + + let entry = &entries[0]; + assert_envelope(entry, &sink, original); + assert_eq!(entry["header_dropped"], json!(["_tags"])); + assert!(entry.get("_tags").is_none(), "{entry}"); + assert_eq!(entry["_raw"].as_str().map(str::len), Some(base64_len(CAP))); +} + +/// The cut counts bytes, so a multibyte character split by it still leaves +/// exactly the cap's worth, and a record with no header keeps all of it. +#[test] +fn the_payload_is_cut_by_bytes_not_characters() { + common::skip_if_no_vector!(); + let sink = json_sink(); + let event = json!({"blob": "\u{e9}".repeat(60)}); + let original = event.to_string().len(); + assert!(original > CAP * 2, "the record has to be over the cap"); + + let entries = run_remap(&sink, &[event]); + + assert_envelope(&entries[0], &sink, original); + assert_eq!( + entries[0]["_raw"].as_str().map(str::len), + Some(base64_len(CAP)) + ); +} + +/// With `raw_bytes` the record is the `message` field, and the dead letter +/// measures and cuts that, as the size cap does. +#[test] +fn a_raw_bytes_record_is_the_message_field() { + common::skip_if_no_vector!(); + let sink = SinkConfig { + encoding: "raw_bytes".into(), + ..json_sink() + }; + let event = json!({"_org_id": "acme", "message": "m".repeat(100)}); + + let entries = run_remap(&sink, &[event]); + + let entry = &entries[0]; + assert_envelope(entry, &sink, 100); + assert_eq!(entry["_org_id"], "acme"); + assert_eq!( + entry["_raw"].as_str().map(str::len), + Some(base64_len(CAP - 6)) + ); + assert!( + entry["_raw"] + .as_str() + .is_some_and(|r| r.starts_with("bW1t")) + ); +} diff --git a/tests/integration/mod.rs b/tests/integration/mod.rs index a0e09e8..168277f 100644 --- a/tests/integration/mod.rs +++ b/tests/integration/mod.rs @@ -14,6 +14,7 @@ mod config_reach; #[allow(clippy::panic)] mod container_hygiene; mod credential_reassembly; +mod dead_letter_vrl; mod deployment; mod direct_transport; mod fixtures; diff --git a/tests/integration/vector_validate.rs b/tests/integration/vector_validate.rs index 888c502..070fddb 100644 --- a/tests/integration/vector_validate.rs +++ b/tests/integration/vector_validate.rs @@ -607,3 +607,22 @@ fn vector_validate_routes_with_named_outputs() { String::from_utf8_lossy(&output.stderr) ); } + +/// With `dlq.enabled` false the size cap stays the filter that drops, and +/// Vector accepts that shape too. +#[test] +#[ignore] // requires Vector binary +fn vector_validate_full_chain_with_the_dead_letter_queue_off() { + common::skip_if_no_vector!(); + + let (mut config, work_dir) = build_config(BufferConfig::default()); + config.dlq.0.enabled = false; + let output = assemble_and_validate(&config, work_dir.path()); + + assert!( + output.status.success(), + "vector validate failed with dlq.enabled false:\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +}