From 1effba2bcf5780ea69e83fe45db5f7e79c59e7bb Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 13:57:49 +1100 Subject: [PATCH] fix(sink): dead-letter events over the per-message budget An event over sink.max_message_bytes after serialisation was dropped. It was logged and counted on events_oversize_total. Enrichment grows an event about 1.5 times. A near-cap input could go over the cap and be lost, breaking at-least-once delivery. Each oversize event goes to the DLQ as its whole serialised body. The reason names its size and the budget. The entry is keyed to the sink topic like the not-JSON dead letters. The DLQ cuts the body to dlq.max_payload_bytes and records original_bytes. The source is released once the DLQ holds the entry. events_oversize_total still counts each oversize event. With dlq.enabled false, oversize events are dropped and counted in pipeline_dead_letters_dropped_total, like the not-JSON ones. serialise_events keeps its signature for tests and offline tools. The service calls serialise_events_with_oversize, which hands the oversize bodies back. --- docs/architecture.md | 2 +- docs/config-schema.json | 2 +- docs/config-schema.yaml | 3 +- docs/configuration.md | 2 +- docs/deployment-contract.json | 2 +- docs/metrics-manifest.json | 2 +- src/config.rs | 3 +- src/metrics.rs | 6 +-- src/pipeline.rs | 93 ++++++++++++++++++++++++++-------- src/service.rs | 44 +++++++++++----- tests/direct_transport.rs | 95 ++++++++++++++++++++++++++++++++++- 11 files changed, 208 insertions(+), 46 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 5451c013b..a15c62b0e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -195,7 +195,7 @@ A record the sink transport would refuse -- over its `message.max.bytes`, or mat JSON is the only payload format. MessagePack, supported in DFE/XDR 2.0 and 2.1, is deprecated in DFE 2.2 and no longer accepted: the JSON path (SIMD parsing with sonic-rs, zstd on the wire) is fast enough that MessagePack gave no CPU saving. -**One record per event.** `pipeline.rs::serialise_events` gives each event its own payload and `service.rs::send_blocks` sends them via `TransportSender::send_batch`, because dfe-loader parses exactly one JSON document per message ([#67](https://github.com/hyperi-io/dfe-transform-elastic/issues/67)). `sink.max_message_bytes` (default 8,000,000 bytes, the DFE record cap) bounds each payload under the 16 MiB ceiling the producer and dfe-loader's consumer share, and the `send_batch` block under gRPC's 16 MiB ceiling. An event over the budget is dropped and counted on `events_oversize_total`: no broker would take it, and retrying blocks the partition. +**One record per event.** `pipeline.rs::serialise_events_with_oversize` gives each event its own payload and `service.rs::send_blocks` sends them via `TransportSender::send_batch`, because dfe-loader parses exactly one JSON document per message ([#67](https://github.com/hyperi-io/dfe-transform-elastic/issues/67)). `sink.max_message_bytes` (default 8,000,000 bytes, the DFE record cap) bounds each payload under the 16 MiB ceiling the producer and dfe-loader's consumer share, and the `send_batch` block under gRPC's 16 MiB ceiling. An event over the budget is dead-lettered and counted on `events_oversize_total`: no broker would take it, and retrying blocks the partition. Duplicates are the accepted cost: a retried block re-sends the records that already landed, so every downstream consumer must be idempotent. diff --git a/docs/config-schema.json b/docs/config-schema.json index 2ddd824ab..6a4d9d603 100644 --- a/docs/config-schema.json +++ b/docs/config-schema.json @@ -308,7 +308,7 @@ "default": null }, "max_message_bytes": { - "description": "Ceiling on one outbound record. An event that serialises larger is\ndropped and counted on `events_oversize_total`.\n\nThe default is the DFE record cap, 8,000,000 bytes. Against a broker or\ntopic whose `message.max.bytes` is lower, set this below that limit, or\nthe broker refuses those events instead and they count on\n`send_filtered_dlq_total`. Brokers that carry more can raise it, up to\n16 MiB.", + "description": "Ceiling on one outbound record. An event that serialises larger is\ndead-lettered, with its size and this budget as the reason, and counted\non `events_oversize_total`.\n\nThe default is the DFE record cap, 8,000,000 bytes. Against a broker or\ntopic whose `message.max.bytes` is lower, set this below that limit, or\nthe broker refuses those events instead and they count on\n`send_filtered_dlq_total`. Brokers that carry more can raise it, up to\n16 MiB.", "type": "integer", "format": "uint", "minimum": 0, diff --git a/docs/config-schema.yaml b/docs/config-schema.yaml index c02a22cc5..bed12aca6 100644 --- a/docs/config-schema.yaml +++ b/docs/config-schema.yaml @@ -324,7 +324,8 @@ $defs: max_message_bytes: description: |- Ceiling on one outbound record. An event that serialises larger is - dropped and counted on `events_oversize_total`. + dead-lettered, with its size and this budget as the reason, and counted + on `events_oversize_total`. The default is the DFE record cap, 8,000,000 bytes. Against a broker or topic whose `message.max.bytes` is lower, set this below that limit, or diff --git a/docs/configuration.md b/docs/configuration.md index 7d3357f0a..e1ed1442e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -67,7 +67,7 @@ On a direct source a push is held at most 18 s, so every send it waits on gives ## `sink.max_message_bytes` caps one outbound record -The default is the DFE record cap, 8,000,000 bytes: dfe-receiver refuses a request body over it, and every broker DFE deploys takes a record that size (Confluent Basic's 8,388,608 limit leaves room for the framing). The producer this service builds allows 16 MiB, so a deployment with larger brokers raises the budget up to that. An event that serialises larger is dropped, logged, counted on `events_oversize_total`, and its source is still acknowledged. +The default is the DFE record cap, 8,000,000 bytes: dfe-receiver refuses a request body over it, and every broker DFE deploys takes a record that size (Confluent Basic's 8,388,608 limit leaves room for the framing). The producer this service builds allows 16 MiB, so a deployment with larger brokers raises the budget up to that. An event that serialises larger is logged, counted on `events_oversize_total` and dead-lettered to the DLQ with its size and the budget as the reason. The DLQ cuts the body to `dlq.max_payload_bytes`, so it keeps `original_bytes` and a truncated head. Against a broker or topic whose `message.max.bytes` is lower -- a stock Kafka broker takes about 1 MB -- set `sink.max_message_bytes` below that limit. Left at the default, a broker below 8,000,000 bytes refuses the events between the two instead: each is dropped as a dead letter, counted on `send_filtered_dlq_total`, and the partition moves on. Validation refuses a value under 4096 or over 16,777,216. diff --git a/docs/deployment-contract.json b/docs/deployment-contract.json index 0f5b7e690..d7dbe8aec 100644 --- a/docs/deployment-contract.json +++ b/docs/deployment-contract.json @@ -459,7 +459,7 @@ "default": null }, "max_message_bytes": { - "description": "Ceiling on one outbound record. An event that serialises larger is\ndropped and counted on `events_oversize_total`.\n\nThe default is the DFE record cap, 8,000,000 bytes. Against a broker or\ntopic whose `message.max.bytes` is lower, set this below that limit, or\nthe broker refuses those events instead and they count on\n`send_filtered_dlq_total`. Brokers that carry more can raise it, up to\n16 MiB.", + "description": "Ceiling on one outbound record. An event that serialises larger is\ndead-lettered, with its size and this budget as the reason, and counted\non `events_oversize_total`.\n\nThe default is the DFE record cap, 8,000,000 bytes. Against a broker or\ntopic whose `message.max.bytes` is lower, set this below that limit, or\nthe broker refuses those events instead and they count on\n`send_filtered_dlq_total`. Brokers that carry more can raise it, up to\n16 MiB.", "type": "integer", "format": "uint", "minimum": 0, diff --git a/docs/metrics-manifest.json b/docs/metrics-manifest.json index 0739b6b86..d15fb2d2e 100644 --- a/docs/metrics-manifest.json +++ b/docs/metrics-manifest.json @@ -875,7 +875,7 @@ { "name": "events_oversize_total", "type": "counter", - "description": "Events too large for one Kafka record, dropped", + "description": "Events too large for one Kafka record, dead-lettered", "unit": "", "labels": [], "group": "custom" diff --git a/src/config.rs b/src/config.rs index b456c57f7..faef6c5da 100644 --- a/src/config.rs +++ b/src/config.rs @@ -326,7 +326,8 @@ pub struct SinkConfig { pub brokers: Option>, /// Ceiling on one outbound record. An event that serialises larger is - /// dropped and counted on `events_oversize_total`. + /// dead-lettered, with its size and this budget as the reason, and counted + /// on `events_oversize_total`. /// /// The default is the DFE record cap, 8,000,000 bytes. Against a broker or /// topic whose `message.max.bytes` is lower, set this below that limit, or diff --git a/src/metrics.rs b/src/metrics.rs index 54658a489..1a9ef5c25 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -82,8 +82,8 @@ pub struct TransformMetrics { /// or topic ceiling below `sink.max_message_bytes`. pub send_filtered_dlq: Counter, - /// Events too large for one Kafka record even on their own. Dropped: no - /// broker would ever accept one, and retrying blocks the partition. + /// Events too large for one Kafka record even on their own. Dead-lettered: + /// no broker would ever accept one, and retrying blocks the partition. pub events_oversize: Counter, /// Batches whose events did not look like the envelope the config pinned. @@ -180,7 +180,7 @@ impl TransformMetrics { ), events_oversize: manager.counter( "events_oversize_total", - "Events too large for one Kafka record, dropped", + "Events too large for one Kafka record, dead-lettered", ), envelope_contradicted: manager.counter( "envelope_contradicted_total", diff --git a/src/pipeline.rs b/src/pipeline.rs index 9b77613ec..c0d8d4213 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -241,6 +241,16 @@ pub(crate) fn elided(value: &str) -> Cow<'_, str> { /// The dead-letter reason for input that is not JSON. pub const NOT_JSON: &str = "payload is not JSON"; +/// The dead-letter reason for an event that serialised over the per-message +/// budget: names its size and the budget it broke. +#[must_use] +pub fn oversize_reason(bytes: usize, budget: usize) -> String { + format!( + "event serialises to {bytes} bytes, over the {budget} byte per-message budget \ + (sink.max_message_bytes)" + ) +} + /// What a payload cost to decode. #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct ParseOutcome { @@ -353,8 +363,8 @@ pub struct SerialiseOutcome { /// Events that became an outbound message. pub serialised: usize, /// Events that on their own exceed the whole per-message budget. No - /// broker will ever accept one, so it is dropped rather than blocking the - /// partition behind a record that can only fail. + /// broker will ever accept one, so it is dead-lettered rather than + /// blocking the partition behind a record that can only fail. pub oversize: usize, /// Events serde refused to serialise. Dropped, on the same footing as an /// event the transform rejected. @@ -394,6 +404,17 @@ pub const fn message_budget(max_bytes: usize) -> usize { } } +/// What [`serialise_events_with_oversize`] produced. +#[derive(Debug, Default)] +pub struct Serialised { + /// One payload per event that fits the budget. + pub payloads: Vec>, + /// The whole serialised body of each event over the budget, for the DLQ. + pub oversize: Vec>, + /// The counts behind both. + pub outcome: SerialiseOutcome, +} + /// Serialise each event into a message payload of its own. /// /// ONE JSON document per payload, never NDJSON. dfe-loader parses exactly one @@ -402,13 +423,16 @@ pub const fn message_budget(max_bytes: usize) -> usize { /// (hyperi-io/dfe-transform-elastic#67). /// /// `max_bytes` bounds each payload on its own, because that is what a broker -/// bounds: an event above the budget is dropped rather than blocking the -/// partition behind a record no broker would take. +/// bounds. An event above the budget is returned in [`Serialised::oversize`] +/// for the caller to dead-letter, rather than blocking the partition behind a +/// record no broker would take. #[must_use] -pub fn serialise_events(events: &[Event], max_bytes: usize) -> (Vec>, SerialiseOutcome) { +pub fn serialise_events_with_oversize(events: &[Event], max_bytes: usize) -> Serialised { let budget = message_budget(max_bytes); - let mut payloads = Vec::with_capacity(events.len()); - let mut outcome = SerialiseOutcome::default(); + let mut serialised = Serialised { + payloads: Vec::with_capacity(events.len()), + ..Serialised::default() + }; // Written once and copied out at its exact length: a payload is held until // its block is delivered, and a buffer sized by a guess holds the guess. let mut scratch: Vec = Vec::with_capacity(1024); @@ -416,25 +440,35 @@ pub fn serialise_events(events: &[Event], max_bytes: usize) -> (Vec>, Se for event in events { scratch.clear(); if let Err(e) = event.write_json(&mut scratch) { - outcome.failed += 1; + serialised.outcome.failed += 1; tracing::warn!(error = %e, "event could not be serialised, dropped"); continue; } if scratch.len() > budget { - outcome.oversize += 1; + serialised.outcome.oversize += 1; tracing::error!( bytes = scratch.len(), budget, - "event exceeds the per-message budget and cannot be produced, dropped" + "event exceeds the per-message budget and cannot be produced, dead-lettered" ); + serialised.oversize.push(scratch.as_slice().to_vec()); continue; } - outcome.serialised += 1; - payloads.push(scratch.as_slice().to_vec()); + serialised.outcome.serialised += 1; + serialised.payloads.push(scratch.as_slice().to_vec()); } - (payloads, outcome) + serialised +} + +/// [`serialise_events_with_oversize`] without the oversize bodies, for tests +/// and offline tools that only count them. The service keeps the bodies and +/// dead-letters them. +#[must_use] +pub fn serialise_events(events: &[Event], max_bytes: usize) -> (Vec>, SerialiseOutcome) { + let serialised = serialise_events_with_oversize(events, max_bytes); + (serialised.payloads, serialised.outcome) } /// Serialise every event into ONE NDJSON payload, whatever its size. @@ -739,9 +773,9 @@ mod tests { } /// The shipped budget is the DFE record cap: a payload of 8,000,000 bytes - /// goes out and one byte more is dropped and counted. + /// goes out and one byte more is counted and held back for the DLQ. #[test] - fn the_shipped_budget_admits_8_000_000_bytes_and_drops_one_more() { + fn the_shipped_budget_admits_8_000_000_bytes_and_holds_back_one_more() { let budget = crate::config::default_max_message_bytes(); assert_eq!(budget, 8_000_000); let overhead = serialise_events(&padded(1, 0), budget).0[0].len(); @@ -755,17 +789,32 @@ mod tests { assert_eq!((outcome.serialised, outcome.oversize), (0, 1)); } - /// A record no broker can accept is dropped and counted, rather than - /// retried forever behind the rest of the partition. + /// A record no broker can accept is counted and handed back whole for the + /// DLQ, rather than retried forever behind the rest of the partition. #[test] - fn an_event_larger_than_the_budget_is_dropped_and_counted() { + fn an_event_larger_than_the_budget_is_counted_and_returned_for_the_dlq() { let mut events = padded(2, 100); events.extend(padded(1, 20_000)); - let (payloads, outcome) = serialise_events(&events, MIN_MESSAGE_BYTES); - assert_eq!(outcome.oversize, 1); - assert_eq!(outcome.serialised, 2); - assert_eq!(payloads.len(), 2); + let serialised = serialise_events_with_oversize(&events, MIN_MESSAGE_BYTES); + assert_eq!(serialised.outcome.oversize, 1); + assert_eq!(serialised.outcome.serialised, 2); + assert_eq!(serialised.payloads.len(), 2); + assert_eq!(serialised.oversize.len(), 1); + assert!( + serialised.oversize[0].len() > 20_000, + "the body was cut before it reached the DLQ" + ); + let body: serde_json::Value = + serde_json::from_slice(&serialised.oversize[0]).expect("a whole document"); + assert_eq!(body.get("i"), Some(&serde_json::json!(0))); + } + + #[test] + fn the_oversize_reason_names_the_size_and_the_budget() { + let reason = oversize_reason(8_000_001, 8_000_000); + assert!(reason.contains("8000001 bytes"), "{reason}"); + assert!(reason.contains("8000000 byte"), "{reason}"); } /// A budget below one ordinary event would drop the entire batch as diff --git a/src/service.rs b/src/service.rs index 059da0104..7ae64f233 100644 --- a/src/service.rs +++ b/src/service.rs @@ -20,12 +20,12 @@ //! downstream must be idempotent, which is the contract every DFE stage carries. //! //! An event that cannot be delivered is dropped and counted, never counted -//! delivered: one that will not transform or serialise, one over -//! `sink.max_message_bytes`, and one the sink transport would refuse, which the -//! pipeline screens out before the send. Input that is not JSON is refused as a -//! dead letter. Dead letters, the screened records among them, go to the DLQ -//! `dlq` configures, and the source is released only once the DLQ holds them. -//! With `dlq.enabled: false` they are dropped and counted in +//! delivered: one that will not transform or serialise, and one the sink +//! transport would refuse, which the pipeline screens out before the send. An +//! event over `sink.max_message_bytes` and input that is not JSON are dead +//! letters. Dead letters, the screened records among them, go to the DLQ `dlq` +//! configures, and the source is released only once the DLQ holds them. With +//! `dlq.enabled: false` they are dropped and counted in //! `pipeline_dead_letters_dropped_total`. use std::sync::Arc; @@ -51,8 +51,8 @@ use crate::agentless::{Agentless, Stopped}; use crate::config::{Config, Transport}; use crate::metrics::TransformMetrics; use crate::pipeline::{ - BatchOutcome, EnvelopeOutcome, NOT_JSON, SerialiseOutcome, message_budget, parse_each, - serialise_events, transform_batch_resolved, + BatchOutcome, EnvelopeOutcome, NOT_JSON, SerialiseOutcome, message_budget, oversize_reason, + parse_each, serialise_events_with_oversize, transform_batch_resolved, }; /// The longest the loop goes without pushing scaling signals. @@ -653,6 +653,8 @@ struct Block { outcome: BatchOutcome, envelopes: EnvelopeOutcome, serialised: SerialiseOutcome, + /// Events over the per-message budget, held for the DLQ. + oversize: Vec, } impl Block { @@ -666,6 +668,7 @@ impl Block { outcome: BatchOutcome::default(), envelopes: EnvelopeOutcome::default(), serialised: SerialiseOutcome::default(), + oversize: Vec::new(), } } } @@ -685,7 +688,8 @@ struct Stage<'a> { impl Stage<'_> { /// Parse, transform and serialise one block, returning a record per event - /// that survived and a dead letter per input refused as not JSON. + /// that survived and a dead letter per input refused as not JSON or event + /// over the per-message budget. /// /// At most `source.batch_size` parsed events are held at once, however many /// NDJSON lines the block's records carry. @@ -735,8 +739,10 @@ impl Stage<'_> { outcome, envelopes, serialised, + oversize, .. } = block; + dead.extend(oversize); // One series, counted once. The source transport counts its own wire // records in `transport_received_*`; these are the events parsed out. @@ -816,17 +822,29 @@ impl Stage<'_> { } => transform_batch_resolved(*transform, resolver, events), Work::Agentless(agentless) => agentless.transform(events)?, }; - let (payloads, serialised) = - serialise_events(&transformed, self.config.sink.max_message_bytes); + let max_bytes = self.config.sink.max_message_bytes; + let budget = message_budget(max_bytes); + let serialised = serialise_events_with_oversize(&transformed, max_bytes); drop(transformed); block.out.extend( - payloads + serialised + .payloads .into_iter() .map(|payload| outbound_record(payload, &self.topic)), ); + block.oversize.extend( + serialised + .oversize + .into_iter() + .map(|payload| FilteredDlqEntry { + reason: oversize_reason(payload.len(), budget), + payload, + key: Some(Arc::clone(&self.topic)), + }), + ); block.outcome.absorb(outcome); block.envelopes.absorb(envelopes); - block.serialised.absorb(serialised); + block.serialised.absorb(serialised.outcome); Ok(()) } diff --git a/tests/direct_transport.rs b/tests/direct_transport.rs index cdac2fda4..a2d350fb6 100644 --- a/tests/direct_transport.rs +++ b/tests/direct_transport.rs @@ -456,6 +456,18 @@ fn push_through( engine: impl FnOnce(&Config) -> BatchEngine, payloads: Vec>, expected: usize, +) -> Pushed { + let budget = dfe_transform_elastic::config::default_max_message_bytes(); + push_through_with_budget(capture, engine, payloads, expected, budget) +} + +/// [`push_through`] with `sink.max_message_bytes` set to `budget`. +fn push_through_with_budget( + capture: &SeriesCapture, + engine: impl FnOnce(&Config) -> BatchEngine, + payloads: Vec>, + expected: usize, + budget: usize, ) -> Pushed { let neighbours = tokio::runtime::Builder::new_multi_thread() .worker_threads(1) @@ -480,7 +492,8 @@ fn push_through( .strip_prefix("http://") .expect("endpoint is http") .to_string(); - let config = direct_config(&listen, &loader_endpoint); + let mut config = direct_config(&listen, &loader_endpoint); + config.sink.max_message_bytes = budget; let consumer = AnyReceiver::Grpc(transform_listener); let manager = MetricsManager::with_config(scalo::metrics::MetricsConfig::offline("")); let metrics = TransformMetrics::register(&manager, "0.0.0-test", "test"); @@ -615,6 +628,86 @@ fn a_messagepack_push_lands_in_a_configured_dlq() { ); } +/// An event that serialises over `sink.max_message_bytes` is dead-lettered +/// whole, with its size and the budget as the reason, and the event beside it +/// still goes. Counting it and dropping it lost it. +#[test] +fn an_oversize_event_lands_in_a_configured_dlq() { + const BUDGET: usize = 16_384; + let dir = tempfile::tempdir().expect("tempdir"); + let dlq_config = scalo::dlq::DlqConfig { + enabled: true, + mode: scalo::dlq::DlqMode::FileOnly, + file: scalo::dlq::FileDlqConfig { + enabled: true, + path: dir.path().to_path_buf(), + compress_rotated: false, + ..scalo::dlq::FileDlqConfig::default() + }, + ..scalo::dlq::DlqConfig::default() + }; + let big = serde_json::json!({ "message": "x".repeat(40_000) }); + let mut oversize = serde_json::to_vec(&big).expect("event serialises"); + oversize.push(b'\n'); + + let capture = SeriesCapture::default(); + let pushed = push_through_with_budget( + &capture, + |config| { + let dlq = scalo::dlq::Dlq::spawn( + &dlq_config, + "dfe-transform-elastic", + None, + CancellationToken::new(), + ) + .expect("the file DLQ starts"); + BatchEngine::new(service::engine_config(config)).with_dlq(Arc::new(dlq)) + }, + vec![oversize, okta_events(1)], + 1, + BUDGET, + ); + + assert!( + pushed + .answers + .iter() + .all(|a| matches!(a, scalo::transport::SendResult::Ok)), + "a dead-lettered push is answered once the DLQ holds it: {:?}", + pushed.answers + ); + assert_eq!(pushed.delivered, 1, "the event that fits still goes"); + + let written = std::fs::read_to_string(dir.path().join("dfe-transform-elastic/dlq.ndjson")) + .expect("the DLQ file exists"); + let entries: Vec = written + .lines() + .map(|line| serde_json::from_str(line).expect("a DLQ line is an entry")) + .collect(); + assert_eq!(entries.len(), 1, "{written}"); + let body: serde_json::Value = + serde_json::from_slice(&entries[0].payload).expect("the event body, whole"); + assert!( + entries[0].payload.len() > BUDGET, + "the DLQ holds the serialised event, {} bytes", + entries[0].payload.len() + ); + assert!(body.is_object()); + assert_eq!( + entries[0].reason, + dfe_transform_elastic::pipeline::oversize_reason(entries[0].payload.len(), BUDGET) + ); + assert_eq!(entries[0].destination.as_deref(), Some(SINK_TOPIC)); + assert_eq!( + ( + capture.total("events_oversize_total"), + capture.total("pipeline_dead_letters_dropped_total"), + ), + (1, 0), + "counted once, and held rather than dropped" + ); +} + #[tokio::test] async fn a_client_pointed_at_nothing_fails_rather_than_hanging() { // A dead endpoint is what a misconfigured `sink.endpoint` produces, and the