diff --git a/docs/architecture.md b/docs/architecture.md index 5451c013b..a15c62b0e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -195,7 +195,7 @@ A record the sink transport would refuse -- over its `message.max.bytes`, or mat JSON is the only payload format. MessagePack, supported in DFE/XDR 2.0 and 2.1, is deprecated in DFE 2.2 and no longer accepted: the JSON path (SIMD parsing with sonic-rs, zstd on the wire) is fast enough that MessagePack gave no CPU saving. -**One record per event.** `pipeline.rs::serialise_events` gives each event its own payload and `service.rs::send_blocks` sends them via `TransportSender::send_batch`, because dfe-loader parses exactly one JSON document per message ([#67](https://github.com/hyperi-io/dfe-transform-elastic/issues/67)). `sink.max_message_bytes` (default 8,000,000 bytes, the DFE record cap) bounds each payload under the 16 MiB ceiling the producer and dfe-loader's consumer share, and the `send_batch` block under gRPC's 16 MiB ceiling. An event over the budget is dropped and counted on `events_oversize_total`: no broker would take it, and retrying blocks the partition. +**One record per event.** `pipeline.rs::serialise_events_with_oversize` gives each event its own payload and `service.rs::send_blocks` sends them via `TransportSender::send_batch`, because dfe-loader parses exactly one JSON document per message ([#67](https://github.com/hyperi-io/dfe-transform-elastic/issues/67)). `sink.max_message_bytes` (default 8,000,000 bytes, the DFE record cap) bounds each payload under the 16 MiB ceiling the producer and dfe-loader's consumer share, and the `send_batch` block under gRPC's 16 MiB ceiling. An event over the budget is dead-lettered and counted on `events_oversize_total`: no broker would take it, and retrying blocks the partition. Duplicates are the accepted cost: a retried block re-sends the records that already landed, so every downstream consumer must be idempotent. diff --git a/docs/config-schema.json b/docs/config-schema.json index 2ddd824ab..6a4d9d603 100644 --- a/docs/config-schema.json +++ b/docs/config-schema.json @@ -308,7 +308,7 @@ "default": null }, "max_message_bytes": { - "description": "Ceiling on one outbound record. An event that serialises larger is\ndropped and counted on `events_oversize_total`.\n\nThe default is the DFE record cap, 8,000,000 bytes. Against a broker or\ntopic whose `message.max.bytes` is lower, set this below that limit, or\nthe broker refuses those events instead and they count on\n`send_filtered_dlq_total`. Brokers that carry more can raise it, up to\n16 MiB.", + "description": "Ceiling on one outbound record. An event that serialises larger is\ndead-lettered, with its size and this budget as the reason, and counted\non `events_oversize_total`.\n\nThe default is the DFE record cap, 8,000,000 bytes. Against a broker or\ntopic whose `message.max.bytes` is lower, set this below that limit, or\nthe broker refuses those events instead and they count on\n`send_filtered_dlq_total`. Brokers that carry more can raise it, up to\n16 MiB.", "type": "integer", "format": "uint", "minimum": 0, diff --git a/docs/config-schema.yaml b/docs/config-schema.yaml index c02a22cc5..bed12aca6 100644 --- a/docs/config-schema.yaml +++ b/docs/config-schema.yaml @@ -324,7 +324,8 @@ $defs: max_message_bytes: description: |- Ceiling on one outbound record. An event that serialises larger is - dropped and counted on `events_oversize_total`. + dead-lettered, with its size and this budget as the reason, and counted + on `events_oversize_total`. The default is the DFE record cap, 8,000,000 bytes. Against a broker or topic whose `message.max.bytes` is lower, set this below that limit, or diff --git a/docs/configuration.md b/docs/configuration.md index 7d3357f0a..e1ed1442e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -67,7 +67,7 @@ On a direct source a push is held at most 18 s, so every send it waits on gives ## `sink.max_message_bytes` caps one outbound record -The default is the DFE record cap, 8,000,000 bytes: dfe-receiver refuses a request body over it, and every broker DFE deploys takes a record that size (Confluent Basic's 8,388,608 limit leaves room for the framing). The producer this service builds allows 16 MiB, so a deployment with larger brokers raises the budget up to that. An event that serialises larger is dropped, logged, counted on `events_oversize_total`, and its source is still acknowledged. +The default is the DFE record cap, 8,000,000 bytes: dfe-receiver refuses a request body over it, and every broker DFE deploys takes a record that size (Confluent Basic's 8,388,608 limit leaves room for the framing). The producer this service builds allows 16 MiB, so a deployment with larger brokers raises the budget up to that. An event that serialises larger is logged, counted on `events_oversize_total` and dead-lettered to the DLQ with its size and the budget as the reason. The DLQ cuts the body to `dlq.max_payload_bytes`, so it keeps `original_bytes` and a truncated head. Against a broker or topic whose `message.max.bytes` is lower -- a stock Kafka broker takes about 1 MB -- set `sink.max_message_bytes` below that limit. Left at the default, a broker below 8,000,000 bytes refuses the events between the two instead: each is dropped as a dead letter, counted on `send_filtered_dlq_total`, and the partition moves on. Validation refuses a value under 4096 or over 16,777,216. diff --git a/docs/deployment-contract.json b/docs/deployment-contract.json index 0f5b7e690..d7dbe8aec 100644 --- a/docs/deployment-contract.json +++ b/docs/deployment-contract.json @@ -459,7 +459,7 @@ "default": null }, "max_message_bytes": { - "description": "Ceiling on one outbound record. An event that serialises larger is\ndropped and counted on `events_oversize_total`.\n\nThe default is the DFE record cap, 8,000,000 bytes. Against a broker or\ntopic whose `message.max.bytes` is lower, set this below that limit, or\nthe broker refuses those events instead and they count on\n`send_filtered_dlq_total`. Brokers that carry more can raise it, up to\n16 MiB.", + "description": "Ceiling on one outbound record. An event that serialises larger is\ndead-lettered, with its size and this budget as the reason, and counted\non `events_oversize_total`.\n\nThe default is the DFE record cap, 8,000,000 bytes. Against a broker or\ntopic whose `message.max.bytes` is lower, set this below that limit, or\nthe broker refuses those events instead and they count on\n`send_filtered_dlq_total`. Brokers that carry more can raise it, up to\n16 MiB.", "type": "integer", "format": "uint", "minimum": 0, diff --git a/docs/metrics-manifest.json b/docs/metrics-manifest.json index 0739b6b86..d15fb2d2e 100644 --- a/docs/metrics-manifest.json +++ b/docs/metrics-manifest.json @@ -875,7 +875,7 @@ { "name": "events_oversize_total", "type": "counter", - "description": "Events too large for one Kafka record, dropped", + "description": "Events too large for one Kafka record, dead-lettered", "unit": "", "labels": [], "group": "custom" diff --git a/src/config.rs b/src/config.rs index b456c57f7..faef6c5da 100644 --- a/src/config.rs +++ b/src/config.rs @@ -326,7 +326,8 @@ pub struct SinkConfig { pub brokers: Option>, /// Ceiling on one outbound record. An event that serialises larger is - /// dropped and counted on `events_oversize_total`. + /// dead-lettered, with its size and this budget as the reason, and counted + /// on `events_oversize_total`. /// /// The default is the DFE record cap, 8,000,000 bytes. Against a broker or /// topic whose `message.max.bytes` is lower, set this below that limit, or diff --git a/src/metrics.rs b/src/metrics.rs index 54658a489..1a9ef5c25 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -82,8 +82,8 @@ pub struct TransformMetrics { /// or topic ceiling below `sink.max_message_bytes`. pub send_filtered_dlq: Counter, - /// Events too large for one Kafka record even on their own. Dropped: no - /// broker would ever accept one, and retrying blocks the partition. + /// Events too large for one Kafka record even on their own. Dead-lettered: + /// no broker would ever accept one, and retrying blocks the partition. pub events_oversize: Counter, /// Batches whose events did not look like the envelope the config pinned. @@ -180,7 +180,7 @@ impl TransformMetrics { ), events_oversize: manager.counter( "events_oversize_total", - "Events too large for one Kafka record, dropped", + "Events too large for one Kafka record, dead-lettered", ), envelope_contradicted: manager.counter( "envelope_contradicted_total", diff --git a/src/pipeline.rs b/src/pipeline.rs index 9b77613ec..c0d8d4213 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -241,6 +241,16 @@ pub(crate) fn elided(value: &str) -> Cow<'_, str> { /// The dead-letter reason for input that is not JSON. pub const NOT_JSON: &str = "payload is not JSON"; +/// The dead-letter reason for an event that serialised over the per-message +/// budget: names its size and the budget it broke. +#[must_use] +pub fn oversize_reason(bytes: usize, budget: usize) -> String { + format!( + "event serialises to {bytes} bytes, over the {budget} byte per-message budget \ + (sink.max_message_bytes)" + ) +} + /// What a payload cost to decode. #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct ParseOutcome { @@ -353,8 +363,8 @@ pub struct SerialiseOutcome { /// Events that became an outbound message. pub serialised: usize, /// Events that on their own exceed the whole per-message budget. No - /// broker will ever accept one, so it is dropped rather than blocking the - /// partition behind a record that can only fail. + /// broker will ever accept one, so it is dead-lettered rather than + /// blocking the partition behind a record that can only fail. pub oversize: usize, /// Events serde refused to serialise. Dropped, on the same footing as an /// event the transform rejected. @@ -394,6 +404,17 @@ pub const fn message_budget(max_bytes: usize) -> usize { } } +/// What [`serialise_events_with_oversize`] produced. +#[derive(Debug, Default)] +pub struct Serialised { + /// One payload per event that fits the budget. + pub payloads: Vec>, + /// The whole serialised body of each event over the budget, for the DLQ. + pub oversize: Vec>, + /// The counts behind both. + pub outcome: SerialiseOutcome, +} + /// Serialise each event into a message payload of its own. /// /// ONE JSON document per payload, never NDJSON. dfe-loader parses exactly one @@ -402,13 +423,16 @@ pub const fn message_budget(max_bytes: usize) -> usize { /// (hyperi-io/dfe-transform-elastic#67). /// /// `max_bytes` bounds each payload on its own, because that is what a broker -/// bounds: an event above the budget is dropped rather than blocking the -/// partition behind a record no broker would take. +/// bounds. An event above the budget is returned in [`Serialised::oversize`] +/// for the caller to dead-letter, rather than blocking the partition behind a +/// record no broker would take. #[must_use] -pub fn serialise_events(events: &[Event], max_bytes: usize) -> (Vec>, SerialiseOutcome) { +pub fn serialise_events_with_oversize(events: &[Event], max_bytes: usize) -> Serialised { let budget = message_budget(max_bytes); - let mut payloads = Vec::with_capacity(events.len()); - let mut outcome = SerialiseOutcome::default(); + let mut serialised = Serialised { + payloads: Vec::with_capacity(events.len()), + ..Serialised::default() + }; // Written once and copied out at its exact length: a payload is held until // its block is delivered, and a buffer sized by a guess holds the guess. let mut scratch: Vec = Vec::with_capacity(1024); @@ -416,25 +440,35 @@ pub fn serialise_events(events: &[Event], max_bytes: usize) -> (Vec>, Se for event in events { scratch.clear(); if let Err(e) = event.write_json(&mut scratch) { - outcome.failed += 1; + serialised.outcome.failed += 1; tracing::warn!(error = %e, "event could not be serialised, dropped"); continue; } if scratch.len() > budget { - outcome.oversize += 1; + serialised.outcome.oversize += 1; tracing::error!( bytes = scratch.len(), budget, - "event exceeds the per-message budget and cannot be produced, dropped" + "event exceeds the per-message budget and cannot be produced, dead-lettered" ); + serialised.oversize.push(scratch.as_slice().to_vec()); continue; } - outcome.serialised += 1; - payloads.push(scratch.as_slice().to_vec()); + serialised.outcome.serialised += 1; + serialised.payloads.push(scratch.as_slice().to_vec()); } - (payloads, outcome) + serialised +} + +/// [`serialise_events_with_oversize`] without the oversize bodies, for tests +/// and offline tools that only count them. The service keeps the bodies and +/// dead-letters them. +#[must_use] +pub fn serialise_events(events: &[Event], max_bytes: usize) -> (Vec>, SerialiseOutcome) { + let serialised = serialise_events_with_oversize(events, max_bytes); + (serialised.payloads, serialised.outcome) } /// Serialise every event into ONE NDJSON payload, whatever its size. @@ -739,9 +773,9 @@ mod tests { } /// The shipped budget is the DFE record cap: a payload of 8,000,000 bytes - /// goes out and one byte more is dropped and counted. + /// goes out and one byte more is counted and held back for the DLQ. #[test] - fn the_shipped_budget_admits_8_000_000_bytes_and_drops_one_more() { + fn the_shipped_budget_admits_8_000_000_bytes_and_holds_back_one_more() { let budget = crate::config::default_max_message_bytes(); assert_eq!(budget, 8_000_000); let overhead = serialise_events(&padded(1, 0), budget).0[0].len(); @@ -755,17 +789,32 @@ mod tests { assert_eq!((outcome.serialised, outcome.oversize), (0, 1)); } - /// A record no broker can accept is dropped and counted, rather than - /// retried forever behind the rest of the partition. + /// A record no broker can accept is counted and handed back whole for the + /// DLQ, rather than retried forever behind the rest of the partition. #[test] - fn an_event_larger_than_the_budget_is_dropped_and_counted() { + fn an_event_larger_than_the_budget_is_counted_and_returned_for_the_dlq() { let mut events = padded(2, 100); events.extend(padded(1, 20_000)); - let (payloads, outcome) = serialise_events(&events, MIN_MESSAGE_BYTES); - assert_eq!(outcome.oversize, 1); - assert_eq!(outcome.serialised, 2); - assert_eq!(payloads.len(), 2); + let serialised = serialise_events_with_oversize(&events, MIN_MESSAGE_BYTES); + assert_eq!(serialised.outcome.oversize, 1); + assert_eq!(serialised.outcome.serialised, 2); + assert_eq!(serialised.payloads.len(), 2); + assert_eq!(serialised.oversize.len(), 1); + assert!( + serialised.oversize[0].len() > 20_000, + "the body was cut before it reached the DLQ" + ); + let body: serde_json::Value = + serde_json::from_slice(&serialised.oversize[0]).expect("a whole document"); + assert_eq!(body.get("i"), Some(&serde_json::json!(0))); + } + + #[test] + fn the_oversize_reason_names_the_size_and_the_budget() { + let reason = oversize_reason(8_000_001, 8_000_000); + assert!(reason.contains("8000001 bytes"), "{reason}"); + assert!(reason.contains("8000000 byte"), "{reason}"); } /// A budget below one ordinary event would drop the entire batch as diff --git a/src/service.rs b/src/service.rs index 059da0104..7ae64f233 100644 --- a/src/service.rs +++ b/src/service.rs @@ -20,12 +20,12 @@ //! downstream must be idempotent, which is the contract every DFE stage carries. //! //! An event that cannot be delivered is dropped and counted, never counted -//! delivered: one that will not transform or serialise, one over -//! `sink.max_message_bytes`, and one the sink transport would refuse, which the -//! pipeline screens out before the send. Input that is not JSON is refused as a -//! dead letter. Dead letters, the screened records among them, go to the DLQ -//! `dlq` configures, and the source is released only once the DLQ holds them. -//! With `dlq.enabled: false` they are dropped and counted in +//! delivered: one that will not transform or serialise, and one the sink +//! transport would refuse, which the pipeline screens out before the send. An +//! event over `sink.max_message_bytes` and input that is not JSON are dead +//! letters. Dead letters, the screened records among them, go to the DLQ `dlq` +//! configures, and the source is released only once the DLQ holds them. With +//! `dlq.enabled: false` they are dropped and counted in //! `pipeline_dead_letters_dropped_total`. use std::sync::Arc; @@ -51,8 +51,8 @@ use crate::agentless::{Agentless, Stopped}; use crate::config::{Config, Transport}; use crate::metrics::TransformMetrics; use crate::pipeline::{ - BatchOutcome, EnvelopeOutcome, NOT_JSON, SerialiseOutcome, message_budget, parse_each, - serialise_events, transform_batch_resolved, + BatchOutcome, EnvelopeOutcome, NOT_JSON, SerialiseOutcome, message_budget, oversize_reason, + parse_each, serialise_events_with_oversize, transform_batch_resolved, }; /// The longest the loop goes without pushing scaling signals. @@ -653,6 +653,8 @@ struct Block { outcome: BatchOutcome, envelopes: EnvelopeOutcome, serialised: SerialiseOutcome, + /// Events over the per-message budget, held for the DLQ. + oversize: Vec, } impl Block { @@ -666,6 +668,7 @@ impl Block { outcome: BatchOutcome::default(), envelopes: EnvelopeOutcome::default(), serialised: SerialiseOutcome::default(), + oversize: Vec::new(), } } } @@ -685,7 +688,8 @@ struct Stage<'a> { impl Stage<'_> { /// Parse, transform and serialise one block, returning a record per event - /// that survived and a dead letter per input refused as not JSON. + /// that survived and a dead letter per input refused as not JSON or event + /// over the per-message budget. /// /// At most `source.batch_size` parsed events are held at once, however many /// NDJSON lines the block's records carry. @@ -735,8 +739,10 @@ impl Stage<'_> { outcome, envelopes, serialised, + oversize, .. } = block; + dead.extend(oversize); // One series, counted once. The source transport counts its own wire // records in `transport_received_*`; these are the events parsed out. @@ -816,17 +822,29 @@ impl Stage<'_> { } => transform_batch_resolved(*transform, resolver, events), Work::Agentless(agentless) => agentless.transform(events)?, }; - let (payloads, serialised) = - serialise_events(&transformed, self.config.sink.max_message_bytes); + let max_bytes = self.config.sink.max_message_bytes; + let budget = message_budget(max_bytes); + let serialised = serialise_events_with_oversize(&transformed, max_bytes); drop(transformed); block.out.extend( - payloads + serialised + .payloads .into_iter() .map(|payload| outbound_record(payload, &self.topic)), ); + block.oversize.extend( + serialised + .oversize + .into_iter() + .map(|payload| FilteredDlqEntry { + reason: oversize_reason(payload.len(), budget), + payload, + key: Some(Arc::clone(&self.topic)), + }), + ); block.outcome.absorb(outcome); block.envelopes.absorb(envelopes); - block.serialised.absorb(serialised); + block.serialised.absorb(serialised.outcome); Ok(()) } diff --git a/tests/direct_transport.rs b/tests/direct_transport.rs index cdac2fda4..a2d350fb6 100644 --- a/tests/direct_transport.rs +++ b/tests/direct_transport.rs @@ -456,6 +456,18 @@ fn push_through( engine: impl FnOnce(&Config) -> BatchEngine, payloads: Vec>, expected: usize, +) -> Pushed { + let budget = dfe_transform_elastic::config::default_max_message_bytes(); + push_through_with_budget(capture, engine, payloads, expected, budget) +} + +/// [`push_through`] with `sink.max_message_bytes` set to `budget`. +fn push_through_with_budget( + capture: &SeriesCapture, + engine: impl FnOnce(&Config) -> BatchEngine, + payloads: Vec>, + expected: usize, + budget: usize, ) -> Pushed { let neighbours = tokio::runtime::Builder::new_multi_thread() .worker_threads(1) @@ -480,7 +492,8 @@ fn push_through( .strip_prefix("http://") .expect("endpoint is http") .to_string(); - let config = direct_config(&listen, &loader_endpoint); + let mut config = direct_config(&listen, &loader_endpoint); + config.sink.max_message_bytes = budget; let consumer = AnyReceiver::Grpc(transform_listener); let manager = MetricsManager::with_config(scalo::metrics::MetricsConfig::offline("")); let metrics = TransformMetrics::register(&manager, "0.0.0-test", "test"); @@ -615,6 +628,86 @@ fn a_messagepack_push_lands_in_a_configured_dlq() { ); } +/// An event that serialises over `sink.max_message_bytes` is dead-lettered +/// whole, with its size and the budget as the reason, and the event beside it +/// still goes. Counting it and dropping it lost it. +#[test] +fn an_oversize_event_lands_in_a_configured_dlq() { + const BUDGET: usize = 16_384; + let dir = tempfile::tempdir().expect("tempdir"); + let dlq_config = scalo::dlq::DlqConfig { + enabled: true, + mode: scalo::dlq::DlqMode::FileOnly, + file: scalo::dlq::FileDlqConfig { + enabled: true, + path: dir.path().to_path_buf(), + compress_rotated: false, + ..scalo::dlq::FileDlqConfig::default() + }, + ..scalo::dlq::DlqConfig::default() + }; + let big = serde_json::json!({ "message": "x".repeat(40_000) }); + let mut oversize = serde_json::to_vec(&big).expect("event serialises"); + oversize.push(b'\n'); + + let capture = SeriesCapture::default(); + let pushed = push_through_with_budget( + &capture, + |config| { + let dlq = scalo::dlq::Dlq::spawn( + &dlq_config, + "dfe-transform-elastic", + None, + CancellationToken::new(), + ) + .expect("the file DLQ starts"); + BatchEngine::new(service::engine_config(config)).with_dlq(Arc::new(dlq)) + }, + vec![oversize, okta_events(1)], + 1, + BUDGET, + ); + + assert!( + pushed + .answers + .iter() + .all(|a| matches!(a, scalo::transport::SendResult::Ok)), + "a dead-lettered push is answered once the DLQ holds it: {:?}", + pushed.answers + ); + assert_eq!(pushed.delivered, 1, "the event that fits still goes"); + + let written = std::fs::read_to_string(dir.path().join("dfe-transform-elastic/dlq.ndjson")) + .expect("the DLQ file exists"); + let entries: Vec = written + .lines() + .map(|line| serde_json::from_str(line).expect("a DLQ line is an entry")) + .collect(); + assert_eq!(entries.len(), 1, "{written}"); + let body: serde_json::Value = + serde_json::from_slice(&entries[0].payload).expect("the event body, whole"); + assert!( + entries[0].payload.len() > BUDGET, + "the DLQ holds the serialised event, {} bytes", + entries[0].payload.len() + ); + assert!(body.is_object()); + assert_eq!( + entries[0].reason, + dfe_transform_elastic::pipeline::oversize_reason(entries[0].payload.len(), BUDGET) + ); + assert_eq!(entries[0].destination.as_deref(), Some(SINK_TOPIC)); + assert_eq!( + ( + capture.total("events_oversize_total"), + capture.total("pipeline_dead_letters_dropped_total"), + ), + (1, 0), + "counted once, and held rather than dropped" + ); +} + #[tokio::test] async fn a_client_pointed_at_nothing_fails_rather_than_hanging() { // A dead endpoint is what a misconfigured `sink.endpoint` produces, and the