From 10405ab0b9a9a369d78051862b856b3c72feacfd Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 14:58:08 +1100 Subject: [PATCH] fix(listeners): dead-letter syslog and GELF frames over the size limit Syslog TCP or TLS messages over syslog.max_message_size and GELF messages over gelf.max_message_size were dropped without a way to read them back. The framing codecs return each skipped message as a separate frame with its head (first bytes) and length. The listener dead-letters it through the same DLQ writer used for refused records. The reason is "frame over bytes on ". original_bytes is set. receiver_records_dropped_total{reason="oversize"} still counts it. - The head is capped at 512 KiB, the DLQ's payload cap. Longer frames are marked truncated. - The connection waits for the DLQ write. This paces oversize floods at the DLQ's write rate. Each codec holds at most one frame's head. - Unconfirmed DLQ writes are retried until confirmation or shutdown, as with held records. - A stream that ends or fails inside an oversize frame hands over what arrived. - With no DLQ configured, the frame is dropped and counted as before. Tests cover codec heads and lengths across split reads. They check DLQ entries for truncated and whole heads. They cover syslog and GELF listeners writing entries to a file DLQ. --- benches/syslog.rs | 5 +- benches/syslog_allocs.rs | 3 +- docs/hardening.md | 2 +- docs/protocols.md | 2 +- src/buffer/rejects.rs | 72 +++++++++++- src/pipeline/mod.rs | 6 + src/server/frame_clock.rs | 10 +- src/server/gelf/mod.rs | 206 +++++++++++++++++++++++++++-------- src/server/hold.rs | 4 +- src/server/mod.rs | 1 + src/server/oversize.rs | 178 ++++++++++++++++++++++++++++++ src/server/syslog/framing.rs | 174 ++++++++++++++++++++++++----- src/server/syslog/tcp.rs | 49 +++++++-- tests/common/mod.rs | 30 +++++ tests/integration/gelf.rs | 15 ++- tests/integration/syslog.rs | 35 +++++- 16 files changed, 691 insertions(+), 101 deletions(-) create mode 100644 src/server/oversize.rs diff --git a/benches/syslog.rs b/benches/syslog.rs index 79aa7e3..e5b4041 100644 --- a/benches/syslog.rs +++ b/benches/syslog.rs @@ -13,6 +13,7 @@ use std::hint::black_box; use bytes::BytesMut; use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; use dfe_receiver::config::RawCapture; +use dfe_receiver::server::oversize::Frame; use dfe_receiver::server::syslog::convert::syslog_to_json; use dfe_receiver::server::syslog::framing::SyslogFrameDecoder; use tokio_util::codec::Decoder; @@ -55,7 +56,9 @@ fn framing(c: &mut Criterion) { let mut buf = BytesMut::from(&stream[..]); let mut n = 0usize; while let Ok(Some(frame)) = decoder.decode(&mut buf) { - n += frame.len(); + if let Frame::Message(message) = frame { + n += message.len(); + } } black_box(n) }); diff --git a/benches/syslog_allocs.rs b/benches/syslog_allocs.rs index 2cd9356..7a4f866 100644 --- a/benches/syslog_allocs.rs +++ b/benches/syslog_allocs.rs @@ -16,6 +16,7 @@ mod common; use bytes::BytesMut; use dfe_receiver::config::RawCapture; +use dfe_receiver::server::oversize::Frame; use dfe_receiver::server::syslog::convert::syslog_to_json; use dfe_receiver::server::syslog::framing::SyslogFrameDecoder; use tokio_util::codec::Decoder; @@ -42,7 +43,7 @@ fn frames_per_stream(stream: &[u8]) -> (f64, f64) { let info = allocation_counter::measure(|| { let mut decoder = SyslogFrameDecoder::default(); while let Ok(Some(frame)) = decoder.decode(&mut buf) { - frames += u64::from(!frame.is_empty()); + frames += u64::from(matches!(&frame, Frame::Message(m) if !m.is_empty())); } }); let frames = frames.max(1) as f64; diff --git a/docs/hardening.md b/docs/hardening.md index 2f38771..ccebe0e 100644 --- a/docs/hardening.md +++ b/docs/hardening.md @@ -36,7 +36,7 @@ Application-level hardening, and cost-conscious fronting on Kubernetes and AWS. | Dead-letter queue | Global | `routing.dlq` | | Memory pressure backpressure | 503 / `UNAVAILABLE` on HTTP and gRPC ingest | `DFE_RECEIVER_MEMORY_PRESSURE_THRESHOLD`, `self_regulation.*` (`buffer.*` bounds the destination queues and nothing else) | | Middleware ordering | RateLimit -> Concurrency -> Timeout -> BodyLimit -> Auth -> Handler, behind the accept loop's IP filter | Cheapest refusal first | -| Frame size validation | GELF 1MB, Syslog 64KB, Fluent 32MB. An oversize syslog or GELF message is skipped to its delimiter and counted (`receiver_records_dropped_total`, reason `oversize`), and its connection goes on. Fluent Forward has no delimiter to resume at, so an oversize frame closes its connection | Per-protocol limits, `syslog.max_message_size`, `gelf.max_message_size` | +| Frame size validation | GELF 1MB, Syslog 64KB, Fluent 32MB. An oversize syslog or GELF message is skipped to its delimiter and counted (`receiver_records_dropped_total`, reason `oversize`), its first bytes go to the DLQ with its length, and its connection goes on. Fluent Forward has no delimiter to resume at, so an oversize frame closes its connection | Per-protocol limits, `syslog.max_message_size`, `gelf.max_message_size` | | Decoded-size budget | An OTLP export, remote-write request or Vector push past 100,000 records, 1,000,000 messages or 64 MiB of resource, scope, metric or series-label bytes copied into its records, or holding a protobuf group, is refused before it decodes: 400 on HTTP, `INVALID_ARGUMENT` on gRPC. A Fluent Forward message past 1,000,000 values, 100,000 entries or 64 MiB of tag copies closes its connection, found by a scan that reads each byte once. A Splunk HEC request past 100,000 events or raw lines, or 64 MiB of `host`, `source`, `sourcetype` and `index` copied into its raw lines, is refused with a 400 | Hard-coded | | Zip-bomb rejection | Lumberjack nested compression rejected. A compressed frame is decompressed one inner frame at a time and refused past 64 MiB | Hard-coded | | Unsettled bytes per connection | Lumberjack settles and ACKs a window in parts past 16 MiB of events | Hard-coded | diff --git a/docs/protocols.md b/docs/protocols.md index b4868b3..3cce7c1 100644 --- a/docs/protocols.md +++ b/docs/protocols.md @@ -16,7 +16,7 @@ An RFC 3164 token is the app name when it reads `name:` or `name[pid]:`, with th RFC 5424 structured data that does not parse, like `[x@1 k=v]`, stays in `message` as sent, and the elements that did parse still land in `structured_data`. A byte order mark at the start of the message and NUL bytes at its end are dropped. -On TCP and TLS a frame is octet-counted (RFC 6587) when it opens with a count, one space and the `<` of a PRI. Anything else is an LF-terminated line, so a line opening with digits, like `2026 Oct 11 ...`, is never read as a count. A message over `syslog.max_message_size` is skipped and counted as dropped (reason `oversize`), and the connection goes on: an octet-counted one by its count, a line up to its LF. A count over `syslog.max_frame_size` (default 256 KiB) is not trusted as a count, and that frame is read as a line. +On TCP and TLS a frame is octet-counted (RFC 6587) when it opens with a count, one space and the `<` of a PRI. Anything else is an LF-terminated line, so a line opening with digits, like `2026 Oct 11 ...`, is never read as a count. A message over `syslog.max_message_size` is skipped and counted as dropped (reason `oversize`), and the connection goes on: an octet-counted one by its count, a line up to its LF. Its first 512 KiB, the DLQ's payload cap, go to the DLQ with the reason `frame over bytes on syslog TCP` (or `TLS`) and its length in `original_bytes`. The connection waits on the DLQ write, and a write the DLQ does not confirm is offered again. A count over `syslog.max_frame_size` (default 256 KiB) is not trusted as a count, and that frame is read as a line. Syslog UDP reads every datagram ready on a wake, up to `syslog.udp_batch_size` (default 64), in batched receive calls, and hands them to the pipeline as one batch. Its socket asks for a `syslog.udp_recv_buffer_size` receive buffer (default 8 MiB), and logs a warning when `net.core.rmem_max` grants less. Datagrams the kernel drops for a full buffer are read from `/proc/net/udp` every 10 seconds and counted as dropped, reason `kernel`. Syslog TCP settles every frame one read brought in with a single hold on the pipeline. diff --git a/src/buffer/rejects.rs b/src/buffer/rejects.rs index bd7d53c..4370920 100644 --- a/src/buffer/rejects.rs +++ b/src/buffer/rejects.rs @@ -159,8 +159,32 @@ impl Rejects { /// [`Disposal::Refused`] leaves the record with the caller, to keep and /// offer again. pub async fn dispose(&self, topic: &str, payload: &Bytes, reason: &str) -> Disposal { + self.dispose_entry(topic, payload, reason, || entry(topic, payload, reason)) + .await + } + + /// [`dispose`](Self::dispose) for the first bytes of a frame a listener + /// skipped for its size, `original_bytes` long in all, which has no + /// destination yet. + pub async fn dispose_head( + &self, + head: &Bytes, + original_bytes: usize, + reason: &str, + ) -> Disposal { + self.dispose_entry("", head, reason, || cut_entry(head, original_bytes, reason)) + .await + } + + async fn dispose_entry( + &self, + topic: &str, + payload: &Bytes, + reason: &str, + make_entry: impl FnOnce() -> DlqEntry, + ) -> Disposal { let disposal = match &self.dlq { - Some(dlq) => match dead_letter_confirmed(dlq, entry(topic, payload, reason)).await { + Some(dlq) => match dead_letter_confirmed(dlq, make_entry()).await { Ok(DeadLetter::Written) => Disposal::DeadLettered, Ok(DeadLetter::Unwritable) => Disposal::Dropped, Err(e) => { @@ -265,6 +289,14 @@ fn entry(topic: &str, payload: &Bytes, reason: &str) -> DlqEntry { } } +/// The DLQ entry for the first bytes of a frame that was `original_bytes` long. +fn cut_entry(head: &Bytes, original_bytes: usize, reason: &str) -> DlqEntry { + let mut entry = DlqEntry::new("receiver", reason, head.to_vec()); + entry.truncated = head.len() < original_bytes; + entry.original_bytes = Some(original_bytes); + entry +} + /// DLQ fixtures shared by the buffer tests. #[cfg(test)] pub(crate) mod test_dlq { @@ -417,6 +449,44 @@ mod tests { .unwrap() } + /// The head of a frame a listener skipped reaches the DLQ with the frame's + /// full length, marked cut when the head is shorter, and with no + /// destination to route it by. + #[tokio::test] + async fn the_head_of_a_skipped_frame_lands_in_the_dlq_with_its_length() { + let dir = tempfile::tempdir().unwrap(); + let dlq = file_dlq(dir.path()); + let metrics = Arc::new(Metrics::default()); + let rejects = Rejects::new(Some(Arc::clone(&dlq)), Some(Arc::clone(&metrics))); + + let cut = Bytes::from_static(b"<14>first bytes"); + for (head, original) in [(&cut, 70_000), (&cut, cut.len())] { + let disposal = rejects + .dispose_head(head, original, "frame over 65536 bytes on syslog tcp") + .await; + assert_eq!(disposal, Disposal::DeadLettered); + } + dlq.flush().await.unwrap(); + assert_eq!(metrics.get_messages_dlq(), 2); + + let written = std::fs::read_to_string(dir.path().join("receiver/dlq.ndjson")).unwrap(); + let entries: Vec = written + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(entries.len(), 2, "{written}"); + for entry in &entries { + assert_eq!(entry["reason"], "frame over 65536 bytes on syslog tcp"); + assert!(entry.get("destination").is_none(), "{entry}"); + } + assert_eq!(entries[0]["truncated"], true); + assert_eq!(entries[0]["original_bytes"], 70_000); + assert_eq!(entries[0]["_raw_encoding"], "base64"); + assert!(entries[1].get("truncated").is_none(), "{}", entries[1]); + assert_eq!(entries[1]["original_bytes"], cut.len()); + assert_eq!(entries[1]["_raw"], "<14>first bytes"); + } + /// A refused record reaches the configured DLQ with its reason, its /// destination and the original bytes, and counts as dead-lettered. #[tokio::test] diff --git a/src/pipeline/mod.rs b/src/pipeline/mod.rs index 1e21095..c45fab3 100644 --- a/src/pipeline/mod.rs +++ b/src/pipeline/mod.rs @@ -1497,6 +1497,12 @@ impl PipelineState { Ok(()) } + /// Where a record refused for good is dead-lettered, for a listener that + /// skips a frame before it is a record. + pub(crate) fn rejects(&self) -> &Rejects { + &self.rejects + } + /// Settle a record a destination refuses for good: dead-letter it, or /// drop it where no DLQ is configured. async fn settle_refused( diff --git a/src/server/frame_clock.rs b/src/server/frame_clock.rs index 299e632..76929dd 100644 --- a/src/server/frame_clock.rs +++ b/src/server/frame_clock.rs @@ -247,10 +247,9 @@ mod tests { use tokio_stream::StreamExt; use tokio_util::codec::FramedRead; - use bytes::Bytes; - use super::*; use crate::server::gelf::GelfFrameDecoder; + use crate::server::oversize::Frame; use crate::server::syslog::framing::SyslogFrameDecoder; const LIMIT: Duration = Duration::from_millis(500); @@ -272,7 +271,7 @@ mod tests { /// for `hold` before it reads on, until a frame times out or the client closes. async fn run(codec: D, steps: Vec, hold: Duration) -> Read where - D: Framing, + D: Framing, { let (mut client, server) = tokio::io::duplex(1024); let socket = Clocked::new(server, LIMIT, Arc::new(Metrics::default()), "test"); @@ -296,6 +295,11 @@ mod tests { read.timed_out = true; return; }; + // A skipped message is no frame to hold, as before the codec + // handed it over. + let Frame::Message(frame) = frame else { + continue; + }; read.frames .push(String::from_utf8_lossy(&frame).into_owned()); tokio::time::sleep(hold).await; diff --git a/src/server/gelf/mod.rs b/src/server/gelf/mod.rs index 8c771ab..acb2b62 100644 --- a/src/server/gelf/mod.rs +++ b/src/server/gelf/mod.rs @@ -17,7 +17,7 @@ use std::io; use std::net::SocketAddr; use std::sync::Arc; -use bytes::{Buf, Bytes, BytesMut}; +use bytes::{Buf, BytesMut}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::net::TcpListener; use tokio::time::timeout; @@ -27,12 +27,13 @@ use tracing::{debug, error, info, warn}; use crate::config::GelfConfig; use crate::error::{Error, Result}; -use crate::metrics::{DropReason, Metrics}; +use crate::metrics::Metrics; use crate::pipeline::{Acks, PipelineState}; use crate::server::frame_clock::Clocked; use crate::server::hold::hold_until_settled; use crate::server::http::{Accept, ConnectionLimits}; use crate::server::ip_filter::IpFilter; +use crate::server::oversize::{self, Frame, Oversize, Skipping}; use crate::server::traits::{BoundAddr, ProtocolHandler}; use crate::tenancy::{Ingest, Principal}; use convert::gelf_to_json; @@ -47,12 +48,13 @@ const TRANSPORT: &str = "gelf"; /// GELF TCP framing decoder -- splits on null bytes (`\0`). /// /// A message over the size limit costs that message and never the connection: -/// it is discarded up to its delimiter and counted for -/// [`GelfFrameDecoder::take_discarded`]. +/// it is skipped up to its delimiter and handed over as [`Frame::Oversize`], +/// with its first bytes and its length, for the listener to dead-letter. pub struct GelfFrameDecoder { max_length: usize, discarding: bool, - discarded: u64, + /// The head and length of the oversize message being skipped. + skipping: Skipping, } impl GelfFrameDecoder { @@ -60,13 +62,14 @@ impl GelfFrameDecoder { Self { max_length, discarding: false, - discarded: 0, + skipping: Skipping::unmeasured(), } } - /// Messages discarded for their size since the last call. - pub fn take_discarded(&mut self) -> u64 { - std::mem::take(&mut self.discarded) + /// The oversize message being skipped, as far as it arrived, for a stream + /// that ends or fails inside it. + pub fn abandon(&mut self) -> Option { + std::mem::take(&mut self.discarding).then(|| self.skipping.finish()) } } @@ -77,7 +80,7 @@ impl crate::server::frame_clock::Framing for GelfFrameDecoder { } impl Decoder for GelfFrameDecoder { - type Item = Bytes; + type Item = Frame; type Error = io::Error; fn decode( @@ -87,11 +90,14 @@ impl Decoder for GelfFrameDecoder { loop { if self.discarding { let Some(null_pos) = memchr::memchr(b'\0', src) else { + self.skipping.grow(src); src.clear(); return Ok(None); }; + self.skipping.grow(&src[..null_pos]); src.advance(null_pos + 1); self.discarding = false; + return Ok(Some(Frame::Oversize(self.skipping.finish()))); } match memchr::memchr(b'\0', src) { @@ -99,14 +105,16 @@ impl Decoder for GelfFrameDecoder { Some(0) => src.advance(1), Some(null_pos) if null_pos <= self.max_length => { let frame = src.split_to(null_pos + 1).freeze(); - return Ok(Some(frame.slice(..null_pos))); + return Ok(Some(Frame::Message(frame.slice(..null_pos)))); } Some(null_pos) => { - self.discarded += 1; + let frame = oversize::whole(&src[..null_pos], null_pos); src.advance(null_pos + 1); + return Ok(Some(Frame::Oversize(frame))); } None if src.len() > self.max_length => { - self.discarded += 1; + self.skipping = Skipping::unmeasured(); + self.skipping.grow(src); src.clear(); self.discarding = true; return Ok(None); @@ -115,6 +123,25 @@ impl Decoder for GelfFrameDecoder { } } } + + /// An oversize message the stream ended inside is handed over as far as it + /// arrived, and any other partial message is an error. + fn decode_eof( + &mut self, + src: &mut BytesMut, + ) -> std::result::Result, Self::Error> { + if let Some(frame) = self.decode(src)? { + return Ok(Some(frame)); + } + if let Some(skipped) = self.abandon() { + return Ok(Some(Frame::Oversize(skipped))); + } + if src.is_empty() { + Ok(None) + } else { + Err(io::Error::other("bytes remaining on stream")) + } + } } // --------------------------------------------------------------------------- @@ -145,10 +172,14 @@ async fn handle_tcp_connection( break; } result = StreamExt::next(&mut framed) => { - let oversize = framed.decoder_mut().get_mut().take_discarded(); - metrics.add_records_dropped(TRANSPORT, DropReason::Oversize, oversize); match result { - Some(Ok(raw)) => { + Some(Ok(Frame::Oversize(skipped))) => { + if !dead_letter_oversize(vec![skipped], &pipeline, &metrics, max_message_size, &shutdown).await { + debug!(peer = %peer_addr, "GELF TCP connection closing (shutdown during a DLQ write)"); + break; + } + } + Some(Ok(Frame::Message(raw))) => { metrics.inc_requests_total("gelf"); metrics.add_bytes_received("gelf", raw.len() as u64); @@ -169,6 +200,8 @@ async fn handle_tcp_connection( } Some(Err(e)) => { warn!(peer = %peer_addr, error = %e, "GELF TCP read error"); + let skipped: Vec = framed.decoder_mut().get_mut().abandon().into_iter().collect(); + dead_letter_oversize(skipped, &pipeline, &metrics, max_message_size, &shutdown).await; break; } None => { @@ -179,8 +212,27 @@ async fn handle_tcp_connection( } } } - let oversize = framed.decoder_mut().get_mut().take_discarded(); - metrics.add_records_dropped(TRANSPORT, DropReason::Oversize, oversize); +} + +/// Dead-letter the messages the framing skipped for their size. Returns false +/// when shutdown came first, and the caller must stop reading. +async fn dead_letter_oversize( + mut skipped: Vec, + pipeline: &PipelineState, + metrics: &Metrics, + limit: usize, + shutdown: &CancellationToken, +) -> bool { + oversize::dead_letter( + &mut skipped, + pipeline, + metrics, + TRANSPORT, + TRANSPORT, + limit, + shutdown, + ) + .await } // --------------------------------------------------------------------------- @@ -372,9 +424,19 @@ impl ProtocolHandler for GelfHandler { #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { + use bytes::Bytes; + use super::*; use crate::server::frame_clock::Framing; + /// The message a frame holds. + fn message(frame: Frame) -> Bytes { + match frame { + Frame::Message(message) => message, + Frame::Oversize(skipped) => panic!("a message was skipped: {skipped:?}"), + } + } + #[test] fn test_default_config() { let config = GelfConfig::default(); @@ -389,7 +451,7 @@ mod tests { let mut decoder = GelfFrameDecoder::new(1024); let mut buf = BytesMut::from(&br#"{"version":"1.1","host":"h","short_message":"m"}"#[..]); buf.extend_from_slice(b"\0"); - let result = decoder.decode(&mut buf).unwrap().unwrap(); + let result = message(decoder.decode(&mut buf).unwrap().unwrap()); let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); assert_eq!(json["host"], "h"); } @@ -403,8 +465,8 @@ mod tests { buf.extend_from_slice(br#"{"version":"1.1","host":"b","short_message":"2"}"#); buf.extend_from_slice(b"\0"); - let msg1 = decoder.decode(&mut buf).unwrap().unwrap(); - let msg2 = decoder.decode(&mut buf).unwrap().unwrap(); + let msg1 = message(decoder.decode(&mut buf).unwrap().unwrap()); + let msg2 = message(decoder.decode(&mut buf).unwrap().unwrap()); let j1: serde_json::Value = serde_json::from_slice(&msg1).unwrap(); let j2: serde_json::Value = serde_json::from_slice(&msg2).unwrap(); assert_eq!(j1["host"], "a"); @@ -418,16 +480,39 @@ mod tests { assert!(decoder.decode(&mut buf).unwrap().is_none()); } + /// What a decoder given `chunks` yields, and whether it is mid-frame after. + fn decode_chunks( + decoder: &mut GelfFrameDecoder, + chunks: &[&[u8]], + ) -> (Vec, Vec, bool) { + let mut buf = BytesMut::new(); + let (mut messages, mut skipped) = (Vec::new(), Vec::new()); + for chunk in chunks { + buf.extend_from_slice(chunk); + while let Some(frame) = decoder.decode(&mut buf).unwrap() { + match frame { + Frame::Message(message) => { + messages.push(String::from_utf8_lossy(&message).into_owned()); + } + Frame::Oversize(frame) => skipped.push(frame), + } + } + } + (messages, skipped, decoder.mid_frame()) + } + /// An oversize message costs itself and nothing after it, whether its - /// delimiter arrives in the same read or several reads later. The decoder - /// says it is mid-frame while it discards, so the frame timeout keeps running. + /// delimiter arrives in the same read or several reads later, and is handed + /// over once its delimiter has arrived. The decoder says it is mid-frame + /// while it skips, so the frame timeout keeps running. #[test] - fn an_oversize_message_is_discarded_and_counted() { + fn an_oversize_message_is_skipped_and_handed_over() { struct Case { name: &'static str, chunks: &'static [&'static [u8]], frames: &'static [&'static str], - discarded: u64, + /// The head and length of each oversize message handed over. + skipped: &'static [(&'static str, usize)], mid_frame: bool, } let cases = [ @@ -435,70 +520,78 @@ mod tests { name: "oversize with its delimiter in the same read", chunks: &[b"a\0this is too long\0b\0"], frames: &["a", "b"], - discarded: 1, + skipped: &[("this is too long", 16)], mid_frame: false, }, Case { name: "oversize split across reads", chunks: &[b"a\0this is too", b" long and still going", b" on\0b\0"], frames: &["a", "b"], - discarded: 1, + skipped: &[("this is too long and still going on", 35)], mid_frame: false, }, Case { name: "oversize still arriving", chunks: &[b"a\0this is too long", b" and no delimiter yet"], frames: &["a"], - discarded: 1, + skipped: &[], mid_frame: true, }, Case { name: "two oversize messages back to back", chunks: &[b"this is too long\0and so is this\0b\0"], frames: &["b"], - discarded: 2, + skipped: &[("this is too long", 16), ("and so is this", 14)], mid_frame: false, }, Case { name: "exactly the limit is kept", chunks: &[b"0123456789\0"], frames: &["0123456789"], - discarded: 0, + skipped: &[], mid_frame: false, }, Case { name: "empty messages are skipped", chunks: &[b"\0\0a\0", b"\0b\0"], frames: &["a", "b"], - discarded: 0, + skipped: &[], mid_frame: false, }, Case { name: "a partial message waits", chunks: &[b"a\0par"], frames: &["a"], - discarded: 0, + skipped: &[], mid_frame: false, }, ]; let mut failed = Vec::new(); for case in cases { - let mut decoder = GelfFrameDecoder::new(10); - let mut buf = BytesMut::new(); - let mut frames = Vec::new(); - for chunk in case.chunks { - buf.extend_from_slice(chunk); - while let Some(frame) = decoder.decode(&mut buf).unwrap() { - frames.push(String::from_utf8_lossy(&frame).into_owned()); - } - } - let got = (frames, decoder.take_discarded(), decoder.mid_frame()); + let (frames, skipped, mid_frame) = + decode_chunks(&mut GelfFrameDecoder::new(10), case.chunks); + let got = ( + frames, + skipped + .iter() + .map(|s| { + ( + String::from_utf8_lossy(&s.head).into_owned(), + s.original_bytes, + ) + }) + .collect::>(), + mid_frame, + ); let want = ( case.frames .iter() .map(ToString::to_string) .collect::>(), - case.discarded, + case.skipped + .iter() + .map(|(head, len)| ((*head).to_string(), *len)) + .collect::>(), case.mid_frame, ); if got != want { @@ -507,4 +600,31 @@ mod tests { } assert!(failed.is_empty(), "{}", failed.join("\n")); } + + /// A stream that ends or fails inside an oversize message still hands over + /// the part that arrived, and a long one keeps only its head. + #[test] + fn an_oversize_message_cut_by_the_stream_is_handed_over_as_far_as_it_arrived() { + let mut decoder = GelfFrameDecoder::new(10); + let mut buf = BytesMut::from(&b"this is too long and no delimiter"[..]); + assert!(decoder.decode(&mut buf).unwrap().is_none()); + let Some(Frame::Oversize(skipped)) = decoder.decode_eof(&mut buf).unwrap() else { + panic!("the oversize message was not handed over"); + }; + assert_eq!(skipped.original_bytes, 33); + assert_eq!(&skipped.head[..], b"this is too long and no delimiter"); + assert!(decoder.decode_eof(&mut buf).unwrap().is_none()); + + let mut decoder = GelfFrameDecoder::new(10); + let chunk = vec![b'x'; oversize::HEAD_BYTES]; + let mut buf = BytesMut::new(); + for _ in 0..3 { + buf.extend_from_slice(&chunk); + assert!(decoder.decode(&mut buf).unwrap().is_none()); + } + let skipped = decoder.abandon().unwrap(); + assert_eq!(skipped.original_bytes, 3 * oversize::HEAD_BYTES); + assert_eq!(skipped.head.len(), oversize::HEAD_BYTES); + assert!(decoder.abandon().is_none()); + } } diff --git a/src/server/hold.rs b/src/server/hold.rs index bf45c5a..0ffe4e0 100644 --- a/src/server/hold.rs +++ b/src/server/hold.rs @@ -27,10 +27,10 @@ use crate::pipeline::{Acks, Offer, PipelineState}; use crate::tenancy::Ingest; /// Wait before the first re-offer. -const FIRST_RETRY: Duration = Duration::from_millis(100); +pub(super) const FIRST_RETRY: Duration = Duration::from_millis(100); /// Longest wait between re-offers, so a pipeline that recovers is found within it. -const LAST_RETRY: Duration = Duration::from_secs(2); +pub(super) const LAST_RETRY: Duration = Duration::from_secs(2); /// How a hold ended. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/src/server/mod.rs b/src/server/mod.rs index d16da1d..6a34212 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -30,6 +30,7 @@ mod metric_record; pub mod netflow; #[cfg(feature = "otlp")] pub mod otlp; +pub mod oversize; pub mod prometheus_rw; pub mod raw_capture; pub mod sflow; diff --git a/src/server/oversize.rs b/src/server/oversize.rs new file mode 100644 index 0000000..455b5e6 --- /dev/null +++ b/src/server/oversize.rs @@ -0,0 +1,178 @@ +// Project: dfe-receiver +// File: src/server/oversize.rs +// Purpose: Dead-letter a frame a raw TCP listener skipped for its size +// Language: Rust +// +// License: BUSL-1.1 +// Copyright: (c) 2026 HYPERI PTY LIMITED + +//! Frames a raw TCP listener skips for their size. +//! +//! A syslog or GELF message over its listener's limit costs that message and +//! never the connection, and the message is not lost: its first bytes go to the +//! DLQ with the reason and its full length, and the listener counts it in +//! `receiver_records_dropped_total{reason="oversize"}` as before. A framing +//! codec hands over each skipped frame as [`Frame::Oversize`] in its place in +//! the stream, so a flood of them backs up into the socket as the DLQ confirms +//! each write, and the codec never holds more than one frame's head. +//! +//! These protocols carry no acknowledgement, so a DLQ write that is not +//! confirmed is offered again until it is, holding the connection as a pipeline +//! that cannot take a record does ([`super::hold`]). + +use bytes::{Bytes, BytesMut}; +use tokio_util::sync::CancellationToken; +use tracing::debug; + +use super::hold::{FIRST_RETRY, LAST_RETRY}; +use crate::buffer::rejects::Disposal; +use crate::metrics::{DropReason, Metrics}; +use crate::pipeline::PipelineState; + +/// The most bytes of a skipped frame a codec keeps for its dead letter: the +/// DLQ's own default payload cap, which it applies again to a smaller one. +pub const HEAD_BYTES: usize = scalo::dlq::DEFAULT_MAX_PAYLOAD_BYTES; + +/// A frame skipped for its size. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Oversize { + /// The frame's first bytes, up to [`HEAD_BYTES`]. A frame cut short by its + /// connection closing holds what arrived. + pub head: Bytes, + /// The frame's length: its octet count, or the bytes up to its delimiter. + pub original_bytes: usize, +} + +/// What a framing codec hands its listener. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Frame { + /// One message, without its framing. + Message(Bytes), + /// A message over the size limit, which the codec skipped. + Oversize(Oversize), +} + +/// The head and length of a frame the codec is still skipping. +#[derive(Debug, Default)] +pub struct Skipping { + head: BytesMut, + original_bytes: usize, +} + +impl Skipping { + /// A frame of a known length, whose bytes arrive as `keep` is given them. + pub fn counted(original_bytes: usize) -> Self { + Self { + head: BytesMut::new(), + original_bytes, + } + } + + /// A frame of unknown length, which `grow` measures. + pub fn unmeasured() -> Self { + Self::default() + } + + /// Keep `chunk` as far as the head has room. + pub fn keep(&mut self, chunk: &[u8]) { + let room = HEAD_BYTES - self.head.len(); + self.head.extend_from_slice(&chunk[..chunk.len().min(room)]); + } + + /// Count `chunk` toward the length and keep what the head has room for. + pub fn grow(&mut self, chunk: &[u8]) { + self.keep(chunk); + self.original_bytes += chunk.len(); + } + + /// The skipped frame, leaving this ready for the next. + pub fn finish(&mut self) -> Oversize { + Oversize { + head: std::mem::take(&mut self.head).freeze(), + original_bytes: std::mem::take(&mut self.original_bytes), + } + } +} + +/// One frame of `head` and `original_bytes`, whole in the codec's buffer. +pub fn whole(frame: &[u8], original_bytes: usize) -> Oversize { + Oversize { + head: Bytes::copy_from_slice(&frame[..frame.len().min(HEAD_BYTES)]), + original_bytes, + } +} + +/// Count `frames` as dropped for their size and dead-letter each, waiting on +/// the DLQ until it has taken or refused every one for good. +/// +/// Returns false when shutdown came first, and the caller must stop reading. +pub async fn dead_letter( + frames: &mut Vec, + pipeline: &PipelineState, + metrics: &Metrics, + transport: &str, + listener: &str, + limit: usize, + shutdown: &CancellationToken, +) -> bool { + metrics.add_records_dropped(transport, DropReason::Oversize, frames.len() as u64); + let reason = format!("frame over {limit} bytes on {listener}"); + for frame in frames.drain(..) { + let mut wait = FIRST_RETRY; + loop { + let disposal = pipeline + .rejects() + .dispose_head(&frame.head, frame.original_bytes, &reason) + .await; + if disposal != Disposal::Refused { + break; + } + debug!( + transport, + listener, "DLQ did not take an oversize frame; holding the connection" + ); + tokio::select! { + biased; + () = shutdown.cancelled() => return false, + () = tokio::time::sleep(wait) => {} + } + wait = (wait * 2).min(LAST_RETRY); + } + } + true +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn a_skipped_frame_keeps_its_head_and_counts_its_length() { + let mut frame = Skipping::unmeasured(); + let chunk = vec![b'x'; HEAD_BYTES]; + frame.grow(&chunk); + frame.grow(&chunk); + let out = frame.finish(); + assert_eq!(out.head.len(), HEAD_BYTES); + assert_eq!(out.original_bytes, 2 * HEAD_BYTES); + assert_eq!(frame.finish().original_bytes, 0, "ready for the next"); + } + + #[test] + fn a_counted_frame_keeps_its_declared_length_whatever_arrived() { + let mut frame = Skipping::counted(100_000); + frame.keep(b"abc"); + let out = frame.finish(); + assert_eq!(&out.head[..], b"abc"); + assert_eq!(out.original_bytes, 100_000); + } + + #[test] + fn a_whole_frame_is_cut_to_the_head_limit() { + let frame = vec![b'y'; HEAD_BYTES + 10]; + let out = whole(&frame, frame.len()); + assert_eq!(out.head.len(), HEAD_BYTES); + assert_eq!(out.original_bytes, HEAD_BYTES + 10); + } +} diff --git a/src/server/syslog/framing.rs b/src/server/syslog/framing.rs index 027f94f..a580c1b 100644 --- a/src/server/syslog/framing.rs +++ b/src/server/syslog/framing.rs @@ -15,8 +15,9 @@ //! //! A message over the size limit costs that message and never the connection: //! an octet-counted one is skipped by its count, a line up to its LF. Each is -//! counted for [`SyslogFrameDecoder::take_discarded`]. A count above the frame -//! limit is not trusted as a count, and that frame is read as a line. +//! handed over as [`Frame::Oversize`], with its first bytes and its length, for +//! the listener to dead-letter. A count above the frame limit is not trusted as +//! a count, and that frame is read as a line. //! //! A frame is its bytes: the encoding is the caller's to check, so a line that //! is not UTF-8 costs that line and not the connection. @@ -26,6 +27,8 @@ use std::io; use bytes::{Buf, Bytes, BytesMut}; use tokio_util::codec::Decoder; +use crate::server::oversize::{self, Frame, Oversize, Skipping}; + /// Default maximum syslog message size (64 KiB). pub const DEFAULT_MAX_LENGTH: usize = 64 * 1024; @@ -58,7 +61,8 @@ pub struct SyslogFrameDecoder { max_length: usize, max_frame: usize, state: State, - discarded: u64, + /// The head and length of the oversize message being skipped. + skipping: Skipping, } impl SyslogFrameDecoder { @@ -75,13 +79,17 @@ impl SyslogFrameDecoder { max_length, max_frame: max_frame.max(max_length), state: State::Start, - discarded: 0, + skipping: Skipping::unmeasured(), } } - /// Messages discarded for their size since the last call. - pub fn take_discarded(&mut self) -> u64 { - std::mem::take(&mut self.discarded) + /// The oversize message being skipped, as far as it arrived, for a stream + /// that ends or fails inside it. + pub fn abandon(&mut self) -> Option { + (self.state != State::Start).then(|| { + self.state = State::Start; + self.skipping.finish() + }) } /// How the frame at the start of `src` opens. @@ -127,7 +135,7 @@ impl Default for SyslogFrameDecoder { } impl Decoder for SyslogFrameDecoder { - type Item = Bytes; + type Item = Frame; type Error = io::Error; fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { @@ -136,20 +144,25 @@ impl Decoder for SyslogFrameDecoder { State::Start => {} State::Skip(left) => { let take = left.min(src.len()); + self.skipping.keep(&src[..take]); src.advance(take); if take < left { self.state = State::Skip(left - take); return Ok(None); } self.state = State::Start; + return Ok(Some(Frame::Oversize(self.skipping.finish()))); } State::Discard => { let Some(lf) = memchr::memchr(b'\n', src) else { + self.skipping.grow(src); src.clear(); return Ok(None); }; + self.skipping.grow(&src[..lf]); src.advance(lf + 1); self.state = State::Start; + return Ok(Some(Frame::Oversize(self.skipping.finish()))); } } @@ -161,8 +174,8 @@ impl Decoder for SyslogFrameDecoder { match self.opening(src) { Opening::Counted { header, count } if count > self.max_length => { - self.discarded += 1; src.advance(header); + self.skipping = Skipping::counted(count); self.state = State::Skip(count); continue; } @@ -173,7 +186,7 @@ impl Decoder for SyslogFrameDecoder { return Ok(None); } let frame = src.split_to(total).freeze(); - return Ok(Some(frame.slice(header..))); + return Ok(Some(Frame::Message(frame.slice(header..)))); } Opening::Undecided => return Ok(None), Opening::Line => {} @@ -182,14 +195,16 @@ impl Decoder for SyslogFrameDecoder { match memchr::memchr(b'\n', src) { Some(lf) if lf <= self.max_length => { let frame = src.split_to(lf + 1).freeze(); - return Ok(Some(frame.slice(..lf))); + return Ok(Some(Frame::Message(frame.slice(..lf)))); } Some(lf) => { - self.discarded += 1; + let frame = oversize::whole(&src[..lf], lf); src.advance(lf + 1); + return Ok(Some(Frame::Oversize(frame))); } None if src.len() > self.max_length => { - self.discarded += 1; + self.skipping = Skipping::unmeasured(); + self.skipping.grow(src); src.clear(); self.state = State::Discard; return Ok(None); @@ -200,20 +215,22 @@ impl Decoder for SyslogFrameDecoder { } /// A sender that closes after a last line with no newline still sent that - /// line, so it is the final frame. A partial octet-counted frame is not. + /// line, so it is the final frame. A partial octet-counted frame is not, and + /// an oversize message the stream ended inside is handed over as far as it + /// arrived. fn decode_eof(&mut self, src: &mut BytesMut) -> Result, Self::Error> { if let Some(frame) = self.decode(src)? { return Ok(Some(frame)); } + if let Some(skipped) = self.abandon() { + return Ok(Some(Frame::Oversize(skipped))); + } let rest = src.split(); let rest = rest.trim_ascii(); - if rest.is_empty() - || self.state != State::Start - || matches!(self.opening(rest), Opening::Counted { .. }) - { + if rest.is_empty() || matches!(self.opening(rest), Opening::Counted { .. }) { return Ok(None); } - Ok(Some(Bytes::copy_from_slice(rest))) + Ok(Some(Frame::Message(Bytes::copy_from_slice(rest)))) } } @@ -225,27 +242,45 @@ impl Decoder for SyslogFrameDecoder { #[allow(clippy::unwrap_used)] mod tests { use super::*; + use crate::server::oversize::HEAD_BYTES; - /// Every frame `data` holds, fed `chunk` bytes at a time and read to the - /// end of the stream when `eof`, with the count of messages discarded. - fn decode_chunked( + /// Every message `data` holds, fed `chunk` bytes at a time and read to the + /// end of the stream when `eof`, with the messages skipped for their size. + fn decode_all( decoder: &mut SyslogFrameDecoder, data: &[u8], chunk: usize, eof: bool, - ) -> (Vec, u64) { + ) -> (Vec, Vec) { let mut buf = BytesMut::new(); - let mut frames = Vec::new(); + let (mut messages, mut skipped) = (Vec::new(), Vec::new()); + let mut take = |frame: Frame| match frame { + Frame::Message(message) => messages.push(message), + Frame::Oversize(frame) => skipped.push(frame), + }; for piece in data.chunks(chunk.max(1)) { buf.extend_from_slice(piece); while let Some(frame) = decoder.decode(&mut buf).unwrap() { - frames.push(frame); + take(frame); } } - if eof && let Some(last) = decoder.decode_eof(&mut buf).unwrap() { - frames.push(last); + if eof { + while let Some(frame) = decoder.decode_eof(&mut buf).unwrap() { + take(frame); + } } - (frames, decoder.take_discarded()) + (messages, skipped) + } + + /// [`decode_all`] with the count of messages skipped. + fn decode_chunked( + decoder: &mut SyslogFrameDecoder, + data: &[u8], + chunk: usize, + eof: bool, + ) -> (Vec, u64) { + let (messages, skipped) = decode_all(decoder, data, chunk, eof); + (messages, skipped.len() as u64) } /// The bytes sent, whether the stream then ends, the frames expected and @@ -462,6 +497,89 @@ mod tests { assert_eq!(discarded, 0); } + /// A skipped message hands over its first bytes and its whole length, + /// however its bytes arrive, so the DLQ holds what was sent and says how + /// much more there was. + #[test] + fn a_skipped_message_hands_over_its_head_and_length() { + let octet = format!("70000 <14>{}", "a".repeat(70_000 - 4)); + let line = format!("<14>{}\n", "b".repeat(70_000)); + let huge = format!("<14>{}\n", "c".repeat(HEAD_BYTES + 1_000)); + let huge_count = HEAD_BYTES + 1_000; + let huge_octet = format!("{huge_count} <14>{}", "d".repeat(huge_count - 4)); + let huge_octet_start = huge_octet.find('<').unwrap(); + // (name, data, head, original bytes) + let cases: Vec<(&str, String, Vec, usize)> = vec![ + ( + "octet", + octet.clone(), + octet.as_bytes()[6..].to_vec(), + 70_000, + ), + ( + "line", + line.clone(), + line.as_bytes()[..70_004].to_vec(), + 70_004, + ), + ( + "huge_line", + huge.clone(), + huge.as_bytes()[..HEAD_BYTES].to_vec(), + HEAD_BYTES + 1_004, + ), + ( + "huge_octet", + huge_octet.clone(), + huge_octet.as_bytes()[huge_octet_start..][..HEAD_BYTES].to_vec(), + huge_count, + ), + ]; + for (name, data, head, original_bytes) in cases { + for chunk in [data.len(), 4_096, 1_000] { + let mut decoder = SyslogFrameDecoder::with_max_frame(64 * 1024, 2 * HEAD_BYTES); + let (messages, skipped) = decode_all(&mut decoder, data.as_bytes(), chunk, false); + assert_eq!(messages.len(), 0, "{name} (chunk {chunk})"); + assert_eq!( + skipped, + vec![Oversize { + head: Bytes::from(head.clone()), + original_bytes + }], + "{name} (chunk {chunk})" + ); + } + } + } + + /// A stream that ends inside a skipped message still hands over the part + /// that arrived, with the length the sender declared for a counted one. + #[test] + fn a_stream_ending_inside_a_skipped_message_hands_over_what_arrived() { + let counted = format!("80000 <14>{}", "e".repeat(1_000)); + let (messages, skipped) = decode_all( + &mut SyslogFrameDecoder::default(), + counted.as_bytes(), + 4_096, + true, + ); + assert_eq!(messages.len(), 0); + assert_eq!(skipped.len(), 1); + assert_eq!(skipped[0].original_bytes, 80_000); + assert_eq!(skipped[0].head.len(), 1_004); + + let line = format!("<14>{}", "f".repeat(70_000)); + let (_, skipped) = decode_all( + &mut SyslogFrameDecoder::default(), + line.as_bytes(), + 4_096, + true, + ); + assert_eq!(skipped.len(), 1); + assert_eq!(skipped[0].original_bytes, 70_004); + assert_eq!(skipped[0].head.len(), 70_004); + } + #[test] fn test_empty_input() { let mut decoder = SyslogFrameDecoder::default(); diff --git a/src/server/syslog/tcp.rs b/src/server/syslog/tcp.rs index 9999215..6a25758 100644 --- a/src/server/syslog/tcp.rs +++ b/src/server/syslog/tcp.rs @@ -33,12 +33,13 @@ use super::convert::Converter; use super::framing::SyslogFrameDecoder; use super::{TRANSPORT, decode_lossy, listener_ingest}; use crate::error::{Error, Result}; -use crate::metrics::{DropReason, Metrics}; +use crate::metrics::Metrics; use crate::pipeline::{Acks, PipelineState}; use crate::server::frame_clock::{Clocked, Framing}; use crate::server::hold::{Held, hold}; use crate::server::http::{Accept, ConnectionLimits}; use crate::server::ip_filter::IpFilter; +use crate::server::oversize::{self, Frame, Oversize}; use crate::server::traits::BoundAddr; /// The idle timer's period with idle closing off, when it is never polled. @@ -121,6 +122,7 @@ impl TcpListenerTask { shutdown: shutdown.clone(), peer_addr, connection: self.connection, + label: self.label, }; let accept = accept.clone(); let tls = self.tls.clone(); @@ -170,6 +172,8 @@ struct Reader { shutdown: CancellationToken, peer_addr: SocketAddr, connection: Connection, + /// `TCP` or `TLS`, the listener the frames arrive on. + label: &'static str, } impl Reader { @@ -182,6 +186,7 @@ impl Reader { )); let mut framed = FramedRead::new(stream, decoder); let mut payloads: Vec = Vec::new(); + let mut skipped: Vec = Vec::new(); let idle_for = c.idle.unwrap_or(NEVER); let idle = tokio::time::sleep(idle_for); tokio::pin!(idle); @@ -196,9 +201,6 @@ impl Reader { () = &mut idle, if c.idle.is_some() => { // A frame part way in is the frame timeout's to close. if framed.read_buffer().is_empty() && !framed.decoder().get_ref().mid_frame() { - let oversize = framed.decoder_mut().get_mut().take_discarded(); - self.metrics - .add_records_dropped(TRANSPORT, DropReason::Oversize, oversize); self.metrics.inc_connection_closed(TRANSPORT, "idle"); debug!(peer = %self.peer_addr, "Syslog TCP connection closing (idle)"); return; @@ -211,16 +213,31 @@ impl Reader { let end = match first { Some(Ok(frame)) => { - self.push(&frame, &mut payloads); - self.drain_buffered(&mut framed, &mut payloads) + self.take(frame, &mut payloads, &mut skipped); + self.drain_buffered(&mut framed, &mut payloads, &mut skipped) } Some(Err(e)) => Some(End::Failed(e)), None => Some(End::Closed), }; - let oversize = framed.decoder_mut().get_mut().take_discarded(); - self.metrics - .add_records_dropped(TRANSPORT, DropReason::Oversize, oversize); + if matches!(end, Some(End::Failed(_))) { + skipped.extend(framed.decoder_mut().get_mut().abandon()); + } + let listener = format!("syslog {}", self.label); + if !oversize::dead_letter( + &mut skipped, + &self.pipeline, + &self.metrics, + TRANSPORT, + &listener, + c.max_message_size, + &self.shutdown, + ) + .await + { + debug!(peer = %self.peer_addr, "Syslog TCP connection closing (shutdown during a DLQ write)"); + return; + } if !payloads.is_empty() { let held = hold( &self.pipeline, @@ -268,16 +285,17 @@ impl Reader { &self, framed: &mut FramedRead, payloads: &mut Vec, + skipped: &mut Vec, ) -> Option where - D: Decoder, + D: Decoder, { // FramedRead lends its buffer and its decoder one at a time, so the // buffer comes out for the drain and goes back after it. let mut buffer = std::mem::take(framed.read_buffer_mut()); let end = loop { match framed.decoder_mut().decode(&mut buffer) { - Ok(Some(frame)) => self.push(&frame, payloads), + Ok(Some(frame)) => self.take(frame, payloads, skipped), Ok(None) => break None, Err(e) => break Some(End::Failed(e)), } @@ -286,6 +304,15 @@ impl Reader { end } + /// Add a message's record to the batch, or keep a message skipped for its + /// size for the DLQ. + fn take(&self, frame: Frame, payloads: &mut Vec, skipped: &mut Vec) { + match frame { + Frame::Message(message) => self.push(&message, payloads), + Frame::Oversize(frame) => skipped.push(frame), + } + } + /// Count one frame and add its record to the batch. fn push(&self, frame: &[u8], payloads: &mut Vec) { self.metrics.inc_requests_total(TRANSPORT); diff --git a/tests/common/mod.rs b/tests/common/mod.rs index c662805..bc8f9be 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -853,3 +853,33 @@ pub async fn stored_within( } records } + +/// Point `config` at a file DLQ under `dir`, so the dead letters it takes can +/// be read back with [`file_dead_letters`]. +pub fn file_dlq(config: &mut dfe_receiver::config::Config, dir: &std::path::Path) { + config.routing.dlq.enabled = true; + config.routing.dlq.mode = "file_only".to_string(); + config.routing.dlq.kafka_enabled = false; + config.routing.dlq.file_path = dir.display().to_string(); +} + +/// Every dead letter written to a file DLQ under `dir`, in file order. +pub fn file_dead_letters(dir: &std::path::Path) -> Vec { + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut letters = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + letters.extend(file_dead_letters(&path)); + } else { + let text = std::fs::read_to_string(&path).unwrap_or_default(); + letters.extend( + text.lines() + .filter_map(|line| serde_json::from_str(line).ok()), + ); + } + } + letters +} diff --git a/tests/integration/gelf.rs b/tests/integration/gelf.rs index f39fb77..3707e27 100644 --- a/tests/integration/gelf.rs +++ b/tests/integration/gelf.rs @@ -32,8 +32,8 @@ fn gelf(marker: &str) -> Vec { } /// An oversize message costs itself and nothing after it on the same -/// connection, whether its delimiter came in the same read or after a pause, -/// and each one is counted dropped. +/// connection, whether its delimiter came in the same read or after a pause. +/// Each one is counted dropped and goes to the DLQ with its length. #[tokio::test] async fn gelf_carries_on_past_oversize_messages() { let (loader_endpoint, loader) = grpc_destination().await; @@ -43,7 +43,8 @@ async fn gelf_carries_on_past_oversize_messages() { config.destinations.default = "loader".into(); config.loader.transport = "grpc".to_string(); config.loader.grpc_endpoint = Some(loader_endpoint); - config.routing.dlq.enabled = false; + let dlq_dir = tempfile::tempdir().unwrap(); + crate::common::file_dlq(&mut config, dlq_dir.path()); config.gelf.enabled = true; config.gelf.bind_address = "127.0.0.1:0".to_string(); config.gelf.accept_unauthenticated = true; @@ -92,6 +93,14 @@ async fn gelf_carries_on_past_oversize_messages() { 2, "both oversize messages counted" ); + let letters = crate::common::file_dead_letters(dlq_dir.path()); + assert_eq!(letters.len(), 2, "dead letters written"); + for (letter, original) in letters.iter().zip([2_000, 3_003]) { + assert_eq!(letter["service"], "receiver"); + assert_eq!(letter["reason"], "frame over 1024 bytes on gelf"); + assert_eq!(letter["original_bytes"], original); + assert_eq!(letter["_raw"].as_str().map(str::len), Some(original)); + } tcp.write_all(&gelf("fourth")).await.unwrap(); let after = receive_records(&loader, 1, Duration::from_secs(10)).await; diff --git a/tests/integration/syslog.rs b/tests/integration/syslog.rs index b7ed314..1e68601 100644 --- a/tests/integration/syslog.rs +++ b/tests/integration/syslog.rs @@ -694,9 +694,10 @@ async fn a_message_that_is_not_utf8_lands_parsed_over_udp_and_tcp() { assert_eq!(syslog.metrics.get_requests_error(), 0); } -/// Over TCP an oversize message costs itself and nothing after it, a line -/// opening with digits is a line, and a stamp with no zone takes -/// `syslog.timezone`. Over UDP a burst larger than a batch arrives whole. +/// Over TCP an oversize message costs itself and nothing after it, and goes to +/// the DLQ with its length. A line opening with digits is a line, and a stamp +/// with no zone takes `syslog.timezone`. Over UDP a burst larger than a batch +/// arrives whole. #[tokio::test] async fn syslog_carries_on_past_oversize_messages_and_reads_bursts_whole() { use tokio::io::AsyncWriteExt; @@ -705,7 +706,9 @@ async fn syslog_carries_on_past_oversize_messages_and_reads_bursts_whole() { let mut config = test_config(); config.loader.transport = "grpc".to_string(); config.loader.grpc_endpoint = Some(loader_endpoint); - config.routing.dlq.enabled = false; + let dlq_dir = tempfile::tempdir().unwrap(); + crate::common::file_dlq(&mut config, dlq_dir.path()); + let limit = config.syslog.max_message_size; config.syslog.timezone = "+10:00".parse().unwrap(); config.syslog.udp_batch_size = 16; let syslog = start_syslog_handler(config).await; @@ -716,6 +719,7 @@ async fn syslog_carries_on_past_oversize_messages_and_reads_bursts_whole() { wire.push_str(&format!("{} {second}", second.len())); wire.push_str("2026 Oct 31 10:00:00 h app: third\n"); wire.push_str(&format!("<14>{}\n", "b".repeat(70_000))); + wire.push_str(&format!("<14>{}\n", "c".repeat(600_000))); wire.push_str("<14>Oct 31 10:00:00 h app: fourth\n"); let mut tcp = tokio::net::TcpStream::connect(syslog.tcp).await.unwrap(); tcp.write_all(wire.as_bytes()).await.unwrap(); @@ -743,8 +747,27 @@ async fn syslog_carries_on_past_oversize_messages_and_reads_bursts_whole() { ); assert_eq!( syslog.metrics.get_records_dropped(), - 2, - "both oversize messages counted" + 3, + "every oversize message counted" + ); + let letters = crate::common::file_dead_letters(dlq_dir.path()); + assert_eq!(letters.len(), 3, "dead letters written"); + for (letter, original) in letters.iter().zip([70_000, 70_004, 600_004]) { + assert_eq!(letter["service"], "receiver"); + assert_eq!( + letter["reason"], + format!("frame over {limit} bytes on syslog TCP") + ); + assert_eq!(letter["original_bytes"], original); + } + assert_eq!(letters[0]["_raw"].as_str().map(str::len), Some(70_000)); + assert_eq!(letters[1]["truncated"], serde_json::Value::Null); + // The 600,004-byte line keeps the DLQ's 512 KiB head, as base64. + assert_eq!(letters[2]["truncated"], true); + assert_eq!(letters[2]["_raw_encoding"], "base64"); + assert_eq!( + letters[2]["_raw"].as_str().map(str::len), + Some(524_288_usize.div_ceil(3) * 4) ); // The connection is still open: another line still lands.