diff --git a/benches/syslog.rs b/benches/syslog.rs index 79aa7e3..e5b4041 100644 --- a/benches/syslog.rs +++ b/benches/syslog.rs @@ -13,6 +13,7 @@ use std::hint::black_box; use bytes::BytesMut; use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; use dfe_receiver::config::RawCapture; +use dfe_receiver::server::oversize::Frame; use dfe_receiver::server::syslog::convert::syslog_to_json; use dfe_receiver::server::syslog::framing::SyslogFrameDecoder; use tokio_util::codec::Decoder; @@ -55,7 +56,9 @@ fn framing(c: &mut Criterion) { let mut buf = BytesMut::from(&stream[..]); let mut n = 0usize; while let Ok(Some(frame)) = decoder.decode(&mut buf) { - n += frame.len(); + if let Frame::Message(message) = frame { + n += message.len(); + } } black_box(n) }); diff --git a/benches/syslog_allocs.rs b/benches/syslog_allocs.rs index 2cd9356..7a4f866 100644 --- a/benches/syslog_allocs.rs +++ b/benches/syslog_allocs.rs @@ -16,6 +16,7 @@ mod common; use bytes::BytesMut; use dfe_receiver::config::RawCapture; +use dfe_receiver::server::oversize::Frame; use dfe_receiver::server::syslog::convert::syslog_to_json; use dfe_receiver::server::syslog::framing::SyslogFrameDecoder; use tokio_util::codec::Decoder; @@ -42,7 +43,7 @@ fn frames_per_stream(stream: &[u8]) -> (f64, f64) { let info = allocation_counter::measure(|| { let mut decoder = SyslogFrameDecoder::default(); while let Ok(Some(frame)) = decoder.decode(&mut buf) { - frames += u64::from(!frame.is_empty()); + frames += u64::from(matches!(&frame, Frame::Message(m) if !m.is_empty())); } }); let frames = frames.max(1) as f64; diff --git a/docs/hardening.md b/docs/hardening.md index 2f38771..ccebe0e 100644 --- a/docs/hardening.md +++ b/docs/hardening.md @@ -36,7 +36,7 @@ Application-level hardening, and cost-conscious fronting on Kubernetes and AWS. | Dead-letter queue | Global | `routing.dlq` | | Memory pressure backpressure | 503 / `UNAVAILABLE` on HTTP and gRPC ingest | `DFE_RECEIVER_MEMORY_PRESSURE_THRESHOLD`, `self_regulation.*` (`buffer.*` bounds the destination queues and nothing else) | | Middleware ordering | RateLimit -> Concurrency -> Timeout -> BodyLimit -> Auth -> Handler, behind the accept loop's IP filter | Cheapest refusal first | -| Frame size validation | GELF 1MB, Syslog 64KB, Fluent 32MB. An oversize syslog or GELF message is skipped to its delimiter and counted (`receiver_records_dropped_total`, reason `oversize`), and its connection goes on. Fluent Forward has no delimiter to resume at, so an oversize frame closes its connection | Per-protocol limits, `syslog.max_message_size`, `gelf.max_message_size` | +| Frame size validation | GELF 1MB, Syslog 64KB, Fluent 32MB. An oversize syslog or GELF message is skipped to its delimiter and counted (`receiver_records_dropped_total`, reason `oversize`), its first bytes go to the DLQ with its length, and its connection goes on. Fluent Forward has no delimiter to resume at, so an oversize frame closes its connection | Per-protocol limits, `syslog.max_message_size`, `gelf.max_message_size` | | Decoded-size budget | An OTLP export, remote-write request or Vector push past 100,000 records, 1,000,000 messages or 64 MiB of resource, scope, metric or series-label bytes copied into its records, or holding a protobuf group, is refused before it decodes: 400 on HTTP, `INVALID_ARGUMENT` on gRPC. A Fluent Forward message past 1,000,000 values, 100,000 entries or 64 MiB of tag copies closes its connection, found by a scan that reads each byte once. A Splunk HEC request past 100,000 events or raw lines, or 64 MiB of `host`, `source`, `sourcetype` and `index` copied into its raw lines, is refused with a 400 | Hard-coded | | Zip-bomb rejection | Lumberjack nested compression rejected. A compressed frame is decompressed one inner frame at a time and refused past 64 MiB | Hard-coded | | Unsettled bytes per connection | Lumberjack settles and ACKs a window in parts past 16 MiB of events | Hard-coded | diff --git a/docs/protocols.md b/docs/protocols.md index b4868b3..3cce7c1 100644 --- a/docs/protocols.md +++ b/docs/protocols.md @@ -16,7 +16,7 @@ An RFC 3164 token is the app name when it reads `name:` or `name[pid]:`, with th RFC 5424 structured data that does not parse, like `[x@1 k=v]`, stays in `message` as sent, and the elements that did parse still land in `structured_data`. A byte order mark at the start of the message and NUL bytes at its end are dropped. -On TCP and TLS a frame is octet-counted (RFC 6587) when it opens with a count, one space and the `<` of a PRI. Anything else is an LF-terminated line, so a line opening with digits, like `2026 Oct 11 ...`, is never read as a count. A message over `syslog.max_message_size` is skipped and counted as dropped (reason `oversize`), and the connection goes on: an octet-counted one by its count, a line up to its LF. A count over `syslog.max_frame_size` (default 256 KiB) is not trusted as a count, and that frame is read as a line. +On TCP and TLS a frame is octet-counted (RFC 6587) when it opens with a count, one space and the `<` of a PRI. Anything else is an LF-terminated line, so a line opening with digits, like `2026 Oct 11 ...`, is never read as a count. A message over `syslog.max_message_size` is skipped and counted as dropped (reason `oversize`), and the connection goes on: an octet-counted one by its count, a line up to its LF. Its first 512 KiB, the DLQ's payload cap, go to the DLQ with the reason `frame over bytes on syslog TCP` (or `TLS`) and its length in `original_bytes`. The connection waits on the DLQ write, and a write the DLQ does not confirm is offered again. A count over `syslog.max_frame_size` (default 256 KiB) is not trusted as a count, and that frame is read as a line. Syslog UDP reads every datagram ready on a wake, up to `syslog.udp_batch_size` (default 64), in batched receive calls, and hands them to the pipeline as one batch. Its socket asks for a `syslog.udp_recv_buffer_size` receive buffer (default 8 MiB), and logs a warning when `net.core.rmem_max` grants less. Datagrams the kernel drops for a full buffer are read from `/proc/net/udp` every 10 seconds and counted as dropped, reason `kernel`. Syslog TCP settles every frame one read brought in with a single hold on the pipeline. diff --git a/src/buffer/rejects.rs b/src/buffer/rejects.rs index bd7d53c..4370920 100644 --- a/src/buffer/rejects.rs +++ b/src/buffer/rejects.rs @@ -159,8 +159,32 @@ impl Rejects { /// [`Disposal::Refused`] leaves the record with the caller, to keep and /// offer again. pub async fn dispose(&self, topic: &str, payload: &Bytes, reason: &str) -> Disposal { + self.dispose_entry(topic, payload, reason, || entry(topic, payload, reason)) + .await + } + + /// [`dispose`](Self::dispose) for the first bytes of a frame a listener + /// skipped for its size, `original_bytes` long in all, which has no + /// destination yet. + pub async fn dispose_head( + &self, + head: &Bytes, + original_bytes: usize, + reason: &str, + ) -> Disposal { + self.dispose_entry("", head, reason, || cut_entry(head, original_bytes, reason)) + .await + } + + async fn dispose_entry( + &self, + topic: &str, + payload: &Bytes, + reason: &str, + make_entry: impl FnOnce() -> DlqEntry, + ) -> Disposal { let disposal = match &self.dlq { - Some(dlq) => match dead_letter_confirmed(dlq, entry(topic, payload, reason)).await { + Some(dlq) => match dead_letter_confirmed(dlq, make_entry()).await { Ok(DeadLetter::Written) => Disposal::DeadLettered, Ok(DeadLetter::Unwritable) => Disposal::Dropped, Err(e) => { @@ -265,6 +289,14 @@ fn entry(topic: &str, payload: &Bytes, reason: &str) -> DlqEntry { } } +/// The DLQ entry for the first bytes of a frame that was `original_bytes` long. +fn cut_entry(head: &Bytes, original_bytes: usize, reason: &str) -> DlqEntry { + let mut entry = DlqEntry::new("receiver", reason, head.to_vec()); + entry.truncated = head.len() < original_bytes; + entry.original_bytes = Some(original_bytes); + entry +} + /// DLQ fixtures shared by the buffer tests. #[cfg(test)] pub(crate) mod test_dlq { @@ -417,6 +449,44 @@ mod tests { .unwrap() } + /// The head of a frame a listener skipped reaches the DLQ with the frame's + /// full length, marked cut when the head is shorter, and with no + /// destination to route it by. + #[tokio::test] + async fn the_head_of_a_skipped_frame_lands_in_the_dlq_with_its_length() { + let dir = tempfile::tempdir().unwrap(); + let dlq = file_dlq(dir.path()); + let metrics = Arc::new(Metrics::default()); + let rejects = Rejects::new(Some(Arc::clone(&dlq)), Some(Arc::clone(&metrics))); + + let cut = Bytes::from_static(b"<14>first bytes"); + for (head, original) in [(&cut, 70_000), (&cut, cut.len())] { + let disposal = rejects + .dispose_head(head, original, "frame over 65536 bytes on syslog tcp") + .await; + assert_eq!(disposal, Disposal::DeadLettered); + } + dlq.flush().await.unwrap(); + assert_eq!(metrics.get_messages_dlq(), 2); + + let written = std::fs::read_to_string(dir.path().join("receiver/dlq.ndjson")).unwrap(); + let entries: Vec = written + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(entries.len(), 2, "{written}"); + for entry in &entries { + assert_eq!(entry["reason"], "frame over 65536 bytes on syslog tcp"); + assert!(entry.get("destination").is_none(), "{entry}"); + } + assert_eq!(entries[0]["truncated"], true); + assert_eq!(entries[0]["original_bytes"], 70_000); + assert_eq!(entries[0]["_raw_encoding"], "base64"); + assert!(entries[1].get("truncated").is_none(), "{}", entries[1]); + assert_eq!(entries[1]["original_bytes"], cut.len()); + assert_eq!(entries[1]["_raw"], "<14>first bytes"); + } + /// A refused record reaches the configured DLQ with its reason, its /// destination and the original bytes, and counts as dead-lettered. #[tokio::test] diff --git a/src/pipeline/mod.rs b/src/pipeline/mod.rs index 1e21095..c45fab3 100644 --- a/src/pipeline/mod.rs +++ b/src/pipeline/mod.rs @@ -1497,6 +1497,12 @@ impl PipelineState { Ok(()) } + /// Where a record refused for good is dead-lettered, for a listener that + /// skips a frame before it is a record. + pub(crate) fn rejects(&self) -> &Rejects { + &self.rejects + } + /// Settle a record a destination refuses for good: dead-letter it, or /// drop it where no DLQ is configured. async fn settle_refused( diff --git a/src/server/frame_clock.rs b/src/server/frame_clock.rs index 299e632..76929dd 100644 --- a/src/server/frame_clock.rs +++ b/src/server/frame_clock.rs @@ -247,10 +247,9 @@ mod tests { use tokio_stream::StreamExt; use tokio_util::codec::FramedRead; - use bytes::Bytes; - use super::*; use crate::server::gelf::GelfFrameDecoder; + use crate::server::oversize::Frame; use crate::server::syslog::framing::SyslogFrameDecoder; const LIMIT: Duration = Duration::from_millis(500); @@ -272,7 +271,7 @@ mod tests { /// for `hold` before it reads on, until a frame times out or the client closes. async fn run(codec: D, steps: Vec, hold: Duration) -> Read where - D: Framing, + D: Framing, { let (mut client, server) = tokio::io::duplex(1024); let socket = Clocked::new(server, LIMIT, Arc::new(Metrics::default()), "test"); @@ -296,6 +295,11 @@ mod tests { read.timed_out = true; return; }; + // A skipped message is no frame to hold, as before the codec + // handed it over. + let Frame::Message(frame) = frame else { + continue; + }; read.frames .push(String::from_utf8_lossy(&frame).into_owned()); tokio::time::sleep(hold).await; diff --git a/src/server/gelf/mod.rs b/src/server/gelf/mod.rs index 8c771ab..acb2b62 100644 --- a/src/server/gelf/mod.rs +++ b/src/server/gelf/mod.rs @@ -17,7 +17,7 @@ use std::io; use std::net::SocketAddr; use std::sync::Arc; -use bytes::{Buf, Bytes, BytesMut}; +use bytes::{Buf, BytesMut}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::net::TcpListener; use tokio::time::timeout; @@ -27,12 +27,13 @@ use tracing::{debug, error, info, warn}; use crate::config::GelfConfig; use crate::error::{Error, Result}; -use crate::metrics::{DropReason, Metrics}; +use crate::metrics::Metrics; use crate::pipeline::{Acks, PipelineState}; use crate::server::frame_clock::Clocked; use crate::server::hold::hold_until_settled; use crate::server::http::{Accept, ConnectionLimits}; use crate::server::ip_filter::IpFilter; +use crate::server::oversize::{self, Frame, Oversize, Skipping}; use crate::server::traits::{BoundAddr, ProtocolHandler}; use crate::tenancy::{Ingest, Principal}; use convert::gelf_to_json; @@ -47,12 +48,13 @@ const TRANSPORT: &str = "gelf"; /// GELF TCP framing decoder -- splits on null bytes (`\0`). /// /// A message over the size limit costs that message and never the connection: -/// it is discarded up to its delimiter and counted for -/// [`GelfFrameDecoder::take_discarded`]. +/// it is skipped up to its delimiter and handed over as [`Frame::Oversize`], +/// with its first bytes and its length, for the listener to dead-letter. pub struct GelfFrameDecoder { max_length: usize, discarding: bool, - discarded: u64, + /// The head and length of the oversize message being skipped. + skipping: Skipping, } impl GelfFrameDecoder { @@ -60,13 +62,14 @@ impl GelfFrameDecoder { Self { max_length, discarding: false, - discarded: 0, + skipping: Skipping::unmeasured(), } } - /// Messages discarded for their size since the last call. - pub fn take_discarded(&mut self) -> u64 { - std::mem::take(&mut self.discarded) + /// The oversize message being skipped, as far as it arrived, for a stream + /// that ends or fails inside it. + pub fn abandon(&mut self) -> Option { + std::mem::take(&mut self.discarding).then(|| self.skipping.finish()) } } @@ -77,7 +80,7 @@ impl crate::server::frame_clock::Framing for GelfFrameDecoder { } impl Decoder for GelfFrameDecoder { - type Item = Bytes; + type Item = Frame; type Error = io::Error; fn decode( @@ -87,11 +90,14 @@ impl Decoder for GelfFrameDecoder { loop { if self.discarding { let Some(null_pos) = memchr::memchr(b'\0', src) else { + self.skipping.grow(src); src.clear(); return Ok(None); }; + self.skipping.grow(&src[..null_pos]); src.advance(null_pos + 1); self.discarding = false; + return Ok(Some(Frame::Oversize(self.skipping.finish()))); } match memchr::memchr(b'\0', src) { @@ -99,14 +105,16 @@ impl Decoder for GelfFrameDecoder { Some(0) => src.advance(1), Some(null_pos) if null_pos <= self.max_length => { let frame = src.split_to(null_pos + 1).freeze(); - return Ok(Some(frame.slice(..null_pos))); + return Ok(Some(Frame::Message(frame.slice(..null_pos)))); } Some(null_pos) => { - self.discarded += 1; + let frame = oversize::whole(&src[..null_pos], null_pos); src.advance(null_pos + 1); + return Ok(Some(Frame::Oversize(frame))); } None if src.len() > self.max_length => { - self.discarded += 1; + self.skipping = Skipping::unmeasured(); + self.skipping.grow(src); src.clear(); self.discarding = true; return Ok(None); @@ -115,6 +123,25 @@ impl Decoder for GelfFrameDecoder { } } } + + /// An oversize message the stream ended inside is handed over as far as it + /// arrived, and any other partial message is an error. + fn decode_eof( + &mut self, + src: &mut BytesMut, + ) -> std::result::Result, Self::Error> { + if let Some(frame) = self.decode(src)? { + return Ok(Some(frame)); + } + if let Some(skipped) = self.abandon() { + return Ok(Some(Frame::Oversize(skipped))); + } + if src.is_empty() { + Ok(None) + } else { + Err(io::Error::other("bytes remaining on stream")) + } + } } // --------------------------------------------------------------------------- @@ -145,10 +172,14 @@ async fn handle_tcp_connection( break; } result = StreamExt::next(&mut framed) => { - let oversize = framed.decoder_mut().get_mut().take_discarded(); - metrics.add_records_dropped(TRANSPORT, DropReason::Oversize, oversize); match result { - Some(Ok(raw)) => { + Some(Ok(Frame::Oversize(skipped))) => { + if !dead_letter_oversize(vec![skipped], &pipeline, &metrics, max_message_size, &shutdown).await { + debug!(peer = %peer_addr, "GELF TCP connection closing (shutdown during a DLQ write)"); + break; + } + } + Some(Ok(Frame::Message(raw))) => { metrics.inc_requests_total("gelf"); metrics.add_bytes_received("gelf", raw.len() as u64); @@ -169,6 +200,8 @@ async fn handle_tcp_connection( } Some(Err(e)) => { warn!(peer = %peer_addr, error = %e, "GELF TCP read error"); + let skipped: Vec = framed.decoder_mut().get_mut().abandon().into_iter().collect(); + dead_letter_oversize(skipped, &pipeline, &metrics, max_message_size, &shutdown).await; break; } None => { @@ -179,8 +212,27 @@ async fn handle_tcp_connection( } } } - let oversize = framed.decoder_mut().get_mut().take_discarded(); - metrics.add_records_dropped(TRANSPORT, DropReason::Oversize, oversize); +} + +/// Dead-letter the messages the framing skipped for their size. Returns false +/// when shutdown came first, and the caller must stop reading. +async fn dead_letter_oversize( + mut skipped: Vec, + pipeline: &PipelineState, + metrics: &Metrics, + limit: usize, + shutdown: &CancellationToken, +) -> bool { + oversize::dead_letter( + &mut skipped, + pipeline, + metrics, + TRANSPORT, + TRANSPORT, + limit, + shutdown, + ) + .await } // --------------------------------------------------------------------------- @@ -372,9 +424,19 @@ impl ProtocolHandler for GelfHandler { #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { + use bytes::Bytes; + use super::*; use crate::server::frame_clock::Framing; + /// The message a frame holds. + fn message(frame: Frame) -> Bytes { + match frame { + Frame::Message(message) => message, + Frame::Oversize(skipped) => panic!("a message was skipped: {skipped:?}"), + } + } + #[test] fn test_default_config() { let config = GelfConfig::default(); @@ -389,7 +451,7 @@ mod tests { let mut decoder = GelfFrameDecoder::new(1024); let mut buf = BytesMut::from(&br#"{"version":"1.1","host":"h","short_message":"m"}"#[..]); buf.extend_from_slice(b"\0"); - let result = decoder.decode(&mut buf).unwrap().unwrap(); + let result = message(decoder.decode(&mut buf).unwrap().unwrap()); let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); assert_eq!(json["host"], "h"); } @@ -403,8 +465,8 @@ mod tests { buf.extend_from_slice(br#"{"version":"1.1","host":"b","short_message":"2"}"#); buf.extend_from_slice(b"\0"); - let msg1 = decoder.decode(&mut buf).unwrap().unwrap(); - let msg2 = decoder.decode(&mut buf).unwrap().unwrap(); + let msg1 = message(decoder.decode(&mut buf).unwrap().unwrap()); + let msg2 = message(decoder.decode(&mut buf).unwrap().unwrap()); let j1: serde_json::Value = serde_json::from_slice(&msg1).unwrap(); let j2: serde_json::Value = serde_json::from_slice(&msg2).unwrap(); assert_eq!(j1["host"], "a"); @@ -418,16 +480,39 @@ mod tests { assert!(decoder.decode(&mut buf).unwrap().is_none()); } + /// What a decoder given `chunks` yields, and whether it is mid-frame after. + fn decode_chunks( + decoder: &mut GelfFrameDecoder, + chunks: &[&[u8]], + ) -> (Vec, Vec, bool) { + let mut buf = BytesMut::new(); + let (mut messages, mut skipped) = (Vec::new(), Vec::new()); + for chunk in chunks { + buf.extend_from_slice(chunk); + while let Some(frame) = decoder.decode(&mut buf).unwrap() { + match frame { + Frame::Message(message) => { + messages.push(String::from_utf8_lossy(&message).into_owned()); + } + Frame::Oversize(frame) => skipped.push(frame), + } + } + } + (messages, skipped, decoder.mid_frame()) + } + /// An oversize message costs itself and nothing after it, whether its - /// delimiter arrives in the same read or several reads later. The decoder - /// says it is mid-frame while it discards, so the frame timeout keeps running. + /// delimiter arrives in the same read or several reads later, and is handed + /// over once its delimiter has arrived. The decoder says it is mid-frame + /// while it skips, so the frame timeout keeps running. #[test] - fn an_oversize_message_is_discarded_and_counted() { + fn an_oversize_message_is_skipped_and_handed_over() { struct Case { name: &'static str, chunks: &'static [&'static [u8]], frames: &'static [&'static str], - discarded: u64, + /// The head and length of each oversize message handed over. + skipped: &'static [(&'static str, usize)], mid_frame: bool, } let cases = [ @@ -435,70 +520,78 @@ mod tests { name: "oversize with its delimiter in the same read", chunks: &[b"a\0this is too long\0b\0"], frames: &["a", "b"], - discarded: 1, + skipped: &[("this is too long", 16)], mid_frame: false, }, Case { name: "oversize split across reads", chunks: &[b"a\0this is too", b" long and still going", b" on\0b\0"], frames: &["a", "b"], - discarded: 1, + skipped: &[("this is too long and still going on", 35)], mid_frame: false, }, Case { name: "oversize still arriving", chunks: &[b"a\0this is too long", b" and no delimiter yet"], frames: &["a"], - discarded: 1, + skipped: &[], mid_frame: true, }, Case { name: "two oversize messages back to back", chunks: &[b"this is too long\0and so is this\0b\0"], frames: &["b"], - discarded: 2, + skipped: &[("this is too long", 16), ("and so is this", 14)], mid_frame: false, }, Case { name: "exactly the limit is kept", chunks: &[b"0123456789\0"], frames: &["0123456789"], - discarded: 0, + skipped: &[], mid_frame: false, }, Case { name: "empty messages are skipped", chunks: &[b"\0\0a\0", b"\0b\0"], frames: &["a", "b"], - discarded: 0, + skipped: &[], mid_frame: false, }, Case { name: "a partial message waits", chunks: &[b"a\0par"], frames: &["a"], - discarded: 0, + skipped: &[], mid_frame: false, }, ]; let mut failed = Vec::new(); for case in cases { - let mut decoder = GelfFrameDecoder::new(10); - let mut buf = BytesMut::new(); - let mut frames = Vec::new(); - for chunk in case.chunks { - buf.extend_from_slice(chunk); - while let Some(frame) = decoder.decode(&mut buf).unwrap() { - frames.push(String::from_utf8_lossy(&frame).into_owned()); - } - } - let got = (frames, decoder.take_discarded(), decoder.mid_frame()); + let (frames, skipped, mid_frame) = + decode_chunks(&mut GelfFrameDecoder::new(10), case.chunks); + let got = ( + frames, + skipped + .iter() + .map(|s| { + ( + String::from_utf8_lossy(&s.head).into_owned(), + s.original_bytes, + ) + }) + .collect::>(), + mid_frame, + ); let want = ( case.frames .iter() .map(ToString::to_string) .collect::>(), - case.discarded, + case.skipped + .iter() + .map(|(head, len)| ((*head).to_string(), *len)) + .collect::>(), case.mid_frame, ); if got != want { @@ -507,4 +600,31 @@ mod tests { } assert!(failed.is_empty(), "{}", failed.join("\n")); } + + /// A stream that ends or fails inside an oversize message still hands over + /// the part that arrived, and a long one keeps only its head. + #[test] + fn an_oversize_message_cut_by_the_stream_is_handed_over_as_far_as_it_arrived() { + let mut decoder = GelfFrameDecoder::new(10); + let mut buf = BytesMut::from(&b"this is too long and no delimiter"[..]); + assert!(decoder.decode(&mut buf).unwrap().is_none()); + let Some(Frame::Oversize(skipped)) = decoder.decode_eof(&mut buf).unwrap() else { + panic!("the oversize message was not handed over"); + }; + assert_eq!(skipped.original_bytes, 33); + assert_eq!(&skipped.head[..], b"this is too long and no delimiter"); + assert!(decoder.decode_eof(&mut buf).unwrap().is_none()); + + let mut decoder = GelfFrameDecoder::new(10); + let chunk = vec![b'x'; oversize::HEAD_BYTES]; + let mut buf = BytesMut::new(); + for _ in 0..3 { + buf.extend_from_slice(&chunk); + assert!(decoder.decode(&mut buf).unwrap().is_none()); + } + let skipped = decoder.abandon().unwrap(); + assert_eq!(skipped.original_bytes, 3 * oversize::HEAD_BYTES); + assert_eq!(skipped.head.len(), oversize::HEAD_BYTES); + assert!(decoder.abandon().is_none()); + } } diff --git a/src/server/hold.rs b/src/server/hold.rs index bf45c5a..0ffe4e0 100644 --- a/src/server/hold.rs +++ b/src/server/hold.rs @@ -27,10 +27,10 @@ use crate::pipeline::{Acks, Offer, PipelineState}; use crate::tenancy::Ingest; /// Wait before the first re-offer. -const FIRST_RETRY: Duration = Duration::from_millis(100); +pub(super) const FIRST_RETRY: Duration = Duration::from_millis(100); /// Longest wait between re-offers, so a pipeline that recovers is found within it. -const LAST_RETRY: Duration = Duration::from_secs(2); +pub(super) const LAST_RETRY: Duration = Duration::from_secs(2); /// How a hold ended. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/src/server/mod.rs b/src/server/mod.rs index d16da1d..6a34212 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -30,6 +30,7 @@ mod metric_record; pub mod netflow; #[cfg(feature = "otlp")] pub mod otlp; +pub mod oversize; pub mod prometheus_rw; pub mod raw_capture; pub mod sflow; diff --git a/src/server/oversize.rs b/src/server/oversize.rs new file mode 100644 index 0000000..455b5e6 --- /dev/null +++ b/src/server/oversize.rs @@ -0,0 +1,178 @@ +// Project: dfe-receiver +// File: src/server/oversize.rs +// Purpose: Dead-letter a frame a raw TCP listener skipped for its size +// Language: Rust +// +// License: BUSL-1.1 +// Copyright: (c) 2026 HYPERI PTY LIMITED + +//! Frames a raw TCP listener skips for their size. +//! +//! A syslog or GELF message over its listener's limit costs that message and +//! never the connection, and the message is not lost: its first bytes go to the +//! DLQ with the reason and its full length, and the listener counts it in +//! `receiver_records_dropped_total{reason="oversize"}` as before. A framing +//! codec hands over each skipped frame as [`Frame::Oversize`] in its place in +//! the stream, so a flood of them backs up into the socket as the DLQ confirms +//! each write, and the codec never holds more than one frame's head. +//! +//! These protocols carry no acknowledgement, so a DLQ write that is not +//! confirmed is offered again until it is, holding the connection as a pipeline +//! that cannot take a record does ([`super::hold`]). + +use bytes::{Bytes, BytesMut}; +use tokio_util::sync::CancellationToken; +use tracing::debug; + +use super::hold::{FIRST_RETRY, LAST_RETRY}; +use crate::buffer::rejects::Disposal; +use crate::metrics::{DropReason, Metrics}; +use crate::pipeline::PipelineState; + +/// The most bytes of a skipped frame a codec keeps for its dead letter: the +/// DLQ's own default payload cap, which it applies again to a smaller one. +pub const HEAD_BYTES: usize = scalo::dlq::DEFAULT_MAX_PAYLOAD_BYTES; + +/// A frame skipped for its size. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Oversize { + /// The frame's first bytes, up to [`HEAD_BYTES`]. A frame cut short by its + /// connection closing holds what arrived. + pub head: Bytes, + /// The frame's length: its octet count, or the bytes up to its delimiter. + pub original_bytes: usize, +} + +/// What a framing codec hands its listener. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Frame { + /// One message, without its framing. + Message(Bytes), + /// A message over the size limit, which the codec skipped. + Oversize(Oversize), +} + +/// The head and length of a frame the codec is still skipping. +#[derive(Debug, Default)] +pub struct Skipping { + head: BytesMut, + original_bytes: usize, +} + +impl Skipping { + /// A frame of a known length, whose bytes arrive as `keep` is given them. + pub fn counted(original_bytes: usize) -> Self { + Self { + head: BytesMut::new(), + original_bytes, + } + } + + /// A frame of unknown length, which `grow` measures. + pub fn unmeasured() -> Self { + Self::default() + } + + /// Keep `chunk` as far as the head has room. + pub fn keep(&mut self, chunk: &[u8]) { + let room = HEAD_BYTES - self.head.len(); + self.head.extend_from_slice(&chunk[..chunk.len().min(room)]); + } + + /// Count `chunk` toward the length and keep what the head has room for. + pub fn grow(&mut self, chunk: &[u8]) { + self.keep(chunk); + self.original_bytes += chunk.len(); + } + + /// The skipped frame, leaving this ready for the next. + pub fn finish(&mut self) -> Oversize { + Oversize { + head: std::mem::take(&mut self.head).freeze(), + original_bytes: std::mem::take(&mut self.original_bytes), + } + } +} + +/// One frame of `head` and `original_bytes`, whole in the codec's buffer. +pub fn whole(frame: &[u8], original_bytes: usize) -> Oversize { + Oversize { + head: Bytes::copy_from_slice(&frame[..frame.len().min(HEAD_BYTES)]), + original_bytes, + } +} + +/// Count `frames` as dropped for their size and dead-letter each, waiting on +/// the DLQ until it has taken or refused every one for good. +/// +/// Returns false when shutdown came first, and the caller must stop reading. +pub async fn dead_letter( + frames: &mut Vec, + pipeline: &PipelineState, + metrics: &Metrics, + transport: &str, + listener: &str, + limit: usize, + shutdown: &CancellationToken, +) -> bool { + metrics.add_records_dropped(transport, DropReason::Oversize, frames.len() as u64); + let reason = format!("frame over {limit} bytes on {listener}"); + for frame in frames.drain(..) { + let mut wait = FIRST_RETRY; + loop { + let disposal = pipeline + .rejects() + .dispose_head(&frame.head, frame.original_bytes, &reason) + .await; + if disposal != Disposal::Refused { + break; + } + debug!( + transport, + listener, "DLQ did not take an oversize frame; holding the connection" + ); + tokio::select! { + biased; + () = shutdown.cancelled() => return false, + () = tokio::time::sleep(wait) => {} + } + wait = (wait * 2).min(LAST_RETRY); + } + } + true +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn a_skipped_frame_keeps_its_head_and_counts_its_length() { + let mut frame = Skipping::unmeasured(); + let chunk = vec![b'x'; HEAD_BYTES]; + frame.grow(&chunk); + frame.grow(&chunk); + let out = frame.finish(); + assert_eq!(out.head.len(), HEAD_BYTES); + assert_eq!(out.original_bytes, 2 * HEAD_BYTES); + assert_eq!(frame.finish().original_bytes, 0, "ready for the next"); + } + + #[test] + fn a_counted_frame_keeps_its_declared_length_whatever_arrived() { + let mut frame = Skipping::counted(100_000); + frame.keep(b"abc"); + let out = frame.finish(); + assert_eq!(&out.head[..], b"abc"); + assert_eq!(out.original_bytes, 100_000); + } + + #[test] + fn a_whole_frame_is_cut_to_the_head_limit() { + let frame = vec![b'y'; HEAD_BYTES + 10]; + let out = whole(&frame, frame.len()); + assert_eq!(out.head.len(), HEAD_BYTES); + assert_eq!(out.original_bytes, HEAD_BYTES + 10); + } +} diff --git a/src/server/syslog/framing.rs b/src/server/syslog/framing.rs index 027f94f..a580c1b 100644 --- a/src/server/syslog/framing.rs +++ b/src/server/syslog/framing.rs @@ -15,8 +15,9 @@ //! //! A message over the size limit costs that message and never the connection: //! an octet-counted one is skipped by its count, a line up to its LF. Each is -//! counted for [`SyslogFrameDecoder::take_discarded`]. A count above the frame -//! limit is not trusted as a count, and that frame is read as a line. +//! handed over as [`Frame::Oversize`], with its first bytes and its length, for +//! the listener to dead-letter. A count above the frame limit is not trusted as +//! a count, and that frame is read as a line. //! //! A frame is its bytes: the encoding is the caller's to check, so a line that //! is not UTF-8 costs that line and not the connection. @@ -26,6 +27,8 @@ use std::io; use bytes::{Buf, Bytes, BytesMut}; use tokio_util::codec::Decoder; +use crate::server::oversize::{self, Frame, Oversize, Skipping}; + /// Default maximum syslog message size (64 KiB). pub const DEFAULT_MAX_LENGTH: usize = 64 * 1024; @@ -58,7 +61,8 @@ pub struct SyslogFrameDecoder { max_length: usize, max_frame: usize, state: State, - discarded: u64, + /// The head and length of the oversize message being skipped. + skipping: Skipping, } impl SyslogFrameDecoder { @@ -75,13 +79,17 @@ impl SyslogFrameDecoder { max_length, max_frame: max_frame.max(max_length), state: State::Start, - discarded: 0, + skipping: Skipping::unmeasured(), } } - /// Messages discarded for their size since the last call. - pub fn take_discarded(&mut self) -> u64 { - std::mem::take(&mut self.discarded) + /// The oversize message being skipped, as far as it arrived, for a stream + /// that ends or fails inside it. + pub fn abandon(&mut self) -> Option { + (self.state != State::Start).then(|| { + self.state = State::Start; + self.skipping.finish() + }) } /// How the frame at the start of `src` opens. @@ -127,7 +135,7 @@ impl Default for SyslogFrameDecoder { } impl Decoder for SyslogFrameDecoder { - type Item = Bytes; + type Item = Frame; type Error = io::Error; fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { @@ -136,20 +144,25 @@ impl Decoder for SyslogFrameDecoder { State::Start => {} State::Skip(left) => { let take = left.min(src.len()); + self.skipping.keep(&src[..take]); src.advance(take); if take < left { self.state = State::Skip(left - take); return Ok(None); } self.state = State::Start; + return Ok(Some(Frame::Oversize(self.skipping.finish()))); } State::Discard => { let Some(lf) = memchr::memchr(b'\n', src) else { + self.skipping.grow(src); src.clear(); return Ok(None); }; + self.skipping.grow(&src[..lf]); src.advance(lf + 1); self.state = State::Start; + return Ok(Some(Frame::Oversize(self.skipping.finish()))); } } @@ -161,8 +174,8 @@ impl Decoder for SyslogFrameDecoder { match self.opening(src) { Opening::Counted { header, count } if count > self.max_length => { - self.discarded += 1; src.advance(header); + self.skipping = Skipping::counted(count); self.state = State::Skip(count); continue; } @@ -173,7 +186,7 @@ impl Decoder for SyslogFrameDecoder { return Ok(None); } let frame = src.split_to(total).freeze(); - return Ok(Some(frame.slice(header..))); + return Ok(Some(Frame::Message(frame.slice(header..)))); } Opening::Undecided => return Ok(None), Opening::Line => {} @@ -182,14 +195,16 @@ impl Decoder for SyslogFrameDecoder { match memchr::memchr(b'\n', src) { Some(lf) if lf <= self.max_length => { let frame = src.split_to(lf + 1).freeze(); - return Ok(Some(frame.slice(..lf))); + return Ok(Some(Frame::Message(frame.slice(..lf)))); } Some(lf) => { - self.discarded += 1; + let frame = oversize::whole(&src[..lf], lf); src.advance(lf + 1); + return Ok(Some(Frame::Oversize(frame))); } None if src.len() > self.max_length => { - self.discarded += 1; + self.skipping = Skipping::unmeasured(); + self.skipping.grow(src); src.clear(); self.state = State::Discard; return Ok(None); @@ -200,20 +215,22 @@ impl Decoder for SyslogFrameDecoder { } /// A sender that closes after a last line with no newline still sent that - /// line, so it is the final frame. A partial octet-counted frame is not. + /// line, so it is the final frame. A partial octet-counted frame is not, and + /// an oversize message the stream ended inside is handed over as far as it + /// arrived. fn decode_eof(&mut self, src: &mut BytesMut) -> Result, Self::Error> { if let Some(frame) = self.decode(src)? { return Ok(Some(frame)); } + if let Some(skipped) = self.abandon() { + return Ok(Some(Frame::Oversize(skipped))); + } let rest = src.split(); let rest = rest.trim_ascii(); - if rest.is_empty() - || self.state != State::Start - || matches!(self.opening(rest), Opening::Counted { .. }) - { + if rest.is_empty() || matches!(self.opening(rest), Opening::Counted { .. }) { return Ok(None); } - Ok(Some(Bytes::copy_from_slice(rest))) + Ok(Some(Frame::Message(Bytes::copy_from_slice(rest)))) } } @@ -225,27 +242,45 @@ impl Decoder for SyslogFrameDecoder { #[allow(clippy::unwrap_used)] mod tests { use super::*; + use crate::server::oversize::HEAD_BYTES; - /// Every frame `data` holds, fed `chunk` bytes at a time and read to the - /// end of the stream when `eof`, with the count of messages discarded. - fn decode_chunked( + /// Every message `data` holds, fed `chunk` bytes at a time and read to the + /// end of the stream when `eof`, with the messages skipped for their size. + fn decode_all( decoder: &mut SyslogFrameDecoder, data: &[u8], chunk: usize, eof: bool, - ) -> (Vec, u64) { + ) -> (Vec, Vec) { let mut buf = BytesMut::new(); - let mut frames = Vec::new(); + let (mut messages, mut skipped) = (Vec::new(), Vec::new()); + let mut take = |frame: Frame| match frame { + Frame::Message(message) => messages.push(message), + Frame::Oversize(frame) => skipped.push(frame), + }; for piece in data.chunks(chunk.max(1)) { buf.extend_from_slice(piece); while let Some(frame) = decoder.decode(&mut buf).unwrap() { - frames.push(frame); + take(frame); } } - if eof && let Some(last) = decoder.decode_eof(&mut buf).unwrap() { - frames.push(last); + if eof { + while let Some(frame) = decoder.decode_eof(&mut buf).unwrap() { + take(frame); + } } - (frames, decoder.take_discarded()) + (messages, skipped) + } + + /// [`decode_all`] with the count of messages skipped. + fn decode_chunked( + decoder: &mut SyslogFrameDecoder, + data: &[u8], + chunk: usize, + eof: bool, + ) -> (Vec, u64) { + let (messages, skipped) = decode_all(decoder, data, chunk, eof); + (messages, skipped.len() as u64) } /// The bytes sent, whether the stream then ends, the frames expected and @@ -462,6 +497,89 @@ mod tests { assert_eq!(discarded, 0); } + /// A skipped message hands over its first bytes and its whole length, + /// however its bytes arrive, so the DLQ holds what was sent and says how + /// much more there was. + #[test] + fn a_skipped_message_hands_over_its_head_and_length() { + let octet = format!("70000 <14>{}", "a".repeat(70_000 - 4)); + let line = format!("<14>{}\n", "b".repeat(70_000)); + let huge = format!("<14>{}\n", "c".repeat(HEAD_BYTES + 1_000)); + let huge_count = HEAD_BYTES + 1_000; + let huge_octet = format!("{huge_count} <14>{}", "d".repeat(huge_count - 4)); + let huge_octet_start = huge_octet.find('<').unwrap(); + // (name, data, head, original bytes) + let cases: Vec<(&str, String, Vec, usize)> = vec![ + ( + "octet", + octet.clone(), + octet.as_bytes()[6..].to_vec(), + 70_000, + ), + ( + "line", + line.clone(), + line.as_bytes()[..70_004].to_vec(), + 70_004, + ), + ( + "huge_line", + huge.clone(), + huge.as_bytes()[..HEAD_BYTES].to_vec(), + HEAD_BYTES + 1_004, + ), + ( + "huge_octet", + huge_octet.clone(), + huge_octet.as_bytes()[huge_octet_start..][..HEAD_BYTES].to_vec(), + huge_count, + ), + ]; + for (name, data, head, original_bytes) in cases { + for chunk in [data.len(), 4_096, 1_000] { + let mut decoder = SyslogFrameDecoder::with_max_frame(64 * 1024, 2 * HEAD_BYTES); + let (messages, skipped) = decode_all(&mut decoder, data.as_bytes(), chunk, false); + assert_eq!(messages.len(), 0, "{name} (chunk {chunk})"); + assert_eq!( + skipped, + vec![Oversize { + head: Bytes::from(head.clone()), + original_bytes + }], + "{name} (chunk {chunk})" + ); + } + } + } + + /// A stream that ends inside a skipped message still hands over the part + /// that arrived, with the length the sender declared for a counted one. + #[test] + fn a_stream_ending_inside_a_skipped_message_hands_over_what_arrived() { + let counted = format!("80000 <14>{}", "e".repeat(1_000)); + let (messages, skipped) = decode_all( + &mut SyslogFrameDecoder::default(), + counted.as_bytes(), + 4_096, + true, + ); + assert_eq!(messages.len(), 0); + assert_eq!(skipped.len(), 1); + assert_eq!(skipped[0].original_bytes, 80_000); + assert_eq!(skipped[0].head.len(), 1_004); + + let line = format!("<14>{}", "f".repeat(70_000)); + let (_, skipped) = decode_all( + &mut SyslogFrameDecoder::default(), + line.as_bytes(), + 4_096, + true, + ); + assert_eq!(skipped.len(), 1); + assert_eq!(skipped[0].original_bytes, 70_004); + assert_eq!(skipped[0].head.len(), 70_004); + } + #[test] fn test_empty_input() { let mut decoder = SyslogFrameDecoder::default(); diff --git a/src/server/syslog/tcp.rs b/src/server/syslog/tcp.rs index 9999215..6a25758 100644 --- a/src/server/syslog/tcp.rs +++ b/src/server/syslog/tcp.rs @@ -33,12 +33,13 @@ use super::convert::Converter; use super::framing::SyslogFrameDecoder; use super::{TRANSPORT, decode_lossy, listener_ingest}; use crate::error::{Error, Result}; -use crate::metrics::{DropReason, Metrics}; +use crate::metrics::Metrics; use crate::pipeline::{Acks, PipelineState}; use crate::server::frame_clock::{Clocked, Framing}; use crate::server::hold::{Held, hold}; use crate::server::http::{Accept, ConnectionLimits}; use crate::server::ip_filter::IpFilter; +use crate::server::oversize::{self, Frame, Oversize}; use crate::server::traits::BoundAddr; /// The idle timer's period with idle closing off, when it is never polled. @@ -121,6 +122,7 @@ impl TcpListenerTask { shutdown: shutdown.clone(), peer_addr, connection: self.connection, + label: self.label, }; let accept = accept.clone(); let tls = self.tls.clone(); @@ -170,6 +172,8 @@ struct Reader { shutdown: CancellationToken, peer_addr: SocketAddr, connection: Connection, + /// `TCP` or `TLS`, the listener the frames arrive on. + label: &'static str, } impl Reader { @@ -182,6 +186,7 @@ impl Reader { )); let mut framed = FramedRead::new(stream, decoder); let mut payloads: Vec = Vec::new(); + let mut skipped: Vec = Vec::new(); let idle_for = c.idle.unwrap_or(NEVER); let idle = tokio::time::sleep(idle_for); tokio::pin!(idle); @@ -196,9 +201,6 @@ impl Reader { () = &mut idle, if c.idle.is_some() => { // A frame part way in is the frame timeout's to close. if framed.read_buffer().is_empty() && !framed.decoder().get_ref().mid_frame() { - let oversize = framed.decoder_mut().get_mut().take_discarded(); - self.metrics - .add_records_dropped(TRANSPORT, DropReason::Oversize, oversize); self.metrics.inc_connection_closed(TRANSPORT, "idle"); debug!(peer = %self.peer_addr, "Syslog TCP connection closing (idle)"); return; @@ -211,16 +213,31 @@ impl Reader { let end = match first { Some(Ok(frame)) => { - self.push(&frame, &mut payloads); - self.drain_buffered(&mut framed, &mut payloads) + self.take(frame, &mut payloads, &mut skipped); + self.drain_buffered(&mut framed, &mut payloads, &mut skipped) } Some(Err(e)) => Some(End::Failed(e)), None => Some(End::Closed), }; - let oversize = framed.decoder_mut().get_mut().take_discarded(); - self.metrics - .add_records_dropped(TRANSPORT, DropReason::Oversize, oversize); + if matches!(end, Some(End::Failed(_))) { + skipped.extend(framed.decoder_mut().get_mut().abandon()); + } + let listener = format!("syslog {}", self.label); + if !oversize::dead_letter( + &mut skipped, + &self.pipeline, + &self.metrics, + TRANSPORT, + &listener, + c.max_message_size, + &self.shutdown, + ) + .await + { + debug!(peer = %self.peer_addr, "Syslog TCP connection closing (shutdown during a DLQ write)"); + return; + } if !payloads.is_empty() { let held = hold( &self.pipeline, @@ -268,16 +285,17 @@ impl Reader { &self, framed: &mut FramedRead, payloads: &mut Vec, + skipped: &mut Vec, ) -> Option where - D: Decoder, + D: Decoder, { // FramedRead lends its buffer and its decoder one at a time, so the // buffer comes out for the drain and goes back after it. let mut buffer = std::mem::take(framed.read_buffer_mut()); let end = loop { match framed.decoder_mut().decode(&mut buffer) { - Ok(Some(frame)) => self.push(&frame, payloads), + Ok(Some(frame)) => self.take(frame, payloads, skipped), Ok(None) => break None, Err(e) => break Some(End::Failed(e)), } @@ -286,6 +304,15 @@ impl Reader { end } + /// Add a message's record to the batch, or keep a message skipped for its + /// size for the DLQ. + fn take(&self, frame: Frame, payloads: &mut Vec, skipped: &mut Vec) { + match frame { + Frame::Message(message) => self.push(&message, payloads), + Frame::Oversize(frame) => skipped.push(frame), + } + } + /// Count one frame and add its record to the batch. fn push(&self, frame: &[u8], payloads: &mut Vec) { self.metrics.inc_requests_total(TRANSPORT); diff --git a/tests/common/mod.rs b/tests/common/mod.rs index c662805..bc8f9be 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -853,3 +853,33 @@ pub async fn stored_within( } records } + +/// Point `config` at a file DLQ under `dir`, so the dead letters it takes can +/// be read back with [`file_dead_letters`]. +pub fn file_dlq(config: &mut dfe_receiver::config::Config, dir: &std::path::Path) { + config.routing.dlq.enabled = true; + config.routing.dlq.mode = "file_only".to_string(); + config.routing.dlq.kafka_enabled = false; + config.routing.dlq.file_path = dir.display().to_string(); +} + +/// Every dead letter written to a file DLQ under `dir`, in file order. +pub fn file_dead_letters(dir: &std::path::Path) -> Vec { + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut letters = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + letters.extend(file_dead_letters(&path)); + } else { + let text = std::fs::read_to_string(&path).unwrap_or_default(); + letters.extend( + text.lines() + .filter_map(|line| serde_json::from_str(line).ok()), + ); + } + } + letters +} diff --git a/tests/integration/gelf.rs b/tests/integration/gelf.rs index f39fb77..3707e27 100644 --- a/tests/integration/gelf.rs +++ b/tests/integration/gelf.rs @@ -32,8 +32,8 @@ fn gelf(marker: &str) -> Vec { } /// An oversize message costs itself and nothing after it on the same -/// connection, whether its delimiter came in the same read or after a pause, -/// and each one is counted dropped. +/// connection, whether its delimiter came in the same read or after a pause. +/// Each one is counted dropped and goes to the DLQ with its length. #[tokio::test] async fn gelf_carries_on_past_oversize_messages() { let (loader_endpoint, loader) = grpc_destination().await; @@ -43,7 +43,8 @@ async fn gelf_carries_on_past_oversize_messages() { config.destinations.default = "loader".into(); config.loader.transport = "grpc".to_string(); config.loader.grpc_endpoint = Some(loader_endpoint); - config.routing.dlq.enabled = false; + let dlq_dir = tempfile::tempdir().unwrap(); + crate::common::file_dlq(&mut config, dlq_dir.path()); config.gelf.enabled = true; config.gelf.bind_address = "127.0.0.1:0".to_string(); config.gelf.accept_unauthenticated = true; @@ -92,6 +93,14 @@ async fn gelf_carries_on_past_oversize_messages() { 2, "both oversize messages counted" ); + let letters = crate::common::file_dead_letters(dlq_dir.path()); + assert_eq!(letters.len(), 2, "dead letters written"); + for (letter, original) in letters.iter().zip([2_000, 3_003]) { + assert_eq!(letter["service"], "receiver"); + assert_eq!(letter["reason"], "frame over 1024 bytes on gelf"); + assert_eq!(letter["original_bytes"], original); + assert_eq!(letter["_raw"].as_str().map(str::len), Some(original)); + } tcp.write_all(&gelf("fourth")).await.unwrap(); let after = receive_records(&loader, 1, Duration::from_secs(10)).await; diff --git a/tests/integration/syslog.rs b/tests/integration/syslog.rs index b7ed314..1e68601 100644 --- a/tests/integration/syslog.rs +++ b/tests/integration/syslog.rs @@ -694,9 +694,10 @@ async fn a_message_that_is_not_utf8_lands_parsed_over_udp_and_tcp() { assert_eq!(syslog.metrics.get_requests_error(), 0); } -/// Over TCP an oversize message costs itself and nothing after it, a line -/// opening with digits is a line, and a stamp with no zone takes -/// `syslog.timezone`. Over UDP a burst larger than a batch arrives whole. +/// Over TCP an oversize message costs itself and nothing after it, and goes to +/// the DLQ with its length. A line opening with digits is a line, and a stamp +/// with no zone takes `syslog.timezone`. Over UDP a burst larger than a batch +/// arrives whole. #[tokio::test] async fn syslog_carries_on_past_oversize_messages_and_reads_bursts_whole() { use tokio::io::AsyncWriteExt; @@ -705,7 +706,9 @@ async fn syslog_carries_on_past_oversize_messages_and_reads_bursts_whole() { let mut config = test_config(); config.loader.transport = "grpc".to_string(); config.loader.grpc_endpoint = Some(loader_endpoint); - config.routing.dlq.enabled = false; + let dlq_dir = tempfile::tempdir().unwrap(); + crate::common::file_dlq(&mut config, dlq_dir.path()); + let limit = config.syslog.max_message_size; config.syslog.timezone = "+10:00".parse().unwrap(); config.syslog.udp_batch_size = 16; let syslog = start_syslog_handler(config).await; @@ -716,6 +719,7 @@ async fn syslog_carries_on_past_oversize_messages_and_reads_bursts_whole() { wire.push_str(&format!("{} {second}", second.len())); wire.push_str("2026 Oct 31 10:00:00 h app: third\n"); wire.push_str(&format!("<14>{}\n", "b".repeat(70_000))); + wire.push_str(&format!("<14>{}\n", "c".repeat(600_000))); wire.push_str("<14>Oct 31 10:00:00 h app: fourth\n"); let mut tcp = tokio::net::TcpStream::connect(syslog.tcp).await.unwrap(); tcp.write_all(wire.as_bytes()).await.unwrap(); @@ -743,8 +747,27 @@ async fn syslog_carries_on_past_oversize_messages_and_reads_bursts_whole() { ); assert_eq!( syslog.metrics.get_records_dropped(), - 2, - "both oversize messages counted" + 3, + "every oversize message counted" + ); + let letters = crate::common::file_dead_letters(dlq_dir.path()); + assert_eq!(letters.len(), 3, "dead letters written"); + for (letter, original) in letters.iter().zip([70_000, 70_004, 600_004]) { + assert_eq!(letter["service"], "receiver"); + assert_eq!( + letter["reason"], + format!("frame over {limit} bytes on syslog TCP") + ); + assert_eq!(letter["original_bytes"], original); + } + assert_eq!(letters[0]["_raw"].as_str().map(str::len), Some(70_000)); + assert_eq!(letters[1]["truncated"], serde_json::Value::Null); + // The 600,004-byte line keeps the DLQ's 512 KiB head, as base64. + assert_eq!(letters[2]["truncated"], true); + assert_eq!(letters[2]["_raw_encoding"], "base64"); + assert_eq!( + letters[2]["_raw"].as_str().map(str::len), + Some(524_288_usize.div_ceil(3) * 4) ); // The connection is still open: another line still lands.