From 1493ca3e8859652ebe04ed275e6a142cd2a2b53b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 05:20:50 +0000 Subject: [PATCH 1/8] feat(stream)!: align SseStream with eventsource-parser The AI SDK's parseJsonEventStream pipes through eventsource-parser. Replace the blank-line frame splitter with its line-based state machine: - \n, \r and \r\n line endings in any mix; a trailing \r waits for a possible \n in the next chunk - strip a UTF-8 BOM at the start of the stream - a field line without ':' has an empty value; empty event -> None; id containing U+0000 ignored; retry accepted only when all ASCII digits - exceeding the buffer limit (event data + partial line) is fatal and ends the stream, as maxBufferSize does upstream Kept from aimux: strict UTF-8 decoding after reassembly (an invalid line now drops only the event being built) and a default 1 MiB bound. extract_lines already matched upstream; the [DONE]-skipping JSON layer in aimux-provider-utils already matches parseJsonEventStream. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015EYDYDWYcPsDmjBjFeuDVe --- aimux-stream/src/sse.rs | 372 ++++++++++++++++++++------------- aimux-stream/tests/sse_test.rs | 184 +++++++++++++++- 2 files changed, 406 insertions(+), 150 deletions(-) diff --git a/aimux-stream/src/sse.rs b/aimux-stream/src/sse.rs index c53f66e40..73eb360fa 100644 --- a/aimux-stream/src/sse.rs +++ b/aimux-stream/src/sse.rs @@ -1,4 +1,28 @@ //! SSE (Server-Sent Events) parser for streaming model responses. +//! +//! Line-based incremental parser following `eventsource-parser`'s +//! `createParser` (the parser behind the AI SDK's `parseJsonEventStream`) and +//! the WHATWG "parsing an event stream" algorithm: +//! +//! - lines end in `\n`, `\r` or `\r\n`, freely mixed; a `\r` at the end of +//! the buffered input waits for the next chunk in case a `\n` follows; +//! - a UTF-8 BOM at the very start of the stream is stripped; +//! - a blank line dispatches the event, but only if it had at least one +//! `data` line; field state resets after every blank line; +//! - `field: value` removes exactly one leading U+0020 SPACE; a line without +//! `:` is a field with an empty value; `:`-prefixed lines are comments; +//! - an empty `event` value means no event type; an `id` containing U+0000 is +//! ignored; `retry` is accepted only when it is all ASCII digits; unknown +//! fields are ignored; +//! - a partial event at end of stream is not dispatched. +//! +//! Two aimux additions: every line is strictly UTF-8 decoded after it has +//! been reassembled (a code point split across chunks is never corrupted, and +//! invalid UTF-8 surfaces as [`SseError::Utf8`] and drops the event being +//! built), and buffered input is bounded by `max_event_size` (the parser's +//! `maxBufferSize`; exceeding it is fatal and ends the stream, as upstream). + +use std::collections::VecDeque; use bytes::Bytes; use futures::Stream; @@ -7,9 +31,11 @@ use std::pin::Pin; use std::task::{Context, Poll}; use thiserror::Error; -/// Default upper bound on a single buffered SSE event's size (1 MiB). +/// Default upper bound on buffered event data plus any partial line (1 MiB). const DEFAULT_MAX_EVENT_SIZE: usize = 1024 * 1024; +const BOM: &[u8] = b"\xEF\xBB\xBF"; + #[derive(Debug, Error)] pub enum SseError { #[error("utf-8 decode error: {0}")] @@ -23,30 +49,188 @@ pub enum SseError { /// A parsed SSE event. #[derive(Debug, Clone, Default)] pub struct SseEvent { - /// The `event:` field (optional). + /// The `event:` field (`None` when absent or empty). pub event: Option, - /// The `data:` field. + /// The `data:` field; multiple `data` lines are joined with `\n`. pub data: String, /// The `id:` field (optional). pub id: Option, - /// The `retry:` field (optional). + /// The `retry:` field (optional; all-digit values only). pub retry: Option, } +/// Incremental SSE line parser (the `createParser` state machine). +#[derive(Debug)] +struct Parser { + /// Bytes after the last complete line terminator. + pending: Vec, + /// Offset in `pending` from which the next terminator search starts, so a + /// long line fed in many small chunks is scanned once, not quadratically. + scan_from: usize, + bom_checked: bool, + event: Option, + data: String, + data_lines: usize, + id: Option, + retry: Option, + max_size: usize, + terminated: bool, + ready: VecDeque>, +} + +impl Parser { + fn new(max_size: usize) -> Self { + Self { + pending: Vec::new(), + scan_from: 0, + bom_checked: false, + event: None, + data: String::new(), + data_lines: 0, + id: None, + retry: None, + max_size, + terminated: false, + ready: VecDeque::new(), + } + } + + fn feed(&mut self, chunk: &[u8]) { + if self.terminated { + return; + } + self.pending.extend_from_slice(chunk); + + if !self.bom_checked { + // Wait until the first three bytes can rule a BOM in or out. + if self.pending.len() < BOM.len() && BOM.starts_with(&self.pending) { + return; + } + if self.pending.starts_with(BOM) { + self.pending.drain(..BOM.len()); + } + self.bom_checked = true; + } + + let mut line_start = 0; + let mut search = self.scan_from; + while let Some(offset) = self.pending[search..] + .iter() + .position(|&b| b == b'\n' || b == b'\r') + { + let end = search + offset; + let is_cr = self.pending[end] == b'\r'; + // A trailing `\r` may be the first half of a `\r\n` split across + // chunks: defer it until more input arrives. + if is_cr && end + 1 == self.pending.len() { + break; + } + let line = self.pending[line_start..end].to_vec(); + self.parse_line(&line); + if self.terminated { + return; + } + line_start = end + 1; + if is_cr && self.pending.get(line_start) == Some(&b'\n') { + line_start += 1; + } + search = line_start; + } + self.pending.drain(..line_start); + // Resume the next search at the deferred `\r`, if any. + self.scan_from = self.pending.len().saturating_sub(1); + if self.pending.last() != Some(&b'\r') { + self.scan_from = self.pending.len(); + } + self.check_size(self.pending.len()); + } + + fn parse_line(&mut self, line: &[u8]) { + if line.is_empty() { + self.dispatch(); + return; + } + let line = match String::from_utf8(line.to_vec()) { + Ok(line) => line, + Err(error) => { + // The event being built can no longer be trusted. + self.reset_event(); + self.ready.push_back(Err(SseError::Utf8(error))); + return; + } + }; + if line.starts_with(':') { + return; // comment + } + let (field, value) = match line.split_once(':') { + Some((field, value)) => (field, value.strip_prefix(' ').unwrap_or(value)), + None => (line.as_str(), ""), + }; + match field { + "event" => self.event = (!value.is_empty()).then(|| value.to_string()), + "data" => { + if self.data_lines > 0 { + self.data.push('\n'); + } + self.data.push_str(value); + self.data_lines += 1; + // Lines already consumed from `pending` are not buffered any + // more; only the event data counts until the chunk is done. + self.check_size(0); + } + "id" => { + if !value.contains('\0') { + self.id = Some(value.to_string()); + } + } + "retry" => { + if !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()) { + self.retry = value.parse().ok(); + } + } + _ => {} // unknown field + } + } + + fn dispatch(&mut self) { + if self.data_lines > 0 { + self.ready.push_back(Ok(SseEvent { + event: self.event.take(), + data: std::mem::take(&mut self.data), + id: self.id.take(), + retry: self.retry.take(), + })); + } + self.reset_event(); + } + + fn reset_event(&mut self) { + self.event = None; + self.data.clear(); + self.data_lines = 0; + self.id = None; + self.retry = None; + } + + /// `maxBufferSize`: buffered event data plus the partial line. Exceeding + /// it is fatal, as in `EventSourceParserStream`. + fn check_size(&mut self, pending_len: usize) { + if self.terminated || pending_len + self.data.len() <= self.max_size { + return; + } + self.terminated = true; + self.pending.clear(); + self.reset_event(); + self.ready.push_back(Err(SseError::FrameTooLarge)); + } +} + pin_project! { /// An adapter that decodes a byte stream into SSE events. - /// - /// Bytes are accumulated in a raw `Vec` buffer and split on the SSE - /// event terminator (a blank line). Each complete frame is strictly - /// UTF-8 decoded only *after* reassembly, so a multi-byte character split - /// across two network chunks is never corrupted into replacement chars — - /// unlike a per-chunk `String::from_utf8_lossy` decode, which would emit a - /// `U+FFFD` on each side of the split. pub struct SseStream { #[pin] inner: S, - buffer: Vec, - max_event_size: usize, + parser: Parser, done: bool, _err: std::marker::PhantomData, } @@ -60,16 +244,13 @@ where Self::with_max_event_size(stream, DEFAULT_MAX_EVENT_SIZE) } - /// Create an [`SseStream`] with a custom per-event size limit. A single - /// event frame (the bytes between two terminators, excluding the - /// terminator itself) larger than `max_event_size` bytes yields - /// [`SseError::FrameTooLarge`], as does a buffer that grows past the limit - /// while waiting for a terminator. + /// Create an [`SseStream`] with a custom buffer limit: when buffered event + /// data plus the current partial line exceed `max_event_size` bytes, the + /// stream yields [`SseError::FrameTooLarge`] and ends. pub fn with_max_event_size(stream: S, max_event_size: usize) -> Self { Self { inner: stream, - buffer: Vec::new(), - max_event_size, + parser: Parser::new(max_event_size), done: false, _err: std::marker::PhantomData, } @@ -87,163 +268,58 @@ where let this = self.as_mut().get_mut(); loop { - // Try to emit a complete event from the buffer. - if let Some((frame_len, sep_len)) = find_separator(&this.buffer) { - if frame_len > this.max_event_size { - // Drop the oversized frame so a retried poll makes progress. - this.buffer.drain(..frame_len + sep_len); - return Poll::Ready(Some(Err(SseError::FrameTooLarge))); - } - // Extract the frame bytes and drop the terminator. Decoding the - // fully reassembled frame with `String::from_utf8` (rather than - // `from_utf8_lossy` per chunk) preserves code points split - // across chunks and surfaces invalid UTF-8 as an error. - let frame_bytes: Vec = this.buffer.drain(..frame_len).collect(); - this.buffer.drain(..sep_len); - let frame = match String::from_utf8(frame_bytes) { - Ok(s) => s, - Err(e) => return Poll::Ready(Some(Err(SseError::Utf8(e)))), - }; - let (event, has_data_line) = try_parse_event(&frame); - // Per the SSE spec / eventsource-parser, an event is dispatched - // only if it had at least one `data:` line (dataLines > 0). - // This covers comment-only, `event:`/`id:`/`retry:`-only, and - // blank-line keep-alives (all dispatched as nothing), while an - // explicit empty data line (`data:\n\n`) is still dispatched. - if !has_data_line { - continue; - } - return Poll::Ready(Some(Ok(event))); - } - - // No complete event yet. Guard against unbounded buffer growth when - // a terminator never arrives. The partial can never form a valid - // frame under the limit, so drop it — a retried poll then makes - // progress instead of looping on the same oversized buffer. - if this.buffer.len() > this.max_event_size { - this.buffer.clear(); - return Poll::Ready(Some(Err(SseError::FrameTooLarge))); + if let Some(item) = this.parser.ready.pop_front() { + return Poll::Ready(Some(item)); } - - if this.done { - // No terminating blank line: the buffered partial is not a - // complete event and is dropped. This matches eventsource-parser's - // `EventSourceParserStream`, which has no flush handler — a - // partial event at EOF is simply not dispatched. + // A partial event at end of stream is not dispatched: the + // upstream stream has no flush handler. + if this.done || this.parser.terminated { return Poll::Ready(None); } - - // Read more data. match Pin::new(&mut this.inner).poll_next(cx) { - Poll::Ready(Some(Ok(bytes))) => { - this.buffer.extend_from_slice(&bytes); - } + Poll::Ready(Some(Ok(bytes))) => this.parser.feed(&bytes), Poll::Ready(Some(Err(e))) => { return Poll::Ready(Some(Err(SseError::Stream(e.to_string())))); } - Poll::Ready(None) => { - this.done = true; - } + Poll::Ready(None) => this.done = true, Poll::Pending => return Poll::Pending, } } } } -/// Locate the next SSE event terminator (a blank line) in `buf`. -/// -/// Returns `(frame_len, sep_len)` where `frame_len` is the number of bytes -/// *before* the terminator and `sep_len` is the terminator's length. Prefers -/// `\n\n` (so a pure-CRLF stream — which contains no bare `\n\n` — falls -/// through to `\r\n\r\n`), mirroring the str-based -/// `find("\n\n").or_else(|| find("\r\n\r\n"))`. -fn find_separator(buf: &[u8]) -> Option<(usize, usize)> { - if let Some(pos) = find_subsequence(buf, b"\n\n") { - return Some((pos, 2)); - } - find_subsequence(buf, b"\r\n\r\n").map(|pos| (pos, 4)) -} - -/// First index of `needle` in `haystack`, comparing raw bytes. -fn find_subsequence(haystack: &[u8], needle: &[u8]) -> Option { - haystack.windows(needle.len()).position(|w| w == needle) -} - -/// Parse a single complete SSE event frame (the bytes between two -/// terminators, already strictly UTF-8 decoded) into an [`SseEvent`] plus a -/// flag indicating whether the event contained at least one `data:` line. -/// -/// A complete event is terminated by a blank line: `\n\n` or `\r\n\r\n` (the -/// terminator is consumed by the caller before this runs). Per the SSE spec, -/// exactly one leading U+0020 SPACE after the `:` is removed from each field -/// value (not all leading whitespace). Comment lines (starting with `:`) and -/// unknown fields are ignored. -fn try_parse_event(frame: &str) -> (SseEvent, bool) { - let mut event = SseEvent::default(); - let mut has_data_line = false; - for line in frame.lines() { - if let Some(value) = field_value(line, "data:") { - has_data_line = true; - if event.data.is_empty() { - event.data = value.to_string(); - } else { - event.data.push('\n'); - event.data.push_str(value); - } - } else if let Some(value) = field_value(line, "event:") { - event.event = Some(value.to_string()); - } else if let Some(value) = field_value(line, "id:") { - event.id = Some(value.to_string()); - } else if let Some(value) = field_value(line, "retry:") { - event.retry = value.parse().ok(); - } - // Comment lines (`:` prefix) and unknown fields are ignored. - } - - (event, has_data_line) -} - -/// Strip `prefix` from `line` and remove exactly one leading U+0020 SPACE from -/// the remainder (per the SSE spec). Returns `None` if `line` does not start -/// with `prefix`. -fn field_value<'a>(line: &'a str, prefix: &str) -> Option<&'a str> { - let rest = line.strip_prefix(prefix)?; - // Remove exactly one leading space, if present. - Some(rest.strip_prefix(' ').unwrap_or(rest)) -} - #[cfg(test)] mod tests { use super::*; + fn parse(input: &str) -> Vec { + let mut parser = Parser::new(DEFAULT_MAX_EVENT_SIZE); + parser.feed(input.as_bytes()); + parser.ready.into_iter().map(Result::unwrap).collect() + } + #[test] fn parse_single_event() { - let (event, has_data_line) = try_parse_event("data: hello world"); - assert_eq!(event.data, "hello world"); - assert!(has_data_line); + assert_eq!(parse("data: hello world\n\n")[0].data, "hello world"); } #[test] fn parse_multi_line_data() { - let (event, has_data_line) = try_parse_event("data: line1\ndata: line2"); - assert_eq!(event.data, "line1\nline2"); - assert!(has_data_line); + assert_eq!( + parse("data: line1\ndata: line2\n\n")[0].data, + "line1\nline2" + ); } #[test] fn parse_event_with_type() { - let (event, has_data_line) = try_parse_event("event: message\ndata: payload"); - assert_eq!(event.event.as_deref(), Some("message")); - assert_eq!(event.data, "payload"); - assert!(has_data_line); + let events = parse("event: message\ndata: payload\n\n"); + assert_eq!(events[0].event.as_deref(), Some("message")); + assert_eq!(events[0].data, "payload"); } #[test] - fn parse_event_without_data_has_no_data_line() { - // An `event:`-only event has no data line -> not dispatched. - let (event, has_data_line) = try_parse_event("event: ping"); - assert_eq!(event.event.as_deref(), Some("ping")); - assert!(event.data.is_empty()); - assert!(!has_data_line); + fn parse_event_without_data_is_not_dispatched() { + assert!(parse("event: ping\n\n").is_empty()); } } diff --git a/aimux-stream/tests/sse_test.rs b/aimux-stream/tests/sse_test.rs index 65e8f3440..58be9a19b 100644 --- a/aimux-stream/tests/sse_test.rs +++ b/aimux-stream/tests/sse_test.rs @@ -6,12 +6,17 @@ //! //! The expected behavior mirrors `eventsource-parser`'s `EventSourceParserStream` //! (which the TS `parseJsonEventStream` pipes through): -//! - an event is dispatched only on a terminating blank line (`\n\n`/`\r\n\r\n`); +//! - lines end in `\n`, `\r` or `\r\n` (freely mixed); an event is +//! dispatched only on a blank line; //! - an event with no `data:` line is NOT dispatched (covers comment-only, //! `event:`/`id:`/`retry:`-only, and blank-line keep-alives); //! - exactly one leading U+0020 SPACE after the `:` is stripped from a field //! value (per the SSE spec); -//! - a partial event at EOF (no terminating blank line) is dropped. +//! - a partial event at EOF (no terminating blank line) is dropped; +//! - a UTF-8 BOM at stream start is stripped; a line without `:` is a field +//! with an empty value; an empty `event` means no type; an `id` with +//! U+0000 is ignored; `retry` must be all ASCII digits; +//! - exceeding the buffer limit is fatal and ends the stream. use aimux_stream::{SseError, SseEvent, SseStream}; use bytes::Bytes; @@ -353,3 +358,178 @@ async fn buffer_growing_past_limit_without_terminator_returns_frame_too_large() assert_eq!(results.len(), 1); assert!(matches!(results[0], Err(SseError::FrameTooLarge))); } + +// ── eventsource-parser alignment ───────────────────────────────────────── + +#[tokio::test] +async fn bare_cr_line_endings() { + let events = collect_events(vec![ + "event: a\rdata: one\rdata: two\r\rdata: next\r\r:keep-alive\n", + ]) + .await; + assert_eq!(events.len(), 2); + let first = events[0].as_ref().unwrap(); + assert_eq!(first.event.as_deref(), Some("a")); + assert_eq!(data(first), "one\ntwo"); + assert_eq!(data(events[1].as_ref().unwrap()), "next"); +} + +#[tokio::test] +async fn trailing_cr_at_end_of_stream_does_not_dispatch() { + // A `\r` at the end of the buffered input waits for a possible `\n`; with + // no flush at end of stream (as `EventSourceParserStream`) the event is + // never completed. + let events = collect_events(vec!["data: a\r\rdata: b\r\r"]).await; + assert_eq!(events.len(), 1); + assert_eq!(data(events[0].as_ref().unwrap()), "a"); +} + +#[tokio::test] +async fn mixed_terminators_within_one_event() { + // `\r\n` then a bare `\n` blank line: one event, as eventsource-parser. + let events = collect_events(vec!["data: a\r\ndata: b\n\ndata: c\r\r\n"]).await; + assert_eq!(events.len(), 2); + assert_eq!(data(events[0].as_ref().unwrap()), "a\nb"); + assert_eq!(data(events[1].as_ref().unwrap()), "c"); +} + +#[tokio::test] +async fn crlf_split_between_chunks_is_one_terminator() { + // A `\r` at the end of a chunk must wait: the `\n` in the next chunk + // completes the same terminator, not an extra blank line. + let events = collect_events(vec!["data: a\r", "\ndata: b\r", "\n\r", "\n"]).await; + assert_eq!(events.len(), 1); + assert_eq!(data(events[0].as_ref().unwrap()), "a\nb"); +} + +#[tokio::test] +async fn leading_bom_is_stripped() { + let events = collect_events(vec!["\u{FEFF}data: first\n\n"]).await; + assert_eq!(events.len(), 1); + assert_eq!(data(events[0].as_ref().unwrap()), "first"); +} + +#[tokio::test] +async fn leading_bom_split_across_chunks_is_stripped() { + let bom = "\u{FEFF}".as_bytes(); + let events = collect_events_bytes(vec![ + bom[..1].to_vec(), + bom[1..].to_vec(), + b"data: first\n\n".to_vec(), + ]) + .await; + assert_eq!(events.len(), 1); + assert_eq!(data(events[0].as_ref().unwrap()), "first"); +} + +#[tokio::test] +async fn bom_after_stream_start_is_not_stripped() { + // Only the very start of the stream may carry a BOM; later it is part of + // the field name, which is then unknown. + let events = collect_events(vec!["data: a\n\n\u{FEFF}data: b\n\n"]).await; + assert_eq!(events.len(), 1); + assert_eq!(data(events[0].as_ref().unwrap()), "a"); +} + +#[tokio::test] +async fn field_without_colon_has_empty_value() { + // `data` alone is a data line with an empty value. + let events = collect_events(vec!["data\n\ndata\ndata: x\n\n"]).await; + assert_eq!(events.len(), 2); + assert_eq!(data(events[0].as_ref().unwrap()), ""); + assert_eq!(data(events[1].as_ref().unwrap()), "\nx"); +} + +#[tokio::test] +async fn empty_event_field_means_no_event_type() { + let events = collect_events(vec!["event:\ndata: x\n\n"]).await; + assert_eq!(events[0].as_ref().unwrap().event, None); +} + +#[tokio::test] +async fn event_type_resets_after_each_dispatch() { + let events = collect_events(vec!["event: a\ndata: 1\n\ndata: 2\n\n"]).await; + assert_eq!(events[0].as_ref().unwrap().event.as_deref(), Some("a")); + assert_eq!(events[1].as_ref().unwrap().event, None); +} + +#[tokio::test] +async fn id_containing_null_is_ignored() { + let events = collect_events(vec!["id: ok\nid: bad\u{0}id\ndata: x\n\n"]).await; + assert_eq!(events[0].as_ref().unwrap().id.as_deref(), Some("ok")); +} + +#[tokio::test] +async fn retry_must_be_all_ascii_digits() { + // `retry: 5` loses its one leading space and is valid; a second space is not. + for value in ["+5", "-5", "5s", " 5", ""] { + let input = format!("retry:{value}\ndata: x\n\n"); + let events = collect_events(vec![input.as_str()]).await; + assert_eq!(events[0].as_ref().unwrap().retry, None, "retry {value:?}"); + } + let events = collect_events(vec!["retry:1500\ndata: x\n\n"]).await; + assert_eq!(events[0].as_ref().unwrap().retry, Some(1500)); +} + +#[tokio::test] +async fn field_names_are_case_sensitive() { + let events = collect_events(vec!["Data: x\n\ndata: y\n\n"]).await; + assert_eq!(events.len(), 1); + assert_eq!(data(events[0].as_ref().unwrap()), "y"); +} + +#[tokio::test] +async fn invalid_utf8_drops_only_the_current_event() { + let mut bytes = b"data: ".to_vec(); + bytes.extend_from_slice(&[0xE4, 0xBD]); + bytes.extend_from_slice(b"\ndata: tail\n\ndata: next\n\n"); + let events = collect_events_bytes(vec![bytes]).await; + assert_eq!(events.len(), 3); + assert!(matches!(events[0], Err(SseError::Utf8(_)))); + // The line after the bad one starts a fresh event buffer. + assert_eq!(data(events[1].as_ref().unwrap()), "tail"); + assert_eq!(data(events[2].as_ref().unwrap()), "next"); +} + +#[tokio::test] +async fn exceeding_the_limit_ends_the_stream() { + let stream = SseStream::with_max_event_size( + stream::iter(vec![ + Ok::<_, std::io::Error>(Bytes::from_static(b"data: ok\n\n")), + Ok(Bytes::from_static(b"data: far too long for the limit\n\n")), + Ok(Bytes::from_static(b"data: ok\n\n")), + ]), + 10, + ); + let results: Vec<_> = stream.collect().await; + assert_eq!(results.len(), 2); + assert_eq!(data(results[0].as_ref().unwrap()), "ok"); + assert!(matches!(results[1], Err(SseError::FrameTooLarge))); +} + +#[tokio::test] +async fn multi_event_payload_split_at_every_byte_boundary() { + let payload = "event: e\r\nid: 1\r\ndata: 你好\r\n\r\ndata: a\rdata: b\r\r:c\ndata: z\n\n"; + let bytes = payload.as_bytes(); + let expected = collect_events(vec![payload]).await; + let expected: Vec<(Option, String, Option)> = expected + .into_iter() + .map(|e| { + let e = e.unwrap(); + (e.event, e.data, e.id) + }) + .collect(); + assert_eq!(expected.len(), 3); + for split in 1..bytes.len() { + let (a, b) = bytes.split_at(split); + let got: Vec<_> = collect_events_bytes(vec![a.to_vec(), b.to_vec()]) + .await + .into_iter() + .map(|e| { + let e = e.unwrap_or_else(|err| panic!("split {split}: {err:?}")); + (e.event, e.data, e.id) + }) + .collect(); + assert_eq!(got, expected, "split at byte {split}"); + } +} From 3901994e7dc2bb6396b261a9f6f44416590bb9e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 05:22:09 +0000 Subject: [PATCH 2/8] refactor(stream)!: remove unused NdjsonStream Nothing in the workspace used NdjsonStream / NdjsonError and the AI SDK's provider-utils has no counterpart. Also move tokio to dev-dependencies (only the tests use it), drop the unused direct serde dependency, and update the crate description in README / PROJECT-OVERVIEW. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015EYDYDWYcPsDmjBjFeuDVe --- README.md | 4 +- aimux-stream/Cargo.toml | 7 +- aimux-stream/src/lib.rs | 5 +- aimux-stream/src/ndjson.rs | 146 ------------------------------ aimux-stream/tests/ndjson_test.rs | 98 -------------------- docs/PROJECT-OVERVIEW.md | 2 +- 6 files changed, 9 insertions(+), 253 deletions(-) delete mode 100644 aimux-stream/src/ndjson.rs delete mode 100644 aimux-stream/tests/ndjson_test.rs diff --git a/README.md b/README.md index e636bc1b1..02ee0ed0d 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ middleware, and telemetry per request). aimux/ ├── aimux-core # Core abstractions: LanguageModel / Provider / Message / StreamPart ├── aimux-providers # Provider implementations — registry-backed + typed (docs/api/providers.md) -├── aimux-stream # SSE / NDJSON stream parsing +├── aimux-stream # SSE parsing, streamed tool-call tracking ├── aimux-provider-utils # One-exchange HTTP helpers, response handlers, API-key loading ├── aimux-ffi # C ABI (opaque handles + JSON results + owned aimux_error_t *) for non-native bindings └── tools/ # aimux-cli (cache probe) · aimux-replay · aimux-web (console) @@ -122,7 +122,7 @@ cargo add aimux-core aimux-providers |-------|-------------|-----------| | `aimux-core` | Core abstractions: `LanguageModel` / `Provider` / `Message` / `StreamPart` | [crates.io](https://crates.io/crates/aimux-core) | | `aimux-providers` | Provider implementations — [registry-backed + typed](docs/api/providers.md) | [crates.io](https://crates.io/crates/aimux-providers) | -| `aimux-stream` | SSE / NDJSON stream parsing | [crates.io](https://crates.io/crates/aimux-stream) | +| `aimux-stream` | SSE parsing, streamed tool-call tracking | [crates.io](https://crates.io/crates/aimux-stream) | | `aimux-provider-utils` | One-exchange HTTP helpers and typed response handlers | [crates.io](https://crates.io/crates/aimux-provider-utils) | | `aimux-ffi` | C ABI for non-native bindings | [crates.io](https://crates.io/crates/aimux-ffi) | diff --git a/aimux-stream/Cargo.toml b/aimux-stream/Cargo.toml index 6ee65c240..7c65cfecf 100644 --- a/aimux-stream/Cargo.toml +++ b/aimux-stream/Cargo.toml @@ -4,18 +4,19 @@ version.workspace = true edition.workspace = true license.workspace = true description = "Streaming primitives for aimux" -keywords = ["llm", "streaming", "sse", "ndjson"] +keywords = ["llm", "streaming", "sse", "tool-calls"] categories = ["asynchronous", "parser-implementations"] documentation = "https://docs.rs/aimux-stream" [dependencies] futures = { workspace = true } -tokio = { workspace = true } pin-project-lite = { workspace = true } bytes = { workspace = true } -serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } +[dev-dependencies] +tokio = { workspace = true } + [lints] workspace = true diff --git a/aimux-stream/src/lib.rs b/aimux-stream/src/lib.rs index 8091dbeb7..e023a3204 100644 --- a/aimux-stream/src/lib.rs +++ b/aimux-stream/src/lib.rs @@ -1,16 +1,15 @@ //! # aimux-stream //! -//! Low-level streaming primitives for SSE (Server-Sent Events) and NDJSON parsing, +//! Low-level streaming primitives for SSE (Server-Sent Events) parsing and +//! streamed tool-call tracking, //! used by provider implementations to decode model API response streams. pub mod lines; -pub mod ndjson; pub mod sse; pub mod streaming_tool_call_tracker; // Re-export the most commonly used items. pub use lines::extract_lines; -pub use ndjson::{NdjsonError, NdjsonStream}; pub use sse::{SseError, SseEvent, SseStream}; pub use streaming_tool_call_tracker::{ StreamingToolCallDelta, StreamingToolCallFunction, StreamingToolCallTracker, diff --git a/aimux-stream/src/ndjson.rs b/aimux-stream/src/ndjson.rs deleted file mode 100644 index ca6801a52..000000000 --- a/aimux-stream/src/ndjson.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! NDJSON (newline-delimited JSON) stream decoder. - -use bytes::Bytes; -use futures::Stream; -use pin_project_lite::pin_project; -use serde::de::DeserializeOwned; -use std::pin::Pin; -use std::task::{Context, Poll}; -use thiserror::Error; - -/// Default upper bound on a single buffered NDJSON line's size (1 MiB). -const DEFAULT_MAX_LINE_SIZE: usize = 1024 * 1024; - -#[derive(Debug, Error)] -pub enum NdjsonError { - #[error("utf-8 error: {0}")] - Utf8(#[from] std::string::FromUtf8Error), - #[error("json error: {0}")] - Json(#[from] serde_json::Error), - #[error("stream error: {0}")] - Stream(String), - #[error("NDJSON line exceeded maximum allowed size")] - LineTooLarge, -} - -pin_project! { - /// Decodes a byte stream into parsed NDJSON values. - /// - /// Bytes are accumulated in a raw `Vec` buffer and split on `\n`. Each - /// complete line is strictly UTF-8 decoded only *after* reassembly, so a - /// multi-byte character split across two network chunks is never corrupted - /// into replacement chars — unlike a per-chunk - /// `String::from_utf8_lossy` decode, which would emit a `U+FFFD` on each - /// side of the split. - pub struct NdjsonStream { - #[pin] - inner: S, - buffer: Vec, - max_line_size: usize, - _marker: std::marker::PhantomData<(T, E)>, - } -} - -impl NdjsonStream -where - S: Stream> + Unpin, - T: DeserializeOwned, -{ - pub fn new(stream: S) -> Self { - Self::with_max_line_size(stream, DEFAULT_MAX_LINE_SIZE) - } - - /// Create an [`NdjsonStream`] with a custom per-line size limit. A line - /// longer than `max_line_size` bytes (excluding the `\n`) yields - /// [`NdjsonError::LineTooLarge`], as does a buffer that grows past the - /// limit while waiting for a newline. - pub fn with_max_line_size(stream: S, max_line_size: usize) -> Self { - Self { - inner: stream, - buffer: Vec::new(), - max_line_size, - _marker: std::marker::PhantomData, - } - } -} - -impl Stream for NdjsonStream -where - S: Stream> + Unpin, - T: DeserializeOwned, - E: std::fmt::Display, -{ - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.as_mut().get_mut(); - - loop { - // Try to parse a complete JSON line. - if let Some(pos) = this.buffer.iter().position(|&b| b == b'\n') { - if pos > this.max_line_size { - // Drop the oversized line so a retried poll makes progress. - this.buffer.drain(..pos + 1); - return Poll::Ready(Some(Err(NdjsonError::LineTooLarge))); - } - // Extract the line bytes (without the newline) and decode - // strictly. Decoding the fully reassembled line (rather than - // `from_utf8_lossy` per chunk) preserves code points split - // across chunks and surfaces invalid UTF-8 as an error. - let line_bytes: Vec = this.buffer.drain(..pos).collect(); - this.buffer.drain(..1); // the newline - let line = match String::from_utf8(line_bytes) { - Ok(s) => s, - Err(e) => return Poll::Ready(Some(Err(NdjsonError::Utf8(e)))), - }; - let line = line.trim(); - if line.is_empty() { - continue; - } - match serde_json::from_str::(line) { - Ok(value) => return Poll::Ready(Some(Ok(value))), - Err(e) => return Poll::Ready(Some(Err(NdjsonError::Json(e)))), - } - } - - // No complete line yet. Guard against unbounded buffer growth when - // a newline never arrives. The partial can never form a valid line - // under the limit, so drop it — a retried poll then makes progress - // instead of looping on the same oversized buffer. - if this.buffer.len() > this.max_line_size { - this.buffer.clear(); - return Poll::Ready(Some(Err(NdjsonError::LineTooLarge))); - } - - // Read more bytes. - match Pin::new(&mut this.inner).poll_next(cx) { - Poll::Ready(Some(Ok(bytes))) => { - this.buffer.extend_from_slice(&bytes); - } - Poll::Ready(Some(Err(e))) => { - return Poll::Ready(Some(Err(NdjsonError::Stream(e.to_string())))); - } - Poll::Ready(None) => { - // Stream ended — try to parse any remaining buffer. - if this.buffer.len() > this.max_line_size { - return Poll::Ready(Some(Err(NdjsonError::LineTooLarge))); - } - let remaining_bytes = std::mem::take(&mut this.buffer); - let remaining = match String::from_utf8(remaining_bytes) { - Ok(s) => s, - Err(e) => return Poll::Ready(Some(Err(NdjsonError::Utf8(e)))), - }; - let remaining = remaining.trim(); - if remaining.is_empty() { - return Poll::Ready(None); - } - match serde_json::from_str::(remaining) { - Ok(value) => return Poll::Ready(Some(Ok(value))), - Err(e) => return Poll::Ready(Some(Err(NdjsonError::Json(e)))), - } - } - Poll::Pending => return Poll::Pending, - } - } - } -} diff --git a/aimux-stream/tests/ndjson_test.rs b/aimux-stream/tests/ndjson_test.rs deleted file mode 100644 index 88e75e1ab..000000000 --- a/aimux-stream/tests/ndjson_test.rs +++ /dev/null @@ -1,98 +0,0 @@ -//! NDJSON stream decoder tests — cross-chunk UTF-8 reassembly, invalid-UTF-8 -//! rejection, and bounded-buffer behavior. - -use aimux_stream::{NdjsonError, NdjsonStream}; -use bytes::Bytes; -use futures::stream::{self, StreamExt}; -use serde::Deserialize; - -#[derive(Debug, Deserialize, PartialEq)] -struct Line { - text: String, -} - -/// Feed raw byte chunks into an [`NdjsonStream`] and collect the results. -async fn collect_lines(chunks: Vec>) -> Vec> { - let items: Vec> = - chunks.into_iter().map(|c| Ok(Bytes::from(c))).collect(); - let stream = NdjsonStream::new(stream::iter(items)); - stream.collect::>().await -} - -// ── cross-chunk UTF-8 reassembly (P0-03) ───────────────────────────────── - -#[tokio::test] -async fn emoji_ndjson_line_split_at_every_byte_boundary() { - // An NDJSON line whose value contains multi-byte emoji. Splitting the full - // line at *every* byte position must still reassemble to the intact text. - // The old per-chunk `String::from_utf8_lossy` decoder would corrupt a code - // point split across two chunks into two U+FFFDs. - let payload = "{\"text\":\"😀👋\"}\n"; - let bytes = payload.as_bytes(); - for split in 1..bytes.len() { - let (a, b) = bytes.split_at(split); - let lines = collect_lines(vec![a.to_vec(), b.to_vec()]).await; - assert_eq!( - lines.len(), - 1, - "split at byte {} produced {} results", - split, - lines.len() - ); - let line = lines[0] - .as_ref() - .unwrap_or_else(|e| panic!("split at byte {split} errored: {e:?}")); - assert_eq!( - line.text, "😀👋", - "split at byte {split} corrupted the text" - ); - } -} - -#[tokio::test] -async fn invalid_utf8_line_returns_utf8_error() { - // `{"text":"` followed by 0xF0 0x9F — the first two bytes of 😀 - // (U+1F600 = F0 9F 98 80) without the trailing bytes — is an incomplete - // 4-byte sequence. Strict decoding of the complete line surfaces a `Utf8` - // error instead of silently producing replacement chars. - let chunk = vec![ - b'{', b'"', b't', b'e', b'x', b't', b'"', b':', b'"', 0xF0, 0x9F, b'\n', - ]; - let lines = collect_lines(vec![chunk]).await; - assert_eq!(lines.len(), 1, "expected exactly one result"); - match &lines[0] { - Err(NdjsonError::Utf8(_)) => {} - other => panic!("expected NdjsonError::Utf8, got {other:?}"), - } -} - -// ── bounded buffers (P1-10) ───────────────────────────────────────────── - -#[tokio::test] -async fn oversized_line_returns_line_too_large() { - // A complete line longer than the 5-byte limit is rejected. - let stream = NdjsonStream::with_max_line_size( - stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static( - b"{\"text\":\"hi\"}\n", - ))]), - 5, - ); - let results: Vec> = stream.collect().await; - assert_eq!(results.len(), 1); - assert!(matches!(results[0], Err(NdjsonError::LineTooLarge))); -} - -#[tokio::test] -async fn buffer_growing_past_limit_without_newline_returns_line_too_large() { - // No newline ever arrives, so the buffer would grow unboundedly; the limit - // trips instead (previously this allocated forever). - let stream = NdjsonStream::with_max_line_size( - stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static( - b"{\"text\":\"no newline here\"", - ))]), - 5, - ); - let results: Vec> = stream.collect().await; - assert_eq!(results.len(), 1); - assert!(matches!(results[0], Err(NdjsonError::LineTooLarge))); -} diff --git a/docs/PROJECT-OVERVIEW.md b/docs/PROJECT-OVERVIEW.md index a5442aca3..a53a672eb 100644 --- a/docs/PROJECT-OVERVIEW.md +++ b/docs/PROJECT-OVERVIEW.md @@ -165,7 +165,7 @@ aimux/ │ ├── native protocols # standalone model + convert, handles provider-specific differences │ ├── OpenAI compatible # registry-backed: provider-registry.json + provider(name, ...) entry (RFC-0017 phase 4) │ └── modalities/search # voice / image / video / search implementations -├── aimux-stream # SSE / NDJSON streaming parsing +├── aimux-stream # SSE parsing, streamed tool-call tracking ├── aimux-provider-utils # One-exchange HTTP helpers, response handlers, API-key loading ├── aimux-ffi # C ABI (FFI infrastructure, shared by all bindings) └── bindings/ # 6 language bindings From d36dc6eb8287904182c5058c1cec3b9f9322a2b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 05:28:25 +0000 Subject: [PATCH 3/8] fix(stream): discard the rest of an SSE event after a UTF-8 error A line that failed UTF-8 decoding reset the event, but the lines after it (up to the blank line) were then dispatched as a separate fragment event, producing a second downstream error. Mark the event poisoned until the next blank line so one bad event yields exactly one SseError::Utf8. Also update the aimux-stream description in CONTRIBUTING. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015EYDYDWYcPsDmjBjFeuDVe --- CONTRIBUTING.md | 2 +- aimux-stream/src/sse.rs | 16 +++++++++++++--- aimux-stream/tests/sse_test.rs | 8 ++++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce84ddbbc..ab11942a7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,7 +9,7 @@ to set up a development environment, run the tests, and submit changes. aimux/ ├── aimux-core/ # Core abstractions: LanguageModel / Provider / Message / StreamPart ├── aimux-providers/ # provider implementations + cassettes (counts: docs/api/providers.md) -├── aimux-stream/ # SSE / NDJSON stream parsing +├── aimux-stream/ # SSE parsing, streamed tool-call tracking ├── aimux-provider-utils/ # HTTP utilities: retry, backoff, error parsing, API-key loading ├── aimux-ffi/ # C ABI (opaque handle + JSON + push callback) for non-native bindings ├── bindings/ # Node, Python, Swift, Kotlin, Flutter, Go, C — share one Rust core diff --git a/aimux-stream/src/sse.rs b/aimux-stream/src/sse.rs index 73eb360fa..dacfc7046 100644 --- a/aimux-stream/src/sse.rs +++ b/aimux-stream/src/sse.rs @@ -18,8 +18,8 @@ //! //! Two aimux additions: every line is strictly UTF-8 decoded after it has //! been reassembled (a code point split across chunks is never corrupted, and -//! invalid UTF-8 surfaces as [`SseError::Utf8`] and drops the event being -//! built), and buffered input is bounded by `max_event_size` (the parser's +//! invalid UTF-8 surfaces as [`SseError::Utf8`] and discards the rest of +//! that event up to the next blank line), and buffered input is bounded by `max_event_size` (the parser's //! `maxBufferSize`; exceeding it is fatal and ends the stream, as upstream). use std::collections::VecDeque; @@ -73,6 +73,9 @@ struct Parser { data_lines: usize, id: Option, retry: Option, + /// Set after a line fails UTF-8 decoding: the rest of the event (up to + /// the next blank line) is discarded rather than dispatched as a fragment. + poisoned: bool, max_size: usize, terminated: bool, ready: VecDeque>, @@ -89,6 +92,7 @@ impl Parser { data_lines: 0, id: None, retry: None, + poisoned: false, max_size, terminated: false, ready: VecDeque::new(), @@ -150,11 +154,16 @@ impl Parser { self.dispatch(); return; } + if self.poisoned { + return; + } let line = match String::from_utf8(line.to_vec()) { Ok(line) => line, Err(error) => { - // The event being built can no longer be trusted. + // The event being built can no longer be trusted: drop it and + // everything up to the next blank line. self.reset_event(); + self.poisoned = true; self.ready.push_back(Err(SseError::Utf8(error))); return; } @@ -205,6 +214,7 @@ impl Parser { } fn reset_event(&mut self) { + self.poisoned = false; self.event = None; self.data.clear(); self.data_lines = 0; diff --git a/aimux-stream/tests/sse_test.rs b/aimux-stream/tests/sse_test.rs index 58be9a19b..80f46365f 100644 --- a/aimux-stream/tests/sse_test.rs +++ b/aimux-stream/tests/sse_test.rs @@ -484,11 +484,11 @@ async fn invalid_utf8_drops_only_the_current_event() { bytes.extend_from_slice(&[0xE4, 0xBD]); bytes.extend_from_slice(b"\ndata: tail\n\ndata: next\n\n"); let events = collect_events_bytes(vec![bytes]).await; - assert_eq!(events.len(), 3); + // The rest of the poisoned event (`tail`) is discarded, not dispatched as + // a fragment; the following event is unaffected. + assert_eq!(events.len(), 2); assert!(matches!(events[0], Err(SseError::Utf8(_)))); - // The line after the bad one starts a fresh event buffer. - assert_eq!(data(events[1].as_ref().unwrap()), "tail"); - assert_eq!(data(events[2].as_ref().unwrap()), "next"); + assert_eq!(data(events[1].as_ref().unwrap()), "next"); } #[tokio::test] From d199324444ea5752ea48bb60a5547f8b5f9c8433 Mon Sep 17 00:00:00 2001 From: chenhaonan Date: Tue, 29 Sep 2026 14:06:23 +0800 Subject: [PATCH 4/8] fix(stream): collapse the SSE retry arm into a match guard clippy::collapsible_match rejects the nested `if` under the CI `-D warnings` gate (stable 1.98); behaviour is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- aimux-stream/src/sse.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/aimux-stream/src/sse.rs b/aimux-stream/src/sse.rs index dacfc7046..7a5a49544 100644 --- a/aimux-stream/src/sse.rs +++ b/aimux-stream/src/sse.rs @@ -192,10 +192,8 @@ impl Parser { self.id = Some(value.to_string()); } } - "retry" => { - if !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()) { - self.retry = value.parse().ok(); - } + "retry" if !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()) => { + self.retry = value.parse().ok(); } _ => {} // unknown field } From 9b1e4810a710abd0391e2be098feffd54239e008 Mon Sep 17 00:00:00 2001 From: chenhaonan Date: Tue, 29 Sep 2026 22:33:35 +0800 Subject: [PATCH 5/8] test(stream): replace hand-written SSE tests with eventsource-parser's suite sse_test.rs was a behaviour list written from scratch, and its header claimed the AI SDK has no SSE parser tests. The parser behind parseJsonEventStream, eventsource-parser, ships its own suite. Port test/parse.test.ts and test/stream.test.ts (v3.1.1, MIT) case for case, driven by the same fixtures; test/multibyte.ts is checked in as JSON and read with include_str!. Cases with no Rust counterpart (onComment counts, reset(), onError payloads) are listed in the file header, as are the adaptations (retry is reported on the event, maxBufferSize maps to with_max_event_size). Four aimux-only tests remain: strict UTF-8, byte-level chunk splitting, transport errors, and the default limit. The four unit tests inside sse.rs are dropped; the port covers them. sha2 (already used by aimux-providers) is a dev-dependency for the upstream SHA-256 check on the 4.8M-character event. The tracker and argument-state tests were already case-for-case ports of the pinned upstream tests; only a comment is added. Co-Authored-By: Claude Sonnet 5.5 --- aimux-stream/Cargo.toml | 1 + aimux-stream/src/sse.rs | 36 - .../eventsource_parser_multibyte.json | 1 + aimux-stream/tests/sse_test.rs | 1117 +++++++++++------ 4 files changed, 723 insertions(+), 432 deletions(-) create mode 100644 aimux-stream/tests/fixtures/eventsource_parser_multibyte.json diff --git a/aimux-stream/Cargo.toml b/aimux-stream/Cargo.toml index 7c65cfecf..2d51d4d69 100644 --- a/aimux-stream/Cargo.toml +++ b/aimux-stream/Cargo.toml @@ -16,6 +16,7 @@ serde_json = { workspace = true } thiserror = { workspace = true } [dev-dependencies] +sha2 = "0.10" tokio = { workspace = true } [lints] diff --git a/aimux-stream/src/sse.rs b/aimux-stream/src/sse.rs index 7a5a49544..a9be7a66f 100644 --- a/aimux-stream/src/sse.rs +++ b/aimux-stream/src/sse.rs @@ -295,39 +295,3 @@ where } } } - -#[cfg(test)] -mod tests { - use super::*; - - fn parse(input: &str) -> Vec { - let mut parser = Parser::new(DEFAULT_MAX_EVENT_SIZE); - parser.feed(input.as_bytes()); - parser.ready.into_iter().map(Result::unwrap).collect() - } - - #[test] - fn parse_single_event() { - assert_eq!(parse("data: hello world\n\n")[0].data, "hello world"); - } - - #[test] - fn parse_multi_line_data() { - assert_eq!( - parse("data: line1\ndata: line2\n\n")[0].data, - "line1\nline2" - ); - } - - #[test] - fn parse_event_with_type() { - let events = parse("event: message\ndata: payload\n\n"); - assert_eq!(events[0].event.as_deref(), Some("message")); - assert_eq!(events[0].data, "payload"); - } - - #[test] - fn parse_event_without_data_is_not_dispatched() { - assert!(parse("event: ping\n\n").is_empty()); - } -} diff --git a/aimux-stream/tests/fixtures/eventsource_parser_multibyte.json b/aimux-stream/tests/fixtures/eventsource_parser_multibyte.json new file mode 100644 index 000000000..f427553c5 --- /dev/null +++ b/aimux-stream/tests/fixtures/eventsource_parser_multibyte.json @@ -0,0 +1 @@ +{"lines":["Blåbærsyltetøy må da være mulig å oppdrive?","በእርግጥ አንድ ሰው ብሉቤሪ ጃምን መግዛት መቻል አለበት?","بالتأكيد يجب أن يكون المرء قادرا على شراء مربى التوت؟","一定能买到蓝莓果酱吗?","ચોક્કસ એક બ્લુબેરી જામ મેળવવા માટે સમર્થ હોવા જ જોઈએ?","בטח אפשר להשיג ריבת אוכמניות?","निश्चित रूप से किसी को ब्लूबेरी जैम खरीदने में सक्षम होना चाहिए?","Bê guman pêdivî ye ku meriv karibe jama şînê peyda bike?","ប្រាកដណាស់ មនុស្សម្នាក់ត្រូវតែអាចទិញយៈសាពូនមី blueberry បានទេ?","ಖಂಡಿತವಾಗಿಯೂ ಒಬ್ಬರು ಬ್ಲೂಬೆರ್ರಿ ಜಾಮ್ ಅನ್ನು ಸಂಗ್ರಹಿಸಲು ಶಕ್ತರಾಗಿರಬೇಕು?","確かにブルーベリージャムを調達できなければなりませんか?","Невже треба вміти добути варення з чорниці?","แน่นอนว่าจะต้องสามารถจัดหาแยมบลูเบอร์รี่ได้?","Сигурно некој мора да може да набави џем од боровинки?"],"emojis":["😶‍🌫️","😮‍💨","😵‍💫","❤️‍🔥","❤️‍🩹","👁️‍🗨️","🫱🏻‍🫲🏼","🫱🏻‍🫲🏽","🫱🏻‍🫲🏾","🫱🏻‍🫲🏿","🫱🏼‍🫲🏻","🫱🏼‍🫲🏽","🫱🏼‍🫲🏾","🫱🏼‍🫲🏿","🫱🏽‍🫲🏻","🫱🏽‍🫲🏼","🫱🏽‍🫲🏾","🫱🏽‍🫲🏿","🫱🏾‍🫲🏻","🫱🏾‍🫲🏼","🫱🏾‍🫲🏽","🫱🏾‍🫲🏿","🫱🏿‍🫲🏻","🫱🏿‍🫲🏼","🫱🏿‍🫲🏽","🫱🏿‍🫲🏾","🧔‍♂️","🧔🏻‍♂️","🧔🏼‍♂️","🧔🏽‍♂️","🧔🏾‍♂️","🧔🏿‍♂️","🧔‍♀️","🧔🏻‍♀️","🧔🏼‍♀️","🧔🏽‍♀️","🧔🏾‍♀️","🧔🏿‍♀️","👨‍🦰","👨🏻‍🦰","👨🏼‍🦰","👨🏽‍🦰","👨🏾‍🦰","👨🏿‍🦰","👨‍🦱","👨🏻‍🦱","👨🏼‍🦱","👨🏽‍🦱","👨🏾‍🦱","👨🏿‍🦱","👨‍🦳","👨🏻‍🦳","👨🏼‍🦳","👨🏽‍🦳","👨🏾‍🦳","👨🏿‍🦳","👨‍🦲","👨🏻‍🦲","👨🏼‍🦲","👨🏽‍🦲","👨🏾‍🦲","👨🏿‍🦲","👩‍🦰","👩🏻‍🦰","👩🏼‍🦰","👩🏽‍🦰","👩🏾‍🦰","👩🏿‍🦰","🧑‍🦰","🧑🏻‍🦰","🧑🏼‍🦰","🧑🏽‍🦰","🧑🏾‍🦰","🧑🏿‍🦰","👩‍🦱","👩🏻‍🦱","👩🏼‍🦱","👩🏽‍🦱","👩🏾‍🦱","👩🏿‍🦱","🧑‍🦱","🧑🏻‍🦱","🧑🏼‍🦱","🧑🏽‍🦱","🧑🏾‍🦱","🧑🏿‍🦱","👩‍🦳","👩🏻‍🦳","👩🏼‍🦳","👩🏽‍🦳","👩🏾‍🦳","👩🏿‍🦳","🧑‍🦳","🧑🏻‍🦳","🧑🏼‍🦳","🧑🏽‍🦳","🧑🏾‍🦳","🧑🏿‍🦳","👩‍🦲","👩🏻‍🦲","👩🏼‍🦲","👩🏽‍🦲","👩🏾‍🦲","👩🏿‍🦲","🧑‍🦲","🧑🏻‍🦲","🧑🏼‍🦲","🧑🏽‍🦲","🧑🏾‍🦲","🧑🏿‍🦲","👱‍♀️","👱🏻‍♀️","👱🏼‍♀️","👱🏽‍♀️","👱🏾‍♀️","👱🏿‍♀️","👱‍♂️","👱🏻‍♂️","👱🏼‍♂️","👱🏽‍♂️","👱🏾‍♂️","👱🏿‍♂️","🙍‍♂️","🙍🏻‍♂️","🙍🏼‍♂️","🙍🏽‍♂️","🙍🏾‍♂️","🙍🏿‍♂️","🙍‍♀️","🙍🏻‍♀️","🙍🏼‍♀️","🙍🏽‍♀️","🙍🏾‍♀️","🙍🏿‍♀️","🙎‍♂️","🙎🏻‍♂️","🙎🏼‍♂️","🙎🏽‍♂️","🙎🏾‍♂️","🙎🏿‍♂️","🙎‍♀️","🙎🏻‍♀️","🙎🏼‍♀️","🙎🏽‍♀️","🙎🏾‍♀️","🙎🏿‍♀️","🙅‍♂️","🙅🏻‍♂️","🙅🏼‍♂️","🙅🏽‍♂️","🙅🏾‍♂️","🙅🏿‍♂️","🙅‍♀️","🙅🏻‍♀️","🙅🏼‍♀️","🙅🏽‍♀️","🙅🏾‍♀️","🙅🏿‍♀️","🙆‍♂️","🙆🏻‍♂️","🙆🏼‍♂️","🙆🏽‍♂️","🙆🏾‍♂️","🙆🏿‍♂️","🙆‍♀️","🙆🏻‍♀️","🙆🏼‍♀️","🙆🏽‍♀️","🙆🏾‍♀️","🙆🏿‍♀️","💁‍♂️","💁🏻‍♂️","💁🏼‍♂️","💁🏽‍♂️","💁🏾‍♂️","💁🏿‍♂️","💁‍♀️","💁🏻‍♀️","💁🏼‍♀️","💁🏽‍♀️","💁🏾‍♀️","💁🏿‍♀️","🙋‍♂️","🙋🏻‍♂️","🙋🏼‍♂️","🙋🏽‍♂️","🙋🏾‍♂️","🙋🏿‍♂️","🙋‍♀️","🙋🏻‍♀️","🙋🏼‍♀️","🙋🏽‍♀️","🙋🏾‍♀️","🙋🏿‍♀️","🧏‍♂️","🧏🏻‍♂️","🧏🏼‍♂️","🧏🏽‍♂️","🧏🏾‍♂️","🧏🏿‍♂️","🧏‍♀️","🧏🏻‍♀️","🧏🏼‍♀️","🧏🏽‍♀️","🧏🏾‍♀️","🧏🏿‍♀️","🙇‍♂️","🙇🏻‍♂️","🙇🏼‍♂️","🙇🏽‍♂️","🙇🏾‍♂️","🙇🏿‍♂️","🙇‍♀️","🙇🏻‍♀️","🙇🏼‍♀️","🙇🏽‍♀️","🙇🏾‍♀️","🙇🏿‍♀️","🤦‍♂️","🤦🏻‍♂️","🤦🏼‍♂️","🤦🏽‍♂️","🤦🏾‍♂️","🤦🏿‍♂️","🤦‍♀️","🤦🏻‍♀️","🤦🏼‍♀️","🤦🏽‍♀️","🤦🏾‍♀️","🤦🏿‍♀️","🤷‍♂️","🤷🏻‍♂️","🤷🏼‍♂️","🤷🏽‍♂️","🤷🏾‍♂️","🤷🏿‍♂️","🤷‍♀️","🤷🏻‍♀️","🤷🏼‍♀️","🤷🏽‍♀️","🤷🏾‍♀️","🤷🏿‍♀️","🧑‍⚕️","🧑🏻‍⚕️","🧑🏼‍⚕️","🧑🏽‍⚕️","🧑🏾‍⚕️","🧑🏿‍⚕️","👨‍⚕️","👨🏻‍⚕️","👨🏼‍⚕️","👨🏽‍⚕️","👨🏾‍⚕️","👨🏿‍⚕️","👩‍⚕️","👩🏻‍⚕️","👩🏼‍⚕️","👩🏽‍⚕️","👩🏾‍⚕️","👩🏿‍⚕️","🧑‍🎓","🧑🏻‍🎓","🧑🏼‍🎓","🧑🏽‍🎓","🧑🏾‍🎓","🧑🏿‍🎓","👨‍🎓","👨🏻‍🎓","👨🏼‍🎓","👨🏽‍🎓","👨🏾‍🎓","👨🏿‍🎓","👩‍🎓","👩🏻‍🎓","👩🏼‍🎓","👩🏽‍🎓","👩🏾‍🎓","👩🏿‍🎓","🧑‍🏫","🧑🏻‍🏫","🧑🏼‍🏫","🧑🏽‍🏫","🧑🏾‍🏫","🧑🏿‍🏫","👨‍🏫","👨🏻‍🏫","👨🏼‍🏫","👨🏽‍🏫","👨🏾‍🏫","👨🏿‍🏫","👩‍🏫","👩🏻‍🏫","👩🏼‍🏫","👩🏽‍🏫","👩🏾‍🏫","👩🏿‍🏫","🧑‍⚖️","🧑🏻‍⚖️","🧑🏼‍⚖️","🧑🏽‍⚖️","🧑🏾‍⚖️","🧑🏿‍⚖️","👨‍⚖️","👨🏻‍⚖️","👨🏼‍⚖️","👨🏽‍⚖️","👨🏾‍⚖️","👨🏿‍⚖️","👩‍⚖️","👩🏻‍⚖️","👩🏼‍⚖️","👩🏽‍⚖️","👩🏾‍⚖️","👩🏿‍⚖️","🧑‍🌾","🧑🏻‍🌾","🧑🏼‍🌾","🧑🏽‍🌾","🧑🏾‍🌾","🧑🏿‍🌾","👨‍🌾","👨🏻‍🌾","👨🏼‍🌾","👨🏽‍🌾","👨🏾‍🌾","👨🏿‍🌾","👩‍🌾","👩🏻‍🌾","👩🏼‍🌾","👩🏽‍🌾","👩🏾‍🌾","👩🏿‍🌾","🧑‍🍳","🧑🏻‍🍳","🧑🏼‍🍳","🧑🏽‍🍳","🧑🏾‍🍳","🧑🏿‍🍳","👨‍🍳","👨🏻‍🍳","👨🏼‍🍳","👨🏽‍🍳","👨🏾‍🍳","👨🏿‍🍳","👩‍🍳","👩🏻‍🍳","👩🏼‍🍳","👩🏽‍🍳","👩🏾‍🍳","👩🏿‍🍳","🧑‍🔧","🧑🏻‍🔧","🧑🏼‍🔧","🧑🏽‍🔧","🧑🏾‍🔧","🧑🏿‍🔧","👨‍🔧","👨🏻‍🔧","👨🏼‍🔧","👨🏽‍🔧","👨🏾‍🔧","👨🏿‍🔧","👩‍🔧","👩🏻‍🔧","👩🏼‍🔧","👩🏽‍🔧","👩🏾‍🔧","👩🏿‍🔧","🧑‍🏭","🧑🏻‍🏭","🧑🏼‍🏭","🧑🏽‍🏭","🧑🏾‍🏭","🧑🏿‍🏭","👨‍🏭","👨🏻‍🏭","👨🏼‍🏭","👨🏽‍🏭","👨🏾‍🏭","👨🏿‍🏭","👩‍🏭","👩🏻‍🏭","👩🏼‍🏭","👩🏽‍🏭","👩🏾‍🏭","👩🏿‍🏭","🧑‍💼","🧑🏻‍💼","🧑🏼‍💼","🧑🏽‍💼","🧑🏾‍💼","🧑🏿‍💼","👨‍💼","👨🏻‍💼","👨🏼‍💼","👨🏽‍💼","👨🏾‍💼","👨🏿‍💼","👩‍💼","👩🏻‍💼","👩🏼‍💼","👩🏽‍💼","👩🏾‍💼","👩🏿‍💼","🧑‍🔬","🧑🏻‍🔬","🧑🏼‍🔬","🧑🏽‍🔬","🧑🏾‍🔬","🧑🏿‍🔬","👨‍🔬","👨🏻‍🔬","👨🏼‍🔬","👨🏽‍🔬","👨🏾‍🔬","👨🏿‍🔬","👩‍🔬","👩🏻‍🔬","👩🏼‍🔬","👩🏽‍🔬","👩🏾‍🔬","👩🏿‍🔬","🧑‍💻","🧑🏻‍💻","🧑🏼‍💻","🧑🏽‍💻","🧑🏾‍💻","🧑🏿‍💻","👨‍💻","👨🏻‍💻","👨🏼‍💻","👨🏽‍💻","👨🏾‍💻","👨🏿‍💻","👩‍💻","👩🏻‍💻","👩🏼‍💻","👩🏽‍💻","👩🏾‍💻","👩🏿‍💻","🧑‍🎤","🧑🏻‍🎤","🧑🏼‍🎤","🧑🏽‍🎤","🧑🏾‍🎤","🧑🏿‍🎤","👨‍🎤","👨🏻‍🎤","👨🏼‍🎤","👨🏽‍🎤","👨🏾‍🎤","👨🏿‍🎤","👩‍🎤","👩🏻‍🎤","👩🏼‍🎤","👩🏽‍🎤","👩🏾‍🎤","👩🏿‍🎤","🧑‍🎨","🧑🏻‍🎨","🧑🏼‍🎨","🧑🏽‍🎨","🧑🏾‍🎨","🧑🏿‍🎨","👨‍🎨","👨🏻‍🎨","👨🏼‍🎨","👨🏽‍🎨","👨🏾‍🎨","👨🏿‍🎨","👩‍🎨","👩🏻‍🎨","👩🏼‍🎨","👩🏽‍🎨","👩🏾‍🎨","👩🏿‍🎨","🧑‍✈️","🧑🏻‍✈️","🧑🏼‍✈️","🧑🏽‍✈️","🧑🏾‍✈️","🧑🏿‍✈️","👨‍✈️","👨🏻‍✈️","👨🏼‍✈️","👨🏽‍✈️","👨🏾‍✈️","👨🏿‍✈️","👩‍✈️","👩🏻‍✈️","👩🏼‍✈️","👩🏽‍✈️","👩🏾‍✈️","👩🏿‍✈️","🧑‍🚀","🧑🏻‍🚀","🧑🏼‍🚀","🧑🏽‍🚀","🧑🏾‍🚀","🧑🏿‍🚀","👨‍🚀","👨🏻‍🚀","👨🏼‍🚀","👨🏽‍🚀","👨🏾‍🚀","👨🏿‍🚀","👩‍🚀","👩🏻‍🚀","👩🏼‍🚀","👩🏽‍🚀","👩🏾‍🚀","👩🏿‍🚀","🧑‍🚒","🧑🏻‍🚒","🧑🏼‍🚒","🧑🏽‍🚒","🧑🏾‍🚒","🧑🏿‍🚒","👨‍🚒","👨🏻‍🚒","👨🏼‍🚒","👨🏽‍🚒","👨🏾‍🚒","👨🏿‍🚒","👩‍🚒","👩🏻‍🚒","👩🏼‍🚒","👩🏽‍🚒","👩🏾‍🚒","👩🏿‍🚒","👮‍♂️","👮🏻‍♂️","👮🏼‍♂️","👮🏽‍♂️","👮🏾‍♂️","👮🏿‍♂️","👮‍♀️","👮🏻‍♀️","👮🏼‍♀️","👮🏽‍♀️","👮🏾‍♀️","👮🏿‍♀️","🕵️‍♂️","🕵🏻‍♂️","🕵🏼‍♂️","🕵🏽‍♂️","🕵🏾‍♂️","🕵🏿‍♂️","🕵️‍♀️","🕵🏻‍♀️","🕵🏼‍♀️","🕵🏽‍♀️","🕵🏾‍♀️","🕵🏿‍♀️","💂‍♂️","💂🏻‍♂️","💂🏼‍♂️","💂🏽‍♂️","💂🏾‍♂️","💂🏿‍♂️","💂‍♀️","💂🏻‍♀️","💂🏼‍♀️","💂🏽‍♀️","💂🏾‍♀️","💂🏿‍♀️","👷‍♂️","👷🏻‍♂️","👷🏼‍♂️","👷🏽‍♂️","👷🏾‍♂️","👷🏿‍♂️","👷‍♀️","👷🏻‍♀️","👷🏼‍♀️","👷🏽‍♀️","👷🏾‍♀️","👷🏿‍♀️","👳‍♂️","👳🏻‍♂️","👳🏼‍♂️","👳🏽‍♂️","👳🏾‍♂️","👳🏿‍♂️","👳‍♀️","👳🏻‍♀️","👳🏼‍♀️","👳🏽‍♀️","👳🏾‍♀️","👳🏿‍♀️","🤵‍♂️","🤵🏻‍♂️","🤵🏼‍♂️","🤵🏽‍♂️","🤵🏾‍♂️","🤵🏿‍♂️","🤵‍♀️","🤵🏻‍♀️","🤵🏼‍♀️","🤵🏽‍♀️","🤵🏾‍♀️","🤵🏿‍♀️","👰‍♂️","👰🏻‍♂️","👰🏼‍♂️","👰🏽‍♂️","👰🏾‍♂️","👰🏿‍♂️","👰‍♀️","👰🏻‍♀️","👰🏼‍♀️","👰🏽‍♀️","👰🏾‍♀️","👰🏿‍♀️","👩‍🍼","👩🏻‍🍼","👩🏼‍🍼","👩🏽‍🍼","👩🏾‍🍼","👩🏿‍🍼","👨‍🍼","👨🏻‍🍼","👨🏼‍🍼","👨🏽‍🍼","👨🏾‍🍼","👨🏿‍🍼","🧑‍🍼","🧑🏻‍🍼","🧑🏼‍🍼","🧑🏽‍🍼","🧑🏾‍🍼","🧑🏿‍🍼","🧑‍🎄","🧑🏻‍🎄","🧑🏼‍🎄","🧑🏽‍🎄","🧑🏾‍🎄","🧑🏿‍🎄","🦸‍♂️","🦸🏻‍♂️","🦸🏼‍♂️","🦸🏽‍♂️","🦸🏾‍♂️","🦸🏿‍♂️","🦸‍♀️","🦸🏻‍♀️","🦸🏼‍♀️","🦸🏽‍♀️","🦸🏾‍♀️","🦸🏿‍♀️","🦹‍♂️","🦹🏻‍♂️","🦹🏼‍♂️","🦹🏽‍♂️","🦹🏾‍♂️","🦹🏿‍♂️","🦹‍♀️","🦹🏻‍♀️","🦹🏼‍♀️","🦹🏽‍♀️","🦹🏾‍♀️","🦹🏿‍♀️","🧙‍♂️","🧙🏻‍♂️","🧙🏼‍♂️","🧙🏽‍♂️","🧙🏾‍♂️","🧙🏿‍♂️","🧙‍♀️","🧙🏻‍♀️","🧙🏼‍♀️","🧙🏽‍♀️","🧙🏾‍♀️","🧙🏿‍♀️","🧚‍♂️","🧚🏻‍♂️","🧚🏼‍♂️","🧚🏽‍♂️","🧚🏾‍♂️","🧚🏿‍♂️","🧚‍♀️","🧚🏻‍♀️","🧚🏼‍♀️","🧚🏽‍♀️","🧚🏾‍♀️","🧚🏿‍♀️","🧛‍♂️","🧛🏻‍♂️","🧛🏼‍♂️","🧛🏽‍♂️","🧛🏾‍♂️","🧛🏿‍♂️","🧛‍♀️","🧛🏻‍♀️","🧛🏼‍♀️","🧛🏽‍♀️","🧛🏾‍♀️","🧛🏿‍♀️","🧜‍♂️","🧜🏻‍♂️","🧜🏼‍♂️","🧜🏽‍♂️","🧜🏾‍♂️","🧜🏿‍♂️","🧜‍♀️","🧜🏻‍♀️","🧜🏼‍♀️","🧜🏽‍♀️","🧜🏾‍♀️","🧜🏿‍♀️","🧝‍♂️","🧝🏻‍♂️","🧝🏼‍♂️","🧝🏽‍♂️","🧝🏾‍♂️","🧝🏿‍♂️","🧝‍♀️","🧝🏻‍♀️","🧝🏼‍♀️","🧝🏽‍♀️","🧝🏾‍♀️","🧝🏿‍♀️","🧞‍♂️","🧞‍♀️","🧟‍♂️","🧟‍♀️","💆‍♂️","💆🏻‍♂️","💆🏼‍♂️","💆🏽‍♂️","💆🏾‍♂️","💆🏿‍♂️","💆‍♀️","💆🏻‍♀️","💆🏼‍♀️","💆🏽‍♀️","💆🏾‍♀️","💆🏿‍♀️","💇‍♂️","💇🏻‍♂️","💇🏼‍♂️","💇🏽‍♂️","💇🏾‍♂️","💇🏿‍♂️","💇‍♀️","💇🏻‍♀️","💇🏼‍♀️","💇🏽‍♀️","💇🏾‍♀️","💇🏿‍♀️","🚶‍♂️","🚶🏻‍♂️","🚶🏼‍♂️","🚶🏽‍♂️","🚶🏾‍♂️","🚶🏿‍♂️","🚶‍♀️","🚶🏻‍♀️","🚶🏼‍♀️","🚶🏽‍♀️","🚶🏾‍♀️","🚶🏿‍♀️","🧍‍♂️","🧍🏻‍♂️","🧍🏼‍♂️","🧍🏽‍♂️","🧍🏾‍♂️","🧍🏿‍♂️","🧍‍♀️","🧍🏻‍♀️","🧍🏼‍♀️","🧍🏽‍♀️","🧍🏾‍♀️","🧍🏿‍♀️","🧎‍♂️","🧎🏻‍♂️","🧎🏼‍♂️","🧎🏽‍♂️","🧎🏾‍♂️","🧎🏿‍♂️","🧎‍♀️","🧎🏻‍♀️","🧎🏼‍♀️","🧎🏽‍♀️","🧎🏾‍♀️","🧎🏿‍♀️","🧑‍🦯","🧑🏻‍🦯","🧑🏼‍🦯","🧑🏽‍🦯","🧑🏾‍🦯","🧑🏿‍🦯","👨‍🦯","👨🏻‍🦯","👨🏼‍🦯","👨🏽‍🦯","👨🏾‍🦯","👨🏿‍🦯","👩‍🦯","👩🏻‍🦯","👩🏼‍🦯","👩🏽‍🦯","👩🏾‍🦯","👩🏿‍🦯","🧑‍🦼","🧑🏻‍🦼","🧑🏼‍🦼","🧑🏽‍🦼","🧑🏾‍🦼","🧑🏿‍🦼","👨‍🦼","👨🏻‍🦼","👨🏼‍🦼","👨🏽‍🦼","👨🏾‍🦼","👨🏿‍🦼","👩‍🦼","👩🏻‍🦼","👩🏼‍🦼","👩🏽‍🦼","👩🏾‍🦼","👩🏿‍🦼","🧑‍🦽","🧑🏻‍🦽","🧑🏼‍🦽","🧑🏽‍🦽","🧑🏾‍🦽","🧑🏿‍🦽","👨‍🦽","👨🏻‍🦽","👨🏼‍🦽","👨🏽‍🦽","👨🏾‍🦽","👨🏿‍🦽","👩‍🦽","👩🏻‍🦽","👩🏼‍🦽","👩🏽‍🦽","👩🏾‍🦽","👩🏿‍🦽","🏃‍♂️","🏃🏻‍♂️","🏃🏼‍♂️","🏃🏽‍♂️","🏃🏾‍♂️","🏃🏿‍♂️","🏃‍♀️","🏃🏻‍♀️","🏃🏼‍♀️","🏃🏽‍♀️","🏃🏾‍♀️","🏃🏿‍♀️","👯‍♂️","👯‍♀️","🧖‍♂️","🧖🏻‍♂️","🧖🏼‍♂️","🧖🏽‍♂️","🧖🏾‍♂️","🧖🏿‍♂️","🧖‍♀️","🧖🏻‍♀️","🧖🏼‍♀️","🧖🏽‍♀️","🧖🏾‍♀️","🧖🏿‍♀️","🧗‍♂️","🧗🏻‍♂️","🧗🏼‍♂️","🧗🏽‍♂️","🧗🏾‍♂️","🧗🏿‍♂️","🧗‍♀️","🧗🏻‍♀️","🧗🏼‍♀️","🧗🏽‍♀️","🧗🏾‍♀️","🧗🏿‍♀️","🏌️‍♂️","🏌🏻‍♂️","🏌🏼‍♂️","🏌🏽‍♂️","🏌🏾‍♂️","🏌🏿‍♂️","🏌️‍♀️","🏌🏻‍♀️","🏌🏼‍♀️","🏌🏽‍♀️","🏌🏾‍♀️","🏌🏿‍♀️","🏄‍♂️","🏄🏻‍♂️","🏄🏼‍♂️","🏄🏽‍♂️","🏄🏾‍♂️","🏄🏿‍♂️","🏄‍♀️","🏄🏻‍♀️","🏄🏼‍♀️","🏄🏽‍♀️","🏄🏾‍♀️","🏄🏿‍♀️","🚣‍♂️","🚣🏻‍♂️","🚣🏼‍♂️","🚣🏽‍♂️","🚣🏾‍♂️","🚣🏿‍♂️","🚣‍♀️","🚣🏻‍♀️","🚣🏼‍♀️","🚣🏽‍♀️","🚣🏾‍♀️","🚣🏿‍♀️","🏊‍♂️","🏊🏻‍♂️","🏊🏼‍♂️","🏊🏽‍♂️","🏊🏾‍♂️","🏊🏿‍♂️","🏊‍♀️","🏊🏻‍♀️","🏊🏼‍♀️","🏊🏽‍♀️","🏊🏾‍♀️","🏊🏿‍♀️","⛹️‍♂️","⛹🏻‍♂️","⛹🏼‍♂️","⛹🏽‍♂️","⛹🏾‍♂️","⛹🏿‍♂️","⛹️‍♀️","⛹🏻‍♀️","⛹🏼‍♀️","⛹🏽‍♀️","⛹🏾‍♀️","⛹🏿‍♀️","🏋️‍♂️","🏋🏻‍♂️","🏋🏼‍♂️","🏋🏽‍♂️","🏋🏾‍♂️","🏋🏿‍♂️","🏋️‍♀️","🏋🏻‍♀️","🏋🏼‍♀️","🏋🏽‍♀️","🏋🏾‍♀️","🏋🏿‍♀️","🚴‍♂️","🚴🏻‍♂️","🚴🏼‍♂️","🚴🏽‍♂️","🚴🏾‍♂️","🚴🏿‍♂️","🚴‍♀️","🚴🏻‍♀️","🚴🏼‍♀️","🚴🏽‍♀️","🚴🏾‍♀️","🚴🏿‍♀️","🚵‍♂️","🚵🏻‍♂️","🚵🏼‍♂️","🚵🏽‍♂️","🚵🏾‍♂️","🚵🏿‍♂️","🚵‍♀️","🚵🏻‍♀️","🚵🏼‍♀️","🚵🏽‍♀️","🚵🏾‍♀️","🚵🏿‍♀️","🤸‍♂️","🤸🏻‍♂️","🤸🏼‍♂️","🤸🏽‍♂️","🤸🏾‍♂️","🤸🏿‍♂️","🤸‍♀️","🤸🏻‍♀️","🤸🏼‍♀️","🤸🏽‍♀️","🤸🏾‍♀️","🤸🏿‍♀️","🤼‍♂️","🤼‍♀️","🤽‍♂️","🤽🏻‍♂️","🤽🏼‍♂️","🤽🏽‍♂️","🤽🏾‍♂️","🤽🏿‍♂️","🤽‍♀️","🤽🏻‍♀️","🤽🏼‍♀️","🤽🏽‍♀️","🤽🏾‍♀️","🤽🏿‍♀️","🤾‍♂️","🤾🏻‍♂️","🤾🏼‍♂️","🤾🏽‍♂️","🤾🏾‍♂️","🤾🏿‍♂️","🤾‍♀️","🤾🏻‍♀️","🤾🏼‍♀️","🤾🏽‍♀️","🤾🏾‍♀️","🤾🏿‍♀️","🤹‍♂️","🤹🏻‍♂️","🤹🏼‍♂️","🤹🏽‍♂️","🤹🏾‍♂️","🤹🏿‍♂️","🤹‍♀️","🤹🏻‍♀️","🤹🏼‍♀️","🤹🏽‍♀️","🤹🏾‍♀️","🤹🏿‍♀️","🧘‍♂️","🧘🏻‍♂️","🧘🏼‍♂️","🧘🏽‍♂️","🧘🏾‍♂️","🧘🏿‍♂️","🧘‍♀️","🧘🏻‍♀️","🧘🏼‍♀️","🧘🏽‍♀️","🧘🏾‍♀️","🧘🏿‍♀️","🧑‍🤝‍🧑","🧑🏻‍🤝‍🧑🏻","🧑🏻‍🤝‍🧑🏼","🧑🏻‍🤝‍🧑🏽","🧑🏻‍🤝‍🧑🏾","🧑🏻‍🤝‍🧑🏿","🧑🏼‍🤝‍🧑🏻","🧑🏼‍🤝‍🧑🏼","🧑🏼‍🤝‍🧑🏽","🧑🏼‍🤝‍🧑🏾","🧑🏼‍🤝‍🧑🏿","🧑🏽‍🤝‍🧑🏻","🧑🏽‍🤝‍🧑🏼","🧑🏽‍🤝‍🧑🏽","🧑🏽‍🤝‍🧑🏾","🧑🏽‍🤝‍🧑🏿","🧑🏾‍🤝‍🧑🏻","🧑🏾‍🤝‍🧑🏼","🧑🏾‍🤝‍🧑🏽","🧑🏾‍🤝‍🧑🏾","🧑🏾‍🤝‍🧑🏿","🧑🏿‍🤝‍🧑🏻","🧑🏿‍🤝‍🧑🏼","🧑🏿‍🤝‍🧑🏽","🧑🏿‍🤝‍🧑🏾","🧑🏿‍🤝‍🧑🏿","👩🏻‍🤝‍👩🏼","👩🏻‍🤝‍👩🏽","👩🏻‍🤝‍👩🏾","👩🏻‍🤝‍👩🏿","👩🏼‍🤝‍👩🏻","👩🏼‍🤝‍👩🏽","👩🏼‍🤝‍👩🏾","👩🏼‍🤝‍👩🏿","👩🏽‍🤝‍👩🏻","👩🏽‍🤝‍👩🏼","👩🏽‍🤝‍👩🏾","👩🏽‍🤝‍👩🏿","👩🏾‍🤝‍👩🏻","👩🏾‍🤝‍👩🏼","👩🏾‍🤝‍👩🏽","👩🏾‍🤝‍👩🏿","👩🏿‍🤝‍👩🏻","👩🏿‍🤝‍👩🏼","👩🏿‍🤝‍👩🏽","👩🏿‍🤝‍👩🏾","👩🏻‍🤝‍👨🏼","👩🏻‍🤝‍👨🏽","👩🏻‍🤝‍👨🏾","👩🏻‍🤝‍👨🏿","👩🏼‍🤝‍👨🏻","👩🏼‍🤝‍👨🏽","👩🏼‍🤝‍👨🏾","👩🏼‍🤝‍👨🏿","👩🏽‍🤝‍👨🏻","👩🏽‍🤝‍👨🏼","👩🏽‍🤝‍👨🏾","👩🏽‍🤝‍👨🏿","👩🏾‍🤝‍👨🏻","👩🏾‍🤝‍👨🏼","👩🏾‍🤝‍👨🏽","👩🏾‍🤝‍👨🏿","👩🏿‍🤝‍👨🏻","👩🏿‍🤝‍👨🏼","👩🏿‍🤝‍👨🏽","👩🏿‍🤝‍👨🏾","👨🏻‍🤝‍👨🏼","👨🏻‍🤝‍👨🏽","👨🏻‍🤝‍👨🏾","👨🏻‍🤝‍👨🏿","👨🏼‍🤝‍👨🏻","👨🏼‍🤝‍👨🏽","👨🏼‍🤝‍👨🏾","👨🏼‍🤝‍👨🏿","👨🏽‍🤝‍👨🏻","👨🏽‍🤝‍👨🏼","👨🏽‍🤝‍👨🏾","👨🏽‍🤝‍👨🏿","👨🏾‍🤝‍👨🏻","👨🏾‍🤝‍👨🏼","👨🏾‍🤝‍👨🏽","👨🏾‍🤝‍👨🏿","👨🏿‍🤝‍👨🏻","👨🏿‍🤝‍👨🏼","👨🏿‍🤝‍👨🏽","👨🏿‍🤝‍👨🏾","💏","🧑🏻‍❤️‍💋‍🧑🏼","🧑🏻‍❤️‍💋‍🧑🏽","🧑🏻‍❤️‍💋‍🧑🏾","🧑🏻‍❤️‍💋‍🧑🏿","🧑🏼‍❤️‍💋‍🧑🏻","🧑🏼‍❤️‍💋‍🧑🏽","🧑🏼‍❤️‍💋‍🧑🏾","🧑🏼‍❤️‍💋‍🧑🏿","🧑🏽‍❤️‍💋‍🧑🏻","🧑🏽‍❤️‍💋‍🧑🏼","🧑🏽‍❤️‍💋‍🧑🏾","🧑🏽‍❤️‍💋‍🧑🏿","🧑🏾‍❤️‍💋‍🧑🏻","🧑🏾‍❤️‍💋‍🧑🏼","🧑🏾‍❤️‍💋‍🧑🏽","🧑🏾‍❤️‍💋‍🧑🏿","🧑🏿‍❤️‍💋‍🧑🏻","🧑🏿‍❤️‍💋‍🧑🏼","🧑🏿‍❤️‍💋‍🧑🏽","🧑🏿‍❤️‍💋‍🧑🏾","👩‍❤️‍💋‍👨","👩🏻‍❤️‍💋‍👨🏻","👩🏻‍❤️‍💋‍👨🏼","👩🏻‍❤️‍💋‍👨🏽","👩🏻‍❤️‍💋‍👨🏾","👩🏻‍❤️‍💋‍👨🏿","👩🏼‍❤️‍💋‍👨🏻","👩🏼‍❤️‍💋‍👨🏼","👩🏼‍❤️‍💋‍👨🏽","👩🏼‍❤️‍💋‍👨🏾","👩🏼‍❤️‍💋‍👨🏿","👩🏽‍❤️‍💋‍👨🏻","👩🏽‍❤️‍💋‍👨🏼","👩🏽‍❤️‍💋‍👨🏽","👩🏽‍❤️‍💋‍👨🏾","👩🏽‍❤️‍💋‍👨🏿","👩🏾‍❤️‍💋‍👨🏻","👩🏾‍❤️‍💋‍👨🏼","👩🏾‍❤️‍💋‍👨🏽","👩🏾‍❤️‍💋‍👨🏾","👩🏾‍❤️‍💋‍👨🏿","👩🏿‍❤️‍💋‍👨🏻","👩🏿‍❤️‍💋‍👨🏼","👩🏿‍❤️‍💋‍👨🏽","👩🏿‍❤️‍💋‍👨🏾","👩🏿‍❤️‍💋‍👨🏿","👨‍❤️‍💋‍👨","👨🏻‍❤️‍💋‍👨🏻","👨🏻‍❤️‍💋‍👨🏼","👨🏻‍❤️‍💋‍👨🏽","👨🏻‍❤️‍💋‍👨🏾","👨🏻‍❤️‍💋‍👨🏿","👨🏼‍❤️‍💋‍👨🏻","👨🏼‍❤️‍💋‍👨🏼","👨🏼‍❤️‍💋‍👨🏽","👨🏼‍❤️‍💋‍👨🏾","👨🏼‍❤️‍💋‍👨🏿","👨🏽‍❤️‍💋‍👨🏻","👨🏽‍❤️‍💋‍👨🏼","👨🏽‍❤️‍💋‍👨🏽","👨🏽‍❤️‍💋‍👨🏾","👨🏽‍❤️‍💋‍👨🏿","👨🏾‍❤️‍💋‍👨🏻","👨🏾‍❤️‍💋‍👨🏼","👨🏾‍❤️‍💋‍👨🏽","👨🏾‍❤️‍💋‍👨🏾","👨🏾‍❤️‍💋‍👨🏿","👨🏿‍❤️‍💋‍👨🏻","👨🏿‍❤️‍💋‍👨🏼","👨🏿‍❤️‍💋‍👨🏽","👨🏿‍❤️‍💋‍👨🏾","👨🏿‍❤️‍💋‍👨🏿","👩‍❤️‍💋‍👩","👩🏻‍❤️‍💋‍👩🏻","👩🏻‍❤️‍💋‍👩🏼","👩🏻‍❤️‍💋‍👩🏽","👩🏻‍❤️‍💋‍👩🏾","👩🏻‍❤️‍💋‍👩🏿","👩🏼‍❤️‍💋‍👩🏻","👩🏼‍❤️‍💋‍👩🏼","👩🏼‍❤️‍💋‍👩🏽","👩🏼‍❤️‍💋‍👩🏾","👩🏼‍❤️‍💋‍👩🏿","👩🏽‍❤️‍💋‍👩🏻","👩🏽‍❤️‍💋‍👩🏼","👩🏽‍❤️‍💋‍👩🏽","👩🏽‍❤️‍💋‍👩🏾","👩🏽‍❤️‍💋‍👩🏿","👩🏾‍❤️‍💋‍👩🏻","👩🏾‍❤️‍💋‍👩🏼","👩🏾‍❤️‍💋‍👩🏽","👩🏾‍❤️‍💋‍👩🏾","👩🏾‍❤️‍💋‍👩🏿","👩🏿‍❤️‍💋‍👩🏻","👩🏿‍❤️‍💋‍👩🏼","👩🏿‍❤️‍💋‍👩🏽","👩🏿‍❤️‍💋‍👩🏾","👩🏿‍❤️‍💋‍👩🏿","💑","🧑🏻‍❤️‍🧑🏼","🧑🏻‍❤️‍🧑🏽","🧑🏻‍❤️‍🧑🏾","🧑🏻‍❤️‍🧑🏿","🧑🏼‍❤️‍🧑🏻","🧑🏼‍❤️‍🧑🏽","🧑🏼‍❤️‍🧑🏾","🧑🏼‍❤️‍🧑🏿","🧑🏽‍❤️‍🧑🏻","🧑🏽‍❤️‍🧑🏼","🧑🏽‍❤️‍🧑🏾","🧑🏽‍❤️‍🧑🏿","🧑🏾‍❤️‍🧑🏻","🧑🏾‍❤️‍🧑🏼","🧑🏾‍❤️‍🧑🏽","🧑🏾‍❤️‍🧑🏿","🧑🏿‍❤️‍🧑🏻","🧑🏿‍❤️‍🧑🏼","🧑🏿‍❤️‍🧑🏽","🧑🏿‍❤️‍🧑🏾","👩‍❤️‍👨","👩🏻‍❤️‍👨🏻","👩🏻‍❤️‍👨🏼","👩🏻‍❤️‍👨🏽","👩🏻‍❤️‍👨🏾","👩🏻‍❤️‍👨🏿","👩🏼‍❤️‍👨🏻","👩🏼‍❤️‍👨🏼","👩🏼‍❤️‍👨🏽","👩🏼‍❤️‍👨🏾","👩🏼‍❤️‍👨🏿","👩🏽‍❤️‍👨🏻","👩🏽‍❤️‍👨🏼","👩🏽‍❤️‍👨🏽","👩🏽‍❤️‍👨🏾","👩🏽‍❤️‍👨🏿","👩🏾‍❤️‍👨🏻","👩🏾‍❤️‍👨🏼","👩🏾‍❤️‍👨🏽","👩🏾‍❤️‍👨🏾","👩🏾‍❤️‍👨🏿","👩🏿‍❤️‍👨🏻","👩🏿‍❤️‍👨🏼","👩🏿‍❤️‍👨🏽","👩🏿‍❤️‍👨🏾","👩🏿‍❤️‍👨🏿","👨‍❤️‍👨","👨🏻‍❤️‍👨🏻","👨🏻‍❤️‍👨🏼","👨🏻‍❤️‍👨🏽","👨🏻‍❤️‍👨🏾","👨🏻‍❤️‍👨🏿","👨🏼‍❤️‍👨🏻","👨🏼‍❤️‍👨🏼","👨🏼‍❤️‍👨🏽","👨🏼‍❤️‍👨🏾","👨🏼‍❤️‍👨🏿","👨🏽‍❤️‍👨🏻","👨🏽‍❤️‍👨🏼","👨🏽‍❤️‍👨🏽","👨🏽‍❤️‍👨🏾","👨🏽‍❤️‍👨🏿","👨🏾‍❤️‍👨🏻","👨🏾‍❤️‍👨🏼","👨🏾‍❤️‍👨🏽","👨🏾‍❤️‍👨🏾","👨🏾‍❤️‍👨🏿","👨🏿‍❤️‍👨🏻","👨🏿‍❤️‍👨🏼","👨🏿‍❤️‍👨🏽","👨🏿‍❤️‍👨🏾","👨🏿‍❤️‍👨🏿","👩‍❤️‍👩","👩🏻‍❤️‍👩🏻","👩🏻‍❤️‍👩🏼","👩🏻‍❤️‍👩🏽","👩🏻‍❤️‍👩🏾","👩🏻‍❤️‍👩🏿","👩🏼‍❤️‍👩🏻","👩🏼‍❤️‍👩🏼","👩🏼‍❤️‍👩🏽","👩🏼‍❤️‍👩🏾","👩🏼‍❤️‍👩🏿","👩🏽‍❤️‍👩🏻","👩🏽‍❤️‍👩🏼","👩🏽‍❤️‍👩🏽","👩🏽‍❤️‍👩🏾","👩🏽‍❤️‍👩🏿","👩🏾‍❤️‍👩🏻","👩🏾‍❤️‍👩🏼","👩🏾‍❤️‍👩🏽","👩🏾‍❤️‍👩🏾","👩🏾‍❤️‍👩🏿","👩🏿‍❤️‍👩🏻","👩🏿‍❤️‍👩🏼","👩🏿‍❤️‍👩🏽","👩🏿‍❤️‍👩🏾","👩🏿‍❤️‍👩🏿","👪","👨‍👩‍👦","👨‍👩‍👧","👨‍👩‍👧‍👦","👨‍👩‍👦‍👦","👨‍👩‍👧‍👧","👨‍👨‍👦","👨‍👨‍👧","👨‍👨‍👧‍👦","👨‍👨‍👦‍👦","👨‍👨‍👧‍👧","👩‍👩‍👦","👩‍👩‍👧","👩‍👩‍👧‍👦","👩‍👩‍👦‍👦","👩‍👩‍👧‍👧","👨‍👦","👨‍👦‍👦","👨‍👧","👨‍👧‍👦","👨‍👧‍👧","👩‍👦","👩‍👦‍👦","👩‍👧","👩‍👧‍👦","👩‍👧‍👧","🐕‍🦺","🐈‍⬛","🐻‍❄️","🏳️‍🌈","🏳️‍⚧️","🏴‍☠️"],"expected":[{"id":"0","event":null,"data":"Blåbæ\nrsyltetøy må da være mulig å oppdrive? 😶‍🌫️ 😮‍💨 😵‍💫 ❤️‍🔥 ❤️‍🩹 👁️‍🗨️ 🫱🏻‍🫲🏼 🫱🏻‍🫲🏽 🫱🏻‍🫲🏾 🫱🏻‍🫲🏿 🫱🏼‍🫲🏻 🫱🏼‍🫲🏽 🫱🏼‍🫲🏾 🫱🏼‍🫲🏿 🫱🏽‍🫲🏻 🫱🏽‍🫲🏼 🫱🏽‍🫲🏾 🫱🏽‍🫲🏿 🫱🏾‍🫲🏻 🫱🏾‍🫲🏼 🫱🏾‍🫲🏽 🫱🏾‍🫲🏿 🫱🏿‍🫲🏻 🫱🏿‍🫲🏼 🫱🏿‍🫲🏽 🫱🏿‍🫲🏾 🧔‍♂️ 🧔🏻‍♂️ 🧔🏼‍♂️ 🧔🏽‍♂️ 🧔🏾‍♂️ 🧔🏿‍♂️ 🧔‍♀️ 🧔🏻‍♀️ 🧔🏼‍♀️ 🧔🏽‍♀️ 🧔🏾‍♀️ 🧔🏿‍♀️ 👨‍🦰 👨🏻‍🦰 👨🏼‍🦰 👨🏽‍🦰 👨🏾‍🦰 👨🏿‍🦰 👨‍🦱 👨🏻‍🦱 👨🏼‍🦱 👨🏽‍🦱 👨🏾‍🦱 👨🏿‍🦱 👨‍🦳 👨🏻‍🦳 👨🏼‍🦳 👨🏽‍🦳 👨🏾‍🦳 👨🏿‍🦳 👨‍🦲 👨🏻‍🦲 👨🏼‍🦲 👨🏽‍🦲 👨🏾‍🦲 👨🏿‍🦲 👩‍🦰 👩🏻‍🦰 👩🏼‍🦰 👩🏽‍🦰 👩🏾‍🦰 👩🏿‍🦰 🧑‍🦰 🧑🏻‍🦰 🧑🏼‍🦰 🧑🏽‍🦰 🧑🏾‍🦰 🧑🏿‍🦰 👩‍🦱 👩🏻‍🦱 👩🏼‍🦱 👩🏽‍🦱 👩🏾‍🦱 👩🏿‍🦱 🧑‍🦱 🧑🏻‍🦱 🧑🏼‍🦱 🧑🏽‍🦱 🧑🏾‍🦱 🧑🏿‍🦱 👩‍🦳 👩🏻‍🦳 👩🏼‍🦳 👩🏽‍🦳 👩🏾‍🦳 👩🏿‍🦳 🧑‍🦳 🧑🏻‍🦳 🧑🏼‍🦳 🧑🏽‍🦳 🧑🏾‍🦳"},{"id":"1","event":null,"data":"በእርግጥ አንድ ሰው ብሉቤሪ ጃምን መግዛት መቻል አለበት? 🧑🏿‍🦳 👩‍🦲 👩🏻‍🦲 👩🏼‍🦲 👩🏽‍🦲 👩🏾‍🦲 👩🏿‍🦲 🧑‍🦲 🧑🏻‍🦲 🧑🏼‍🦲 🧑🏽‍🦲 🧑🏾‍🦲 🧑🏿‍🦲 👱‍♀️ 👱🏻‍♀️ 👱🏼‍♀️ 👱🏽‍♀️ 👱🏾‍♀️ 👱🏿‍♀️ 👱‍♂️ 👱🏻‍♂️ 👱🏼‍♂️ 👱🏽‍♂️ 👱🏾‍♂️ 👱🏿‍♂️ 🙍‍♂️ 🙍🏻‍♂️ 🙍🏼‍♂️ 🙍🏽‍♂️ 🙍🏾‍♂️ 🙍🏿‍♂️ 🙍‍♀️ 🙍🏻‍♀️ 🙍🏼‍♀️ 🙍🏽‍♀️ 🙍🏾‍♀️ 🙍🏿‍♀️ 🙎‍♂️ 🙎🏻‍♂️ 🙎🏼‍♂️ 🙎🏽‍♂️ 🙎🏾‍♂️ 🙎🏿‍♂️ 🙎‍♀️ 🙎🏻‍♀️ 🙎🏼‍♀️ 🙎🏽‍♀️ 🙎🏾‍♀️ 🙎🏿‍♀️ 🙅‍♂️ 🙅🏻‍♂️ 🙅🏼‍♂️ 🙅🏽‍♂️ 🙅🏾‍♂️ 🙅🏿‍♂️ 🙅‍♀️ 🙅🏻‍♀️ 🙅🏼‍♀️ 🙅🏽‍♀️ 🙅🏾‍♀️ 🙅🏿‍♀️ 🙆‍♂️ 🙆🏻‍♂️ 🙆🏼‍♂️ 🙆🏽‍♂️ 🙆🏾‍♂️ 🙆🏿‍♂️ 🙆‍♀️ 🙆🏻‍♀️ 🙆🏼‍♀️ 🙆🏽‍♀️ 🙆🏾‍♀️ 🙆🏿‍♀️ 💁‍♂️ 💁🏻‍♂️ 💁🏼‍♂️ 💁🏽‍♂️ 💁🏾‍♂️ 💁🏿‍♂️ 💁‍♀️ 💁🏻‍♀️ 💁🏼‍♀️ 💁🏽‍♀️ 💁🏾‍♀️ 💁🏿‍♀️ 🙋‍♂️ 🙋🏻‍♂️ 🙋🏼‍♂️ 🙋🏽‍♂️ 🙋🏾‍♂️ 🙋🏿‍♂️ 🙋‍♀️ 🙋🏻‍♀️ 🙋🏼‍♀️ 🙋🏽‍♀️ 🙋🏾‍♀️ 🙋🏿‍♀️"},{"id":"2","event":null,"data":"بالتأ\nكيد يجب أن يكون المرء قادرا على شراء مربى التوت؟ 🧏‍♂️ 🧏🏻‍♂️ 🧏🏼‍♂️ 🧏🏽‍♂️ 🧏🏾‍♂️ 🧏🏿‍♂️ 🧏‍♀️ 🧏🏻‍♀️ 🧏🏼‍♀️ 🧏🏽‍♀️ 🧏🏾‍♀️ 🧏🏿‍♀️ 🙇‍♂️ 🙇🏻‍♂️ 🙇🏼‍♂️ 🙇🏽‍♂️ 🙇🏾‍♂️ 🙇🏿‍♂️ 🙇‍♀️ 🙇🏻‍♀️ 🙇🏼‍♀️ 🙇🏽‍♀️ 🙇🏾‍♀️ 🙇🏿‍♀️ 🤦‍♂️ 🤦🏻‍♂️ 🤦🏼‍♂️ 🤦🏽‍♂️ 🤦🏾‍♂️ 🤦🏿‍♂️ 🤦‍♀️ 🤦🏻‍♀️ 🤦🏼‍♀️ 🤦🏽‍♀️ 🤦🏾‍♀️ 🤦🏿‍♀️ 🤷‍♂️ 🤷🏻‍♂️ 🤷🏼‍♂️ 🤷🏽‍♂️ 🤷🏾‍♂️ 🤷🏿‍♂️ 🤷‍♀️ 🤷🏻‍♀️ 🤷🏼‍♀️ 🤷🏽‍♀️ 🤷🏾‍♀️ 🤷🏿‍♀️ 🧑‍⚕️ 🧑🏻‍⚕️ 🧑🏼‍⚕️ 🧑🏽‍⚕️ 🧑🏾‍⚕️ 🧑🏿‍⚕️ 👨‍⚕️ 👨🏻‍⚕️ 👨🏼‍⚕️ 👨🏽‍⚕️ 👨🏾‍⚕️ 👨🏿‍⚕️ 👩‍⚕️ 👩🏻‍⚕️ 👩🏼‍⚕️ 👩🏽‍⚕️ 👩🏾‍⚕️ 👩🏿‍⚕️ 🧑‍🎓 🧑🏻‍🎓 🧑🏼‍🎓 🧑🏽‍🎓 🧑🏾‍🎓 🧑🏿‍🎓 👨‍🎓 👨🏻‍🎓 👨🏼‍🎓 👨🏽‍🎓 👨🏾‍🎓 👨🏿‍🎓 👩‍🎓 👩🏻‍🎓 👩🏼‍🎓 👩🏽‍🎓 👩🏾‍🎓 👩🏿‍🎓 🧑‍🏫 🧑🏻‍🏫 🧑🏼‍🏫 🧑🏽‍🏫 🧑🏾‍🏫 🧑🏿‍🏫 👨‍🏫 👨🏻‍🏫 👨🏼‍🏫 👨🏽‍🏫 👨🏾‍🏫 👨🏿‍🏫 👩‍🏫"},{"id":"3","event":null,"data":"一定能买到蓝莓果酱吗? 👩🏻‍🏫 👩🏼‍🏫 👩🏽‍🏫 👩🏾‍🏫 👩🏿‍🏫 🧑‍⚖️ 🧑🏻‍⚖️ 🧑🏼‍⚖️ 🧑🏽‍⚖️ 🧑🏾‍⚖️ 🧑🏿‍⚖️ 👨‍⚖️ 👨🏻‍⚖️ 👨🏼‍⚖️ 👨🏽‍⚖️ 👨🏾‍⚖️ 👨🏿‍⚖️ 👩‍⚖️ 👩🏻‍⚖️ 👩🏼‍⚖️ 👩🏽‍⚖️ 👩🏾‍⚖️ 👩🏿‍⚖️ 🧑‍🌾 🧑🏻‍🌾 🧑🏼‍🌾 🧑🏽‍🌾 🧑🏾‍🌾 🧑🏿‍🌾 👨‍🌾 👨🏻‍🌾 👨🏼‍🌾 👨🏽‍🌾 👨🏾‍🌾 👨🏿‍🌾 👩‍🌾 👩🏻‍🌾 👩🏼‍🌾 👩🏽‍🌾 👩🏾‍🌾 👩🏿‍🌾 🧑‍🍳 🧑🏻‍🍳 🧑🏼‍🍳 🧑🏽‍🍳 🧑🏾‍🍳 🧑🏿‍🍳 👨‍🍳 👨🏻‍🍳 👨🏼‍🍳 👨🏽‍🍳 👨🏾‍🍳 👨🏿‍🍳 👩‍🍳 👩🏻‍🍳 👩🏼‍🍳 👩🏽‍🍳 👩🏾‍🍳 👩🏿‍🍳 🧑‍🔧 🧑🏻‍🔧 🧑🏼‍🔧 🧑🏽‍🔧 🧑🏾‍🔧 🧑🏿‍🔧 👨‍🔧 👨🏻‍🔧 👨🏼‍🔧 👨🏽‍🔧 👨🏾‍🔧 👨🏿‍🔧 👩‍🔧 👩🏻‍🔧 👩🏼‍🔧 👩🏽‍🔧 👩🏾‍🔧 👩🏿‍🔧 🧑‍🏭 🧑🏻‍🏭 🧑🏼‍🏭 🧑🏽‍🏭 🧑🏾‍🏭 🧑🏿‍🏭 👨‍🏭 👨🏻‍🏭 👨🏼‍🏭 👨🏽‍🏭 👨🏾‍🏭 👨🏿‍🏭 👩‍🏭 👩🏻‍🏭 👩🏼‍🏭 👩🏽‍🏭 👩🏾‍🏭 👩🏿‍🏭 🧑‍💼 🧑🏻‍💼"},{"id":"4","event":null,"data":"ચોક્ક\nસ એક બ્લુબેરી જામ મેળવવા માટે સમર્થ હોવા જ જોઈએ? 🧑🏼‍💼 🧑🏽‍💼 🧑🏾‍💼 🧑🏿‍💼 👨‍💼 👨🏻‍💼 👨🏼‍💼 👨🏽‍💼 👨🏾‍💼 👨🏿‍💼 👩‍💼 👩🏻‍💼 👩🏼‍💼 👩🏽‍💼 👩🏾‍💼 👩🏿‍💼 🧑‍🔬 🧑🏻‍🔬 🧑🏼‍🔬 🧑🏽‍🔬 🧑🏾‍🔬 🧑🏿‍🔬 👨‍🔬 👨🏻‍🔬 👨🏼‍🔬 👨🏽‍🔬 👨🏾‍🔬 👨🏿‍🔬 👩‍🔬 👩🏻‍🔬 👩🏼‍🔬 👩🏽‍🔬 👩🏾‍🔬 👩🏿‍🔬 🧑‍💻 🧑🏻‍💻 🧑🏼‍💻 🧑🏽‍💻 🧑🏾‍💻 🧑🏿‍💻 👨‍💻 👨🏻‍💻 👨🏼‍💻 👨🏽‍💻 👨🏾‍💻 👨🏿‍💻 👩‍💻 👩🏻‍💻 👩🏼‍💻 👩🏽‍💻 👩🏾‍💻 👩🏿‍💻 🧑‍🎤 🧑🏻‍🎤 🧑🏼‍🎤 🧑🏽‍🎤 🧑🏾‍🎤 🧑🏿‍🎤 👨‍🎤 👨🏻‍🎤 👨🏼‍🎤 👨🏽‍🎤 👨🏾‍🎤 👨🏿‍🎤 👩‍🎤 👩🏻‍🎤 👩🏼‍🎤 👩🏽‍🎤 👩🏾‍🎤 👩🏿‍🎤 🧑‍🎨 🧑🏻‍🎨 🧑🏼‍🎨 🧑🏽‍🎨 🧑🏾‍🎨 🧑🏿‍🎨 👨‍🎨 👨🏻‍🎨 👨🏼‍🎨 👨🏽‍🎨 👨🏾‍🎨 👨🏿‍🎨 👩‍🎨 👩🏻‍🎨 👩🏼‍🎨 👩🏽‍🎨 👩🏾‍🎨 👩🏿‍🎨 🧑‍✈️ 🧑🏻‍✈️ 🧑🏼‍✈️ 🧑🏽‍✈️ 🧑🏾‍✈️ 🧑🏿‍✈️ 👨‍✈️ 👨🏻‍✈️ 👨🏼‍✈️"},{"id":"5","event":null,"data":"בטח אפשר להשיג ריבת אוכמניות? 👨🏽‍✈️ 👨🏾‍✈️ 👨🏿‍✈️ 👩‍✈️ 👩🏻‍✈️ 👩🏼‍✈️ 👩🏽‍✈️ 👩🏾‍✈️ 👩🏿‍✈️ 🧑‍🚀 🧑🏻‍🚀 🧑🏼‍🚀 🧑🏽‍🚀 🧑🏾‍🚀 🧑🏿‍🚀 👨‍🚀 👨🏻‍🚀 👨🏼‍🚀 👨🏽‍🚀 👨🏾‍🚀 👨🏿‍🚀 👩‍🚀 👩🏻‍🚀 👩🏼‍🚀 👩🏽‍🚀 👩🏾‍🚀 👩🏿‍🚀 🧑‍🚒 🧑🏻‍🚒 🧑🏼‍🚒 🧑🏽‍🚒 🧑🏾‍🚒 🧑🏿‍🚒 👨‍🚒 👨🏻‍🚒 👨🏼‍🚒 👨🏽‍🚒 👨🏾‍🚒 👨🏿‍🚒 👩‍🚒 👩🏻‍🚒 👩🏼‍🚒 👩🏽‍🚒 👩🏾‍🚒 👩🏿‍🚒 👮‍♂️ 👮🏻‍♂️ 👮🏼‍♂️ 👮🏽‍♂️ 👮🏾‍♂️ 👮🏿‍♂️ 👮‍♀️ 👮🏻‍♀️ 👮🏼‍♀️ 👮🏽‍♀️ 👮🏾‍♀️ 👮🏿‍♀️ 🕵️‍♂️ 🕵🏻‍♂️ 🕵🏼‍♂️ 🕵🏽‍♂️ 🕵🏾‍♂️ 🕵🏿‍♂️ 🕵️‍♀️ 🕵🏻‍♀️ 🕵🏼‍♀️ 🕵🏽‍♀️ 🕵🏾‍♀️ 🕵🏿‍♀️ 💂‍♂️ 💂🏻‍♂️ 💂🏼‍♂️ 💂🏽‍♂️ 💂🏾‍♂️ 💂🏿‍♂️ 💂‍♀️ 💂🏻‍♀️ 💂🏼‍♀️ 💂🏽‍♀️ 💂🏾‍♀️ 💂🏿‍♀️ 👷‍♂️ 👷🏻‍♂️ 👷🏼‍♂️ 👷🏽‍♂️ 👷🏾‍♂️ 👷🏿‍♂️ 👷‍♀️ 👷🏻‍♀️ 👷🏼‍♀️ 👷🏽‍♀️ 👷🏾‍♀️ 👷🏿‍♀️ 👳‍♂️ 👳🏻‍♂️ 👳🏼‍♂️ 👳🏽‍♂️"},{"id":"6","event":null,"data":"निश्च\nित रूप से किसी को ब्लूबेरी जैम खरीदने में सक्षम होना चाहिए? 👳🏾‍♂️ 👳🏿‍♂️ 👳‍♀️ 👳🏻‍♀️ 👳🏼‍♀️ 👳🏽‍♀️ 👳🏾‍♀️ 👳🏿‍♀️ 🤵‍♂️ 🤵🏻‍♂️ 🤵🏼‍♂️ 🤵🏽‍♂️ 🤵🏾‍♂️ 🤵🏿‍♂️ 🤵‍♀️ 🤵🏻‍♀️ 🤵🏼‍♀️ 🤵🏽‍♀️ 🤵🏾‍♀️ 🤵🏿‍♀️ 👰‍♂️ 👰🏻‍♂️ 👰🏼‍♂️ 👰🏽‍♂️ 👰🏾‍♂️ 👰🏿‍♂️ 👰‍♀️ 👰🏻‍♀️ 👰🏼‍♀️ 👰🏽‍♀️ 👰🏾‍♀️ 👰🏿‍♀️ 👩‍🍼 👩🏻‍🍼 👩🏼‍🍼 👩🏽‍🍼 👩🏾‍🍼 👩🏿‍🍼 👨‍🍼 👨🏻‍🍼 👨🏼‍🍼 👨🏽‍🍼 👨🏾‍🍼 👨🏿‍🍼 🧑‍🍼 🧑🏻‍🍼 🧑🏼‍🍼 🧑🏽‍🍼 🧑🏾‍🍼 🧑🏿‍🍼 🧑‍🎄 🧑🏻‍🎄 🧑🏼‍🎄 🧑🏽‍🎄 🧑🏾‍🎄 🧑🏿‍🎄 🦸‍♂️ 🦸🏻‍♂️ 🦸🏼‍♂️ 🦸🏽‍♂️ 🦸🏾‍♂️ 🦸🏿‍♂️ 🦸‍♀️ 🦸🏻‍♀️ 🦸🏼‍♀️ 🦸🏽‍♀️ 🦸🏾‍♀️ 🦸🏿‍♀️ 🦹‍♂️ 🦹🏻‍♂️ 🦹🏼‍♂️ 🦹🏽‍♂️ 🦹🏾‍♂️ 🦹🏿‍♂️ 🦹‍♀️ 🦹🏻‍♀️ 🦹🏼‍♀️ 🦹🏽‍♀️ 🦹🏾‍♀️ 🦹🏿‍♀️ 🧙‍♂️ 🧙🏻‍♂️ 🧙🏼‍♂️ 🧙🏽‍♂️ 🧙🏾‍♂️ 🧙🏿‍♂️ 🧙‍♀️ 🧙🏻‍♀️ 🧙🏼‍♀️ 🧙🏽‍♀️ 🧙🏾‍♀️ 🧙🏿‍♀️ 🧚‍♂️ 🧚🏻‍♂️ 🧚🏼‍♂️ 🧚🏽‍♂️ 🧚🏾‍♂️"},{"id":"7","event":null,"data":"Bê guman pêdivî ye ku meriv karibe jama şînê peyda bike? 🧚🏿‍♂️ 🧚‍♀️ 🧚🏻‍♀️ 🧚🏼‍♀️ 🧚🏽‍♀️ 🧚🏾‍♀️ 🧚🏿‍♀️ 🧛‍♂️ 🧛🏻‍♂️ 🧛🏼‍♂️ 🧛🏽‍♂️ 🧛🏾‍♂️ 🧛🏿‍♂️ 🧛‍♀️ 🧛🏻‍♀️ 🧛🏼‍♀️ 🧛🏽‍♀️ 🧛🏾‍♀️ 🧛🏿‍♀️ 🧜‍♂️ 🧜🏻‍♂️ 🧜🏼‍♂️ 🧜🏽‍♂️ 🧜🏾‍♂️ 🧜🏿‍♂️ 🧜‍♀️ 🧜🏻‍♀️ 🧜🏼‍♀️ 🧜🏽‍♀️ 🧜🏾‍♀️ 🧜🏿‍♀️ 🧝‍♂️ 🧝🏻‍♂️ 🧝🏼‍♂️ 🧝🏽‍♂️ 🧝🏾‍♂️ 🧝🏿‍♂️ 🧝‍♀️ 🧝🏻‍♀️ 🧝🏼‍♀️ 🧝🏽‍♀️ 🧝🏾‍♀️ 🧝🏿‍♀️ 🧞‍♂️ 🧞‍♀️ 🧟‍♂️ 🧟‍♀️ 💆‍♂️ 💆🏻‍♂️ 💆🏼‍♂️ 💆🏽‍♂️ 💆🏾‍♂️ 💆🏿‍♂️ 💆‍♀️ 💆🏻‍♀️ 💆🏼‍♀️ 💆🏽‍♀️ 💆🏾‍♀️ 💆🏿‍♀️ 💇‍♂️ 💇🏻‍♂️ 💇🏼‍♂️ 💇🏽‍♂️ 💇🏾‍♂️ 💇🏿‍♂️ 💇‍♀️ 💇🏻‍♀️ 💇🏼‍♀️ 💇🏽‍♀️ 💇🏾‍♀️ 💇🏿‍♀️ 🚶‍♂️ 🚶🏻‍♂️ 🚶🏼‍♂️ 🚶🏽‍♂️ 🚶🏾‍♂️ 🚶🏿‍♂️ 🚶‍♀️ 🚶🏻‍♀️ 🚶🏼‍♀️ 🚶🏽‍♀️ 🚶🏾‍♀️ 🚶🏿‍♀️ 🧍‍♂️ 🧍🏻‍♂️ 🧍🏼‍♂️ 🧍🏽‍♂️ 🧍🏾‍♂️ 🧍🏿‍♂️ 🧍‍♀️ 🧍🏻‍♀️ 🧍🏼‍♀️ 🧍🏽‍♀️ 🧍🏾‍♀️ 🧍🏿‍♀️ 🧎‍♂️ 🧎🏻‍♂️"},{"id":"8","event":null,"data":"ប្រាក\nដណាស់ មនុស្សម្នាក់ត្រូវតែអាចទិញយៈសាពូនមី blueberry បានទេ? 🧎🏼‍♂️ 🧎🏽‍♂️ 🧎🏾‍♂️ 🧎🏿‍♂️ 🧎‍♀️ 🧎🏻‍♀️ 🧎🏼‍♀️ 🧎🏽‍♀️ 🧎🏾‍♀️ 🧎🏿‍♀️ 🧑‍🦯 🧑🏻‍🦯 🧑🏼‍🦯 🧑🏽‍🦯 🧑🏾‍🦯 🧑🏿‍🦯 👨‍🦯 👨🏻‍🦯 👨🏼‍🦯 👨🏽‍🦯 👨🏾‍🦯 👨🏿‍🦯 👩‍🦯 👩🏻‍🦯 👩🏼‍🦯 👩🏽‍🦯 👩🏾‍🦯 👩🏿‍🦯 🧑‍🦼 🧑🏻‍🦼 🧑🏼‍🦼 🧑🏽‍🦼 🧑🏾‍🦼 🧑🏿‍🦼 👨‍🦼 👨🏻‍🦼 👨🏼‍🦼 👨🏽‍🦼 👨🏾‍🦼 👨🏿‍🦼 👩‍🦼 👩🏻‍🦼 👩🏼‍🦼 👩🏽‍🦼 👩🏾‍🦼 👩🏿‍🦼 🧑‍🦽 🧑🏻‍🦽 🧑🏼‍🦽 🧑🏽‍🦽 🧑🏾‍🦽 🧑🏿‍🦽 👨‍🦽 👨🏻‍🦽 👨🏼‍🦽 👨🏽‍🦽 👨🏾‍🦽 👨🏿‍🦽 👩‍🦽 👩🏻‍🦽 👩🏼‍🦽 👩🏽‍🦽 👩🏾‍🦽 👩🏿‍🦽 🏃‍♂️ 🏃🏻‍♂️ 🏃🏼‍♂️ 🏃🏽‍♂️ 🏃🏾‍♂️ 🏃🏿‍♂️ 🏃‍♀️ 🏃🏻‍♀️ 🏃🏼‍♀️ 🏃🏽‍♀️ 🏃🏾‍♀️ 🏃🏿‍♀️ 👯‍♂️ 👯‍♀️ 🧖‍♂️ 🧖🏻‍♂️ 🧖🏼‍♂️ 🧖🏽‍♂️ 🧖🏾‍♂️ 🧖🏿‍♂️ 🧖‍♀️ 🧖🏻‍♀️ 🧖🏼‍♀️ 🧖🏽‍♀️ 🧖🏾‍♀️ 🧖🏿‍♀️ 🧗‍♂️ 🧗🏻‍♂️ 🧗🏼‍♂️ 🧗🏽‍♂️ 🧗🏾‍♂️ 🧗🏿‍♂️ 🧗‍♀️"},{"id":"9","event":null,"data":"ಖಂಡಿತವಾಗಿಯೂ ಒಬ್ಬರು ಬ್ಲೂಬೆರ್ರಿ ಜಾಮ್ ಅನ್ನು ಸಂಗ್ರಹಿಸಲು ಶಕ್ತರಾಗಿರಬೇಕು? 🧗🏻‍♀️ 🧗🏼‍♀️ 🧗🏽‍♀️ 🧗🏾‍♀️ 🧗🏿‍♀️ 🏌️‍♂️ 🏌🏻‍♂️ 🏌🏼‍♂️ 🏌🏽‍♂️ 🏌🏾‍♂️ 🏌🏿‍♂️ 🏌️‍♀️ 🏌🏻‍♀️ 🏌🏼‍♀️ 🏌🏽‍♀️ 🏌🏾‍♀️ 🏌🏿‍♀️ 🏄‍♂️ 🏄🏻‍♂️ 🏄🏼‍♂️ 🏄🏽‍♂️ 🏄🏾‍♂️ 🏄🏿‍♂️ 🏄‍♀️ 🏄🏻‍♀️ 🏄🏼‍♀️ 🏄🏽‍♀️ 🏄🏾‍♀️ 🏄🏿‍♀️ 🚣‍♂️ 🚣🏻‍♂️ 🚣🏼‍♂️ 🚣🏽‍♂️ 🚣🏾‍♂️ 🚣🏿‍♂️ 🚣‍♀️ 🚣🏻‍♀️ 🚣🏼‍♀️ 🚣🏽‍♀️ 🚣🏾‍♀️ 🚣🏿‍♀️ 🏊‍♂️ 🏊🏻‍♂️ 🏊🏼‍♂️ 🏊🏽‍♂️ 🏊🏾‍♂️ 🏊🏿‍♂️ 🏊‍♀️ 🏊🏻‍♀️ 🏊🏼‍♀️ 🏊🏽‍♀️ 🏊🏾‍♀️ 🏊🏿‍♀️ ⛹️‍♂️ ⛹🏻‍♂️ ⛹🏼‍♂️ ⛹🏽‍♂️ ⛹🏾‍♂️ ⛹🏿‍♂️ ⛹️‍♀️ ⛹🏻‍♀️ ⛹🏼‍♀️ ⛹🏽‍♀️ ⛹🏾‍♀️ ⛹🏿‍♀️ 🏋️‍♂️ 🏋🏻‍♂️ 🏋🏼‍♂️ 🏋🏽‍♂️ 🏋🏾‍♂️ 🏋🏿‍♂️ 🏋️‍♀️ 🏋🏻‍♀️ 🏋🏼‍♀️ 🏋🏽‍♀️ 🏋🏾‍♀️ 🏋🏿‍♀️ 🚴‍♂️ 🚴🏻‍♂️ 🚴🏼‍♂️ 🚴🏽‍♂️ 🚴🏾‍♂️ 🚴🏿‍♂️ 🚴‍♀️ 🚴🏻‍♀️ 🚴🏼‍♀️ 🚴🏽‍♀️ 🚴🏾‍♀️ 🚴🏿‍♀️ 🚵‍♂️ 🚵🏻‍♂️ 🚵🏼‍♂️ 🚵🏽‍♂️ 🚵🏾‍♂️ 🚵🏿‍♂️ 🚵‍♀️ 🚵🏻‍♀️"},{"id":"10","event":null,"data":"確かにブル\nーベリージャムを調達できなければなりませんか? 🚵🏼‍♀️ 🚵🏽‍♀️ 🚵🏾‍♀️ 🚵🏿‍♀️ 🤸‍♂️ 🤸🏻‍♂️ 🤸🏼‍♂️ 🤸🏽‍♂️ 🤸🏾‍♂️ 🤸🏿‍♂️ 🤸‍♀️ 🤸🏻‍♀️ 🤸🏼‍♀️ 🤸🏽‍♀️ 🤸🏾‍♀️ 🤸🏿‍♀️ 🤼‍♂️ 🤼‍♀️ 🤽‍♂️ 🤽🏻‍♂️ 🤽🏼‍♂️ 🤽🏽‍♂️ 🤽🏾‍♂️ 🤽🏿‍♂️ 🤽‍♀️ 🤽🏻‍♀️ 🤽🏼‍♀️ 🤽🏽‍♀️ 🤽🏾‍♀️ 🤽🏿‍♀️ 🤾‍♂️ 🤾🏻‍♂️ 🤾🏼‍♂️ 🤾🏽‍♂️ 🤾🏾‍♂️ 🤾🏿‍♂️ 🤾‍♀️ 🤾🏻‍♀️ 🤾🏼‍♀️ 🤾🏽‍♀️ 🤾🏾‍♀️ 🤾🏿‍♀️ 🤹‍♂️ 🤹🏻‍♂️ 🤹🏼‍♂️ 🤹🏽‍♂️ 🤹🏾‍♂️ 🤹🏿‍♂️ 🤹‍♀️ 🤹🏻‍♀️ 🤹🏼‍♀️ 🤹🏽‍♀️ 🤹🏾‍♀️ 🤹🏿‍♀️ 🧘‍♂️ 🧘🏻‍♂️ 🧘🏼‍♂️ 🧘🏽‍♂️ 🧘🏾‍♂️ 🧘🏿‍♂️ 🧘‍♀️ 🧘🏻‍♀️ 🧘🏼‍♀️ 🧘🏽‍♀️ 🧘🏾‍♀️ 🧘🏿‍♀️ 🧑‍🤝‍🧑 🧑🏻‍🤝‍🧑🏻 🧑🏻‍🤝‍🧑🏼 🧑🏻‍🤝‍🧑🏽 🧑🏻‍🤝‍🧑🏾 🧑🏻‍🤝‍🧑🏿 🧑🏼‍🤝‍🧑🏻 🧑🏼‍🤝‍🧑🏼 🧑🏼‍🤝‍🧑🏽 🧑🏼‍🤝‍🧑🏾 🧑🏼‍🤝‍🧑🏿 🧑🏽‍🤝‍🧑🏻 🧑🏽‍🤝‍🧑🏼 🧑🏽‍🤝‍🧑🏽 🧑🏽‍🤝‍🧑🏾 🧑🏽‍🤝‍🧑🏿 🧑🏾‍🤝‍🧑🏻 🧑🏾‍🤝‍🧑🏼 🧑🏾‍🤝‍🧑🏽 🧑🏾‍🤝‍🧑🏾 🧑🏾‍🤝‍🧑🏿 🧑🏿‍🤝‍🧑🏻 🧑🏿‍🤝‍🧑🏼 🧑🏿‍🤝‍🧑🏽 🧑🏿‍🤝‍🧑🏾 🧑🏿‍🤝‍🧑🏿 👩🏻‍🤝‍👩🏼 👩🏻‍🤝‍👩🏽 👩🏻‍🤝‍👩🏾 👩🏻‍🤝‍👩🏿 👩🏼‍🤝‍👩🏻"},{"id":"11","event":null,"data":"Невже треба вміти добути варення з чорниці? 👩🏼‍🤝‍👩🏽 👩🏼‍🤝‍👩🏾 👩🏼‍🤝‍👩🏿 👩🏽‍🤝‍👩🏻 👩🏽‍🤝‍👩🏼 👩🏽‍🤝‍👩🏾 👩🏽‍🤝‍👩🏿 👩🏾‍🤝‍👩🏻 👩🏾‍🤝‍👩🏼 👩🏾‍🤝‍👩🏽 👩🏾‍🤝‍👩🏿 👩🏿‍🤝‍👩🏻 👩🏿‍🤝‍👩🏼 👩🏿‍🤝‍👩🏽 👩🏿‍🤝‍👩🏾 👩🏻‍🤝‍👨🏼 👩🏻‍🤝‍👨🏽 👩🏻‍🤝‍👨🏾 👩🏻‍🤝‍👨🏿 👩🏼‍🤝‍👨🏻 👩🏼‍🤝‍👨🏽 👩🏼‍🤝‍👨🏾 👩🏼‍🤝‍👨🏿 👩🏽‍🤝‍👨🏻 👩🏽‍🤝‍👨🏼 👩🏽‍🤝‍👨🏾 👩🏽‍🤝‍👨🏿 👩🏾‍🤝‍👨🏻 👩🏾‍🤝‍👨🏼 👩🏾‍🤝‍👨🏽 👩🏾‍🤝‍👨🏿 👩🏿‍🤝‍👨🏻 👩🏿‍🤝‍👨🏼 👩🏿‍🤝‍👨🏽 👩🏿‍🤝‍👨🏾 👨🏻‍🤝‍👨🏼 👨🏻‍🤝‍👨🏽 👨🏻‍🤝‍👨🏾 👨🏻‍🤝‍👨🏿 👨🏼‍🤝‍👨🏻 👨🏼‍🤝‍👨🏽 👨🏼‍🤝‍👨🏾 👨🏼‍🤝‍👨🏿 👨🏽‍🤝‍👨🏻 👨🏽‍🤝‍👨🏼 👨🏽‍🤝‍👨🏾 👨🏽‍🤝‍👨🏿 👨🏾‍🤝‍👨🏻 👨🏾‍🤝‍👨🏼 👨🏾‍🤝‍👨🏽 👨🏾‍🤝‍👨🏿 👨🏿‍🤝‍👨🏻 👨🏿‍🤝‍👨🏼 👨🏿‍🤝‍👨🏽 👨🏿‍🤝‍👨🏾 💏 🧑🏻‍❤️‍💋‍🧑🏼 🧑🏻‍❤️‍💋‍🧑🏽 🧑🏻‍❤️‍💋‍🧑🏾 🧑🏻‍❤️‍💋‍🧑🏿 🧑🏼‍❤️‍💋‍🧑🏻 🧑🏼‍❤️‍💋‍🧑🏽 🧑🏼‍❤️‍💋‍🧑🏾 🧑🏼‍❤️‍💋‍🧑🏿 🧑🏽‍❤️‍💋‍🧑🏻 🧑🏽‍❤️‍💋‍🧑🏼 🧑🏽‍❤️‍💋‍🧑🏾 🧑🏽‍❤️‍💋‍🧑🏿 🧑🏾‍❤️‍💋‍🧑🏻 🧑🏾‍❤️‍💋‍🧑🏼 🧑🏾‍❤️‍💋‍🧑🏽 🧑🏾‍❤️‍💋‍🧑🏿 🧑🏿‍❤️‍💋‍🧑🏻 🧑🏿‍❤️‍💋‍🧑🏼 🧑🏿‍❤️‍💋‍🧑🏽 🧑🏿‍❤️‍💋‍🧑🏾 👩‍❤️‍💋‍👨 👩🏻‍❤️‍💋‍👨🏻 👩🏻‍❤️‍💋‍👨🏼 👩🏻‍❤️‍💋‍👨🏽 👩🏻‍❤️‍💋‍👨🏾 👩🏻‍❤️‍💋‍👨🏿 👩🏼‍❤️‍💋‍👨🏻 👩🏼‍❤️‍💋‍👨🏼 👩🏼‍❤️‍💋‍👨🏽 👩🏼‍❤️‍💋‍👨🏾 👩🏼‍❤️‍💋‍👨🏿 👩🏽‍❤️‍💋‍👨🏻 👩🏽‍❤️‍💋‍👨🏼 👩🏽‍❤️‍💋‍👨🏽 👩🏽‍❤️‍💋‍👨🏾 👩🏽‍❤️‍💋‍👨🏿 👩🏾‍❤️‍💋‍👨🏻 👩🏾‍❤️‍💋‍👨🏼 👩🏾‍❤️‍💋‍👨🏽 👩🏾‍❤️‍💋‍👨🏾 👩🏾‍❤️‍💋‍👨🏿"},{"id":"12","event":null,"data":"แน่นอ\nนว่าจะต้องสามารถจัดหาแยมบลูเบอร์รี่ได้? 👩🏿‍❤️‍💋‍👨🏻 👩🏿‍❤️‍💋‍👨🏼 👩🏿‍❤️‍💋‍👨🏽 👩🏿‍❤️‍💋‍👨🏾 👩🏿‍❤️‍💋‍👨🏿 👨‍❤️‍💋‍👨 👨🏻‍❤️‍💋‍👨🏻 👨🏻‍❤️‍💋‍👨🏼 👨🏻‍❤️‍💋‍👨🏽 👨🏻‍❤️‍💋‍👨🏾 👨🏻‍❤️‍💋‍👨🏿 👨🏼‍❤️‍💋‍👨🏻 👨🏼‍❤️‍💋‍👨🏼 👨🏼‍❤️‍💋‍👨🏽 👨🏼‍❤️‍💋‍👨🏾 👨🏼‍❤️‍💋‍👨🏿 👨🏽‍❤️‍💋‍👨🏻 👨🏽‍❤️‍💋‍👨🏼 👨🏽‍❤️‍💋‍👨🏽 👨🏽‍❤️‍💋‍👨🏾 👨🏽‍❤️‍💋‍👨🏿 👨🏾‍❤️‍💋‍👨🏻 👨🏾‍❤️‍💋‍👨🏼 👨🏾‍❤️‍💋‍👨🏽 👨🏾‍❤️‍💋‍👨🏾 👨🏾‍❤️‍💋‍👨🏿 👨🏿‍❤️‍💋‍👨🏻 👨🏿‍❤️‍💋‍👨🏼 👨🏿‍❤️‍💋‍👨🏽 👨🏿‍❤️‍💋‍👨🏾 👨🏿‍❤️‍💋‍👨🏿 👩‍❤️‍💋‍👩 👩🏻‍❤️‍💋‍👩🏻 👩🏻‍❤️‍💋‍👩🏼 👩🏻‍❤️‍💋‍👩🏽 👩🏻‍❤️‍💋‍👩🏾 👩🏻‍❤️‍💋‍👩🏿 👩🏼‍❤️‍💋‍👩🏻 👩🏼‍❤️‍💋‍👩🏼 👩🏼‍❤️‍💋‍👩🏽 👩🏼‍❤️‍💋‍👩🏾 👩🏼‍❤️‍💋‍👩🏿 👩🏽‍❤️‍💋‍👩🏻 👩🏽‍❤️‍💋‍👩🏼 👩🏽‍❤️‍💋‍👩🏽 👩🏽‍❤️‍💋‍👩🏾 👩🏽‍❤️‍💋‍👩🏿 👩🏾‍❤️‍💋‍👩🏻 👩🏾‍❤️‍💋‍👩🏼 👩🏾‍❤️‍💋‍👩🏽 👩🏾‍❤️‍💋‍👩🏾 👩🏾‍❤️‍💋‍👩🏿 👩🏿‍❤️‍💋‍👩🏻 👩🏿‍❤️‍💋‍👩🏼 👩🏿‍❤️‍💋‍👩🏽 👩🏿‍❤️‍💋‍👩🏾 👩🏿‍❤️‍💋‍👩🏿 💑 🧑🏻‍❤️‍🧑🏼 🧑🏻‍❤️‍🧑🏽 🧑🏻‍❤️‍🧑🏾 🧑🏻‍❤️‍🧑🏿 🧑🏼‍❤️‍🧑🏻 🧑🏼‍❤️‍🧑🏽 🧑🏼‍❤️‍🧑🏾 🧑🏼‍❤️‍🧑🏿 🧑🏽‍❤️‍🧑🏻 🧑🏽‍❤️‍🧑🏼 🧑🏽‍❤️‍🧑🏾 🧑🏽‍❤️‍🧑🏿 🧑🏾‍❤️‍🧑🏻 🧑🏾‍❤️‍🧑🏼 🧑🏾‍❤️‍🧑🏽 🧑🏾‍❤️‍🧑🏿 🧑🏿‍❤️‍🧑🏻 🧑🏿‍❤️‍🧑🏼 🧑🏿‍❤️‍🧑🏽 🧑🏿‍❤️‍🧑🏾 👩‍❤️‍👨 👩🏻‍❤️‍👨🏻 👩🏻‍❤️‍👨🏼 👩🏻‍❤️‍👨🏽 👩🏻‍❤️‍👨🏾 👩🏻‍❤️‍👨🏿 👩🏼‍❤️‍👨🏻 👩🏼‍❤️‍👨🏼 👩🏼‍❤️‍👨🏽 👩🏼‍❤️‍👨🏾 👩🏼‍❤️‍👨🏿 👩🏽‍❤️‍👨🏻 👩🏽‍❤️‍👨🏼 👩🏽‍❤️‍👨🏽 👩🏽‍❤️‍👨🏾 👩🏽‍❤️‍👨🏿 👩🏾‍❤️‍👨🏻 👩🏾‍❤️‍👨🏼 👩🏾‍❤️‍👨🏽"},{"id":"13","event":null,"data":"Сигурно некој мора да може да набави џем од боровинки? 👩🏾‍❤️‍👨🏾 👩🏾‍❤️‍👨🏿 👩🏿‍❤️‍👨🏻 👩🏿‍❤️‍👨🏼 👩🏿‍❤️‍👨🏽 👩🏿‍❤️‍👨🏾 👩🏿‍❤️‍👨🏿 👨‍❤️‍👨 👨🏻‍❤️‍👨🏻 👨🏻‍❤️‍👨🏼 👨🏻‍❤️‍👨🏽 👨🏻‍❤️‍👨🏾 👨🏻‍❤️‍👨🏿 👨🏼‍❤️‍👨🏻 👨🏼‍❤️‍👨🏼 👨🏼‍❤️‍👨🏽 👨🏼‍❤️‍👨🏾 👨🏼‍❤️‍👨🏿 👨🏽‍❤️‍👨🏻 👨🏽‍❤️‍👨🏼 👨🏽‍❤️‍👨🏽 👨🏽‍❤️‍👨🏾 👨🏽‍❤️‍👨🏿 👨🏾‍❤️‍👨🏻 👨🏾‍❤️‍👨🏼 👨🏾‍❤️‍👨🏽 👨🏾‍❤️‍👨🏾 👨🏾‍❤️‍👨🏿 👨🏿‍❤️‍👨🏻 👨🏿‍❤️‍👨🏼 👨🏿‍❤️‍👨🏽 👨🏿‍❤️‍👨🏾 👨🏿‍❤️‍👨🏿 👩‍❤️‍👩 👩🏻‍❤️‍👩🏻 👩🏻‍❤️‍👩🏼 👩🏻‍❤️‍👩🏽 👩🏻‍❤️‍👩🏾 👩🏻‍❤️‍👩🏿 👩🏼‍❤️‍👩🏻 👩🏼‍❤️‍👩🏼 👩🏼‍❤️‍👩🏽 👩🏼‍❤️‍👩🏾 👩🏼‍❤️‍👩🏿 👩🏽‍❤️‍👩🏻 👩🏽‍❤️‍👩🏼 👩🏽‍❤️‍👩🏽 👩🏽‍❤️‍👩🏾 👩🏽‍❤️‍👩🏿 👩🏾‍❤️‍👩🏻 👩🏾‍❤️‍👩🏼 👩🏾‍❤️‍👩🏽 👩🏾‍❤️‍👩🏾 👩🏾‍❤️‍👩🏿 👩🏿‍❤️‍👩🏻 👩🏿‍❤️‍👩🏼 👩🏿‍❤️‍👩🏽 👩🏿‍❤️‍👩🏾 👩🏿‍❤️‍👩🏿 👪 👨‍👩‍👦 👨‍👩‍👧 👨‍👩‍👧‍👦 👨‍👩‍👦‍👦 👨‍👩‍👧‍👧 👨‍👨‍👦 👨‍👨‍👧 👨‍👨‍👧‍👦 👨‍👨‍👦‍👦 👨‍👨‍👧‍👧 👩‍👩‍👦 👩‍👩‍👧 👩‍👩‍👧‍👦 👩‍👩‍👦‍👦 👩‍👩‍👧‍👧 👨‍👦 👨‍👦‍👦 👨‍👧 👨‍👧‍👦 👨‍👧‍👧 👩‍👦 👩‍👦‍👦 👩‍👧 👩‍👧‍👦 👩‍👧‍👧 🐕‍🦺 🐈‍⬛ 🐻‍❄️ 🏳️‍🌈 🏳️‍⚧️ 🏴‍☠️"},{"id":null,"event":"done","data":"✔"}]} \ No newline at end of file diff --git a/aimux-stream/tests/sse_test.rs b/aimux-stream/tests/sse_test.rs index 80f46365f..cc1f595c0 100644 --- a/aimux-stream/tests/sse_test.rs +++ b/aimux-stream/tests/sse_test.rs @@ -1,535 +1,860 @@ -//! Independent SSE parsing tests. +//! Port of the `eventsource-parser` test suite against [`aimux_stream::SseStream`]. //! -//! The TS SDK has no standalone SSE parser tests — `parseJsonEventStream` is -//! only exercised indirectly through provider tests. These tests fill that gap, -//! covering the edge cases listed in the task against [`aimux_stream::SseStream`]. +//! `eventsource-parser` is the parser behind the AI SDK's `parseJsonEventStream`. +//! This file ports `test/parse.test.ts` and `test/stream.test.ts` from v3.1.1 +//! (MIT), driven by the same fixtures (`test/fixtures.ts`; `test/multibyte.ts` +//! is checked in as `tests/fixtures/eventsource_parser_multibyte.json`). Test +//! names follow the upstream titles. //! -//! The expected behavior mirrors `eventsource-parser`'s `EventSourceParserStream` -//! (which the TS `parseJsonEventStream` pipes through): -//! - lines end in `\n`, `\r` or `\r\n` (freely mixed); an event is -//! dispatched only on a blank line; -//! - an event with no `data:` line is NOT dispatched (covers comment-only, -//! `event:`/`id:`/`retry:`-only, and blank-line keep-alives); -//! - exactly one leading U+0020 SPACE after the `:` is stripped from a field -//! value (per the SSE spec); -//! - a partial event at EOF (no terminating blank line) is dropped; -//! - a UTF-8 BOM at stream start is stripped; a line without `:` is a field -//! with an empty value; an empty `event` means no type; an `id` with -//! U+0000 is ignored; `retry` must be all ASCII digits; -//! - exceeding the buffer limit is fatal and ends the stream. +//! Adaptations forced by the Rust API (`SseStream` yields whole events, has no +//! callbacks, and works on bytes): +//! - `retry` is reported on the dispatched event instead of through `onRetry`, +//! so a `retry:` block with no `data:` is dropped, and upstream's +//! `reconnect-interval` events become `SseEvent::retry`. +//! - Upstream's `onError` for an invalid `retry` value has no counterpart: the +//! value is ignored, which is what is asserted. +//! - `maxBufferSize` maps to `with_max_event_size`; overflow yields +//! `SseError::FrameTooLarge` and ends the stream. +//! - A leading U+FEFF is the UTF-8 BOM at byte level. Upstream's +//! "invalid byte-order mark" case feeds a decoded U+FEFF that the JS +//! `TextDecoderStream` would already have stripped, so only the "multiple +//! places" case is ported. +//! +//! Not ported (no Rust counterpart): `onComment` call counts, `reset()` +//! (3 tests), `onError` `ParseError` payloads (3 tests), the "function passed +//! to `createParser`" guard, and the `onError: 'terminate'` variants of the +//! stream tests (covered by the single `FrameTooLarge` case). +//! +//! The `aimux` module holds the few tests for behaviour upstream does not have. use aimux_stream::{SseError, SseEvent, SseStream}; use bytes::Bytes; use futures::stream::{self, StreamExt}; +use serde_json::Value; +use sha2::{Digest, Sha256}; -/// Feed `chunks` (in arrival order) into a [`SseStream`] and collect all -/// emitted events. Each chunk simulates one `Result` item from a -/// real byte stream, so splitting a single event across chunks exercises the -/// cross-chunk buffering path. -async fn collect_events(chunks: Vec<&str>) -> Vec> { - let items: Vec> = chunks - .into_iter() - .map(|s| Ok(Bytes::copy_from_slice(s.as_bytes()))) - .collect(); - let stream = SseStream::new(stream::iter(items)); - stream.collect::>().await -} - -fn data(event: &SseEvent) -> &str { - &event.data -} +// ── harness ────────────────────────────────────────────────────────────── -// ── the 12 required edge cases ──────────────────────────────────────────── +type Item = Result; +type Triple<'a> = (Option<&'a str>, Option<&'a str>, &'a str); -#[tokio::test] -async fn single_complete_event() { - let events = collect_events(vec!["data: hello\n\n"]).await; - assert_eq!(events.len(), 1); - let event = events[0].as_ref().unwrap(); - assert_eq!(data(event), "hello"); - assert!(event.event.is_none()); - assert!(event.id.is_none()); - assert!(event.retry.is_none()); +async fn run_bytes(chunks: Vec>, max_event_size: Option) -> Vec { + let items: Vec> = + chunks.into_iter().map(|c| Ok(Bytes::from(c))).collect(); + match max_event_size { + Some(max) => { + SseStream::with_max_event_size(stream::iter(items), max) + .collect::>() + .await + } + None => { + SseStream::new(stream::iter(items)) + .collect::>() + .await + } + } } -#[tokio::test] -async fn multiple_consecutive_events() { - let events = collect_events(vec!["data: first\n\ndata: second\n\n"]).await; - assert_eq!(events.len(), 2); - assert_eq!(data(events[0].as_ref().unwrap()), "first"); - assert_eq!(data(events[1].as_ref().unwrap()), "second"); +async fn run(chunks: Vec, max_event_size: Option) -> Vec { + run_bytes( + chunks.into_iter().map(String::into_bytes).collect(), + max_event_size, + ) + .await } -#[tokio::test] -async fn event_spanning_two_chunks_half_line_split() { - // A single event whose data line arrives in two pieces. - let events = collect_events(vec!["data: hel", "lo\n\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "hello"); +/// Feed `chunks` and return the events; any error item fails the test. +async fn events(chunks: Vec) -> Vec { + run(chunks, None) + .await + .into_iter() + .map(|item| item.unwrap_or_else(|e| panic!("unexpected error item: {e:?}"))) + .collect() } -#[tokio::test] -async fn event_spanning_many_tiny_chunks() { - // The same event byte-split across many tiny chunks. - let events = collect_events(vec!["da", "ta: ", "wor", "ld", "\n", "\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "world"); +fn s(chunks: &[&str]) -> Vec { + chunks.iter().map(|c| (*c).to_string()).collect() } -#[tokio::test] -async fn multi_line_data_field() { - let events = collect_events(vec!["data: line1\ndata: line2\n\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "line1\nline2"); +/// `(id, event, data)` of each event. +fn triples(events: &[SseEvent]) -> Vec> { + events + .iter() + .map(|e| (e.id.as_deref(), e.event.as_deref(), e.data.as_str())) + .collect() } -#[tokio::test] -async fn event_field() { - let events = collect_events(vec!["event: message\ndata: payload\n\n"]).await; - assert_eq!(events.len(), 1); - let event = events[0].as_ref().unwrap(); - assert_eq!(event.event.as_deref(), Some("message")); - assert_eq!(data(event), "payload"); -} +// ── `eventsource-encoder` (used by upstream's fixtures) ────────────────── -#[tokio::test] -async fn id_field() { - let events = collect_events(vec!["id: 42\ndata: payload\n\n"]).await; - assert_eq!(events.len(), 1); - let event = events[0].as_ref().unwrap(); - assert_eq!(event.id.as_deref(), Some("42")); - assert_eq!(data(event), "payload"); +#[derive(Default)] +struct Msg<'a> { + event: Option<&'a str>, + retry: Option, + id: Option<&'a str>, + data: Option<&'a str>, } -#[tokio::test] -async fn retry_field() { - let events = collect_events(vec!["retry: 5000\ndata: payload\n\n"]).await; - assert_eq!(events.len(), 1); - let event = events[0].as_ref().unwrap(); - assert_eq!(event.retry, Some(5000)); - assert_eq!(data(event), "payload"); +fn encode_data(text: &str) -> String { + let normalized = text.replace("\r\n", "\n").replace('\r', "\n"); + let lines: Vec<&str> = normalized.split('\n').collect(); + let mut out = String::new(); + for (i, line) in lines.iter().enumerate() { + out.push_str("data: "); + out.push_str(line); + out.push_str(if i + 1 == lines.len() { "\n\n" } else { "\n" }); + } + out } -#[tokio::test] -async fn comment_lines_starting_with_colon_are_ignored() { - // A comment line within an event block does not affect the event. - let events = collect_events(vec![": this is a comment\ndata: hello\n\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "hello"); +fn encode_comment(comment: &str) -> String { + let normalized = comment.replace("\r\n", "\n").replace('\r', "\n"); + format!(": {}\n\n", normalized.replace('\n', "\n: ")) } -#[tokio::test] -async fn comment_only_event_is_not_emitted() { - // An event consisting solely of a comment has no data line → not dispatched. - let events = collect_events(vec![": just a comment\n\n"]).await; - assert!(events.is_empty()); +fn encode(msg: &Msg<'_>) -> String { + let mut out = String::new(); + if let Some(event) = msg.event.filter(|e| !e.is_empty()) { + out.push_str(&format!("event: {event}\n")); + } + if let Some(retry) = msg.retry { + out.push_str(&format!("retry: {retry}\n")); + } + if let Some(id) = msg.id { + out.push_str(&format!("id: {id}\n")); + } + if let Some(data) = msg.data { + out.push_str(&encode_data(data)); + } else if !out.is_empty() { + out.push_str("\n\n"); + } + out +} + +fn done() -> String { + encode(&Msg { + event: Some("done"), + data: Some("✔"), + ..Msg::default() + }) +} + +fn data_only(data: &str) -> String { + encode(&Msg { + data: Some(data), + ..Msg::default() + }) +} + +// ── fixtures (`test/fixtures.ts`) ──────────────────────────────────────── + +struct Multibyte { + lines: Vec, + emojis: Vec, + expected: Vec<(Option, Option, String)>, +} + +fn multibyte() -> Multibyte { + let root: Value = + serde_json::from_str(include_str!("fixtures/eventsource_parser_multibyte.json")).unwrap(); + let strings = |key: &str| -> Vec { + root[key] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap().to_string()) + .collect() + }; + let optional = |v: &Value| v.as_str().map(str::to_string); + Multibyte { + lines: strings("lines"), + emojis: strings("emojis"), + expected: root["expected"] + .as_array() + .unwrap() + .iter() + .map(|e| { + ( + optional(&e["id"]), + optional(&e["event"]), + e["data"].as_str().unwrap().to_string(), + ) + }) + .collect(), + } } -#[tokio::test] -async fn done_sentinel_is_emitted_as_data() { - // The parser emits `data: [DONE]` like any other event; higher layers - // (e.g. the OpenAI provider) filter the sentinel. This mirrors the TS - // `parseJsonEventStream`, which drops `[DONE]` after parsing — the SSE - // parser itself does not special-case it. - let events = collect_events(vec!["data: [DONE]\n\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "[DONE]"); +/// Split `text` after `units` UTF-16 code units (JS `slice(0, n)` / `slice(n)`). +fn split_utf16(text: &str, units: usize) -> (&str, &str) { + let mut seen = 0; + for (index, ch) in text.char_indices() { + if seen == units { + return text.split_at(index); + } + seen += ch.len_utf16(); + assert!(seen <= units, "UTF-16 cut inside a surrogate pair"); + } + (text, "") +} + +fn multibyte_chunks(mb: &Multibyte) -> Vec { + let per_message = mb.emojis.len().div_ceil(mb.lines.len()); + let mut chunks = Vec::new(); + for (i, line) in mb.lines.iter().enumerate() { + let start = (per_message * i).min(mb.emojis.len()); + let end = (per_message * i + per_message).min(mb.emojis.len()); + let line = format!("{line} {}", mb.emojis[start..end].join(" ")); + chunks.push(format!("id: {i}\n")); + if i % 2 == 0 { + // Even lines are split into two `data:` lines. + let (head, tail) = split_utf16(&line, 5); + chunks.push(format!("data:{head}\n")); + chunks.push(format!("data:{tail}\n\n")); + } else { + chunks.push(format!("data:{line}\n\n")); + } + } + chunks.push(done()); + chunks } -#[tokio::test] -async fn blank_lines_and_heartbeat_emit_nothing() { - // Bare blank lines are keep-alives / heartbeats with no data → not emitted. - let events = collect_events(vec!["\n\n\n\n"]).await; - assert!(events.is_empty()); +fn iso_timestamp(i: usize) -> String { + format!("2026-09-29T05:51:{:02}.{:03}Z", i % 60, i) } -#[tokio::test] -async fn crlf_vs_lf_line_endings() { - // CRLF-terminated single event. - let events = collect_events(vec!["data: hello\r\n\r\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "hello"); +/// `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$` +fn is_iso_timestamp(value: &str) -> bool { + const SHAPE: &[u8] = b"dddd-dd-ddTdd:dd:dd.dddZ"; + value.len() == SHAPE.len() + && value.bytes().zip(SHAPE).all(|(b, shape)| match shape { + b'd' => b.is_ascii_digit(), + other => b == *other, + }) } -#[tokio::test] -async fn crlf_multi_line_data() { - // CRLF-terminated multi-line data joins with a single `\n`. - let events = collect_events(vec!["data: line1\r\ndata: line2\r\n\r\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "line1\nline2"); -} +// ── parse.test.ts ──────────────────────────────────────────────────────── #[tokio::test] -async fn incomplete_event_at_eof_without_blank_line_is_dropped() { - // No terminating blank line → the buffered partial is not a complete event - // and is dropped (matches eventsource-parser's EventSourceParserStream, - // which has no flush handler). - let events = collect_events(vec!["data: hello"]).await; - assert!(events.is_empty()); +async fn basic_unnamed_events_stream() { + let mut chunks: Vec = (0..5).map(|i| data_only(&i.to_string())).collect(); + chunks.push(done()); + let got = events(chunks).await; + assert_eq!( + triples(&got), + vec![ + (None, None, "0"), + (None, None, "1"), + (None, None, "2"), + (None, None, "3"), + (None, None, "4"), + (None, Some("done"), "✔"), + ] + ); } #[tokio::test] -async fn incomplete_multi_line_event_at_eof_is_dropped() { - // A buffered data line that never receives its terminating blank line is - // dropped, even if the data line itself looks complete. - let events = collect_events(vec!["data: hello\n"]).await; - assert!(events.is_empty()); +async fn stream_of_time_event_name() { + let mut chunks: Vec = (0..5) + .map(|i| { + encode(&Msg { + event: Some("time"), + data: Some(&iso_timestamp(i)), + ..Msg::default() + }) + }) + .collect(); + chunks.push(done()); + let got = events(chunks).await; + assert_eq!(got.len(), 6); + for event in &got[..5] { + assert_eq!(event.event.as_deref(), Some("time")); + assert!(is_iso_timestamp(&event.data), "{}", event.data); + } } -// ── additional spec-faithfulness coverage ──────────────────────────────── - #[tokio::test] -async fn data_without_space_after_colon() { - // `data:hello` (no space) → value is `hello`. - let events = collect_events(vec!["data:hello\n\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "hello"); +async fn stream_of_time_event_names_unbalanced_chunks() { + let mut chunks = Vec::new(); + let mut ids = Vec::new(); + for i in 0..30 { + let id = (100_000_000_000_u64 + i as u64 * 7919).to_string(); + let message = encode(&Msg { + id: Some(&id), + event: Some("time"), + data: Some(&iso_timestamp(i)), + ..Msg::default() + }); + // Upstream splits at a random offset; the offset is varied + // deterministically here (the message is ASCII). + let split = (i * 11 + 3) % message.len(); + chunks.push(message[..split].to_string()); + chunks.push(message[split..].to_string()); + ids.push(id); + } + let got = events(chunks).await; + assert_eq!(got.len(), 30); + for (event, id) in got.iter().zip(&ids) { + assert_eq!(event.event.as_deref(), Some("time")); + assert_eq!(event.id.as_deref(), Some(id.as_str())); + assert!(is_iso_timestamp(&event.data), "{}", event.data); + } } #[tokio::test] -async fn only_one_leading_space_is_removed() { - // Per the SSE spec exactly ONE leading U+0020 SPACE is removed; the rest - // of the value is preserved verbatim. - let events = collect_events(vec!["data: two leading spaces\n\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), " two leading spaces"); +async fn stream_of_identified_messages_and_retry_interval() { + let chunks = (1337..1339) + .map(|id| { + let id = id.to_string(); + encode(&Msg { + event: Some("tick"), + data: Some(&id), + id: Some(&id), + retry: Some(50), + }) + }) + .collect(); + let got = events(chunks).await; + assert_eq!( + triples(&got), + vec![ + (Some("1337"), Some("tick"), "1337"), + (Some("1338"), Some("tick"), "1338"), + ] + ); + assert!(got.iter().all(|e| e.retry == Some(50))); } #[tokio::test] -async fn empty_data_value() { - // `data:` with nothing after it contributes an empty data line. - let events = collect_events(vec!["data:\n\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), ""); +async fn stream_of_heartbeat_comments_unnamed_events() { + let mut chunks = Vec::new(); + for letter in ['A', 'B', 'C', 'D', 'E'] { + chunks.push(encode_comment(" ♥")); + chunks.push(data_only(&letter.to_string())); + } + chunks.push(done()); + let got = events(chunks).await; + assert_eq!( + triples(&got), + vec![ + (None, None, "A"), + (None, None, "B"), + (None, None, "C"), + (None, None, "D"), + (None, None, "E"), + (None, Some("done"), "✔"), + ] + ); } #[tokio::test] -async fn event_field_without_data_is_not_emitted() { - // `event:` without a `data:` line → not dispatched. - let events = collect_events(vec!["event: ping\n\n"]).await; - assert!(events.is_empty()); +async fn stream_of_multi_line_data_events() { + let mut chunks = s(&[ + "event: stock\n", + "data: YHOO\n", + "data: +2\n", + "data: 10\n\n", + "event: stock\n", + "data: GOOG\n", + "data: -8\n", + "data: 1881\n\n", + ]); + chunks.push(done()); + let got = events(chunks).await; + assert_eq!( + triples(&got)[..2], + [ + (None, Some("stock"), "YHOO\n+2\n10"), + (None, Some("stock"), "GOOG\n-8\n1881"), + ] + ); } #[tokio::test] -async fn id_field_without_data_is_not_emitted() { - let events = collect_events(vec!["id: 99\n\n"]).await; - assert!(events.is_empty()); +async fn stream_of_multi_byte_events() { + let mb = multibyte(); + let got = events(multibyte_chunks(&mb)).await; + let got: Vec<_> = got.into_iter().map(|e| (e.id, e.event, e.data)).collect(); + assert_eq!(got, mb.expected); } #[tokio::test] -async fn retry_field_without_data_is_not_emitted() { - let events = collect_events(vec!["retry: 1000\n\n"]).await; - assert!(events.is_empty()); +async fn stream_of_multi_byte_events_with_some_empty_lines_thrown_in() { + let got = events(vec![ + "\n\n\n\nid: 1\ndata: 我現在都看實況不玩遊戲\n\n".to_string(), + done(), + ]) + .await; + assert_eq!( + triples(&got), + vec![ + (Some("1"), None, "我現在都看實況不玩遊戲"), + (None, Some("done"), "✔"), + ] + ); } #[tokio::test] -async fn event_with_all_fields_together() { - let events = collect_events(vec!["event: update\nid: 7\nretry: 3000\ndata: payload\n\n"]).await; - assert_eq!(events.len(), 1); - let e = events[0].as_ref().unwrap(); - assert_eq!(e.event.as_deref(), Some("update")); - assert_eq!(e.id.as_deref(), Some("7")); - assert_eq!(e.retry, Some(3000)); - assert_eq!(data(e), "payload"); +async fn stream_of_leading_bom() { + let got = events(vec![ + "\u{FEFF}data: bomful 1\n\n".to_string(), + "data: bomless 2\n\n".to_string(), + done(), + ]) + .await; + assert_eq!( + triples(&got), + vec![ + (None, None, "bomful 1"), + (None, None, "bomless 2"), + (None, Some("done"), "✔"), + ] + ); } #[tokio::test] -async fn intermixed_comments_between_events() { - let events = collect_events(vec![ - ": heartbeat\ndata: first\n\n", - ": another comment\ndata: second\n\n", +async fn stream_containing_byte_order_mark_multiple_places() { + let got = events(vec![ + "\u{FEFF}data: bomful 1\n\n".to_string(), + "\u{FEFF}data: bomful 2\n\n".to_string(), + "data: bomless 3\n\n".to_string(), + done(), ]) .await; - assert_eq!(events.len(), 2); - assert_eq!(data(events[0].as_ref().unwrap()), "first"); - assert_eq!(data(events[1].as_ref().unwrap()), "second"); + // Only the BOM at the start of the stream is stripped; the second one + // turns the field name into an unknown field, so that event has no data. + assert_eq!( + triples(&got), + vec![ + (None, None, "bomful 1"), + (None, None, "bomless 3"), + (None, Some("done"), "✔"), + ] + ); } #[tokio::test] -async fn invalid_retry_value_is_ignored() { - // A non-numeric `retry:` value is ignored; the event is still dispatched - // because it has a data line. - let events = collect_events(vec!["retry: not-a-number\ndata: payload\n\n"]).await; - assert_eq!(events.len(), 1); - let e = events[0].as_ref().unwrap(); - assert_eq!(e.retry, None); - assert_eq!(data(e), "payload"); +async fn stream_using_carriage_returns() { + let mut chunks = s(&[ + "data: dog\r", + "data: bark\r\r", + "data: cat\r", + "data: meow\r\r", + ]); + chunks.push(done()); + let got = events(chunks).await; + assert_eq!( + triples(&got), + vec![ + (None, None, "dog\nbark"), + (None, None, "cat\nmeow"), + (None, Some("done"), "✔"), + ] + ); } #[tokio::test] -async fn unknown_field_is_ignored() { - let events = collect_events(vec!["foo: bar\ndata: payload\n\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "payload"); +async fn stream_using_line_feeds() { + let mut chunks = s(&[ + "data: cow\n", + "data: moo\n\n", + "data: horse\n", + "data: neigh\n\n", + ]); + chunks.push(done()); + let got = events(chunks).await; + assert_eq!( + triples(&got), + vec![ + (None, None, "cow\nmoo"), + (None, None, "horse\nneigh"), + (None, Some("done"), "✔"), + ] + ); } #[tokio::test] -async fn mixed_lf_and_crlf_events() { - // An LF event followed by a CRLF event in the same stream. - let events = collect_events(vec!["data: lf\n\ndata: crlf\r\n\r\n"]).await; - assert_eq!(events.len(), 2); - assert_eq!(data(events[0].as_ref().unwrap()), "lf"); - assert_eq!(data(events[1].as_ref().unwrap()), "crlf"); -} - -// ── cross-chunk UTF-8 reassembly (P0-03) ───────────────────────────────── - -/// Like [`collect_events`] but takes raw byte chunks, so a chunk that ends in -/// the middle of a multi-byte UTF-8 sequence (not representable as a `&str`) -/// can be fed in. -async fn collect_events_bytes(chunks: Vec>) -> Vec> { - let items: Vec> = - chunks.into_iter().map(|c| Ok(Bytes::from(c))).collect(); - let stream = SseStream::new(stream::iter(items)); - stream.collect::>().await +async fn stream_using_carriage_returns_and_line_feeds() { + let mut chunks = s(&[ + "data: sheep\r\n", + "data: bleat\r\n\r\n", + "data: pig\r\n", + "data: oink\r\n\r\n", + ]); + chunks.push(done()); + let got = events(chunks).await; + assert_eq!( + triples(&got), + vec![ + (None, None, "sheep\nbleat"), + (None, None, "pig\noink"), + (None, Some("done"), "✔"), + ] + ); } #[tokio::test] -async fn chinese_sse_event_split_at_every_byte_boundary() { - // A Chinese SSE event whose data contains multi-byte UTF-8. Splitting the - // full event at *every* byte position must still reassemble to the intact - // text. The old per-chunk `String::from_utf8_lossy` decoder would corrupt - // a code point split across two chunks into two U+FFFDs. - let payload = "data: 你好世界\n\n"; - let bytes = payload.as_bytes(); - for split in 1..bytes.len() { - let (a, b) = bytes.split_at(split); - let events = collect_events_bytes(vec![a.to_vec(), b.to_vec()]).await; - assert_eq!( - events.len(), - 1, - "split at byte {} produced {} results", - split, - events.len() - ); - let event = events[0] - .as_ref() - .unwrap_or_else(|e| panic!("split at byte {split} errored: {e:?}")); - assert_eq!( - event.data, "你好世界", - "split at byte {split} corrupted the data" - ); - } +async fn stream_with_varying_odd_uses_of_comments() { + let mb = multibyte(); + let mut chunks = s(&[": Hello\n\n"]); + chunks.push(":".repeat(300)); + chunks.extend(s(&[ + "\n", + "data: First\n\n", + ": Первый", + ": 第二", + "\n", + "data: Second\n\n", + ])); + chunks.extend(std::iter::repeat_n(": Moop \n".to_string(), 10)); + chunks.extend(s(&[ + ": ثالث", + "\n", + "data: Third\n\n", + ":നാലാമത്തെ", + "\n", + "data: Fourth\n\n", + ])); + chunks.push(format!(": {} :", mb.emojis[..100].join(" "))); + chunks.extend(s(&["\n", "data: Fifth\n\n"])); + chunks.push(done()); + let got = events(chunks).await; + assert_eq!( + triples(&got), + vec![ + (None, None, "First"), + (None, None, "Second"), + (None, None, "Third"), + (None, None, "Fourth"), + (None, None, "Fifth"), + (None, Some("done"), "✔"), + ] + ); } #[tokio::test] -async fn invalid_utf8_frame_returns_utf8_error() { - // `data: ` followed by 0xE4 0xBD — the first two bytes of `你` (U+4F60 = - // E4 BD A0) without the trailing byte — is an incomplete UTF-8 sequence. - // Strict decoding of the complete frame surfaces a `Utf8` error instead - // of silently producing a replacement char. - let chunk = vec![b'd', b'a', b't', b'a', b':', b' ', 0xE4, 0xBD, b'\n', b'\n']; - let events = collect_events_bytes(vec![chunk]).await; - assert_eq!(events.len(), 1, "expected exactly one result"); - match &events[0] { - Err(SseError::Utf8(_)) => {} - other => panic!("expected SseError::Utf8, got {other:?}"), - } +async fn stream_with_even_more_odd_uses_of_comments() { + let long = "x".repeat(2 * 1024 + 1); + let chunks = vec![ + "data:1\r\r:\0\n:\r\ndata:2\n\n:".to_string(), + long.clone(), + "\rdata:3\n\n:data:fail\r:".to_string(), + long, + "\ndata:4\n\n".to_string(), + "data:5".to_string(), + ]; + let got = events(chunks).await; + // No newline after the last message, thus not emitted. + assert_eq!( + triples(&got), + vec![ + (None, None, "1"), + (None, None, "2"), + (None, None, "3"), + (None, None, "4"), + ] + ); } -// ── bounded buffers (P1-10) ───────────────────────────────────────────── - #[tokio::test] -async fn oversized_frame_returns_frame_too_large() { - // A complete event whose frame (`data: hello world`) exceeds the 10-byte - // limit is rejected; the frame+terminator is dropped so the stream ends. - let stream = SseStream::with_max_event_size( - stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static( - b"data: hello world\n\n", - ))]), - 10, +async fn stream_with_empty_event_field() { + let got = events(vec![ + "event:\ndata: Hello 1\n\n".to_string(), + "event:\n\n".to_string(), + done(), + ]) + .await; + assert_eq!( + triples(&got), + vec![(None, None, "Hello 1"), (None, Some("done"), "✔")] ); - let results: Vec<_> = stream.collect().await; - assert_eq!(results.len(), 1); - assert!(matches!(results[0], Err(SseError::FrameTooLarge))); } #[tokio::test] -async fn buffer_growing_past_limit_without_terminator_returns_frame_too_large() { - // No terminator ever arrives, so the buffer would grow unboundedly; the - // limit trips instead (previously this allocated forever). - let stream = SseStream::with_max_event_size( - stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static( - b"data: no terminator here", - ))]), - 10, +async fn stream_with_empty_retry_field() { + let got = events(vec![ + encode(&Msg { + id: Some("1"), + retry: Some(500), + data: Some("🥌"), + ..Msg::default() + }), + "id:2\nretry:\ndata:🧹\n\n".to_string(), + encode(&Msg { + id: Some("3"), + data: Some("✅"), + ..Msg::default() + }), + ]) + .await; + assert_eq!( + triples(&got), + vec![ + (Some("1"), None, "🥌"), + (Some("2"), None, "🧹"), + (Some("3"), None, "✅"), + ] + ); + // The empty `retry` is ignored; `retry: 500` belongs to the first event. + assert_eq!( + got.iter().map(|e| e.retry).collect::>(), + vec![Some(500), None, None] ); - let results: Vec<_> = stream.collect().await; - assert_eq!(results.len(), 1); - assert!(matches!(results[0], Err(SseError::FrameTooLarge))); } -// ── eventsource-parser alignment ───────────────────────────────────────── - #[tokio::test] -async fn bare_cr_line_endings() { - let events = collect_events(vec![ - "event: a\rdata: one\rdata: two\r\rdata: next\r\r:keep-alive\n", +async fn stream_with_oddly_shaped_data_field() { + let got = events(vec![ + "data:\n\ndata\ndata\n\ndata:test\n\n".to_string(), + done(), ]) .await; - assert_eq!(events.len(), 2); - let first = events[0].as_ref().unwrap(); - assert_eq!(first.event.as_deref(), Some("a")); - assert_eq!(data(first), "one\ntwo"); - assert_eq!(data(events[1].as_ref().unwrap()), "next"); + // `data:\n\n` dispatches an event with empty data; `data\ndata\n\n` is two + // empty data lines, joined by a newline. + assert_eq!( + triples(&got), + vec![ + (None, None, ""), + (None, None, "\n"), + (None, None, "test"), + (None, Some("done"), "✔"), + ] + ); } #[tokio::test] -async fn trailing_cr_at_end_of_stream_does_not_dispatch() { - // A `\r` at the end of the buffered input waits for a possible `\n`; with - // no flush at end of stream (as `EventSourceParserStream`) the event is - // never completed. - let events = collect_events(vec!["data: a\r\rdata: b\r\r"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "a"); +async fn stream_with_cr_separating_chunks_of_same_event() { + // A CR at the end of a chunk may be half of a CRLF, so it must not end + // the line yet: otherwise `A\nB` and `C` would be two events. + // https://github.com/rexxars/eventsource-parser/issues/17 + let got = events(s(&["data: A\r\n", "data: B\r", "\n", "data: C\r\n", "\n"])).await; + assert_eq!(triples(&got), vec![(None, None, "A\nB\nC")]); } #[tokio::test] -async fn mixed_terminators_within_one_event() { - // `\r\n` then a bare `\n` blank line: one event, as eventsource-parser. - let events = collect_events(vec!["data: a\r\ndata: b\n\ndata: c\r\r\n"]).await; - assert_eq!(events.len(), 2); - assert_eq!(data(events[0].as_ref().unwrap()), "a\nb"); - assert_eq!(data(events[1].as_ref().unwrap()), "c"); +async fn stream_with_partially_incorrect_retry_fields() { + let got = events(s(&["retry:1000\nretry:2000x\ndata:x\n\n"])).await; + // `2000x` is not all ASCII digits and is ignored; `1000` is kept. + assert_eq!(triples(&got), vec![(None, None, "x")]); + assert_eq!(got[0].retry, Some(1000)); } #[tokio::test] -async fn crlf_split_between_chunks_is_one_terminator() { - // A `\r` at the end of a chunk must wait: the `\n` in the next chunk - // completes the same terminator, not an extra blank line. - let events = collect_events(vec!["data: a\r", "\ndata: b\r", "\n\r", "\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "a\nb"); +async fn stream_with_id_field_containing_a_null_character() { + // An `id` containing U+0000 is ignored, so the earlier `123` survives. + let got = events(s(&["id: 123\nid: bad\0id\ndata: hello\n\n"])).await; + assert_eq!(triples(&got), vec![(Some("123"), None, "hello")]); } #[tokio::test] -async fn leading_bom_is_stripped() { - let events = collect_events(vec!["\u{FEFF}data: first\n\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "first"); +async fn stream_with_incorrect_retry_fields() { + let got = events(s(&[ + "\nretry: 500\n\ndata: first\n\nretry: 50x\n\ndata: second\n\n", + ])) + .await; + assert_eq!( + triples(&got), + vec![(None, None, "first"), (None, None, "second")] + ); } #[tokio::test] -async fn leading_bom_split_across_chunks_is_stripped() { - let bom = "\u{FEFF}".as_bytes(); - let events = collect_events_bytes(vec![ - bom[..1].to_vec(), - bom[1..].to_vec(), - b"data: first\n\n".to_vec(), +async fn stream_with_unknown_fields_in_the_stream() { + let got = events(vec![ + "data:abc\n data\ndata\nfoobar:xxx\njustsometext\n:thisisacommentyay\ndata:123\n\n" + .to_string(), + done(), ]) .await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "first"); + assert_eq!( + triples(&got), + vec![(None, None, "abc\n\n123"), (None, Some("done"), "✔")] + ); } #[tokio::test] -async fn bom_after_stream_start_is_not_stripped() { - // Only the very start of the stream may carry a BOM; later it is part of - // the field name, which is then unknown. - let events = collect_events(vec!["data: a\n\n\u{FEFF}data: b\n\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "a"); +async fn stream_with_huge_data_chunks() { + const TEN_MEGABYTES: usize = 1024 * 1024 * 10; + const EXPECTED_SHA256: &str = + "e094a44a2436226ea9feb04e413a28de012b406012ec0eb6b37ad0a19d403660"; + + let mb = multibyte(); + let data_chunk = encode_data(&format!( + "{}\n{}", + mb.lines.join("\n\n"), + mb.emojis.join(" ") + )) + .trim() + .to_string(); + let mut chunks = s(&[": hello\n\n"]); + let mut written = 0; + while written < TEN_MEGABYTES { + chunks.push(data_chunk.clone()); + written += data_chunk.len(); + } + chunks.extend(s(&["\n\n", ": END-OF-STREAM\n\n"])); + chunks.push(encode(&Msg { + event: Some("done"), + data: Some(EXPECTED_SHA256), + ..Msg::default() + })); + + // The default limit (1 MiB) would end the stream; upstream is unbounded. + let got: Vec = run(chunks, Some(usize::MAX)) + .await + .into_iter() + .map(Result::unwrap) + .collect(); + assert_eq!(got.len(), 2); + // JS `String.length` counts UTF-16 code units. + assert_eq!(got[0].data.encode_utf16().count(), 4_808_512); + assert_eq!( + format!("{:x}", Sha256::digest(got[0].data.as_bytes())), + got[1].data + ); } #[tokio::test] -async fn field_without_colon_has_empty_value() { - // `data` alone is a data line with an empty value. - let events = collect_events(vec!["data\n\ndata\ndata: x\n\n"]).await; - assert_eq!(events.len(), 2); - assert_eq!(data(events[0].as_ref().unwrap()), ""); - assert_eq!(data(events[1].as_ref().unwrap()), "\nx"); +async fn max_buffer_size_triggers_on_pending_fragment_overflow_no_terminator() { + // Under the limit: nothing yet. + assert!(run(s(&["short start"]), Some(16)).await.is_empty()); + // The accumulated fragments pass the limit. + let results = run(s(&["short start", " and now too long"]), Some(16)).await; + assert_eq!(results.len(), 1); + assert!(matches!(results[0], Err(SseError::FrameTooLarge))); } #[tokio::test] -async fn empty_event_field_means_no_event_type() { - let events = collect_events(vec!["event:\ndata: x\n\n"]).await; - assert_eq!(events[0].as_ref().unwrap().event, None); +async fn max_buffer_size_triggers_on_data_buffer_overflow_no_blank_line() { + // Each `data:` line grows the event's data; without a blank line it + // accumulates until the limit trips, and no event is dispatched. + let chunks = (0..50).map(|i| format!("data: chunk-{i}\n")).collect(); + let results = run(chunks, Some(32)).await; + assert_eq!(results.len(), 1); + assert!(matches!(results[0], Err(SseError::FrameTooLarge))); } #[tokio::test] -async fn event_type_resets_after_each_dispatch() { - let events = collect_events(vec!["event: a\ndata: 1\n\ndata: 2\n\n"]).await; - assert_eq!(events[0].as_ref().unwrap().event.as_deref(), Some("a")); - assert_eq!(events[1].as_ref().unwrap().event, None); +async fn max_buffer_size_ends_the_stream_after_overflow() { + // Upstream: `feed` throws after an overflow until `reset()`. Here the + // stream ends, so later events are never yielded. + let results = run( + s(&[ + "this is too long for the buffer", + "data: hello\n\n", + "data: world\n\n", + ]), + Some(8), + ) + .await; + assert_eq!(results.len(), 1); + assert!(matches!(results[0], Err(SseError::FrameTooLarge))); } #[tokio::test] -async fn id_containing_null_is_ignored() { - let events = collect_events(vec!["id: ok\nid: bad\u{0}id\ndata: x\n\n"]).await; - assert_eq!(events[0].as_ref().unwrap().id.as_deref(), Some("ok")); +async fn max_buffer_size_not_triggered_when_events_dispatch_within_the_limit() { + let chunks = (0..100).map(|i| format!("data: {i}\n\n")).collect(); + let got: Vec = run(chunks, Some(64)) + .await + .into_iter() + .map(Result::unwrap) + .collect(); + assert_eq!(got.len(), 100); } #[tokio::test] -async fn retry_must_be_all_ascii_digits() { - // `retry: 5` loses its one leading space and is valid; a second space is not. - for value in ["+5", "-5", "5s", " 5", ""] { - let input = format!("retry:{value}\ndata: x\n\n"); - let events = collect_events(vec![input.as_str()]).await; - assert_eq!(events[0].as_ref().unwrap().retry, None, "retry {value:?}"); - } - let events = collect_events(vec!["retry:1500\ndata: x\n\n"]).await; - assert_eq!(events[0].as_ref().unwrap().retry, Some(1500)); +async fn max_buffer_size_large_input_within_the_default_limit_is_not_an_error() { + // Upstream's default is unbounded; aimux's is 1 MiB (see `aimux` below). + assert!(run(vec!["x".repeat(1_000_000)], None).await.is_empty()); } -#[tokio::test] -async fn field_names_are_case_sensitive() { - let events = collect_events(vec!["Data: x\n\ndata: y\n\n"]).await; - assert_eq!(events.len(), 1); - assert_eq!(data(events[0].as_ref().unwrap()), "y"); -} +// ── stream.test.ts ─────────────────────────────────────────────────────── #[tokio::test] -async fn invalid_utf8_drops_only_the_current_event() { - let mut bytes = b"data: ".to_vec(); - bytes.extend_from_slice(&[0xE4, 0xBD]); - bytes.extend_from_slice(b"\ndata: tail\n\ndata: next\n\n"); - let events = collect_events_bytes(vec![bytes]).await; - // The rest of the poisoned event (`tail`) is discarded, not dispatched as - // a fragment; the following event is unaffected. - assert_eq!(events.len(), 2); - assert!(matches!(events[0], Err(SseError::Utf8(_)))); - assert_eq!(data(events[1].as_ref().unwrap()), "next"); +async fn can_use_event_source_parser_stream() { + let chunks = (0..10) + .map(|i| { + encode(&Msg { + event: Some("foo"), + id: Some(&format!("evt-{i}")), + data: Some(&format!("Hello {i}")), + ..Msg::default() + }) + }) + .collect::>() + .concat(); + let got = events(vec![chunks]).await; + assert_eq!(got.len(), 10); + assert_eq!(triples(&got)[0], (Some("evt-0"), Some("foo"), "Hello 0")); + assert_eq!(triples(&got)[9], (Some("evt-9"), Some("foo"), "Hello 9")); } #[tokio::test] -async fn exceeding_the_limit_ends_the_stream() { - let stream = SseStream::with_max_event_size( - stream::iter(vec![ - Ok::<_, std::io::Error>(Bytes::from_static(b"data: ok\n\n")), - Ok(Bytes::from_static(b"data: far too long for the limit\n\n")), - Ok(Bytes::from_static(b"data: ok\n\n")), - ]), - 10, - ); - let results: Vec<_> = stream.collect().await; - assert_eq!(results.len(), 2); - assert_eq!(data(results[0].as_ref().unwrap()), "ok"); - assert!(matches!(results[1], Err(SseError::FrameTooLarge))); +async fn max_buffer_size_terminates_the_stream() { + // Upstream also runs this with `onError: 'terminate'` and with a custom + // `onError`; overflow is fatal either way. + let results = run(vec!["x".repeat(1024)], Some(64)).await; + assert_eq!(results.len(), 1); + assert!(matches!(results[0], Err(SseError::FrameTooLarge))); } -#[tokio::test] -async fn multi_event_payload_split_at_every_byte_boundary() { - let payload = "event: e\r\nid: 1\r\ndata: 你好\r\n\r\ndata: a\rdata: b\r\r:c\ndata: z\n\n"; - let bytes = payload.as_bytes(); - let expected = collect_events(vec![payload]).await; - let expected: Vec<(Option, String, Option)> = expected - .into_iter() - .map(|e| { - let e = e.unwrap(); - (e.event, e.data, e.id) - }) - .collect(); - assert_eq!(expected.len(), 3); - for split in 1..bytes.len() { - let (a, b) = bytes.split_at(split); - let got: Vec<_> = collect_events_bytes(vec![a.to_vec(), b.to_vec()]) - .await - .into_iter() - .map(|e| { - let e = e.unwrap_or_else(|err| panic!("split {split}: {err:?}")); - (e.event, e.data, e.id) - }) - .collect(); - assert_eq!(got, expected, "split at byte {split}"); +// ── aimux-specific behaviour (not covered upstream) ────────────────────── + +mod aimux { + use super::*; + + #[tokio::test] + async fn invalid_utf8_discards_the_rest_of_the_event_only() { + // Upstream decodes lossily (`TextDecoder`); SseStream decodes strictly. + // `E4 BD` is the start of `你` without its last byte. + let mut bytes = b"data: ".to_vec(); + bytes.extend_from_slice(&[0xE4, 0xBD]); + bytes.extend_from_slice(b"\ndata: tail\n\ndata: next\n\n"); + let results = run_bytes(vec![bytes], None).await; + assert_eq!(results.len(), 2); + assert!(matches!(results[0], Err(SseError::Utf8(_)))); + assert_eq!(results[1].as_ref().unwrap().data, "next"); + } + + #[tokio::test] + async fn multibyte_fixture_split_at_arbitrary_byte_sizes() { + // Chunks can end inside a code point or inside the BOM; the parser + // works on bytes, so the result must not depend on where they end. + let mb = multibyte(); + let mut stream_bytes = "\u{FEFF}".as_bytes().to_vec(); + stream_bytes.extend(multibyte_chunks(&mb).concat().into_bytes()); + for size in [1, 2, 3, 5, 8, 13] { + let chunks = stream_bytes.chunks(size).map(<[u8]>::to_vec).collect(); + let got: Vec<_> = run_bytes(chunks, None) + .await + .into_iter() + .map(|item| { + let e = item.unwrap_or_else(|err| panic!("chunk size {size}: {err:?}")); + (e.id, e.event, e.data) + }) + .collect(); + assert_eq!(got, mb.expected, "chunk size {size}"); + } + } + + #[tokio::test] + async fn transport_error_is_reported_as_a_stream_error() { + let items: Vec> = vec![ + Ok(Bytes::from_static(b"data: a\n\n")), + Err(std::io::Error::other("boom")), + Ok(Bytes::from_static(b"data: b\n\n")), + ]; + let results = SseStream::new(stream::iter(items)) + .collect::>() + .await; + assert_eq!(results.len(), 3); + assert_eq!(results[0].as_ref().unwrap().data, "a"); + assert!(matches!(&results[1], Err(SseError::Stream(m)) if m == "boom")); + assert_eq!(results[2].as_ref().unwrap().data, "b"); + } + + #[tokio::test] + async fn default_limit_is_one_mebibyte() { + let results = run(vec!["x".repeat(1024 * 1024 + 1)], None).await; + assert_eq!(results.len(), 1); + assert!(matches!(results[0], Err(SseError::FrameTooLarge))); } } From f80f5c10047735a35814e3d073cd8ef8aadc7a26 Mon Sep 17 00:00:00 2001 From: cunninghamcard-bit Date: Thu, 1 Oct 2026 15:44:11 +0000 Subject: [PATCH 6/8] feat(stream)!: decode SSE with the sse-stream crate Replace aimux-stream's hand-written SSE parser with a thin adapter over sse-stream 0.3 (WHATWG event-stream parsing), the way the AI SDK's parseJsonEventStream wraps eventsource-parser. The adapter keeps the AI SDK dispatch rules: only blocks with a data line are dispatched, an empty `event:` is None, and `retry` rides on its event. - No event size limit any more, as upstream: `with_max_event_size` and `SseError::FrameTooLarge` are removed. Real recordings of OpenAI Responses and Gemini image generation carry 2-3 MB events; the 1 MiB cap dropped them, and after the previous commit ended the stream on the first one. Both recordings now decode completely (15/15 and 5/5 events). - `SseError::Stream` carries the transport error as its source instead of a String; `Utf8` holds a `str::Utf8Error`; `Decode` covers the rest. Decoder errors end the stream after being reported. - The ported eventsource-parser suite runs unchanged against the adapter (29 cases); the 6 `maxBufferSize` cases go with the limit. - provider-utils: the `SseError::Stream` arm formats the source error. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WGwanvo9WLU9HWD7sRR8TS --- aimux-provider-utils/src/response_handler.rs | 4 +- aimux-stream/Cargo.toml | 1 + aimux-stream/src/sse.rs | 311 ++++--------------- aimux-stream/tests/sse_test.rs | 130 ++------ 4 files changed, 96 insertions(+), 350 deletions(-) diff --git a/aimux-provider-utils/src/response_handler.rs b/aimux-provider-utils/src/response_handler.rs index db568ef8b..2042669df 100644 --- a/aimux-provider-utils/src/response_handler.rs +++ b/aimux-provider-utils/src/response_handler.rs @@ -392,14 +392,14 @@ where match item { Ok(event) if event.data == "[DONE]" => continue, Ok(event) => yield serde_json::from_str::(&event.data).map_err(AiMuxError::from), - Err(aimux_stream::SseError::Stream(message)) => { + Err(aimux_stream::SseError::Stream(error)) => { // Preserve response transport failures as ApiCallError // items. Framing/parser failures remain JsonParse below. yield Err(AiMuxError::ApiCall(Box::new(ApiCallError { response_headers: Some(stream_headers.clone()), is_retryable: true, ..ApiCallError::new( - message, + error.to_string(), stream_url.clone(), stream_request_body_values.clone(), ) diff --git a/aimux-stream/Cargo.toml b/aimux-stream/Cargo.toml index 2d51d4d69..85b6bb205 100644 --- a/aimux-stream/Cargo.toml +++ b/aimux-stream/Cargo.toml @@ -14,6 +14,7 @@ pin-project-lite = { workspace = true } bytes = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } +sse-stream = { version = "0.3.0", default-features = false, features = ["memchr"] } [dev-dependencies] sha2 = "0.10" diff --git a/aimux-stream/src/sse.rs b/aimux-stream/src/sse.rs index a9be7a66f..f32f436cd 100644 --- a/aimux-stream/src/sse.rs +++ b/aimux-stream/src/sse.rs @@ -1,49 +1,60 @@ -//! SSE (Server-Sent Events) parser for streaming model responses. +//! SSE (Server-Sent Events) decoding for streaming model responses. //! -//! Line-based incremental parser following `eventsource-parser`'s -//! `createParser` (the parser behind the AI SDK's `parseJsonEventStream`) and -//! the WHATWG "parsing an event stream" algorithm: +//! The parser is the [`sse-stream`](https://crates.io/crates/sse-stream) crate +//! (the WHATWG "parsing an event stream" algorithm: `\n`, `\r` and `\r\n` +//! line endings in any mix, a leading UTF-8 BOM, `field: value` with one +//! optional leading space, lines without `:` as empty-valued fields, +//! `:`-prefixed comments, an `id` containing U+0000 ignored, `retry` only when +//! all ASCII digits, unknown fields ignored, a partial block at end of stream +//! dropped). //! -//! - lines end in `\n`, `\r` or `\r\n`, freely mixed; a `\r` at the end of -//! the buffered input waits for the next chunk in case a `\n` follows; -//! - a UTF-8 BOM at the very start of the stream is stripped; -//! - a blank line dispatches the event, but only if it had at least one -//! `data` line; field state resets after every blank line; -//! - `field: value` removes exactly one leading U+0020 SPACE; a line without -//! `:` is a field with an empty value; `:`-prefixed lines are comments; -//! - an empty `event` value means no event type; an `id` containing U+0000 is -//! ignored; `retry` is accepted only when it is all ASCII digits; unknown -//! fields are ignored; -//! - a partial event at end of stream is not dispatched. +//! This module is the thin adapter that gives it the AI SDK's dispatch +//! semantics, the way `parseJsonEventStream` wraps `eventsource-parser`: //! -//! Two aimux additions: every line is strictly UTF-8 decoded after it has -//! been reassembled (a code point split across chunks is never corrupted, and -//! invalid UTF-8 surfaces as [`SseError::Utf8`] and discards the rest of -//! that event up to the next blank line), and buffered input is bounded by `max_event_size` (the parser's -//! `maxBufferSize`; exceeding it is fatal and ends the stream, as upstream). +//! - a block is dispatched only if it had at least one `data` line; comment- +//! only and metadata-only blocks are dropped; +//! - an empty `event` value means no event type; +//! - `retry` is reported on the event it was parsed with (upstream reports it +//! through `onRetry`), so a `retry` block without `data` is dropped; +//! - there is no buffer size limit, as in `parseJsonEventStream`. +//! +//! Differences from upstream kept on purpose: a field value that is not valid +//! UTF-8 is a strict [`SseError::Utf8`] and ends the stream (upstream decodes +//! lossily), and a transport error ends the stream after it is reported. -use std::collections::VecDeque; +use std::marker::PhantomData; +use std::pin::Pin; +use std::task::{Context, Poll, ready}; use bytes::Bytes; use futures::Stream; use pin_project_lite::pin_project; -use std::pin::Pin; -use std::task::{Context, Poll}; +use sse_stream::SseByteStream; use thiserror::Error; -/// Default upper bound on buffered event data plus any partial line (1 MiB). -const DEFAULT_MAX_EVENT_SIZE: usize = 1024 * 1024; - -const BOM: &[u8] = b"\xEF\xBB\xBF"; - +/// A terminal decoding error: after yielding one, the stream ends. #[derive(Debug, Error)] pub enum SseError { + /// A recognized field value (`data`, `event`, `id`, `retry`) is not valid + /// UTF-8. #[error("utf-8 decode error: {0}")] - Utf8(#[from] std::string::FromUtf8Error), + Utf8(#[source] std::str::Utf8Error), + /// The underlying byte stream failed; the source is the transport error. #[error("stream error: {0}")] - Stream(String), - #[error("SSE frame exceeded maximum allowed size")] - FrameTooLarge, + Stream(#[source] Box), + /// Any other decoder error. + #[error("SSE decode error: {0}")] + Decode(#[source] sse_stream::Error), +} + +impl From for SseError { + fn from(error: sse_stream::Error) -> Self { + match error { + sse_stream::Error::Body(source) => Self::Stream(source), + sse_stream::Error::Utf8Parse(source) => Self::Utf8(source), + other => Self::Decode(other), + } + } } /// A parsed SSE event. @@ -59,238 +70,54 @@ pub struct SseEvent { pub retry: Option, } -/// Incremental SSE line parser (the `createParser` state machine). -#[derive(Debug)] -struct Parser { - /// Bytes after the last complete line terminator. - pending: Vec, - /// Offset in `pending` from which the next terminator search starts, so a - /// long line fed in many small chunks is scanned once, not quadratically. - scan_from: usize, - bom_checked: bool, - event: Option, - data: String, - data_lines: usize, - id: Option, - retry: Option, - /// Set after a line fails UTF-8 decoding: the rest of the event (up to - /// the next blank line) is discarded rather than dispatched as a fragment. - poisoned: bool, - max_size: usize, - terminated: bool, - ready: VecDeque>, -} - -impl Parser { - fn new(max_size: usize) -> Self { - Self { - pending: Vec::new(), - scan_from: 0, - bom_checked: false, - event: None, - data: String::new(), - data_lines: 0, - id: None, - retry: None, - poisoned: false, - max_size, - terminated: false, - ready: VecDeque::new(), - } - } - - fn feed(&mut self, chunk: &[u8]) { - if self.terminated { - return; - } - self.pending.extend_from_slice(chunk); - - if !self.bom_checked { - // Wait until the first three bytes can rule a BOM in or out. - if self.pending.len() < BOM.len() && BOM.starts_with(&self.pending) { - return; - } - if self.pending.starts_with(BOM) { - self.pending.drain(..BOM.len()); - } - self.bom_checked = true; - } - - let mut line_start = 0; - let mut search = self.scan_from; - while let Some(offset) = self.pending[search..] - .iter() - .position(|&b| b == b'\n' || b == b'\r') - { - let end = search + offset; - let is_cr = self.pending[end] == b'\r'; - // A trailing `\r` may be the first half of a `\r\n` split across - // chunks: defer it until more input arrives. - if is_cr && end + 1 == self.pending.len() { - break; - } - let line = self.pending[line_start..end].to_vec(); - self.parse_line(&line); - if self.terminated { - return; - } - line_start = end + 1; - if is_cr && self.pending.get(line_start) == Some(&b'\n') { - line_start += 1; - } - search = line_start; - } - self.pending.drain(..line_start); - // Resume the next search at the deferred `\r`, if any. - self.scan_from = self.pending.len().saturating_sub(1); - if self.pending.last() != Some(&b'\r') { - self.scan_from = self.pending.len(); - } - self.check_size(self.pending.len()); - } - - fn parse_line(&mut self, line: &[u8]) { - if line.is_empty() { - self.dispatch(); - return; - } - if self.poisoned { - return; - } - let line = match String::from_utf8(line.to_vec()) { - Ok(line) => line, - Err(error) => { - // The event being built can no longer be trusted: drop it and - // everything up to the next blank line. - self.reset_event(); - self.poisoned = true; - self.ready.push_back(Err(SseError::Utf8(error))); - return; - } - }; - if line.starts_with(':') { - return; // comment - } - let (field, value) = match line.split_once(':') { - Some((field, value)) => (field, value.strip_prefix(' ').unwrap_or(value)), - None => (line.as_str(), ""), - }; - match field { - "event" => self.event = (!value.is_empty()).then(|| value.to_string()), - "data" => { - if self.data_lines > 0 { - self.data.push('\n'); - } - self.data.push_str(value); - self.data_lines += 1; - // Lines already consumed from `pending` are not buffered any - // more; only the event data counts until the chunk is done. - self.check_size(0); - } - "id" => { - if !value.contains('\0') { - self.id = Some(value.to_string()); - } - } - "retry" if !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()) => { - self.retry = value.parse().ok(); - } - _ => {} // unknown field - } - } - - fn dispatch(&mut self) { - if self.data_lines > 0 { - self.ready.push_back(Ok(SseEvent { - event: self.event.take(), - data: std::mem::take(&mut self.data), - id: self.id.take(), - retry: self.retry.take(), - })); - } - self.reset_event(); - } - - fn reset_event(&mut self) { - self.poisoned = false; - self.event = None; - self.data.clear(); - self.data_lines = 0; - self.id = None; - self.retry = None; - } - - /// `maxBufferSize`: buffered event data plus the partial line. Exceeding - /// it is fatal, as in `EventSourceParserStream`. - fn check_size(&mut self, pending_len: usize) { - if self.terminated || pending_len + self.data.len() <= self.max_size { - return; - } - self.terminated = true; - self.pending.clear(); - self.reset_event(); - self.ready.push_back(Err(SseError::FrameTooLarge)); - } -} - pin_project! { /// An adapter that decodes a byte stream into SSE events. - pub struct SseStream { + pub struct SseStream + where + S: Stream>, + { #[pin] - inner: S, - parser: Parser, - done: bool, - _err: std::marker::PhantomData, + inner: SseByteStream, + _err: PhantomData, } } impl SseStream where - S: Stream> + Unpin, + S: Stream>, { pub fn new(stream: S) -> Self { - Self::with_max_event_size(stream, DEFAULT_MAX_EVENT_SIZE) - } - - /// Create an [`SseStream`] with a custom buffer limit: when buffered event - /// data plus the current partial line exceed `max_event_size` bytes, the - /// stream yields [`SseError::FrameTooLarge`] and ends. - pub fn with_max_event_size(stream: S, max_event_size: usize) -> Self { Self { - inner: stream, - parser: Parser::new(max_event_size), - done: false, - _err: std::marker::PhantomData, + inner: SseByteStream::new(stream), + _err: PhantomData, } } } impl Stream for SseStream where - S: Stream> + Unpin, - E: std::fmt::Display, + S: Stream>, + E: std::error::Error + Send + Sync + 'static, { type Item = Result; - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.as_mut().get_mut(); - + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut this = self.project(); loop { - if let Some(item) = this.parser.ready.pop_front() { - return Poll::Ready(Some(item)); - } - // A partial event at end of stream is not dispatched: the - // upstream stream has no flush handler. - if this.done || this.parser.terminated { - return Poll::Ready(None); - } - match Pin::new(&mut this.inner).poll_next(cx) { - Poll::Ready(Some(Ok(bytes))) => this.parser.feed(&bytes), - Poll::Ready(Some(Err(e))) => { - return Poll::Ready(Some(Err(SseError::Stream(e.to_string())))); + match ready!(this.inner.as_mut().poll_next(cx)) { + None => return Poll::Ready(None), + Some(Err(error)) => return Poll::Ready(Some(Err(error.into()))), + Some(Ok(block)) => { + // Dispatch only blocks that carried at least one `data` + // line (`dataLines > 0` upstream). + let Some(data) = block.data else { continue }; + return Poll::Ready(Some(Ok(SseEvent { + event: block.event.filter(|event| !event.is_empty()), + data, + id: block.id, + retry: block.retry, + }))); } - Poll::Ready(None) => this.done = true, - Poll::Pending => return Poll::Pending, } } } diff --git a/aimux-stream/tests/sse_test.rs b/aimux-stream/tests/sse_test.rs index cc1f595c0..2672a962d 100644 --- a/aimux-stream/tests/sse_test.rs +++ b/aimux-stream/tests/sse_test.rs @@ -13,8 +13,8 @@ //! `reconnect-interval` events become `SseEvent::retry`. //! - Upstream's `onError` for an invalid `retry` value has no counterpart: the //! value is ignored, which is what is asserted. -//! - `maxBufferSize` maps to `with_max_event_size`; overflow yields -//! `SseError::FrameTooLarge` and ends the stream. +//! - There is no `maxBufferSize`: like `parseJsonEventStream`, the stream is +//! unbounded, so the upstream overflow cases are not ported. //! - A leading U+FEFF is the UTF-8 BOM at byte level. Upstream's //! "invalid byte-order mark" case feeds a decoded U+FEFF that the JS //! `TextDecoderStream` would already have stripped, so only the "multiple @@ -22,8 +22,8 @@ //! //! Not ported (no Rust counterpart): `onComment` call counts, `reset()` //! (3 tests), `onError` `ParseError` payloads (3 tests), the "function passed -//! to `createParser`" guard, and the `onError: 'terminate'` variants of the -//! stream tests (covered by the single `FrameTooLarge` case). +//! to `createParser`" guard, and the `maxBufferSize` / `onError: 'terminate'` +//! stream tests (no size limit here). //! //! The `aimux` module holds the few tests for behaviour upstream does not have. @@ -38,34 +38,21 @@ use sha2::{Digest, Sha256}; type Item = Result; type Triple<'a> = (Option<&'a str>, Option<&'a str>, &'a str); -async fn run_bytes(chunks: Vec>, max_event_size: Option) -> Vec { +async fn run_bytes(chunks: Vec>) -> Vec { let items: Vec> = chunks.into_iter().map(|c| Ok(Bytes::from(c))).collect(); - match max_event_size { - Some(max) => { - SseStream::with_max_event_size(stream::iter(items), max) - .collect::>() - .await - } - None => { - SseStream::new(stream::iter(items)) - .collect::>() - .await - } - } + SseStream::new(stream::iter(items)) + .collect::>() + .await } -async fn run(chunks: Vec, max_event_size: Option) -> Vec { - run_bytes( - chunks.into_iter().map(String::into_bytes).collect(), - max_event_size, - ) - .await +async fn run(chunks: Vec) -> Vec { + run_bytes(chunks.into_iter().map(String::into_bytes).collect()).await } /// Feed `chunks` and return the events; any error item fails the test. async fn events(chunks: Vec) -> Vec { - run(chunks, None) + run(chunks) .await .into_iter() .map(|item| item.unwrap_or_else(|e| panic!("unexpected error item: {e:?}"))) @@ -697,12 +684,7 @@ async fn stream_with_huge_data_chunks() { ..Msg::default() })); - // The default limit (1 MiB) would end the stream; upstream is unbounded. - let got: Vec = run(chunks, Some(usize::MAX)) - .await - .into_iter() - .map(Result::unwrap) - .collect(); + let got: Vec = run(chunks).await.into_iter().map(Result::unwrap).collect(); assert_eq!(got.len(), 2); // JS `String.length` counts UTF-16 code units. assert_eq!(got[0].data.encode_utf16().count(), 4_808_512); @@ -713,57 +695,9 @@ async fn stream_with_huge_data_chunks() { } #[tokio::test] -async fn max_buffer_size_triggers_on_pending_fragment_overflow_no_terminator() { - // Under the limit: nothing yet. - assert!(run(s(&["short start"]), Some(16)).await.is_empty()); - // The accumulated fragments pass the limit. - let results = run(s(&["short start", " and now too long"]), Some(16)).await; - assert_eq!(results.len(), 1); - assert!(matches!(results[0], Err(SseError::FrameTooLarge))); -} - -#[tokio::test] -async fn max_buffer_size_triggers_on_data_buffer_overflow_no_blank_line() { - // Each `data:` line grows the event's data; without a blank line it - // accumulates until the limit trips, and no event is dispatched. - let chunks = (0..50).map(|i| format!("data: chunk-{i}\n")).collect(); - let results = run(chunks, Some(32)).await; - assert_eq!(results.len(), 1); - assert!(matches!(results[0], Err(SseError::FrameTooLarge))); -} - -#[tokio::test] -async fn max_buffer_size_ends_the_stream_after_overflow() { - // Upstream: `feed` throws after an overflow until `reset()`. Here the - // stream ends, so later events are never yielded. - let results = run( - s(&[ - "this is too long for the buffer", - "data: hello\n\n", - "data: world\n\n", - ]), - Some(8), - ) - .await; - assert_eq!(results.len(), 1); - assert!(matches!(results[0], Err(SseError::FrameTooLarge))); -} - -#[tokio::test] -async fn max_buffer_size_not_triggered_when_events_dispatch_within_the_limit() { - let chunks = (0..100).map(|i| format!("data: {i}\n\n")).collect(); - let got: Vec = run(chunks, Some(64)) - .await - .into_iter() - .map(Result::unwrap) - .collect(); - assert_eq!(got.len(), 100); -} - -#[tokio::test] -async fn max_buffer_size_large_input_within_the_default_limit_is_not_an_error() { - // Upstream's default is unbounded; aimux's is 1 MiB (see `aimux` below). - assert!(run(vec!["x".repeat(1_000_000)], None).await.is_empty()); +async fn large_input_without_a_terminator_is_not_an_error() { + // Unbounded, as upstream: a partial block is simply never dispatched. + assert!(run(vec!["x".repeat(4 * 1024 * 1024)]).await.is_empty()); } // ── stream.test.ts ─────────────────────────────────────────────────────── @@ -787,31 +721,22 @@ async fn can_use_event_source_parser_stream() { assert_eq!(triples(&got)[9], (Some("evt-9"), Some("foo"), "Hello 9")); } -#[tokio::test] -async fn max_buffer_size_terminates_the_stream() { - // Upstream also runs this with `onError: 'terminate'` and with a custom - // `onError`; overflow is fatal either way. - let results = run(vec!["x".repeat(1024)], Some(64)).await; - assert_eq!(results.len(), 1); - assert!(matches!(results[0], Err(SseError::FrameTooLarge))); -} - // ── aimux-specific behaviour (not covered upstream) ────────────────────── mod aimux { use super::*; #[tokio::test] - async fn invalid_utf8_discards_the_rest_of_the_event_only() { - // Upstream decodes lossily (`TextDecoder`); SseStream decodes strictly. + async fn invalid_utf8_is_a_terminal_error() { + // Upstream decodes lossily (`TextDecoder`); SseStream decodes strictly + // and, like every decoder error, ends the stream after reporting it. // `E4 BD` is the start of `你` without its last byte. let mut bytes = b"data: ".to_vec(); bytes.extend_from_slice(&[0xE4, 0xBD]); bytes.extend_from_slice(b"\ndata: tail\n\ndata: next\n\n"); - let results = run_bytes(vec![bytes], None).await; - assert_eq!(results.len(), 2); + let results = run_bytes(vec![bytes]).await; + assert_eq!(results.len(), 1); assert!(matches!(results[0], Err(SseError::Utf8(_)))); - assert_eq!(results[1].as_ref().unwrap().data, "next"); } #[tokio::test] @@ -823,7 +748,7 @@ mod aimux { stream_bytes.extend(multibyte_chunks(&mb).concat().into_bytes()); for size in [1, 2, 3, 5, 8, 13] { let chunks = stream_bytes.chunks(size).map(<[u8]>::to_vec).collect(); - let got: Vec<_> = run_bytes(chunks, None) + let got: Vec<_> = run_bytes(chunks) .await .into_iter() .map(|item| { @@ -845,16 +770,9 @@ mod aimux { let results = SseStream::new(stream::iter(items)) .collect::>() .await; - assert_eq!(results.len(), 3); + // The transport error is terminal: nothing after it is read. + assert_eq!(results.len(), 2); assert_eq!(results[0].as_ref().unwrap().data, "a"); - assert!(matches!(&results[1], Err(SseError::Stream(m)) if m == "boom")); - assert_eq!(results[2].as_ref().unwrap().data, "b"); - } - - #[tokio::test] - async fn default_limit_is_one_mebibyte() { - let results = run(vec!["x".repeat(1024 * 1024 + 1)], None).await; - assert_eq!(results.len(), 1); - assert!(matches!(results[0], Err(SseError::FrameTooLarge))); + assert!(matches!(&results[1], Err(SseError::Stream(e)) if e.to_string() == "boom")); } } From 28bdff6b5094c37c266746639257c809f6c56b4e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 04:49:26 +0000 Subject: [PATCH 7/8] docs(changelog): record the SseStream rewrite and NdjsonStream removal One entry for the final state of the branch: the per-commit changelog hunks described intermediate APIs that no longer exist. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WGwanvo9WLU9HWD7sRR8TS --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84c468191..4f23ca3be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,31 @@ All notable changes to aimux are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Breaking + +**Rust (aimux-stream)** + +- `SseStream` is now a thin adapter over the `sse-stream` crate (WHATWG + event-stream parsing), mirroring how the AI SDK's `parseJsonEventStream` + wraps `eventsource-parser`: `\n`, `\r` and `\r\n` line endings in any + mix, a leading UTF-8 BOM is stripped, a field line without `:` counts as an + empty value, an empty `event:` is `None`, an `id` containing U+0000 is + ignored, `retry` must be all ASCII digits, and a block is dispatched only + when it had a `data` line. There is no event size limit any more (as + upstream): `with_max_event_size` and `SseError::FrameTooLarge` are gone, + which fixes streams carrying multi-megabyte events (OpenAI Responses and + Gemini image generation). `SseError::Stream` carries the transport error + as its source instead of a `String`; `SseError::Utf8` holds a + `str::Utf8Error`; `SseError::Decode` is added. Decoder errors (invalid + UTF-8, transport failure) now end the stream after being reported. + `SseStream` requires the body error type to implement `std::error::Error + + Send + Sync + 'static`. +- Removed `NdjsonStream` / `NdjsonError`: nothing in the workspace used them + and the AI SDK has no counterpart. `tokio` is now a dev-dependency only and + the unused direct `serde` dependency is dropped. + ## [0.5.0] - 2026-09-27 **Breaking release.** 13 PRs since 0.3.0: the cross-language error model From 7c0933b100b12b0edddd675c2342cb078200af30 Mon Sep 17 00:00:00 2001 From: chenhaonan Date: Wed, 7 Oct 2026 18:14:26 +0800 Subject: [PATCH 8/8] fix(provider-utils): propagate terminal SSE decoding failures --- CHANGELOG.md | 4 + aimux-provider-utils/src/response_handler.rs | 21 +++- .../tests/sse_terminal_error_test.rs | 98 +++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 aimux-providers/tests/sse_terminal_error_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f23ca3be..fe45c4d4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 as its source instead of a `String`; `SseError::Utf8` holds a `str::Utf8Error`; `SseError::Decode` is added. Decoder errors (invalid UTF-8, transport failure) now end the stream after being reported. + The response handler propagates terminal SSE decoding failures as + non-retryable `ApiCall` errors, preventing a normal finish or tool-call + finalization after truncated input. Invalid JSON inside an otherwise valid + SSE event remains a recoverable `JsonParse` error. `SseStream` requires the body error type to implement `std::error::Error + Send + Sync + 'static`. - Removed `NdjsonStream` / `NdjsonError`: nothing in the workspace used them diff --git a/aimux-provider-utils/src/response_handler.rs b/aimux-provider-utils/src/response_handler.rs index 2042669df..0d8917bd3 100644 --- a/aimux-provider-utils/src/response_handler.rs +++ b/aimux-provider-utils/src/response_handler.rs @@ -340,6 +340,9 @@ pub fn create_binary_response_handler() -> ResponseHandler { } /// Parse an SSE response and deserialize each `data:` field as `T`. +/// +/// Invalid JSON in a complete event remains a recoverable frame error. SSE +/// decoding failures terminate the byte stream and are non-retryable API errors. #[must_use] pub fn create_event_source_response_handler() -> ResponseHandler>> @@ -394,7 +397,7 @@ where Ok(event) => yield serde_json::from_str::(&event.data).map_err(AiMuxError::from), Err(aimux_stream::SseError::Stream(error)) => { // Preserve response transport failures as ApiCallError - // items. Framing/parser failures remain JsonParse below. + // items, eligible for retry before output is emitted. yield Err(AiMuxError::ApiCall(Box::new(ApiCallError { response_headers: Some(stream_headers.clone()), is_retryable: true, @@ -406,7 +409,21 @@ where }))); break; } - Err(error) => yield Err(AiMuxError::JsonParse(error.to_string())), + Err(error) => { + // The SSE decoder ends after this error. Reporting a + // recoverable JsonParse would let providers treat that + // EOF as success and flush incomplete output/tool calls. + yield Err(AiMuxError::ApiCall(Box::new(ApiCallError { + response_headers: Some(stream_headers.clone()), + is_retryable: false, + ..ApiCallError::new( + error.to_string(), + stream_url.clone(), + stream_request_body_values.clone(), + ) + }))); + break; + } } } }); diff --git a/aimux-providers/tests/sse_terminal_error_test.rs b/aimux-providers/tests/sse_terminal_error_test.rs new file mode 100644 index 000000000..2d110c389 --- /dev/null +++ b/aimux-providers/tests/sse_terminal_error_test.rs @@ -0,0 +1,98 @@ +//! Integration checks for aimux's strict UTF-8 decoding (unlike upstream's +//! lossy decoder): a terminal byte error must stay terminal through the +//! response handler and provider, without a successful finish or tool flush. + +use aimux_core::error::AiMuxError; +use aimux_core::language_model::LanguageModel; +use aimux_core::options::CallOptions; +use aimux_core::stream_part::StreamPart; +use aimux_providers::{OpenAIConfig, OpenAIProvider}; +use futures::StreamExt; +use serde_json::{Value, json}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +async fn corrupted_stream(prefix: Value) -> Vec> { + let server = MockServer::start().await; + let suffix = json!({ + "choices": [{"delta": {"content": "world"}, "finish_reason": "stop"}] + }); + let mut body = format!("data: {prefix}\n\n").into_bytes(); + body.extend_from_slice(b"data: \xff\n\n"); + body.extend_from_slice(format!("data: {suffix}\n\ndata: [DONE]\n\n").as_bytes()); + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_bytes(body), + ) + .mount(&server) + .await; + let provider = OpenAIProvider::new(OpenAIConfig::new("test-key").with_base_url(server.uri())); + provider + .model("gpt-4o") + .do_stream(&CallOptions::new(vec![])) + .await + .expect("the first event is valid") + .stream + .collect() + .await +} + +fn assert_terminal_decode_error(parts: &[Result]) { + let errors: Vec<_> = parts + .iter() + .filter_map(|part| part.as_ref().err()) + .collect(); + assert_eq!(errors.len(), 1); + assert!(matches!(errors[0], AiMuxError::ApiCall(_))); + assert!(!errors[0].is_recoverable_stream_error()); + assert!(!errors[0].is_retryable()); + assert!(parts.last().is_some_and(Result::is_err)); + assert!( + !parts + .iter() + .any(|part| matches!(part, Ok(StreamPart::Finish { .. }))) + ); +} + +#[tokio::test] +async fn invalid_utf8_after_text_does_not_emit_a_successful_finish() { + let parts = corrupted_stream(json!({ + "choices": [{"delta": {"content": "hello "}}] + })) + .await; + let text: String = parts + .iter() + .filter_map(|part| match part { + Ok(StreamPart::TextDelta { delta, .. }) => Some(delta.as_str()), + _ => None, + }) + .collect(); + assert_eq!(text, "hello "); + assert_terminal_decode_error(&parts); +} + +#[tokio::test] +async fn invalid_utf8_does_not_finalize_an_incomplete_tool_call() { + let parts = corrupted_stream(json!({ + "choices": [{"delta": {"tool_calls": [{ + "index": 0, + "id": "call-1", + "type": "function", + "function": {"name": "lookup", "arguments": "{"} + }]}}] + })) + .await; + assert!( + parts + .iter() + .any(|part| matches!(part, Ok(StreamPart::ToolInputStart { .. }))) + ); + assert!(!parts.iter().any(|part| matches!( + part, + Ok(StreamPart::ToolCall { .. } | StreamPart::ToolInputEnd { .. }) + ))); + assert_terminal_decode_error(&parts); +}