From 99879a5a5552aae62025f7f68c28c30b2231b60c Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 11 Jun 2026 18:40:16 +0200 Subject: [PATCH 1/7] feat: Add initial version of deich --- Cargo.lock | 12 ++ Cargo.toml | 8 +- breakwater-parser/Cargo.toml | 1 + breakwater-parser/src/framebuffer/mod.rs | 5 + .../src/framebuffer/shared_memory.rs | 5 + breakwater-parser/src/framebuffer/simple.rs | 5 + .../src/framebuffer/time_tracking.rs | 177 ++++++++++++++++++ breakwater-parser/src/lib.rs | 9 + breakwater-parser/src/original.rs | 40 ++++ breakwater/src/lib.rs | 28 +++ breakwater/src/main.rs | 44 +---- deich/Cargo.toml | 24 +++ deich/src/main.rs | 59 ++++++ 13 files changed, 380 insertions(+), 37 deletions(-) create mode 100644 breakwater-parser/src/framebuffer/time_tracking.rs create mode 100644 breakwater/src/lib.rs create mode 100644 deich/Cargo.toml create mode 100644 deich/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 79f9b7f..2c7a6ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1053,6 +1053,18 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "deich" +version = "0.20.0" +dependencies = [ + "breakwater", + "breakwater-parser", + "color-eyre", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "deranged" version = "0.5.8" diff --git a/Cargo.toml b/Cargo.toml index a3a7cf9..a37cc59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,10 @@ [workspace] members = [ "breakwater-parser", - "breakwater", "breakwater-egui-overlay", - "breakwater-parser-c-bindings" + "breakwater-parser-c-bindings", + "breakwater", + "deich", ] resolver = "2" @@ -51,8 +52,9 @@ winit = "0.30" # Uses the given path when used locally, and uses the specified version from crates.io when published. breakwater-core = { path = "breakwater-core", version = "0.20.0" } -breakwater-parser = { path = "breakwater-parser", version = "0.20.0" } +breakwater-parser = { path = "breakwater-parser", version = "0.20.0", default-features = false } breakwater-egui-overlay = { path = "breakwater-egui-overlay", version = "0.20.0" } +breakwater = { path = "breakwater", version = "0.20.0", default-features = false } [profile.dev] opt-level = 3 diff --git a/breakwater-parser/Cargo.toml b/breakwater-parser/Cargo.toml index 7fe8018..d28a202 100644 --- a/breakwater-parser/Cargo.toml +++ b/breakwater-parser/Cargo.toml @@ -27,6 +27,7 @@ rstest.workspace = true alpha = [] binary-set-pixel = [] binary-sync-pixels = [] +time-tracking = [] default = [] diff --git a/breakwater-parser/src/framebuffer/mod.rs b/breakwater-parser/src/framebuffer/mod.rs index bff496f..423c4d6 100644 --- a/breakwater-parser/src/framebuffer/mod.rs +++ b/breakwater-parser/src/framebuffer/mod.rs @@ -1,5 +1,7 @@ pub mod shared_memory; pub mod simple; +#[cfg(feature = "time-tracking")] +pub mod time_tracking; pub const FB_BYTES_PER_PIXEL: usize = std::mem::size_of::(); @@ -55,4 +57,7 @@ pub trait FrameBuffer { /// As the pixel memory doesn't necessarily need to be aligned (think of using shared memory for /// that), we can only return it as a list of bytes, not a list of pixels. fn as_bytes(&self) -> &[u8]; + + #[cfg(feature = "time-tracking")] + fn set_with_ns_since_unix_epoch(&self, x: usize, y: usize, rgba: u32, ns_since_unix_epoch: u64); } diff --git a/breakwater-parser/src/framebuffer/shared_memory.rs b/breakwater-parser/src/framebuffer/shared_memory.rs index 81977e0..1fc681c 100644 --- a/breakwater-parser/src/framebuffer/shared_memory.rs +++ b/breakwater-parser/src/framebuffer/shared_memory.rs @@ -222,4 +222,9 @@ impl FrameBuffer for SharedMemoryFrameBuffer { let base_ptr: *const u8 = self.buffer.as_ptr().cast(); unsafe { slice::from_raw_parts(base_ptr, self.bytes) } } + + #[cfg(feature = "time-tracking")] + fn set_with_ns_since_unix_epoch(&self, _: usize, _: usize, _: u32, _: u64) { + panic!("The shared memory framebuffer does not support time tracking"); + } } diff --git a/breakwater-parser/src/framebuffer/simple.rs b/breakwater-parser/src/framebuffer/simple.rs index d723111..83cc621 100644 --- a/breakwater-parser/src/framebuffer/simple.rs +++ b/breakwater-parser/src/framebuffer/simple.rs @@ -82,6 +82,11 @@ impl FrameBuffer for SimpleFrameBuffer { let ptr = self.buffer.as_ptr() as *const u8; unsafe { std::slice::from_raw_parts(ptr, len) } } + + #[cfg(feature = "time-tracking")] + fn set_with_ns_since_unix_epoch(&self, _: usize, _: usize, _: u32, _: u64) { + panic!("The simply framebuffer does not support time tracking"); + } } #[cfg(test)] diff --git a/breakwater-parser/src/framebuffer/time_tracking.rs b/breakwater-parser/src/framebuffer/time_tracking.rs new file mode 100644 index 0000000..cad7a99 --- /dev/null +++ b/breakwater-parser/src/framebuffer/time_tracking.rs @@ -0,0 +1,177 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +use tracing::debug; + +use crate::FrameBuffer; + +pub struct TimeTrackingFrameBuffer { + width: usize, + height: usize, + buffer: Vec, + base_ns_since_unix_epoch: u64, +} + +/// Number of low bits we drop from `ns_since_base` before storing it. We only have 3 bytes +/// (24 bits) of storage for it, so dropping 5 bits (i.e. dividing by 32) lets us cover +/// `0xff_ffff * 32` ns ≈ 536 ms before we have to clamp. That's plenty given the base +/// timestamp is re-based frequently, and 32 ns of resolution is way more than we need. +const NS_SHIFT: u32 = 5; + +/// Largest value we can store in the 3 `ns` bytes. +const NS_MAX: u32 = 0x00ff_ffff; + +/// A single pixel, laid out as exactly 6 bytes (`align = 1`): +/// +/// ```text +/// byte: 0 1 2 3 4 5 +/// [ r ] [ g ] [ b ] [ns0] [ns1] [ns2] +/// ``` +/// +/// The `ns` bytes hold `ns_since_base >> NS_SHIFT` (see [`NS_SHIFT`]), clamped to [`NS_MAX`]. +/// +/// We deliberately *don't* expose the fields as `u32`s (which would force `align = 4` and 8 +/// bytes per pixel). The trade-off of going to 6 bytes: 6 does not divide 64, so ~10% of +/// pixels straddle a cache-line boundary. The *write* path (random, hot) therefore uses +/// byte/`[u8; 3]` stores rather than wide `u32`/`u16` stores — a wide store that crosses the +/// boundary triggers the x86 split-store penalty on every such pixel, which measured slower +/// (~65G vs ~76G). +/// +/// Both reads and writes only ever touch the bytes of their own pixel, so no pixel can tear a +/// neighbour and no access runs past the end of the buffer — no padding is required. +/// +/// Note: 6-byte cells don't quite match the original 8-byte *aligned* layout (~90G vs ~86G +/// here) on a write-bound load — that one is fast precisely because every write stays within +/// one cache line. The ~4% is the cost of shrinking the per-pixel sync footprint by 25%. +#[derive(Debug, Default, Clone, Copy)] +#[repr(C)] +pub struct TimeTrackingPixel { + pub rgb: [u8; 3], + /// `ns_since_base >> NS_SHIFT`, little-endian, clamped to `NS_MAX`. + pub coarse_ns_since_base: [u8; 3], +} + +impl TimeTrackingFrameBuffer { + pub fn new(width: usize, height: usize, base_ns_since_unix_epoch: u64) -> Self { + let mut buffer = Vec::with_capacity(width * height); + buffer.resize_with(width * height, TimeTrackingPixel::default); + + debug!( + size = buffer.len(), + bytes = buffer.len() * size_of::(), + "Allocated time tracking framebuffer" + ); + Self { + width, + height, + buffer, + base_ns_since_unix_epoch, + } + } + + #[inline(always)] + fn pixel_index(&self, x: usize, y: usize) -> usize { + x + y * self.width + } +} + +impl FrameBuffer for TimeTrackingFrameBuffer { + #[inline(always)] + fn get_width(&self) -> usize { + self.width + } + + #[inline(always)] + fn get_height(&self) -> usize { + self.height + } + + #[inline(always)] + unsafe fn get_unchecked(&self, x: usize, y: usize) -> u32 { + let pixel_index = self.pixel_index(x, y); + // Assemble the RGB bytes; the 4th byte (alpha) is always zero. This is the sequential + // scanout path, which is bandwidth-bound, so the byte assembly is hidden under the + // memory stream and reads no wider than the 3 RGB bytes. + let rgb = unsafe { self.buffer.get_unchecked(pixel_index).rgb }; + u32::from_le_bytes([rgb[0], rgb[1], rgb[2], 0]) + } + + fn set(&self, _: usize, _: usize, _: u32) { + panic!( + "The time tracking framebuffer requires you to use the set_with_ns_since_unix_epoch function!" + ); + } + + fn set_with_ns_since_unix_epoch( + &self, + x: usize, + y: usize, + rgba: u32, + ns_since_unix_epoch: u64, + ) { + if x < self.width && y < self.height { + let pixel_index = self.pixel_index(x, y); + + // Drop the low NS_SHIFT bits and clamp into our 3 ns bytes. If the base timestamp is + // more than ~536 ms in the past (should not happen, it gets re-based regularly) we + // saturate to NS_MAX. + let raw_coarse_ns = (ns_since_unix_epoch - self.base_ns_since_unix_epoch) >> NS_SHIFT; + + // Commented out while benchmarking on a debug build (the warning would otherwise fire + // on the hot path). Re-enable to check whether the ~536 ms window is ever exceeded. + // #[cfg(debug_assertions)] + // if raw_coarse_ns > u64::from(NS_MAX) { + // tracing::warn!( + // base_ns_since_unix_epoch = self.base_ns_since_unix_epoch, + // ns_since_unix_epoch, + // raw_coarse_ns, + // ns_max = NS_MAX, + // "A pixel was set more than ~536ms after the last base timestamp; this should \ + // not happen. Clamping it to ~536ms" + // ); + // } + + let coarse_ns_since_base = u32::try_from(raw_coarse_ns) + .unwrap_or(u32::MAX) + .min(NS_MAX) + .to_le_bytes(); + + // Write via the byte-array fields, *not* via wide unaligned u32/u16 stores. A 6-byte + // stride means ~10% of pixels straddle a 64-byte cache line, and a single wide store + // that crosses the boundary hits the x86 split-store penalty on every such write + // (random writes are the hot path here). Byte/u16-sized stores rarely split, which is + // why this is meaningfully faster than packing into one u32 + one u16. + unsafe { + let ptr: *mut TimeTrackingPixel = self.buffer.as_ptr().add(pixel_index).cast_mut(); + (*ptr).rgb = [rgba as u8, (rgba >> 8) as u8, (rgba >> 16) as u8]; + (*ptr).coarse_ns_since_base = [ + coarse_ns_since_base[0], + coarse_ns_since_base[1], + coarse_ns_since_base[2], + ]; + } + } + } + + fn set_multi_from_start_index(&self, _: usize, _: &[u8]) -> usize { + panic!("The time tracking framebuffer does not implement set_multi_from_start_index"); + } + + fn as_bytes(&self) -> &[u8] { + todo!("We might actually can and want to implement TimeTrackingFrameBuffer::as_bytes") + } +} + +/// Don't call too often, there is a cost involved! +pub fn get_current_ns_since_unix_epoch() -> u64 { + let ns_since_unix_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + // TODO: If this turns out to be correct, convert to `unwrap_unchecked` + .expect("your system clock must be after UNIX EPOCH (so greater than 0)") + .as_nanos(); + + // u64::MAX allows us 18446744073709551615 ns since UNIX_EPOCH, which is + // some time in the year 2554, well beyond any reasonable timestamp. + u64::try_from(ns_since_unix_epoch).expect( + "your system time is >= year 2554. I'm developing this in 2026, I'm very likely dead now. And did no one write a better server to use in all that years?", + ) +} diff --git a/breakwater-parser/src/lib.rs b/breakwater-parser/src/lib.rs index dceadd5..af885e8 100644 --- a/breakwater-parser/src/lib.rs +++ b/breakwater-parser/src/lib.rs @@ -12,6 +12,10 @@ mod refactored; #[cfg(target_arch = "x86_64")] pub use assembler::AssemblerParser; +#[cfg(feature = "time-tracking")] +pub use framebuffer::time_tracking::{ + TimeTrackingFrameBuffer, TimeTrackingPixel, get_current_ns_since_unix_epoch, +}; pub use framebuffer::{ FB_BYTES_PER_PIXEL, FrameBuffer, shared_memory::SharedMemoryFrameBuffer, simple::SimpleFrameBuffer, @@ -57,3 +61,8 @@ pub trait Parser { // Sadly this cant be const (yet?) (https://github.com/rust-lang/rust/issues/71971 and https://github.com/rust-lang/rfcs/pull/2632) fn parser_lookahead(&self) -> usize; } + +#[cfg(all(feature = "time-tracking", feature = "alpha"))] +compile_error!("The features time-tracking and alpha can not be combined"); +#[cfg(all(feature = "time-tracking", feature = "binary-sync-pixels"))] +compile_error!("The features time-tracking and binary-sync-pixels can not be combined"); diff --git a/breakwater-parser/src/original.rs b/breakwater-parser/src/original.rs index 2103645..e687898 100644 --- a/breakwater-parser/src/original.rs +++ b/breakwater-parser/src/original.rs @@ -53,6 +53,12 @@ impl Parser for OriginalParser { let mut i = 0; // We can't use a for loop here because Rust don't lets use skip characters by incrementing i let loop_end = buffer.len().saturating_sub(PARSER_LOOKAHEAD); // Let's extract the .len() call and the subtraction into it's own variable so we only compute it once + // As this is a potentially(?) expensive operation we only call it one in this parsing loop + // All the pixels likely where in the same TCP packets (+- 1/2 or so) it doesn't matter after all + #[cfg(feature = "time-tracking")] + let ns_since_unix_epoch = + crate::framebuffer::time_tracking::get_current_ns_since_unix_epoch(); + #[cfg(feature = "binary-sync-pixels")] if let Some(remaining) = &self.remaining_pixel_sync { let buffer = &buffer[0..loop_end]; @@ -118,7 +124,16 @@ impl Parser for OriginalParser { let rgba: u32 = simd_unhex(unsafe { buffer.as_ptr().add(i - 7) }); + #[cfg(not(feature = "time-tracking"))] self.fb.set(x, y, rgba & 0x00ff_ffff); + #[cfg(feature = "time-tracking")] + self.fb.set_with_ns_since_unix_epoch( + x, + y, + rgba & 0x00ff_ffff, + ns_since_unix_epoch, + ); + continue; } @@ -130,7 +145,16 @@ impl Parser for OriginalParser { let rgba: u32 = simd_unhex(unsafe { buffer.as_ptr().add(i - 9) }); + #[cfg(not(feature = "time-tracking"))] self.fb.set(x, y, rgba & 0x00ff_ffff); + #[cfg(feature = "time-tracking")] + self.fb.set_with_ns_since_unix_epoch( + x, + y, + rgba & 0x00ff_ffff, + ns_since_unix_epoch, + ); + continue; } #[cfg(feature = "alpha")] @@ -156,6 +180,7 @@ impl Parser for OriginalParser { let g: u32 = (((current >> 16) & 0xff) * alpha_comp + g * alpha) / 0xff; let b: u32 = (((current >> 8) & 0xff) * alpha_comp + b * alpha) / 0xff; + // Note: We have a compile check to make sure time-tracking and alpha can not be combined self.fb.set(x, y, (r << 16) | (g << 8) | b); continue; } @@ -169,7 +194,11 @@ impl Parser for OriginalParser { let rgba: u32 = (base << 16) | (base << 8) | base; + #[cfg(not(feature = "time-tracking"))] self.fb.set(x, y, rgba); + #[cfg(feature = "time-tracking")] + self.fb + .set_with_ns_since_unix_epoch(x, y, rgba, ns_since_unix_epoch); continue; } @@ -205,7 +234,16 @@ impl Parser for OriginalParser { let rgba = u32::from_le((command_bytes >> 32) as u32); // TODO: Support alpha channel (behind alpha feature flag) + #[cfg(not(feature = "time-tracking"))] self.fb.set(x as usize, y as usize, rgba & 0x00ff_ffff); + #[cfg(feature = "time-tracking")] + self.fb.set_with_ns_since_unix_epoch( + x as usize, + y as usize, + rgba & 0x00ff_ffff, + ns_since_unix_epoch, + ); + // P B XX YY RGBA last_byte_parsed = i + 1 + 2 + 2 + 4; i += 10; @@ -225,6 +263,7 @@ impl Parser for OriginalParser { if len_in_bytes <= bytes_left_in_buffer { // Easy going here + // Note: We have a compile check to make sure time-tracking and binary-sync-pixels can not be combined self.fb .set_multi(start_x as usize, start_y as usize, unsafe { slice::from_raw_parts(buffer.as_ptr().add(i), len_in_bytes) @@ -241,6 +280,7 @@ impl Parser for OriginalParser { // what the client is doing. let mut current_index = start_x as usize + start_y as usize * self.fb.get_width(); + // Note: We have a compile check to make sure time-tracking and binary-sync-pixels can not be combined current_index += self.fb.set_multi_from_start_index(current_index, unsafe { slice::from_raw_parts(buffer.as_ptr().add(i), pixel_bytes) }); diff --git a/breakwater/src/lib.rs b/breakwater/src/lib.rs new file mode 100644 index 0000000..b20edd3 --- /dev/null +++ b/breakwater/src/lib.rs @@ -0,0 +1,28 @@ +use color_eyre::eyre::{self, Context}; +use tokio::sync::broadcast; + +pub mod cli_args; +pub mod prometheus_exporter; +pub mod server; +pub mod sinks; +pub mod statistics; + +mod connection_buffer; + +#[cfg(test)] +mod test_helpers; + +#[cfg(test)] +mod tests; + +pub async fn handle_ctrl_c(terminate_signal_tx: broadcast::Sender<()>) -> eyre::Result<()> { + tokio::signal::ctrl_c() + .await + .context("failed to wait for ctrl + c")?; + + terminate_signal_tx + .send(()) + .context("failed to signal termination")?; + + Ok(()) +} diff --git a/breakwater/src/main.rs b/breakwater/src/main.rs index fef3f70..1ca3def 100644 --- a/breakwater/src/main.rs +++ b/breakwater/src/main.rs @@ -1,29 +1,17 @@ use std::sync::Arc; -use breakwater_parser::SharedMemoryFrameBuffer; -use clap::Parser; -use color_eyre::eyre::{self, Context}; -use server::Server; -use tokio::sync::{broadcast, mpsc}; - -use crate::{ +use breakwater::{ cli_args::CliArgs, + handle_ctrl_c, prometheus_exporter::PrometheusExporter, + server::Server, sinks::{DisplaySink, ffmpeg::FfmpegSink}, statistics::{Statistics, StatisticsEvent, StatisticsInformationEvent, StatisticsSaveMode}, }; - -mod cli_args; -mod connection_buffer; -mod prometheus_exporter; -mod server; -mod sinks; -mod statistics; -#[cfg(test)] -mod test_helpers; - -#[cfg(test)] -mod tests; +use breakwater_parser::SharedMemoryFrameBuffer; +use clap::Parser; +use color_eyre::eyre::{self, Context}; +use tokio::sync::{broadcast, mpsc}; #[tokio::main] #[allow(clippy::too_many_lines)] @@ -106,7 +94,7 @@ async fn main() -> eyre::Result<()> { #[cfg(all(feature = "native-display", not(feature = "egui")))] { - use crate::sinks::native_display::NativeDisplaySink; + use breakwater::sinks::native_display::NativeDisplaySink; if let Some(native_display_sink) = NativeDisplaySink::new( fb.clone(), @@ -124,7 +112,7 @@ async fn main() -> eyre::Result<()> { #[cfg(feature = "vnc")] { - use crate::sinks::vnc::VncSink; + use breakwater::sinks::vnc::VncSink; if let Some(vnc_sink) = VncSink::new( fb.clone(), @@ -165,7 +153,7 @@ async fn main() -> eyre::Result<()> { #[cfg(feature = "egui")] { - use sinks::egui::EguiSink; + use breakwater::sinks::egui::EguiSink; match EguiSink::new( fb.clone(), @@ -217,15 +205,3 @@ async fn main() -> eyre::Result<()> { Ok(()) } - -async fn handle_ctrl_c(terminate_signal_tx: broadcast::Sender<()>) -> eyre::Result<()> { - tokio::signal::ctrl_c() - .await - .context("failed to wait for ctrl + c")?; - - terminate_signal_tx - .send(()) - .context("failed to signal termination")?; - - Ok(()) -} diff --git a/deich/Cargo.toml b/deich/Cargo.toml new file mode 100644 index 0000000..99a3b44 --- /dev/null +++ b/deich/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "deich" +description = "Distributed Pixelflut server" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +repository.workspace = true + +[[bin]] +name = "deich" +path = "src/main.rs" + +[dependencies] +breakwater-parser = { workspace = true, features = ["time-tracking"] } +breakwater.workspace = true + +color-eyre.workspace = true +tokio.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true + +[lints] +workspace = true diff --git a/deich/src/main.rs b/deich/src/main.rs new file mode 100644 index 0000000..8cfe141 --- /dev/null +++ b/deich/src/main.rs @@ -0,0 +1,59 @@ +use std::{net::SocketAddr, sync::Arc}; + +use breakwater::{ + cli_args::DEFAULT_NETWORK_BUFFER_SIZE, handle_ctrl_c, server::Server, + statistics::StatisticsEvent, +}; +use breakwater_parser::{TimeTrackingFrameBuffer, get_current_ns_since_unix_epoch}; +use color_eyre::eyre::{self, Context}; +use tokio::sync::{broadcast, mpsc}; + +#[tokio::main] +async fn main() -> eyre::Result<()> { + color_eyre::install()?; + + let filter = tracing_subscriber::EnvFilter::builder() + .with_default_directive(if cfg!(debug_assertions) { + tracing::Level::DEBUG.into() + } else { + tracing::Level::INFO.into() + }) + .from_env()?; + tracing_subscriber::fmt().with_env_filter(filter).init(); + + let fb = Arc::new(TimeTrackingFrameBuffer::new( + 1920, + 1080, + get_current_ns_since_unix_epoch(), + )); + let (statistics_tx, statistics_rx) = mpsc::channel::(100); + let (terminate_signal_tx, _terminate_signal_rx) = broadcast::channel::<()>(1); + + let mut server = Server::new( + &["[::]:1234".parse::().unwrap()], + fb.clone(), + statistics_tx.clone(), + DEFAULT_NETWORK_BUFFER_SIZE, + None, + ) + .await + .context("failed to start pixelflut server")?; + + let server_listener_thread = tokio::spawn(async move { server.start().await }); + let stats_drain_thread = tokio::spawn(async move { drain_stats(statistics_rx).await }); + + handle_ctrl_c(terminate_signal_tx).await?; + server_listener_thread.abort(); + stats_drain_thread.abort(); + + Ok(()) +} + +/// Currently we don't care about stats, so let's just drain them +async fn drain_stats(mut statistics_rx: mpsc::Receiver) { + loop { + if statistics_rx.recv().await.is_none() { + return; + } + } +} From 7a868a2df73436500469df3eb9ac26635e929686 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 11 Jun 2026 20:40:39 +0200 Subject: [PATCH 2/7] Add a job to reset the framebuffer base timestamp at 30 FPS --- breakwater-parser/src/framebuffer/mod.rs | 18 +++- .../src/framebuffer/shared_memory.rs | 7 +- breakwater-parser/src/framebuffer/simple.rs | 9 +- .../src/framebuffer/time_tracking.rs | 85 ++++++++++++------- breakwater-parser/src/original.rs | 22 +++-- deich/src/main.rs | 32 ++++++- 6 files changed, 128 insertions(+), 45 deletions(-) diff --git a/breakwater-parser/src/framebuffer/mod.rs b/breakwater-parser/src/framebuffer/mod.rs index 423c4d6..c7e6c0d 100644 --- a/breakwater-parser/src/framebuffer/mod.rs +++ b/breakwater-parser/src/framebuffer/mod.rs @@ -58,6 +58,22 @@ pub trait FrameBuffer { /// that), we can only return it as a list of bytes, not a list of pixels. fn as_bytes(&self) -> &[u8]; + /// Encode `ns_since_unix_epoch` into the compact, per-pixel `coarse_ns_since_base` + /// representation. This is the only place that touches the (atomic) base timestamp, so call + /// it **once per parse call** and pass the result to [`Self::set_with_coarse_ns_since_base`] + /// for every pixel. That keeps the base load (and the encoding arithmetic) out of the hot + /// per-pixel path, where it would otherwise re-run on every single write. #[cfg(feature = "time-tracking")] - fn set_with_ns_since_unix_epoch(&self, x: usize, y: usize, rgba: u32, ns_since_unix_epoch: u64); + fn coarse_ns_since_base(&self, ns_since_unix_epoch: u64) -> u32; + + /// Like [`Self::set`], but also records *when* the pixel was written, as the precomputed + /// `coarse_ns_since_base` from [`Self::coarse_ns_since_base`]. + #[cfg(feature = "time-tracking")] + fn set_with_coarse_ns_since_base( + &self, + x: usize, + y: usize, + rgba: u32, + coarse_ns_since_base: u32, + ); } diff --git a/breakwater-parser/src/framebuffer/shared_memory.rs b/breakwater-parser/src/framebuffer/shared_memory.rs index 1fc681c..0b75a76 100644 --- a/breakwater-parser/src/framebuffer/shared_memory.rs +++ b/breakwater-parser/src/framebuffer/shared_memory.rs @@ -224,7 +224,12 @@ impl FrameBuffer for SharedMemoryFrameBuffer { } #[cfg(feature = "time-tracking")] - fn set_with_ns_since_unix_epoch(&self, _: usize, _: usize, _: u32, _: u64) { + fn coarse_ns_since_base(&self, _: u64) -> u32 { + panic!("The shared memory framebuffer does not support time tracking"); + } + + #[cfg(feature = "time-tracking")] + fn set_with_coarse_ns_since_base(&self, _: usize, _: usize, _: u32, _: u32) { panic!("The shared memory framebuffer does not support time tracking"); } } diff --git a/breakwater-parser/src/framebuffer/simple.rs b/breakwater-parser/src/framebuffer/simple.rs index 83cc621..eafdb1a 100644 --- a/breakwater-parser/src/framebuffer/simple.rs +++ b/breakwater-parser/src/framebuffer/simple.rs @@ -84,8 +84,13 @@ impl FrameBuffer for SimpleFrameBuffer { } #[cfg(feature = "time-tracking")] - fn set_with_ns_since_unix_epoch(&self, _: usize, _: usize, _: u32, _: u64) { - panic!("The simply framebuffer does not support time tracking"); + fn coarse_ns_since_base(&self, _: u64) -> u32 { + panic!("The simple framebuffer does not support time tracking"); + } + + #[cfg(feature = "time-tracking")] + fn set_with_coarse_ns_since_base(&self, _: usize, _: usize, _: u32, _: u32) { + panic!("The simple framebuffer does not support time tracking"); } } diff --git a/breakwater-parser/src/framebuffer/time_tracking.rs b/breakwater-parser/src/framebuffer/time_tracking.rs index cad7a99..0c4f9e3 100644 --- a/breakwater-parser/src/framebuffer/time_tracking.rs +++ b/breakwater-parser/src/framebuffer/time_tracking.rs @@ -1,6 +1,9 @@ -use std::time::{SystemTime, UNIX_EPOCH}; +use std::{ + sync::atomic::{AtomicU64, Ordering}, + time::{SystemTime, UNIX_EPOCH}, +}; -use tracing::debug; +use tracing::{debug, warn}; use crate::FrameBuffer; @@ -8,7 +11,11 @@ pub struct TimeTrackingFrameBuffer { width: usize, height: usize, buffer: Vec, - base_ns_since_unix_epoch: u64, + /// All per-pixel timestamps are stored relative to this base. A background task re-bases it + /// (to the current time) at a fixed rate, which keeps `ns_since_base` small enough to fit in + /// our 3 bytes. Read on the hot write path and written by the re-basing task, hence atomic. + /// A `Relaxed` load compiles to a plain `mov` on x86_64, so it doesn't cost us anything. + base_ns_since_unix_epoch: AtomicU64, } /// Number of low bits we drop from `ns_since_base` before storing it. We only have 3 bytes @@ -64,7 +71,7 @@ impl TimeTrackingFrameBuffer { width, height, buffer, - base_ns_since_unix_epoch, + base_ns_since_unix_epoch: AtomicU64::new(base_ns_since_unix_epoch), } } @@ -72,6 +79,18 @@ impl TimeTrackingFrameBuffer { fn pixel_index(&self, x: usize, y: usize) -> usize { x + y * self.width } + + /// Re-base all future per-pixel timestamps to `base_ns_since_unix_epoch`. Call this + /// regularly (faster than the ~536 ms window) so `ns_since_base` keeps fitting in 3 bytes. + pub fn set_base_ns_since_unix_epoch(&self, base_ns_since_unix_epoch: u64) { + self.base_ns_since_unix_epoch + .store(base_ns_since_unix_epoch, Ordering::Relaxed); + } + + /// Number of bytes the framebuffer occupies (i.e. how much we'd sync). + pub fn num_bytes(&self) -> usize { + self.buffer.len() * size_of::() + } } impl FrameBuffer for TimeTrackingFrameBuffer { @@ -97,43 +116,47 @@ impl FrameBuffer for TimeTrackingFrameBuffer { fn set(&self, _: usize, _: usize, _: u32) { panic!( - "The time tracking framebuffer requires you to use the set_with_ns_since_unix_epoch function!" + "The time tracking framebuffer requires you to use the set_with_coarse_ns_since_base function!" ); } - fn set_with_ns_since_unix_epoch( + fn coarse_ns_since_base(&self, ns_since_unix_epoch: u64) -> u32 { + // The single point that reads the (atomic) base, called once per parse call. Drop the + // low NS_SHIFT bits and clamp into our 3 ns bytes; if the base is more than ~536 ms in + // the past (shouldn't happen, it gets re-based regularly) we saturate to NS_MAX. + // + // `saturating_sub`, not `-`: the parser captures `ns_since_unix_epoch` once at the start + // of a parse call, but the background task may re-base `base` to a newer time in between. + // In that case the pixels were effectively written at the new base, so a difference of 0 + // is exactly right. + let base_ns_since_unix_epoch = self.base_ns_since_unix_epoch.load(Ordering::Relaxed); + let raw_coarse_ns = + ns_since_unix_epoch.saturating_sub(base_ns_since_unix_epoch) >> NS_SHIFT; + + if raw_coarse_ns > u64::from(NS_MAX) { + warn!( + base_ns_since_unix_epoch, + ns_since_unix_epoch, + raw_coarse_ns, + ns_max = NS_MAX, + "Pixels were written more than ~536ms after the last base timestamp; clamping to \ + ~536ms. Is the framebuffer re-basing task still running?" + ); + } + + u32::try_from(raw_coarse_ns).unwrap_or(u32::MAX).min(NS_MAX) + } + + fn set_with_coarse_ns_since_base( &self, x: usize, y: usize, rgba: u32, - ns_since_unix_epoch: u64, + coarse_ns_since_base: u32, ) { if x < self.width && y < self.height { let pixel_index = self.pixel_index(x, y); - - // Drop the low NS_SHIFT bits and clamp into our 3 ns bytes. If the base timestamp is - // more than ~536 ms in the past (should not happen, it gets re-based regularly) we - // saturate to NS_MAX. - let raw_coarse_ns = (ns_since_unix_epoch - self.base_ns_since_unix_epoch) >> NS_SHIFT; - - // Commented out while benchmarking on a debug build (the warning would otherwise fire - // on the hot path). Re-enable to check whether the ~536 ms window is ever exceeded. - // #[cfg(debug_assertions)] - // if raw_coarse_ns > u64::from(NS_MAX) { - // tracing::warn!( - // base_ns_since_unix_epoch = self.base_ns_since_unix_epoch, - // ns_since_unix_epoch, - // raw_coarse_ns, - // ns_max = NS_MAX, - // "A pixel was set more than ~536ms after the last base timestamp; this should \ - // not happen. Clamping it to ~536ms" - // ); - // } - - let coarse_ns_since_base = u32::try_from(raw_coarse_ns) - .unwrap_or(u32::MAX) - .min(NS_MAX) - .to_le_bytes(); + let coarse_ns_since_base = coarse_ns_since_base.to_le_bytes(); // Write via the byte-array fields, *not* via wide unaligned u32/u16 stores. A 6-byte // stride means ~10% of pixels straddle a 64-byte cache line, and a single wide store diff --git a/breakwater-parser/src/original.rs b/breakwater-parser/src/original.rs index e687898..c50064a 100644 --- a/breakwater-parser/src/original.rs +++ b/breakwater-parser/src/original.rs @@ -55,9 +55,13 @@ impl Parser for OriginalParser { // As this is a potentially(?) expensive operation we only call it one in this parsing loop // All the pixels likely where in the same TCP packets (+- 1/2 or so) it doesn't matter after all + // Encode the timestamp exactly once here, not per pixel: reading the framebuffer's + // (atomic) base and doing the encoding arithmetic on every write costs measurable + // throughput, and it's constant for the whole parse call anyway. #[cfg(feature = "time-tracking")] - let ns_since_unix_epoch = - crate::framebuffer::time_tracking::get_current_ns_since_unix_epoch(); + let coarse_ns_since_base = self.fb.coarse_ns_since_base( + crate::framebuffer::time_tracking::get_current_ns_since_unix_epoch(), + ); #[cfg(feature = "binary-sync-pixels")] if let Some(remaining) = &self.remaining_pixel_sync { @@ -127,11 +131,11 @@ impl Parser for OriginalParser { #[cfg(not(feature = "time-tracking"))] self.fb.set(x, y, rgba & 0x00ff_ffff); #[cfg(feature = "time-tracking")] - self.fb.set_with_ns_since_unix_epoch( + self.fb.set_with_coarse_ns_since_base( x, y, rgba & 0x00ff_ffff, - ns_since_unix_epoch, + coarse_ns_since_base, ); continue; @@ -148,11 +152,11 @@ impl Parser for OriginalParser { #[cfg(not(feature = "time-tracking"))] self.fb.set(x, y, rgba & 0x00ff_ffff); #[cfg(feature = "time-tracking")] - self.fb.set_with_ns_since_unix_epoch( + self.fb.set_with_coarse_ns_since_base( x, y, rgba & 0x00ff_ffff, - ns_since_unix_epoch, + coarse_ns_since_base, ); continue; @@ -198,7 +202,7 @@ impl Parser for OriginalParser { self.fb.set(x, y, rgba); #[cfg(feature = "time-tracking")] self.fb - .set_with_ns_since_unix_epoch(x, y, rgba, ns_since_unix_epoch); + .set_with_coarse_ns_since_base(x, y, rgba, coarse_ns_since_base); continue; } @@ -237,11 +241,11 @@ impl Parser for OriginalParser { #[cfg(not(feature = "time-tracking"))] self.fb.set(x as usize, y as usize, rgba & 0x00ff_ffff); #[cfg(feature = "time-tracking")] - self.fb.set_with_ns_since_unix_epoch( + self.fb.set_with_coarse_ns_since_base( x as usize, y as usize, rgba & 0x00ff_ffff, - ns_since_unix_epoch, + coarse_ns_since_base, ); // P B XX YY RGBA diff --git a/deich/src/main.rs b/deich/src/main.rs index 8cfe141..29008ef 100644 --- a/deich/src/main.rs +++ b/deich/src/main.rs @@ -1,4 +1,4 @@ -use std::{net::SocketAddr, sync::Arc}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; use breakwater::{ cli_args::DEFAULT_NETWORK_BUFFER_SIZE, handle_ctrl_c, server::Server, @@ -7,6 +7,10 @@ use breakwater::{ use breakwater_parser::{TimeTrackingFrameBuffer, get_current_ns_since_unix_epoch}; use color_eyre::eyre::{self, Context}; use tokio::sync::{broadcast, mpsc}; +use tracing::debug; + +/// How often we sync the framebuffer +const SYNC_FPS: u64 = 30; #[tokio::main] async fn main() -> eyre::Result<()> { @@ -41,14 +45,40 @@ async fn main() -> eyre::Result<()> { let server_listener_thread = tokio::spawn(async move { server.start().await }); let stats_drain_thread = tokio::spawn(async move { drain_stats(statistics_rx).await }); + let sync_thread = { + let fb = fb.clone(); + tokio::spawn(async move { sync_framebuffer(fb).await }) + }; handle_ctrl_c(terminate_signal_tx).await?; server_listener_thread.abort(); stats_drain_thread.abort(); + sync_thread.abort(); Ok(()) } +/// Periodically syncs the framebuffer and re-bases its per-pixel timestamps to the current time. +/// +/// The re-basing has to happen faster than the ~536 ms window the 3-byte `ns_since_base` can +/// represent; 30 fps (~33 ms) leaves plenty of headroom. +async fn sync_framebuffer(fb: Arc) { + let mut interval = tokio::time::interval(Duration::from_nanos(1_000_000_000 / SYNC_FPS)); + // Let's try to never skip a frame and catch up to make it easy for the sync collection server + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Burst); + + loop { + interval.tick().await; + + // TODO: Actually sync the framebuffer somewhere instead of just logging its size. + debug!(bytes = fb.num_bytes(), "Syncing framebuffer"); + + // Re-base *after* the sync, so the bytes we just synced are still relative to the old + // base. New writes from here on are relative to "now". + fb.set_base_ns_since_unix_epoch(get_current_ns_since_unix_epoch()); + } +} + /// Currently we don't care about stats, so let's just drain them async fn drain_stats(mut statistics_rx: mpsc::Receiver) { loop { From 39c1de747baed7c65035d620736cdf7a4a1476f8 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Sat, 13 Jun 2026 13:34:02 +0200 Subject: [PATCH 3/7] WIP --- .gitignore | 1 + Cargo.lock | 273 ++++++++++- Cargo.toml | 2 + .../src/framebuffer/time_tracking.rs | 62 ++- breakwater-parser/src/lib.rs | 1 + deich/Cargo.toml | 4 + deich/src/cli_args.rs | 69 +++ deich/src/collector.rs | 426 ++++++++++++++++++ deich/src/lib.rs | 30 ++ deich/src/main.rs | 91 +--- deich/src/sync.rs | 316 +++++++++++++ deich/src/worker.rs | 126 ++++++ 12 files changed, 1302 insertions(+), 99 deletions(-) create mode 100644 deich/src/cli_args.rs create mode 100644 deich/src/collector.rs create mode 100644 deich/src/lib.rs create mode 100644 deich/src/sync.rs create mode 100644 deich/src/worker.rs diff --git a/.gitignore b/.gitignore index d84f564..da79167 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .vscode/ statistics.json /pixelflut/ +worker-id diff --git a/Cargo.lock b/Cargo.lock index 2c7a6ae..553b61e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -271,6 +271,15 @@ dependencies = [ "syn", ] +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -774,6 +783,15 @@ dependencies = [ "error-code", ] +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "codespan-reporting" version = "0.13.1" @@ -966,6 +984,12 @@ dependencies = [ "itertools 0.13.0", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1059,10 +1083,14 @@ version = "0.20.0" dependencies = [ "breakwater", "breakwater-parser", + "clap", "color-eyre", + "postcard", + "serde", "tokio", "tracing", "tracing-subscriber", + "uuid", ] [[package]] @@ -1352,6 +1380,18 @@ dependencies = [ "serde", ] +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "enum-as-inner" version = "0.6.1" @@ -1555,6 +1595,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -1730,10 +1776,23 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + [[package]] name = "getset" version = "0.1.6" @@ -1869,13 +1928,31 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -1884,6 +1961,20 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32", + "rustc_version", + "serde", + "spin", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.5.0" @@ -2017,6 +2108,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -2108,6 +2205,8 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -2338,6 +2437,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "lebe" version = "0.5.3" @@ -3424,6 +3529,19 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless", + "serde", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -3590,6 +3708,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.8.6" @@ -4267,6 +4391,15 @@ dependencies = [ "x11rb", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -4795,7 +4928,10 @@ version = "1.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" dependencies = [ + "getrandom 0.4.2", + "js-sys", "serde_core", + "wasm-bindgen", ] [[package]] @@ -4885,7 +5021,16 @@ version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] @@ -4943,6 +5088,40 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.1", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "wayland-backend" version = "0.3.15" @@ -5731,12 +5910,100 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.1", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index a37cc59..38f4777 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ memchr = "2.7" number_prefix = "0.4" page_size = "0.6" pixelbomber = "1.1" +postcard = { version = "1", features = ["use-std"] } prometheus_exporter = "0.8" rstest = "0.26" rusttype = "0.9" @@ -47,6 +48,7 @@ tokio-stream = { version = "0.1", features = ["net"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } trait-variant = "0.1" +uuid = { version = "1", features = ["v4", "serde"] } vncserver = "0.2" winit = "0.30" diff --git a/breakwater-parser/src/framebuffer/time_tracking.rs b/breakwater-parser/src/framebuffer/time_tracking.rs index 0c4f9e3..07f4d04 100644 --- a/breakwater-parser/src/framebuffer/time_tracking.rs +++ b/breakwater-parser/src/framebuffer/time_tracking.rs @@ -3,7 +3,7 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; -use tracing::{debug, warn}; +use tracing::warn; use crate::FrameBuffer; @@ -31,7 +31,7 @@ const NS_MAX: u32 = 0x00ff_ffff; /// /// ```text /// byte: 0 1 2 3 4 5 -/// [ r ] [ g ] [ b ] [ns0] [ns1] [ns2] +/// [ r ] [ g ] [ b ] [ns0] [ns1] [ns2] /// ``` /// /// The `ns` bytes hold `ns_since_base >> NS_SHIFT` (see [`NS_SHIFT`]), clamped to [`NS_MAX`]. @@ -57,16 +57,45 @@ pub struct TimeTrackingPixel { pub coarse_ns_since_base: [u8; 3], } +/// Views a slice of pixels as their raw bytes (6 per pixel), e.g. to read a frame straight off the +/// wire into a `Vec`. Same layout as [`FrameBuffer::as_bytes`]. +pub fn pixels_as_bytes_mut(pixels: &mut [TimeTrackingPixel]) -> &mut [u8] { + let len = size_of_val(pixels); + let ptr = pixels.as_mut_ptr().cast::(); + // SAFETY: `TimeTrackingPixel` is `repr(C)`, `align = 1`, all-bytes-valid (just `[u8; 3]`s), so + // its `len * 6` bytes are a valid `[u8]` of the same lifetime and exclusive borrow. + unsafe { std::slice::from_raw_parts_mut(ptr, len) } +} + +impl TimeTrackingPixel { + /// The stored `coarse_ns_since_base` as a `u32` (the implicit high byte is always zero). + fn coarse_ns_since_base(self) -> u32 { + u32::from_le_bytes([ + self.coarse_ns_since_base[0], + self.coarse_ns_since_base[1], + self.coarse_ns_since_base[2], + 0, + ]) + } + + /// The absolute UNIX-epoch time this pixel was last written, given the framebuffer's + /// `base_ns_since_unix_epoch`, reconstructed at `1 << NS_SHIFT` ns resolution. + /// + /// Returns `None` when the pixel carries no recent-write information (`coarse_ns == 0`): the + /// framebuffer re-bases every frame, so a pixel not written within the last window saturates to + /// 0 and is indistinguishable from "never written". Callers merging framebuffers should treat + /// `None` as "no update from this pixel" so stale/blank pixels don't clobber fresher content. + pub fn written_ns_since_unix_epoch(self, base_ns_since_unix_epoch: u64) -> Option { + let coarse = self.coarse_ns_since_base(); + (coarse != 0).then(|| base_ns_since_unix_epoch + (u64::from(coarse) << NS_SHIFT)) + } +} + impl TimeTrackingFrameBuffer { pub fn new(width: usize, height: usize, base_ns_since_unix_epoch: u64) -> Self { let mut buffer = Vec::with_capacity(width * height); buffer.resize_with(width * height, TimeTrackingPixel::default); - debug!( - size = buffer.len(), - bytes = buffer.len() * size_of::(), - "Allocated time tracking framebuffer" - ); Self { width, height, @@ -80,17 +109,18 @@ impl TimeTrackingFrameBuffer { x + y * self.width } + /// The current base all per-pixel timestamps are relative to. Capture this *before* syncing a + /// frame, so the consumer can interpret that frame's `coarse_ns_since_base` values. + pub fn base_ns_since_unix_epoch(&self) -> u64 { + self.base_ns_since_unix_epoch.load(Ordering::Relaxed) + } + /// Re-base all future per-pixel timestamps to `base_ns_since_unix_epoch`. Call this /// regularly (faster than the ~536 ms window) so `ns_since_base` keeps fitting in 3 bytes. pub fn set_base_ns_since_unix_epoch(&self, base_ns_since_unix_epoch: u64) { self.base_ns_since_unix_epoch .store(base_ns_since_unix_epoch, Ordering::Relaxed); } - - /// Number of bytes the framebuffer occupies (i.e. how much we'd sync). - pub fn num_bytes(&self) -> usize { - self.buffer.len() * size_of::() - } } impl FrameBuffer for TimeTrackingFrameBuffer { @@ -180,7 +210,13 @@ impl FrameBuffer for TimeTrackingFrameBuffer { } fn as_bytes(&self) -> &[u8] { - todo!("We might actually can and want to implement TimeTrackingFrameBuffer::as_bytes") + // The buffer is a contiguous `Vec` of `align = 1` cells, so the raw + // bytes are exactly the 6-bytes-per-pixel wire layout. Like the other framebuffers, this + // reads memory that writers may be mutating concurrently — fine for a lossy, best-effort + // sync (a torn pixel just gets corrected on the next frame). + let len = self.buffer.len() * size_of::(); + let ptr = self.buffer.as_ptr().cast::(); + unsafe { std::slice::from_raw_parts(ptr, len) } } } diff --git a/breakwater-parser/src/lib.rs b/breakwater-parser/src/lib.rs index af885e8..f4f6245 100644 --- a/breakwater-parser/src/lib.rs +++ b/breakwater-parser/src/lib.rs @@ -15,6 +15,7 @@ pub use assembler::AssemblerParser; #[cfg(feature = "time-tracking")] pub use framebuffer::time_tracking::{ TimeTrackingFrameBuffer, TimeTrackingPixel, get_current_ns_since_unix_epoch, + pixels_as_bytes_mut, }; pub use framebuffer::{ FB_BYTES_PER_PIXEL, FrameBuffer, shared_memory::SharedMemoryFrameBuffer, diff --git a/deich/Cargo.toml b/deich/Cargo.toml index 99a3b44..8d0eae9 100644 --- a/deich/Cargo.toml +++ b/deich/Cargo.toml @@ -15,10 +15,14 @@ path = "src/main.rs" breakwater-parser = { workspace = true, features = ["time-tracking"] } breakwater.workspace = true +clap.workspace = true color-eyre.workspace = true +postcard.workspace = true +serde.workspace = true tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true +uuid.workspace = true [lints] workspace = true diff --git a/deich/src/cli_args.rs b/deich/src/cli_args.rs new file mode 100644 index 0000000..3e744e9 --- /dev/null +++ b/deich/src/cli_args.rs @@ -0,0 +1,69 @@ +use std::{net::SocketAddr, path::PathBuf}; + +use breakwater::cli_args::DEFAULT_NETWORK_BUFFER_SIZE_STR; +use clap::{Args, Parser, Subcommand}; + +/// `deich` is breakwater's distributed mode: many workers each run a Pixelflut server and sync +/// their framebuffer to a single collector. +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +pub struct Cli { + #[command(subcommand)] + pub role: Role, +} + +#[derive(Subcommand, Debug)] +pub enum Role { + /// Run a worker: a Pixelflut server that continuously syncs its framebuffer to the collector. + /// Canvas geometry and frame rate are dictated by the collector. + Worker(WorkerArgs), + + /// Run the collector: gathers and merges the framebuffers of all workers. + Collector(CollectorArgs), +} + +#[derive(Args, Debug)] +pub struct WorkerArgs { + /// Listen address to bind the Pixelflut server to (multiple can be specified). + /// The default value will listen on all interfaces for IPv4 and IPv6 packets. + #[clap(short, long = "listen-address", default_value = "[::]:1234")] + pub listen_addresses: Vec, + + /// The size in bytes of the network buffer used for each open TCP connection. + /// Please use at least 64 KB (64_000 bytes). + #[clap( + long, + default_value = DEFAULT_NETWORK_BUFFER_SIZE_STR, + value_parser = 64_000..100_000_000, + )] + pub network_buffer_size: i64, + + /// Address of the collector to fetch the config from and stream the framebuffer to. + #[clap(long, default_value = "127.0.0.1:9999")] + pub collector_address: String, + + /// File storing this worker's persistent UUID, created on first run. The UUID identifies the + /// worker to the collector across restarts, which keeps per-worker stats stable over time. + #[clap(long, default_value = "worker-id")] + pub worker_id_file: PathBuf, +} + +#[derive(Args, Debug)] +pub struct CollectorArgs { + /// Listen address the workers connect to. + #[clap(short, long, default_value = "[::]:9999")] + pub listen_address: SocketAddr, + + /// Width of the canvas. Sent to every worker as part of its config. + #[clap(long, default_value_t = 1920)] + pub width: u32, + + /// Height of the canvas. Sent to every worker as part of its config. + #[clap(long, default_value_t = 1080)] + pub height: u32, + + /// Frames per second the workers should sync at. Sent to every worker as part of its config. + /// Must stay well below the ~536 ms window the per-pixel timestamp can represent. + #[clap(long, default_value_t = 30, value_parser = clap::value_parser!(u32).range(3..=60))] + pub fps: u32, +} diff --git a/deich/src/collector.rs b/deich/src/collector.rs new file mode 100644 index 0000000..9278482 --- /dev/null +++ b/deich/src/collector.rs @@ -0,0 +1,426 @@ +//! The collector role: identifies each connecting worker (via its [`Hello`]), tracks the connected +//! set, and stores the frames they stream. +//! +//! A **master** task ticks on the shared [`FrameSchedule`]. Each tick it publishes the window of +//! frame numbers it currently wants — `[render_frame ..= current]`, where `render_frame` is the +//! oldest frame that has had the full streaming budget to arrive — into the shared [`FrameStore`]. +//! Worker connections check that window per incoming frame: in-window frames are stored, others are +//! warned about and discarded. The master then merges `render_frame`'s stored frames into a +//! long-term [`Canvas`] by exact (full-`u64`-timestamp) last-write-per-pixel. +//! +//! [`Hello`]: crate::sync::WorkerMessage::Hello + +use std::{ + collections::HashMap, + io, + net::SocketAddr, + ops::RangeInclusive, + sync::{Arc, Mutex, RwLock}, + time::Duration, +}; + +use breakwater_parser::{TimeTrackingPixel, get_current_ns_since_unix_epoch, pixels_as_bytes_mut}; +use color_eyre::eyre::{self, Context}; +use tokio::net::{TcpListener, TcpStream}; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +use crate::{ + cli_args::CollectorArgs, + sync::{self, FrameSchedule, WorkerConfig}, +}; + +/// How long we assume a worker needs to stream a frame to us. The master keeps the last +/// `⌈budget / frame_period⌉` frames "interesting" and renders the oldest of them, so it only +/// renders a frame once every worker has had this long to deliver it. +const FRAME_STREAM_BUDGET: Duration = Duration::from_millis(50); + +/// The workers currently connected, keyed by their UUID (value: the connection's peer address, for +/// logging). A duplicate UUID is refused at connect time, so each entry maps to exactly one live +/// connection. +/// +/// Reads happen every master tick while writes happen only on (dis)connect, hence a read-optimized +/// `RwLock`. It's a `std` (sync) lock on purpose: the critical sections are a single map op and +/// never cross an `.await`, so an async `tokio::sync::RwLock` would only add overhead. +type ConnectedWorkers = Arc>>; + +/// One worker's framebuffer for one frame, plus the timestamp base it's relative to. Stored as +/// [`TimeTrackingPixel`]s (rather than raw bytes) to keep the layout tied to the canonical type; +/// the wire bytes are read straight into this buffer via a `bytemuck` cast. +struct ReceivedFrame { + /// The base the pixels' `coarse_ns_since_base` values are relative to. + base_ns_since_unix_epoch: u64, + buffer: Vec, +} + +/// The long-term merged canvas, held by the master for the whole process lifetime. +/// +/// Unlike a worker's framebuffer (whose 3-byte `coarse_ns` only spans ~536 ms), each pixel here +/// keeps a full `u64` absolute write timestamp, so last-write-wins is *exact* over arbitrary time +/// and survives traffic gaps and worker restarts. It's a flat pixel vector — no width/height, +/// merging is purely per-index. +struct Canvas { + pixels: Vec, +} + +#[derive(Clone, Copy, Default)] +struct CanvasPixel { + /// The winning color. Written by the merge; read by the (future) renderer and the tests. + // Not read in the binary yet (no rendering); `expect` can't be used as the tests *do* read it. + #[allow(dead_code)] + rgb: [u8; 3], + /// Absolute UNIX-epoch time the winning write happened; `0` means "never written". + written_ns_since_unix_epoch: u64, +} + +impl Canvas { + fn new(pixel_count: usize) -> Self { + Self { + pixels: vec![CanvasPixel::default(); pixel_count], + } + } + + /// Folds `frame` in, keeping for each pixel the write with the latest absolute timestamp. + /// Pixels the frame carries no recent-write info for (`coarse_ns == 0`) are skipped, so a blank + /// or stale frame never clobbers fresher content. + fn merge(&mut self, frame: &ReceivedFrame) { + for (canvas_pixel, frame_pixel) in self.pixels.iter_mut().zip(&frame.buffer) { + if let Some(written_ns) = + frame_pixel.written_ns_since_unix_epoch(frame.base_ns_since_unix_epoch) + && written_ns > canvas_pixel.written_ns_since_unix_epoch + { + canvas_pixel.rgb = frame_pixel.rgb; + canvas_pixel.written_ns_since_unix_epoch = written_ns; + } + } + } +} + +/// Shared between the master task and the worker-connection tasks. +struct FrameStore { + /// Frame numbers the master currently wants: the inclusive window `[render_frame ..= current]`. + /// Read by every worker connection per frame; written by the master once per tick. `None` + /// until the master's first tick. + interesting: RwLock>>, + + /// Stored frames: `frame_number -> (worker_id -> frame)`. Written by workers, read and evicted + /// by the master. + frames: Mutex>>, +} + +impl FrameStore { + fn new() -> Self { + Self { + interesting: RwLock::new(None), + frames: Mutex::new(HashMap::new()), + } + } + + /// Whether the master currently wants frame `frame_number`. + fn is_interested(&self, frame_number: u64) -> bool { + self.interesting + .read() + .expect("interesting-window lock poisoned") + .as_ref() + .is_some_and(|window| window.contains(&frame_number)) + } + + /// Stores a worker's frame (overwriting any previous frame from the same worker for that slot). + fn store(&self, frame_number: u64, worker_id: Uuid, frame: ReceivedFrame) { + self.frames + .lock() + .expect("frame store lock poisoned") + .entry(frame_number) + .or_default() + .insert(worker_id, frame); + } + + /// Publishes the new interesting `window`, evicts frames below it (they missed their render), + /// and removes & returns `render_frame`'s frames so the caller can merge them without holding + /// the lock. + fn advance( + &self, + window: RangeInclusive, + render_frame: u64, + ) -> HashMap { + *self + .interesting + .write() + .expect("interesting-window lock poisoned") = Some(window); + + let mut frames = self.frames.lock().expect("frame store lock poisoned"); + frames.retain(|&frame_number, _| frame_number >= render_frame); + frames.remove(&render_frame).unwrap_or_default() + } +} + +/// Runs the collector until Ctrl-C: accepts worker connections, configures them, stores frames, and +/// runs the master render-scheduling task. +pub async fn run(args: CollectorArgs) -> eyre::Result<()> { + let config = WorkerConfig { + width: args.width, + height: args.height, + sync_fps: args.fps, + }; + + let listener = TcpListener::bind(args.listen_address) + .await + .with_context(|| format!("failed to bind collector to {}", args.listen_address))?; + info!( + listen_address = %args.listen_address, + ?config, + frame_size = config.frame_size_bytes(), + "Collector listening for workers" + ); + + let connected_workers: ConnectedWorkers = Arc::new(RwLock::new(HashMap::new())); + let frame_store = Arc::new(FrameStore::new()); + + { + let frame_store = frame_store.clone(); + let connected_workers = connected_workers.clone(); + tokio::spawn(async move { run_master(&frame_store, &connected_workers, config).await }); + } + + loop { + let (stream, peer) = listener + .accept() + .await + .context("failed to accept worker connection")?; + + let connected_workers = connected_workers.clone(); + let frame_store = frame_store.clone(); + tokio::spawn(async move { + match handle_worker(stream, peer, config, &connected_workers, &frame_store).await { + // The read loop only ever returns on error; a clean disconnect shows up as EOF. + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => { + debug!(%peer, "Worker connection closed"); + } + Err(error) => warn!(%peer, %error, "Worker connection error"), + Ok(()) => {} + } + }); + } +} + +/// The master render-scheduling task. Ticks on the shared schedule; each tick it takes the workers' +/// frames for the frame it is rendering and merges them (latest write per pixel) into the long-term +/// canvas. +/// +/// The canvas is held here for the whole process lifetime, so it survives traffic gaps and worker +/// restarts: with no new frames it simply keeps its last contents, and a restarted worker's blank +/// frame can't overwrite live pixels (a blank pixel is never "newer" than real content). +async fn run_master( + frame_store: &FrameStore, + connected_workers: &ConnectedWorkers, + config: WorkerConfig, +) { + let schedule = FrameSchedule::new(config.sync_fps); + // Render the oldest frame that has had `FRAME_STREAM_BUDGET` to arrive. + let margin = schedule.frames_spanning(FRAME_STREAM_BUDGET); + + let mut canvas = Canvas::new(config.width as usize * config.height as usize); + + let mut current_frame = schedule.frame_number_at(get_current_ns_since_unix_epoch()); + loop { + // Tick at the start of `current_frame`'s slot. + let now = get_current_ns_since_unix_epoch(); + let slot_start = schedule.frame_start_ns(current_frame); + tokio::time::sleep(Duration::from_nanos(slot_start.saturating_sub(now))).await; + + let render_frame = current_frame.saturating_sub(margin); + let frames = frame_store.advance(render_frame..=current_frame, render_frame); + let connected = connected_workers + .read() + .expect("connected workers lock poisoned") + .len(); + + // Fold each worker's frame for this slot into the canvas (exact latest-write-per-pixel). + for frame in frames.values() { + canvas.merge(frame); + } + + // TODO: render `canvas` to the screen / output sink. + info!( + render_frame, + frames_merged = frames.len(), + connected_workers = connected, + "Master tick: merged frames into the canvas" + ); + + // Advance to the current slot, skipping any we fell behind on, always moving forward. + current_frame = schedule + .frame_number_at(get_current_ns_since_unix_epoch()) + .max(current_frame + 1); + } +} + +/// Reads the worker's hello, registers it, then stores its frames until the connection drops, +/// finally deregistering it. +async fn handle_worker( + mut stream: TcpStream, + peer: SocketAddr, + config: WorkerConfig, + connected_workers: &ConnectedWorkers, + frame_store: &FrameStore, +) -> io::Result<()> { + let worker_id = sync::accept_worker(&mut stream).await?; + + if !register(connected_workers, worker_id, peer) { + warn!( + %worker_id, + %peer, + "Worker id is already connected; refusing this duplicate connection" + ); + // Returning drops `stream`, which closes (slams) the connection. + return Ok(()); + } + + let result = serve_frames(&mut stream, peer, worker_id, config, frame_store).await; + + deregister(connected_workers, worker_id, peer); + result +} + +/// Sends the worker its config, then reads its frames forever, storing the ones the master wants. +async fn serve_frames( + stream: &mut TcpStream, + peer: SocketAddr, + worker_id: Uuid, + config: WorkerConfig, + frame_store: &FrameStore, +) -> io::Result<()> { + sync::send_config(stream, config).await?; + + let pixel_count = config.width as usize * config.height as usize; + // Reused only for discarding frames the master doesn't want. + let mut discard = vec![0u8; config.frame_size_bytes()]; + + loop { + let header = sync::receive_frame_header(stream).await?; + + if frame_store.is_interested(header.frame_number) { + // Read straight into a fresh pixel buffer (outside any lock) so we can hand ownership to + // the store; `read_exact` writes directly into its bytes, no extra copy. + let mut buffer = vec![TimeTrackingPixel::default(); pixel_count]; + sync::receive_frame_body(stream, pixels_as_bytes_mut(&mut buffer)).await?; + frame_store.store( + header.frame_number, + worker_id, + ReceivedFrame { + base_ns_since_unix_epoch: header.base_ns_since_unix_epoch, + buffer, + }, + ); + } else { + // Still consume the blob so the stream stays aligned for the next message. + sync::receive_frame_body(stream, &mut discard).await?; + warn!( + %peer, + frame_number = header.frame_number, + "Master is not interested in this frame (outside the window); dropping it" + ); + } + } +} + +/// Registers a worker. Returns `false` (without inserting) if its UUID is already connected. +fn register(connected_workers: &ConnectedWorkers, worker_id: Uuid, peer: SocketAddr) -> bool { + let mut workers = connected_workers + .write() + .expect("connected workers lock poisoned"); + if workers.contains_key(&worker_id) { + return false; + } + + workers.insert(worker_id, peer); + info!( + %worker_id, + %peer, + connected_workers = workers.len(), + "Worker connected" + ); + true +} + +fn deregister(connected_workers: &ConnectedWorkers, worker_id: Uuid, peer: SocketAddr) { + let mut workers = connected_workers + .write() + .expect("connected workers lock poisoned"); + // Duplicates are refused at connect time, so the entry is guaranteed to be ours to remove. + workers.remove(&worker_id); + info!( + %worker_id, + %peer, + connected_workers = workers.len(), + "Worker disconnected" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Builds a `ReceivedFrame` from `(rgb, coarse_ns)` pairs. + fn frame(base_ns_since_unix_epoch: u64, pixels: &[([u8; 3], u32)]) -> ReceivedFrame { + let buffer = pixels + .iter() + .map(|&(rgb, coarse)| { + let coarse = coarse.to_le_bytes(); + TimeTrackingPixel { + rgb, + coarse_ns_since_base: [coarse[0], coarse[1], coarse[2]], + } + }) + .collect(); + ReceivedFrame { + base_ns_since_unix_epoch, + buffer, + } + } + + #[test] + fn merges_latest_absolute_write_per_pixel() { + let mut canvas = Canvas::new(2); + + // Pixel 0: A at 1000 + (10 << 5) = 1320; B at 5000 + (1 << 5) = 5032 -> B wins. + // Pixel 1: A at 1000 + (100 << 5) = 4200; B has coarse 0 -> skipped -> A stays. + let a = frame(1_000, &[([0xaa, 0, 0], 10), ([0xaa, 0, 1], 100)]); + let b = frame(5_000, &[([0, 0, 0xbb], 1), ([0, 0, 0xbc], 0)]); + + canvas.merge(&a); + canvas.merge(&b); + + assert_eq!(canvas.pixels[0].rgb, [0, 0, 0xbb]); + assert_eq!(canvas.pixels[0].written_ns_since_unix_epoch, 5_032); + assert_eq!(canvas.pixels[1].rgb, [0xaa, 0, 1]); + assert_eq!(canvas.pixels[1].written_ns_since_unix_epoch, 4_200); + } + + #[test] + fn blank_or_stale_frame_never_overwrites_live_content() { + let mut canvas = Canvas::new(1); + canvas.merge(&frame(1_000, &[([0x12, 0x34, 0x56], 50)])); + + // A frame with coarse 0 carries no recent-write info — even with a far-later base it must + // not clobber the live pixel (this is the restarted-worker / blank-canvas case). + canvas.merge(&frame(9_000_000_000, &[([0, 0, 0], 0)])); + + assert_eq!(canvas.pixels[0].rgb, [0x12, 0x34, 0x56]); + assert_eq!( + canvas.pixels[0].written_ns_since_unix_epoch, + 1_000 + (50 << 5) + ); + } + + #[test] + fn older_write_does_not_replace_newer() { + let mut canvas = Canvas::new(1); + // Merge the newer write first, then an older one — order must not matter. + canvas.merge(&frame(5_000, &[([0, 0, 0xbb], 1)])); // 5032 + canvas.merge(&frame(1_000, &[([0xaa, 0, 0], 10)])); // 1320, older + + assert_eq!(canvas.pixels[0].rgb, [0, 0, 0xbb]); + assert_eq!(canvas.pixels[0].written_ns_since_unix_epoch, 5_032); + } +} diff --git a/deich/src/lib.rs b/deich/src/lib.rs new file mode 100644 index 0000000..c0c5a0b --- /dev/null +++ b/deich/src/lib.rs @@ -0,0 +1,30 @@ +//! `deich` is breakwater's distributed mode: a fleet of **workers**, each running a Pixelflut +//! server, that continuously sync their framebuffer to a central **collector** which merges them +//! into the final picture. +//! +//! A single binary runs either role (see [`cli_args::Role`]). Right now only the [`worker`] is +//! functional. The [`sync`] protocol is still just a raw framebuffer blob, and the [`collector`] +//! is not implemented yet — both are next on the list. + +use color_eyre::eyre; + +pub mod cli_args; +pub mod collector; +pub mod sync; +pub mod worker; + +/// Shared process setup for the `deich` binaries: error reporting and logging. +pub fn init_telemetry() -> eyre::Result<()> { + color_eyre::install()?; + + let filter = tracing_subscriber::EnvFilter::builder() + .with_default_directive(if cfg!(debug_assertions) { + tracing::Level::DEBUG.into() + } else { + tracing::Level::INFO.into() + }) + .from_env()?; + tracing_subscriber::fmt().with_env_filter(filter).init(); + + Ok(()) +} diff --git a/deich/src/main.rs b/deich/src/main.rs index 29008ef..2d1b16b 100644 --- a/deich/src/main.rs +++ b/deich/src/main.rs @@ -1,89 +1,14 @@ -use std::{net::SocketAddr, sync::Arc, time::Duration}; - -use breakwater::{ - cli_args::DEFAULT_NETWORK_BUFFER_SIZE, handle_ctrl_c, server::Server, - statistics::StatisticsEvent, -}; -use breakwater_parser::{TimeTrackingFrameBuffer, get_current_ns_since_unix_epoch}; -use color_eyre::eyre::{self, Context}; -use tokio::sync::{broadcast, mpsc}; -use tracing::debug; - -/// How often we sync the framebuffer -const SYNC_FPS: u64 = 30; +use clap::Parser; +use color_eyre::eyre; +use deich::cli_args::{Cli, Role}; #[tokio::main] async fn main() -> eyre::Result<()> { - color_eyre::install()?; - - let filter = tracing_subscriber::EnvFilter::builder() - .with_default_directive(if cfg!(debug_assertions) { - tracing::Level::DEBUG.into() - } else { - tracing::Level::INFO.into() - }) - .from_env()?; - tracing_subscriber::fmt().with_env_filter(filter).init(); - - let fb = Arc::new(TimeTrackingFrameBuffer::new( - 1920, - 1080, - get_current_ns_since_unix_epoch(), - )); - let (statistics_tx, statistics_rx) = mpsc::channel::(100); - let (terminate_signal_tx, _terminate_signal_rx) = broadcast::channel::<()>(1); - - let mut server = Server::new( - &["[::]:1234".parse::().unwrap()], - fb.clone(), - statistics_tx.clone(), - DEFAULT_NETWORK_BUFFER_SIZE, - None, - ) - .await - .context("failed to start pixelflut server")?; - - let server_listener_thread = tokio::spawn(async move { server.start().await }); - let stats_drain_thread = tokio::spawn(async move { drain_stats(statistics_rx).await }); - let sync_thread = { - let fb = fb.clone(); - tokio::spawn(async move { sync_framebuffer(fb).await }) - }; - - handle_ctrl_c(terminate_signal_tx).await?; - server_listener_thread.abort(); - stats_drain_thread.abort(); - sync_thread.abort(); - - Ok(()) -} - -/// Periodically syncs the framebuffer and re-bases its per-pixel timestamps to the current time. -/// -/// The re-basing has to happen faster than the ~536 ms window the 3-byte `ns_since_base` can -/// represent; 30 fps (~33 ms) leaves plenty of headroom. -async fn sync_framebuffer(fb: Arc) { - let mut interval = tokio::time::interval(Duration::from_nanos(1_000_000_000 / SYNC_FPS)); - // Let's try to never skip a frame and catch up to make it easy for the sync collection server - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Burst); - - loop { - interval.tick().await; - - // TODO: Actually sync the framebuffer somewhere instead of just logging its size. - debug!(bytes = fb.num_bytes(), "Syncing framebuffer"); - - // Re-base *after* the sync, so the bytes we just synced are still relative to the old - // base. New writes from here on are relative to "now". - fb.set_base_ns_since_unix_epoch(get_current_ns_since_unix_epoch()); - } -} + let cli = Cli::parse(); + deich::init_telemetry()?; -/// Currently we don't care about stats, so let's just drain them -async fn drain_stats(mut statistics_rx: mpsc::Receiver) { - loop { - if statistics_rx.recv().await.is_none() { - return; - } + match cli.role { + Role::Worker(args) => deich::worker::run(args).await, + Role::Collector(args) => deich::collector::run(args).await, } } diff --git a/deich/src/sync.rs b/deich/src/sync.rs new file mode 100644 index 0000000..1e36e7d --- /dev/null +++ b/deich/src/sync.rs @@ -0,0 +1,316 @@ +//! Worker ↔ collector sync protocol. +//! +//! Flow: a worker connects to the collector and announces itself with a handshake (magic) plus a +//! [`WorkerMessage::Hello`] carrying its persistent UUID, so the collector knows which worker the +//! connection belongs to. The collector replies with a [`CollectorMessage::Config`] — it owns +//! canvas geometry and frame rate — and the worker then streams frames until the connection drops. +//! +//! Control messages are serialized with [`postcard`] and length-delimited (a little-endian `u32` +//! length prefix), so the message set can grow into richer enums over time. The 12 MB framebuffer +//! blob is deliberately *not* serialized: a [`WorkerMessage::Frame`] header is immediately followed +//! on the wire by `frame_size` raw bytes, keeping serialization off the hot payload. + +use std::{io, sync::Arc, time::Duration}; + +use breakwater_parser::{ + FrameBuffer, TimeTrackingFrameBuffer, TimeTrackingPixel, get_current_ns_since_unix_epoch, +}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, + net::TcpStream, +}; +use tracing::{info, warn}; +use uuid::Uuid; + +/// Identifies the deich sync protocol on the wire ("deic" in ASCII). The collector and workers are +/// always deployed together, so a single magic to reject obviously-wrong connections is enough; we +/// don't bother with version negotiation. +const MAGIC: u32 = 0x6465_6963; + +/// Upper bound on a (length-delimited) control message, to reject garbage before allocating. +/// Control messages are tiny; the big framebuffer blob is sent raw, outside this path. +const MAX_MESSAGE_SIZE: usize = 1024 * 1024; + +/// How long a worker waits between attempts to (re)connect to the collector. +const RECONNECT_BACKOFF: Duration = Duration::from_secs(1); + +/// Messages the collector sends to a worker. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum CollectorMessage { + /// Reply to the worker's [`WorkerMessage::Hello`]; tells the worker how to configure itself. + Config(WorkerConfig), +} + +/// Messages a worker sends to the collector. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum WorkerMessage { + /// First message after the handshake: identifies which worker this connection belongs to. + Hello { worker_id: Uuid }, + + /// A framebuffer frame. On the wire this header is immediately followed by + /// [`WorkerConfig::frame_size`] raw framebuffer bytes (not part of this serialized message). + Frame(FrameHeader), +} + +/// Metadata sent ahead of each raw framebuffer blob. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct FrameHeader { + /// Which scheduled slot this frame is for (see [`FrameSchedule`]). Because every node derives + /// it from the same wall-clock schedule, the collector can tell when all workers' frames for a + /// slot have arrived, and discard frames for slots it has already rendered. + pub frame_number: u64, + + /// The base the frame's per-pixel `coarse_ns_since_base` values are relative to. This is the + /// worker's *actual* re-base time, deliberately not `frame_number * period`: a temporarily + /// overloaded worker may re-base a slot late, and the timestamps must stay correct regardless. + pub base_ns_since_unix_epoch: u64, +} + +/// The wall-clock frame schedule shared by every worker and the collector. Frames are numbered by +/// how many `1/fps`-long slots have elapsed since the UNIX epoch, so each node maps an instant to a +/// slot independently and still agrees on the number — no node is the master clock. (Server clocks +/// here are within ~µs of each other, far below a tens-of-milliseconds slot.) +#[derive(Debug, Clone, Copy)] +pub struct FrameSchedule { + frame_period_ns: u64, +} + +impl FrameSchedule { + pub fn new(fps: u32) -> Self { + Self { + frame_period_ns: (1_000_000_000 / u64::from(fps)).max(1), + } + } + + /// The frame slot the given UNIX-epoch instant falls into. + pub fn frame_number_at(&self, ns_since_unix_epoch: u64) -> u64 { + ns_since_unix_epoch / self.frame_period_ns + } + + /// The UNIX-epoch nanosecond timestamp at which `frame_number` begins. + pub fn frame_start_ns(&self, frame_number: u64) -> u64 { + frame_number * self.frame_period_ns + } + + /// How many whole frames `duration` spans, rounded up. Used to size delay margins. + pub fn frames_spanning(&self, duration: Duration) -> u64 { + let ns = u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX); + ns.div_ceil(self.frame_period_ns) + } +} + +/// Configuration the collector hands to each worker. The collector is the single source of truth +/// for canvas geometry and frame rate; workers configure themselves from it. Extend this when +/// workers need more, e.g. their offset within a larger canvas. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct WorkerConfig { + pub width: u32, + pub height: u32, + pub sync_fps: u32, +} + +impl WorkerConfig { + /// Number of bytes in one framebuffer blob. + pub fn frame_size_bytes(&self) -> usize { + self.width as usize * self.height as usize * size_of::() + } +} + +/// Worker side: connect to the collector, announce ourselves with `worker_id`, and read the config +/// the collector replies with. +pub async fn connect( + collector_address: &str, + worker_id: Uuid, +) -> io::Result<(TcpStream, WorkerConfig)> { + let mut stream = TcpStream::connect(collector_address).await?; + write_handshake(&mut stream).await?; + write_message(&mut stream, &WorkerMessage::Hello { worker_id }).await?; + + let CollectorMessage::Config(config) = read_message(&mut stream).await?; + Ok((stream, config)) +} + +/// Collector side: validate the handshake and read the worker's [`WorkerMessage::Hello`], returning +/// the worker's UUID so the connection can be attributed to it. +pub async fn accept_worker(reader: &mut R) -> io::Result { + read_handshake(reader).await?; + match read_message(reader).await? { + WorkerMessage::Hello { worker_id } => Ok(worker_id), + WorkerMessage::Frame { .. } => Err(io::Error::new( + io::ErrorKind::InvalidData, + "expected a hello message but got a frame", + )), + } +} + +/// Collector side: send the worker its config in reply to the hello. +pub async fn send_config( + writer: &mut W, + config: WorkerConfig, +) -> io::Result<()> { + write_message(writer, &CollectorMessage::Config(config)).await +} + +/// Collector side: read a frame's [`FrameHeader`]. The caller must then read exactly +/// [`WorkerConfig::frame_size_bytes`] bytes with [`receive_frame_body`] before the next message, +/// or discard them — splitting the two lets the caller decide whether to keep the blob. +pub async fn receive_frame_header(reader: &mut R) -> io::Result { + match read_message(reader).await? { + WorkerMessage::Frame(header) => Ok(header), + WorkerMessage::Hello { worker_id } => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("expected a frame but got another hello from {worker_id}"), + )), + } +} + +/// Collector side: read a frame's raw blob into `pixels` (which must be +/// [`WorkerConfig::frame_size_bytes`] long), following a [`receive_frame_header`]. +pub async fn receive_frame_body( + reader: &mut R, + pixels: &mut [u8], +) -> io::Result<()> { + reader.read_exact(pixels).await?; + Ok(()) +} + +/// Worker side: stream the framebuffer to the collector forever, reconnecting as needed. +/// +/// `stream`/`config` are the live connection and config from the initial [`connect`]. On every +/// (re)connect the collector re-sends the config; we honour the frame rate, but since the +/// framebuffer is already allocated we only warn if the geometry changed (live resize is not +/// supported yet). +pub async fn sync_framebuffer( + fb: Arc, + collector_address: String, + worker_id: Uuid, + mut stream: TcpStream, + mut config: WorkerConfig, +) { + loop { + let schedule = FrameSchedule::new(config.sync_fps); + if let Err(error) = stream_frames(&fb, &mut stream, schedule).await { + warn!(%error, "Framebuffer sync stream failed, reconnecting"); + } + + (stream, config) = reconnect(&fb, &collector_address, worker_id).await; + } +} + +/// Streams frames aligned to the shared [`FrameSchedule`] until a write fails (returns the error). +/// +/// Each iteration sleeps until the current slot ends, sends the frame accumulated during it, then +/// re-bases the framebuffer for the next slot. The slot the worker enters is recomputed from the +/// clock each time, so a worker that falls behind (e.g. temporarily overloaded) skips the slots it +/// missed instead of sending a backlog of stale frames. +async fn stream_frames( + fb: &TimeTrackingFrameBuffer, + stream: &mut TcpStream, + schedule: FrameSchedule, +) -> io::Result<()> { + // Start accumulating the current slot, basing pixels at the actual time we started. + fb.set_base_ns_since_unix_epoch(get_current_ns_since_unix_epoch()); + let mut frame_number = schedule.frame_number_at(get_current_ns_since_unix_epoch()); + + loop { + // Sleep until the slot we're accumulating ends. + let slot_end = schedule.frame_start_ns(frame_number + 1); + let now = get_current_ns_since_unix_epoch(); + tokio::time::sleep(Duration::from_nanos(slot_end.saturating_sub(now))).await; + + // Send the just-completed slot's frame. `base_ns` is the framebuffer's *actual* base (set + // below for this slot), so the collector interprets `coarse_ns` correctly even if we + // re-based late; `frame_number` is the schedule slot, for the collector's barrier. + let header = FrameHeader { + frame_number, + base_ns_since_unix_epoch: fb.base_ns_since_unix_epoch(), + }; + write_message(stream, &WorkerMessage::Frame(header)).await?; + stream.write_all(fb.as_bytes()).await?; + stream.flush().await?; + + // Re-base for the next slot at the actual current time, and advance to whichever slot that + // is — skipping any we fell behind on, while always moving forward by at least one. + let now = get_current_ns_since_unix_epoch(); + fb.set_base_ns_since_unix_epoch(now); + frame_number = schedule.frame_number_at(now).max(frame_number + 1); + } +} + +/// Reconnects to the collector (retrying with a backoff), returning the new stream and config. +async fn reconnect( + fb: &TimeTrackingFrameBuffer, + collector_address: &str, + worker_id: Uuid, +) -> (TcpStream, WorkerConfig) { + loop { + match connect(collector_address, worker_id).await { + Ok((stream, config)) => { + if config.width as usize != fb.get_width() + || config.height as usize != fb.get_height() + { + warn!( + ?config, + fb_width = fb.get_width(), + fb_height = fb.get_height(), + "Collector changed canvas geometry; live resize is not supported, keeping the current framebuffer" + ); + } + info!(collector_address, "Reconnected to collector"); + return (stream, config); + } + Err(error) => { + warn!(collector_address, %error, "Failed to reconnect to collector, retrying"); + tokio::time::sleep(RECONNECT_BACKOFF).await; + } + } + } +} + +async fn write_handshake(writer: &mut W) -> io::Result<()> { + writer.write_u32_le(MAGIC).await?; + writer.flush().await +} + +async fn read_handshake(reader: &mut R) -> io::Result<()> { + let magic = reader.read_u32_le().await?; + if magic != MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unexpected protocol magic {magic:#010x}, expected {MAGIC:#010x}"), + )); + } + + Ok(()) +} + +/// Writes a length-delimited, postcard-encoded control message. +async fn write_message( + writer: &mut W, + message: &M, +) -> io::Result<()> { + let bytes = postcard::to_allocvec(message) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + let len = u32::try_from(bytes.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "control message too large"))?; + + writer.write_u32_le(len).await?; + writer.write_all(&bytes).await?; + writer.flush().await +} + +/// Reads a length-delimited, postcard-encoded control message. +async fn read_message(reader: &mut R) -> io::Result { + let len = reader.read_u32_le().await? as usize; + if len > MAX_MESSAGE_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("control message of {len} bytes exceeds the {MAX_MESSAGE_SIZE} byte cap"), + )); + } + + let mut bytes = vec![0; len]; + reader.read_exact(&mut bytes).await?; + postcard::from_bytes(&bytes).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) +} diff --git a/deich/src/worker.rs b/deich/src/worker.rs new file mode 100644 index 0000000..ccbcdd5 --- /dev/null +++ b/deich/src/worker.rs @@ -0,0 +1,126 @@ +//! The worker role: runs a Pixelflut server into a time-tracking framebuffer and spawns the +//! background task that syncs that framebuffer to the collector. +//! +//! The collector owns the canvas geometry and frame rate, so the worker fetches its config from +//! the collector before it can allocate the framebuffer and start serving. + +use std::{fs, io, path::Path, sync::Arc, time::Duration}; + +use breakwater::{handle_ctrl_c, server::Server, statistics::StatisticsEvent}; +use breakwater_parser::{TimeTrackingFrameBuffer, get_current_ns_since_unix_epoch}; +use color_eyre::eyre::{self, Context}; +use tokio::{ + net::TcpStream, + sync::{broadcast, mpsc}, +}; +use tracing::{info, warn}; +use uuid::Uuid; + +use crate::{ + cli_args::WorkerArgs, + sync::{self, WorkerConfig}, +}; + +/// Backoff between attempts to reach the collector during startup. +const COLLECTOR_CONNECT_BACKOFF: Duration = Duration::from_secs(1); + +/// Runs the worker until Ctrl-C: a Pixelflut server plus the background framebuffer sync. +pub async fn run(args: WorkerArgs) -> eyre::Result<()> { + let worker_id = load_or_create_worker_id(&args.worker_id_file)?; + info!(%worker_id, "Starting worker"); + + // Fetch the canvas config from the collector (retrying until it's reachable) before we can + // allocate anything. + let (stream, config) = connect_to_collector(&args.collector_address, worker_id).await; + info!(?config, "Received configuration from collector"); + + let fb = Arc::new(TimeTrackingFrameBuffer::new( + config.width as usize, + config.height as usize, + get_current_ns_since_unix_epoch(), + )); + let (statistics_tx, statistics_rx) = mpsc::channel::(100); + let (terminate_signal_tx, _terminate_signal_rx) = broadcast::channel::<()>(1); + + let network_buffer_size = args + .network_buffer_size + .try_into() + // This should never happen as clap checks the range for us + .with_context(|| format!("invalid network buffer size: {}", args.network_buffer_size))?; + + let mut server = Server::new( + &args.listen_addresses, + fb.clone(), + statistics_tx.clone(), + network_buffer_size, + None, + ) + .await + .context("failed to start pixelflut server")?; + + let server_listener_thread = tokio::spawn(async move { server.start().await }); + let stats_drain_thread = tokio::spawn(async move { drain_stats(statistics_rx).await }); + let sync_thread = { + let fb = fb.clone(); + let collector_address = args.collector_address; + tokio::spawn(async move { + sync::sync_framebuffer(fb, collector_address, worker_id, stream, config).await; + }) + }; + + handle_ctrl_c(terminate_signal_tx).await?; + server_listener_thread.abort(); + stats_drain_thread.abort(); + sync_thread.abort(); + + Ok(()) +} + +/// Connects to the collector, retrying with a backoff until it succeeds. +async fn connect_to_collector( + collector_address: &str, + worker_id: Uuid, +) -> (TcpStream, WorkerConfig) { + loop { + match sync::connect(collector_address, worker_id).await { + Ok(result) => return result, + Err(error) => { + warn!( + collector_address, + %error, + "Waiting for the collector to become reachable" + ); + tokio::time::sleep(COLLECTOR_CONNECT_BACKOFF).await; + } + } + } +} + +/// Loads this worker's persistent UUID from `path`, generating and saving a fresh one on first run. +fn load_or_create_worker_id(path: &Path) -> eyre::Result { + match fs::read_to_string(path) { + Ok(contents) => contents + .trim() + .parse() + .with_context(|| format!("failed to parse worker id from {}", path.display())), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + let worker_id = Uuid::new_v4(); + fs::write(path, worker_id.to_string()) + .with_context(|| format!("failed to persist worker id to {}", path.display()))?; + info!(%worker_id, path = %path.display(), "Generated and persisted a new worker id"); + Ok(worker_id) + } + Err(error) => { + Err(error).with_context(|| format!("failed to read worker id from {}", path.display())) + } + } +} + +/// Currently we don't care about stats, so let's just drain them +async fn drain_stats(mut statistics_rx: mpsc::Receiver) { + loop { + if statistics_rx.recv().await.is_none() { + return; + } + } +} From 9b29924f56da7d3dc4193d34ca53ed6ef6a5bf6d Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Sat, 13 Jun 2026 14:32:56 +0200 Subject: [PATCH 4/7] fix: Store pixel timestampos as absolute value of qs since collector startup --- breakwater-parser/src/framebuffer/mod.rs | 20 +- .../src/framebuffer/shared_memory.rs | 4 +- breakwater-parser/src/framebuffer/simple.rs | 4 +- .../src/framebuffer/time_tracking.rs | 236 +++++++----------- breakwater-parser/src/original.rs | 21 +- deich/src/cli_args.rs | 2 +- deich/src/collector.rs | 141 ++++------- deich/src/sync.rs | 147 ++++------- deich/src/worker.rs | 88 ++++--- 9 files changed, 256 insertions(+), 407 deletions(-) diff --git a/breakwater-parser/src/framebuffer/mod.rs b/breakwater-parser/src/framebuffer/mod.rs index c7e6c0d..1ebeb8c 100644 --- a/breakwater-parser/src/framebuffer/mod.rs +++ b/breakwater-parser/src/framebuffer/mod.rs @@ -58,22 +58,14 @@ pub trait FrameBuffer { /// that), we can only return it as a list of bytes, not a list of pixels. fn as_bytes(&self) -> &[u8]; - /// Encode `ns_since_unix_epoch` into the compact, per-pixel `coarse_ns_since_base` - /// representation. This is the only place that touches the (atomic) base timestamp, so call - /// it **once per parse call** and pass the result to [`Self::set_with_coarse_ns_since_base`] - /// for every pixel. That keeps the base load (and the encoding arithmetic) out of the hot - /// per-pixel path, where it would otherwise re-run on every single write. + /// Encode `ns_since_unix_epoch` into the per-pixel timestamp representation. Compute it **once + /// per parse call** and pass the result to [`Self::set_with_pixel_timestamp`] for every pixel + /// (it's the same for all pixels in a call), keeping the encoding out of the hot per-pixel path. #[cfg(feature = "time-tracking")] - fn coarse_ns_since_base(&self, ns_since_unix_epoch: u64) -> u32; + fn pixel_timestamp(&self, ns_since_unix_epoch: u64) -> u64; /// Like [`Self::set`], but also records *when* the pixel was written, as the precomputed - /// `coarse_ns_since_base` from [`Self::coarse_ns_since_base`]. + /// `pixel_timestamp` from [`Self::pixel_timestamp`]. #[cfg(feature = "time-tracking")] - fn set_with_coarse_ns_since_base( - &self, - x: usize, - y: usize, - rgba: u32, - coarse_ns_since_base: u32, - ); + fn set_with_pixel_timestamp(&self, x: usize, y: usize, rgba: u32, pixel_timestamp: u64); } diff --git a/breakwater-parser/src/framebuffer/shared_memory.rs b/breakwater-parser/src/framebuffer/shared_memory.rs index 0b75a76..02b4630 100644 --- a/breakwater-parser/src/framebuffer/shared_memory.rs +++ b/breakwater-parser/src/framebuffer/shared_memory.rs @@ -224,12 +224,12 @@ impl FrameBuffer for SharedMemoryFrameBuffer { } #[cfg(feature = "time-tracking")] - fn coarse_ns_since_base(&self, _: u64) -> u32 { + fn pixel_timestamp(&self, _: u64) -> u64 { panic!("The shared memory framebuffer does not support time tracking"); } #[cfg(feature = "time-tracking")] - fn set_with_coarse_ns_since_base(&self, _: usize, _: usize, _: u32, _: u32) { + fn set_with_pixel_timestamp(&self, _: usize, _: usize, _: u32, _: u64) { panic!("The shared memory framebuffer does not support time tracking"); } } diff --git a/breakwater-parser/src/framebuffer/simple.rs b/breakwater-parser/src/framebuffer/simple.rs index eafdb1a..3c45ca0 100644 --- a/breakwater-parser/src/framebuffer/simple.rs +++ b/breakwater-parser/src/framebuffer/simple.rs @@ -84,12 +84,12 @@ impl FrameBuffer for SimpleFrameBuffer { } #[cfg(feature = "time-tracking")] - fn coarse_ns_since_base(&self, _: u64) -> u32 { + fn pixel_timestamp(&self, _: u64) -> u64 { panic!("The simple framebuffer does not support time tracking"); } #[cfg(feature = "time-tracking")] - fn set_with_coarse_ns_since_base(&self, _: usize, _: usize, _: u32, _: u32) { + fn set_with_pixel_timestamp(&self, _: usize, _: usize, _: u32, _: u64) { panic!("The simple framebuffer does not support time tracking"); } } diff --git a/breakwater-parser/src/framebuffer/time_tracking.rs b/breakwater-parser/src/framebuffer/time_tracking.rs index 07f4d04..89f788d 100644 --- a/breakwater-parser/src/framebuffer/time_tracking.rs +++ b/breakwater-parser/src/framebuffer/time_tracking.rs @@ -1,98 +1,65 @@ -use std::{ - sync::atomic::{AtomicU64, Ordering}, - time::{SystemTime, UNIX_EPOCH}, -}; - -use tracing::warn; +use std::time::{SystemTime, UNIX_EPOCH}; use crate::FrameBuffer; -pub struct TimeTrackingFrameBuffer { - width: usize, - height: usize, - buffer: Vec, - /// All per-pixel timestamps are stored relative to this base. A background task re-bases it - /// (to the current time) at a fixed rate, which keeps `ns_since_base` small enough to fit in - /// our 3 bytes. Read on the hot write path and written by the re-basing task, hence atomic. - /// A `Relaxed` load compiles to a plain `mov` on x86_64, so it doesn't cost us anything. - base_ns_since_unix_epoch: AtomicU64, -} +/// Number of low bits holding the RGB color; the rest hold the timestamp. +const RGB_BITS: u32 = 24; +const RGB_MASK: u64 = (1 << RGB_BITS) - 1; -/// Number of low bits we drop from `ns_since_base` before storing it. We only have 3 bytes -/// (24 bits) of storage for it, so dropping 5 bits (i.e. dividing by 32) lets us cover -/// `0xff_ffff * 32` ns ≈ 536 ms before we have to clamp. That's plenty given the base -/// timestamp is re-based frequently, and 32 ns of resolution is way more than we need. -const NS_SHIFT: u32 = 5; +/// Largest timestamp we can store in the remaining `64 - RGB_BITS = 40` bits. In microseconds +/// that's ≈ 12.7 days of collector uptime before it saturates — plenty, and a collector restart +/// resets the epoch anyway. +const TIMESTAMP_MAX: u64 = (1 << (u64::BITS - RGB_BITS)) - 1; -/// Largest value we can store in the 3 `ns` bytes. -const NS_MAX: u32 = 0x00ff_ffff; - -/// A single pixel, laid out as exactly 6 bytes (`align = 1`): -/// -/// ```text -/// byte: 0 1 2 3 4 5 -/// [ r ] [ g ] [ b ] [ns0] [ns1] [ns2] -/// ``` -/// -/// The `ns` bytes hold `ns_since_base >> NS_SHIFT` (see [`NS_SHIFT`]), clamped to [`NS_MAX`]. -/// -/// We deliberately *don't* expose the fields as `u32`s (which would force `align = 4` and 8 -/// bytes per pixel). The trade-off of going to 6 bytes: 6 does not divide 64, so ~10% of -/// pixels straddle a cache-line boundary. The *write* path (random, hot) therefore uses -/// byte/`[u8; 3]` stores rather than wide `u32`/`u16` stores — a wide store that crosses the -/// boundary triggers the x86 split-store penalty on every such pixel, which measured slower -/// (~65G vs ~76G). -/// -/// Both reads and writes only ever touch the bytes of their own pixel, so no pixel can tear a -/// neighbour and no access runs past the end of the buffer — no padding is required. +/// A single pixel packed into one `u64`: the low [`RGB_BITS`] bits are the RGB color, the high bits +/// are an opaque timestamp. Packing both into a `u64` means writes are a single aligned 64-bit +/// store (the hot path). /// -/// Note: 6-byte cells don't quite match the original 8-byte *aligned* layout (~90G vs ~86G -/// here) on a write-bound load — that one is fast precisely because every write stays within -/// one cache line. The ~4% is the cost of shrinking the per-pixel sync footprint by 25%. +/// The timestamp is deliberately opaque here — its *meaning* (microseconds since the collector +/// epoch) is assigned solely by [`TimeTrackingFrameBuffer::pixel_timestamp`]. To everyone else it's +/// just a comparable number: larger = more recently written, and `0` (the default) = never written, +/// which compares as the oldest possible, so it never wins a merge. #[derive(Debug, Default, Clone, Copy)] -#[repr(C)] -pub struct TimeTrackingPixel { - pub rgb: [u8; 3], - /// `ns_since_base >> NS_SHIFT`, little-endian, clamped to `NS_MAX`. - pub coarse_ns_since_base: [u8; 3], +#[repr(transparent)] +pub struct TimeTrackingPixel(u64); + +impl TimeTrackingPixel { + pub fn new(rgb: u32, timestamp: u64) -> Self { + Self((timestamp << RGB_BITS) | (u64::from(rgb) & RGB_MASK)) + } + + /// The 24-bit RGB color (the implicit alpha byte is always zero). + pub fn rgb(self) -> u32 { + (self.0 & RGB_MASK) as u32 + } + + /// The opaque write timestamp (see the type docs); larger is newer, `0` is never written. + pub fn timestamp(self) -> u64 { + self.0 >> RGB_BITS + } } -/// Views a slice of pixels as their raw bytes (6 per pixel), e.g. to read a frame straight off the +/// Views a slice of pixels as their raw bytes (8 per pixel), e.g. to read a frame straight off the /// wire into a `Vec`. Same layout as [`FrameBuffer::as_bytes`]. pub fn pixels_as_bytes_mut(pixels: &mut [TimeTrackingPixel]) -> &mut [u8] { let len = size_of_val(pixels); let ptr = pixels.as_mut_ptr().cast::(); - // SAFETY: `TimeTrackingPixel` is `repr(C)`, `align = 1`, all-bytes-valid (just `[u8; 3]`s), so - // its `len * 6` bytes are a valid `[u8]` of the same lifetime and exclusive borrow. + // SAFETY: `TimeTrackingPixel` is `repr(transparent)` over a `u64` (all bit patterns valid), so + // its bytes are a valid `[u8]` of the same lifetime and exclusive borrow. unsafe { std::slice::from_raw_parts_mut(ptr, len) } } -impl TimeTrackingPixel { - /// The stored `coarse_ns_since_base` as a `u32` (the implicit high byte is always zero). - fn coarse_ns_since_base(self) -> u32 { - u32::from_le_bytes([ - self.coarse_ns_since_base[0], - self.coarse_ns_since_base[1], - self.coarse_ns_since_base[2], - 0, - ]) - } - - /// The absolute UNIX-epoch time this pixel was last written, given the framebuffer's - /// `base_ns_since_unix_epoch`, reconstructed at `1 << NS_SHIFT` ns resolution. - /// - /// Returns `None` when the pixel carries no recent-write information (`coarse_ns == 0`): the - /// framebuffer re-bases every frame, so a pixel not written within the last window saturates to - /// 0 and is indistinguishable from "never written". Callers merging framebuffers should treat - /// `None` as "no update from this pixel" so stale/blank pixels don't clobber fresher content. - pub fn written_ns_since_unix_epoch(self, base_ns_since_unix_epoch: u64) -> Option { - let coarse = self.coarse_ns_since_base(); - (coarse != 0).then(|| base_ns_since_unix_epoch + (u64::from(coarse) << NS_SHIFT)) - } +pub struct TimeTrackingFrameBuffer { + width: usize, + height: usize, + buffer: Vec, + /// Per-pixel timestamps are microseconds since this epoch — the collector's startup time, in ns + /// since the UNIX epoch. Fixed for the framebuffer's life, so there is no re-basing. + epoch_ns_since_unix_epoch: u64, } impl TimeTrackingFrameBuffer { - pub fn new(width: usize, height: usize, base_ns_since_unix_epoch: u64) -> Self { + pub fn new(width: usize, height: usize, epoch_ns_since_unix_epoch: u64) -> Self { let mut buffer = Vec::with_capacity(width * height); buffer.resize_with(width * height, TimeTrackingPixel::default); @@ -100,7 +67,7 @@ impl TimeTrackingFrameBuffer { width, height, buffer, - base_ns_since_unix_epoch: AtomicU64::new(base_ns_since_unix_epoch), + epoch_ns_since_unix_epoch, } } @@ -108,19 +75,6 @@ impl TimeTrackingFrameBuffer { fn pixel_index(&self, x: usize, y: usize) -> usize { x + y * self.width } - - /// The current base all per-pixel timestamps are relative to. Capture this *before* syncing a - /// frame, so the consumer can interpret that frame's `coarse_ns_since_base` values. - pub fn base_ns_since_unix_epoch(&self) -> u64 { - self.base_ns_since_unix_epoch.load(Ordering::Relaxed) - } - - /// Re-base all future per-pixel timestamps to `base_ns_since_unix_epoch`. Call this - /// regularly (faster than the ~536 ms window) so `ns_since_base` keeps fitting in 3 bytes. - pub fn set_base_ns_since_unix_epoch(&self, base_ns_since_unix_epoch: u64) { - self.base_ns_since_unix_epoch - .store(base_ns_since_unix_epoch, Ordering::Relaxed); - } } impl FrameBuffer for TimeTrackingFrameBuffer { @@ -137,70 +91,28 @@ impl FrameBuffer for TimeTrackingFrameBuffer { #[inline(always)] unsafe fn get_unchecked(&self, x: usize, y: usize) -> u32 { let pixel_index = self.pixel_index(x, y); - // Assemble the RGB bytes; the 4th byte (alpha) is always zero. This is the sequential - // scanout path, which is bandwidth-bound, so the byte assembly is hidden under the - // memory stream and reads no wider than the 3 RGB bytes. - let rgb = unsafe { self.buffer.get_unchecked(pixel_index).rgb }; - u32::from_le_bytes([rgb[0], rgb[1], rgb[2], 0]) + unsafe { self.buffer.get_unchecked(pixel_index).rgb() } } fn set(&self, _: usize, _: usize, _: u32) { panic!( - "The time tracking framebuffer requires you to use the set_with_coarse_ns_since_base function!" + "The time tracking framebuffer requires you to use the set_with_pixel_timestamp function!" ); } - fn coarse_ns_since_base(&self, ns_since_unix_epoch: u64) -> u32 { - // The single point that reads the (atomic) base, called once per parse call. Drop the - // low NS_SHIFT bits and clamp into our 3 ns bytes; if the base is more than ~536 ms in - // the past (shouldn't happen, it gets re-based regularly) we saturate to NS_MAX. - // - // `saturating_sub`, not `-`: the parser captures `ns_since_unix_epoch` once at the start - // of a parse call, but the background task may re-base `base` to a newer time in between. - // In that case the pixels were effectively written at the new base, so a difference of 0 - // is exactly right. - let base_ns_since_unix_epoch = self.base_ns_since_unix_epoch.load(Ordering::Relaxed); - let raw_coarse_ns = - ns_since_unix_epoch.saturating_sub(base_ns_since_unix_epoch) >> NS_SHIFT; - - if raw_coarse_ns > u64::from(NS_MAX) { - warn!( - base_ns_since_unix_epoch, - ns_since_unix_epoch, - raw_coarse_ns, - ns_max = NS_MAX, - "Pixels were written more than ~536ms after the last base timestamp; clamping to \ - ~536ms. Is the framebuffer re-basing task still running?" - ); - } - - u32::try_from(raw_coarse_ns).unwrap_or(u32::MAX).min(NS_MAX) + fn pixel_timestamp(&self, ns_since_unix_epoch: u64) -> u64 { + // The per-parse-call timestamp, computed once. `saturating_sub` guards against a worker + // whose clock trails the collector's epoch slightly; `.min` clamps the ~12.7 day range. + (ns_since_unix_epoch.saturating_sub(self.epoch_ns_since_unix_epoch) / 1_000).min(TIMESTAMP_MAX) } - fn set_with_coarse_ns_since_base( - &self, - x: usize, - y: usize, - rgba: u32, - coarse_ns_since_base: u32, - ) { + fn set_with_pixel_timestamp(&self, x: usize, y: usize, rgba: u32, pixel_timestamp: u64) { if x < self.width && y < self.height { let pixel_index = self.pixel_index(x, y); - let coarse_ns_since_base = coarse_ns_since_base.to_le_bytes(); - - // Write via the byte-array fields, *not* via wide unaligned u32/u16 stores. A 6-byte - // stride means ~10% of pixels straddle a 64-byte cache line, and a single wide store - // that crosses the boundary hits the x86 split-store penalty on every such write - // (random writes are the hot path here). Byte/u16-sized stores rarely split, which is - // why this is meaningfully faster than packing into one u32 + one u16. + // A single aligned 64-bit store (the whole point of packing the pixel into a `u64`). unsafe { let ptr: *mut TimeTrackingPixel = self.buffer.as_ptr().add(pixel_index).cast_mut(); - (*ptr).rgb = [rgba as u8, (rgba >> 8) as u8, (rgba >> 16) as u8]; - (*ptr).coarse_ns_since_base = [ - coarse_ns_since_base[0], - coarse_ns_since_base[1], - coarse_ns_since_base[2], - ]; + *ptr = TimeTrackingPixel::new(rgba, pixel_timestamp); } } } @@ -210,10 +122,9 @@ impl FrameBuffer for TimeTrackingFrameBuffer { } fn as_bytes(&self) -> &[u8] { - // The buffer is a contiguous `Vec` of `align = 1` cells, so the raw - // bytes are exactly the 6-bytes-per-pixel wire layout. Like the other framebuffers, this - // reads memory that writers may be mutating concurrently — fine for a lossy, best-effort - // sync (a torn pixel just gets corrected on the next frame). + // The buffer is a contiguous `Vec`, so its raw bytes are exactly the + // 8-bytes-per-pixel wire layout. Like the other framebuffers, this reads memory that writers + // may be mutating concurrently — fine for a lossy, best-effort sync. let len = self.buffer.len() * size_of::(); let ptr = self.buffer.as_ptr().cast::(); unsafe { std::slice::from_raw_parts(ptr, len) } @@ -234,3 +145,38 @@ pub fn get_current_ns_since_unix_epoch() -> u64 { "your system time is >= year 2554. I'm developing this in 2026, I'm very likely dead now. And did no one write a better server to use in all that years?", ) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pixel_packs_rgb_and_timestamp() { + let pixel = TimeTrackingPixel::new(0x12_3456, 1_000_000); + assert_eq!(pixel.rgb(), 0x12_3456); + assert_eq!(pixel.timestamp(), 1_000_000); + } + + #[test] + fn pixel_timestamp_is_micros_since_epoch_clamped() { + let fb = TimeTrackingFrameBuffer::new(1, 1, 1_000_000); + + // 1_000_000 ns after the epoch -> 1000 µs. + assert_eq!(fb.pixel_timestamp(2_000_000), 1_000); + // A timestamp before the epoch saturates to 0 (clock skew guard). + assert_eq!(fb.pixel_timestamp(0), 0); + // Beyond the 40-bit range it clamps. + assert_eq!(fb.pixel_timestamp(u64::MAX), TIMESTAMP_MAX); + } + + #[test] + fn set_and_read_back_pixel() { + let fb = TimeTrackingFrameBuffer::new(4, 4, 0); + fb.set_with_pixel_timestamp(1, 2, 0x00_ff00, 42); + + let pixel = fb.buffer[fb.pixel_index(1, 2)]; + assert_eq!(pixel.rgb(), 0x00_ff00); + assert_eq!(pixel.timestamp(), 42); + assert_eq!(fb.get(1, 2), Some(0x00_ff00)); + } +} diff --git a/breakwater-parser/src/original.rs b/breakwater-parser/src/original.rs index c50064a..be9aafe 100644 --- a/breakwater-parser/src/original.rs +++ b/breakwater-parser/src/original.rs @@ -55,11 +55,10 @@ impl Parser for OriginalParser { // As this is a potentially(?) expensive operation we only call it one in this parsing loop // All the pixels likely where in the same TCP packets (+- 1/2 or so) it doesn't matter after all - // Encode the timestamp exactly once here, not per pixel: reading the framebuffer's - // (atomic) base and doing the encoding arithmetic on every write costs measurable - // throughput, and it's constant for the whole parse call anyway. + // Encode the timestamp exactly once here, not per pixel: it's constant for the whole parse + // call, so computing it per write would just waste throughput on the hot path. #[cfg(feature = "time-tracking")] - let coarse_ns_since_base = self.fb.coarse_ns_since_base( + let pixel_timestamp = self.fb.pixel_timestamp( crate::framebuffer::time_tracking::get_current_ns_since_unix_epoch(), ); @@ -131,11 +130,11 @@ impl Parser for OriginalParser { #[cfg(not(feature = "time-tracking"))] self.fb.set(x, y, rgba & 0x00ff_ffff); #[cfg(feature = "time-tracking")] - self.fb.set_with_coarse_ns_since_base( + self.fb.set_with_pixel_timestamp( x, y, rgba & 0x00ff_ffff, - coarse_ns_since_base, + pixel_timestamp, ); continue; @@ -152,11 +151,11 @@ impl Parser for OriginalParser { #[cfg(not(feature = "time-tracking"))] self.fb.set(x, y, rgba & 0x00ff_ffff); #[cfg(feature = "time-tracking")] - self.fb.set_with_coarse_ns_since_base( + self.fb.set_with_pixel_timestamp( x, y, rgba & 0x00ff_ffff, - coarse_ns_since_base, + pixel_timestamp, ); continue; @@ -202,7 +201,7 @@ impl Parser for OriginalParser { self.fb.set(x, y, rgba); #[cfg(feature = "time-tracking")] self.fb - .set_with_coarse_ns_since_base(x, y, rgba, coarse_ns_since_base); + .set_with_pixel_timestamp(x, y, rgba, pixel_timestamp); continue; } @@ -241,11 +240,11 @@ impl Parser for OriginalParser { #[cfg(not(feature = "time-tracking"))] self.fb.set(x as usize, y as usize, rgba & 0x00ff_ffff); #[cfg(feature = "time-tracking")] - self.fb.set_with_coarse_ns_since_base( + self.fb.set_with_pixel_timestamp( x as usize, y as usize, rgba & 0x00ff_ffff, - coarse_ns_since_base, + pixel_timestamp, ); // P B XX YY RGBA diff --git a/deich/src/cli_args.rs b/deich/src/cli_args.rs index 3e744e9..e0b7a63 100644 --- a/deich/src/cli_args.rs +++ b/deich/src/cli_args.rs @@ -64,6 +64,6 @@ pub struct CollectorArgs { /// Frames per second the workers should sync at. Sent to every worker as part of its config. /// Must stay well below the ~536 ms window the per-pixel timestamp can represent. - #[clap(long, default_value_t = 30, value_parser = clap::value_parser!(u32).range(3..=60))] + #[clap(long, default_value_t = 30, value_parser = clap::value_parser!(u32).range(1..=60))] pub fps: u32, } diff --git a/deich/src/collector.rs b/deich/src/collector.rs index 9278482..ad9c564 100644 --- a/deich/src/collector.rs +++ b/deich/src/collector.rs @@ -44,53 +44,29 @@ const FRAME_STREAM_BUDGET: Duration = Duration::from_millis(50); /// never cross an `.await`, so an async `tokio::sync::RwLock` would only add overhead. type ConnectedWorkers = Arc>>; -/// One worker's framebuffer for one frame, plus the timestamp base it's relative to. Stored as -/// [`TimeTrackingPixel`]s (rather than raw bytes) to keep the layout tied to the canonical type; -/// the wire bytes are read straight into this buffer via a `bytemuck` cast. -struct ReceivedFrame { - /// The base the pixels' `coarse_ns_since_base` values are relative to. - base_ns_since_unix_epoch: u64, - buffer: Vec, -} - /// The long-term merged canvas, held by the master for the whole process lifetime. /// -/// Unlike a worker's framebuffer (whose 3-byte `coarse_ns` only spans ~536 ms), each pixel here -/// keeps a full `u64` absolute write timestamp, so last-write-wins is *exact* over arbitrary time -/// and survives traffic gaps and worker restarts. It's a flat pixel vector — no width/height, -/// merging is purely per-index. +/// Each pixel keeps its full write timestamp (the high bits of [`TimeTrackingPixel`]), so +/// last-write-wins is exact over arbitrary time and survives traffic gaps and worker restarts. +/// It's a flat pixel vector — no width/height; merging is purely per-index. struct Canvas { - pixels: Vec, -} - -#[derive(Clone, Copy, Default)] -struct CanvasPixel { - /// The winning color. Written by the merge; read by the (future) renderer and the tests. - // Not read in the binary yet (no rendering); `expect` can't be used as the tests *do* read it. - #[allow(dead_code)] - rgb: [u8; 3], - /// Absolute UNIX-epoch time the winning write happened; `0` means "never written". - written_ns_since_unix_epoch: u64, + pixels: Vec, } impl Canvas { fn new(pixel_count: usize) -> Self { Self { - pixels: vec![CanvasPixel::default(); pixel_count], + pixels: vec![TimeTrackingPixel::default(); pixel_count], } } - /// Folds `frame` in, keeping for each pixel the write with the latest absolute timestamp. - /// Pixels the frame carries no recent-write info for (`coarse_ns == 0`) are skipped, so a blank - /// or stale frame never clobbers fresher content. - fn merge(&mut self, frame: &ReceivedFrame) { - for (canvas_pixel, frame_pixel) in self.pixels.iter_mut().zip(&frame.buffer) { - if let Some(written_ns) = - frame_pixel.written_ns_since_unix_epoch(frame.base_ns_since_unix_epoch) - && written_ns > canvas_pixel.written_ns_since_unix_epoch - { - canvas_pixel.rgb = frame_pixel.rgb; - canvas_pixel.written_ns_since_unix_epoch = written_ns; + /// Folds `frame` in, keeping for each pixel whichever write has the larger timestamp. Since the + /// default timestamp is `0` (oldest possible), a never-written pixel — a blank or restarted + /// worker's frame — never clobbers fresher content. + fn merge(&mut self, frame: &[TimeTrackingPixel]) { + for (canvas_pixel, &frame_pixel) in self.pixels.iter_mut().zip(frame) { + if frame_pixel.timestamp() > canvas_pixel.timestamp() { + *canvas_pixel = frame_pixel; } } } @@ -103,9 +79,9 @@ struct FrameStore { /// until the master's first tick. interesting: RwLock>>, - /// Stored frames: `frame_number -> (worker_id -> frame)`. Written by workers, read and evicted - /// by the master. - frames: Mutex>>, + /// Stored frames: `frame_number -> (worker_id -> framebuffer)`. Written by workers, read and + /// evicted by the master. + frames: Mutex>>>, } impl FrameStore { @@ -126,7 +102,7 @@ impl FrameStore { } /// Stores a worker's frame (overwriting any previous frame from the same worker for that slot). - fn store(&self, frame_number: u64, worker_id: Uuid, frame: ReceivedFrame) { + fn store(&self, frame_number: u64, worker_id: Uuid, frame: Vec) { self.frames .lock() .expect("frame store lock poisoned") @@ -142,7 +118,7 @@ impl FrameStore { &self, window: RangeInclusive, render_frame: u64, - ) -> HashMap { + ) -> HashMap> { *self .interesting .write() @@ -161,6 +137,8 @@ pub async fn run(args: CollectorArgs) -> eyre::Result<()> { width: args.width, height: args.height, sync_fps: args.fps, + // Our startup time: the zero point for every worker's per-pixel timestamps. + epoch_ns_since_unix_epoch: get_current_ns_since_unix_epoch(), }; let listener = TcpListener::bind(args.listen_address) @@ -297,27 +275,20 @@ async fn serve_frames( let mut discard = vec![0u8; config.frame_size_bytes()]; loop { - let header = sync::receive_frame_header(stream).await?; + let frame_number = sync::receive_frame_number(stream).await?; - if frame_store.is_interested(header.frame_number) { + if frame_store.is_interested(frame_number) { // Read straight into a fresh pixel buffer (outside any lock) so we can hand ownership to // the store; `read_exact` writes directly into its bytes, no extra copy. let mut buffer = vec![TimeTrackingPixel::default(); pixel_count]; sync::receive_frame_body(stream, pixels_as_bytes_mut(&mut buffer)).await?; - frame_store.store( - header.frame_number, - worker_id, - ReceivedFrame { - base_ns_since_unix_epoch: header.base_ns_since_unix_epoch, - buffer, - }, - ); + frame_store.store(frame_number, worker_id, buffer); } else { // Still consume the blob so the stream stays aligned for the next message. sync::receive_frame_body(stream, &mut discard).await?; warn!( %peer, - frame_number = header.frame_number, + frame_number, "Master is not interested in this frame (outside the window); dropping it" ); } @@ -361,66 +332,50 @@ fn deregister(connected_workers: &ConnectedWorkers, worker_id: Uuid, peer: Socke mod tests { use super::*; - /// Builds a `ReceivedFrame` from `(rgb, coarse_ns)` pairs. - fn frame(base_ns_since_unix_epoch: u64, pixels: &[([u8; 3], u32)]) -> ReceivedFrame { - let buffer = pixels + /// Builds a frame from `(rgb, timestamp)` pairs. + fn frame(pixels: &[(u32, u64)]) -> Vec { + pixels .iter() - .map(|&(rgb, coarse)| { - let coarse = coarse.to_le_bytes(); - TimeTrackingPixel { - rgb, - coarse_ns_since_base: [coarse[0], coarse[1], coarse[2]], - } - }) - .collect(); - ReceivedFrame { - base_ns_since_unix_epoch, - buffer, - } + .map(|&(rgb, timestamp)| TimeTrackingPixel::new(rgb, timestamp)) + .collect() } #[test] - fn merges_latest_absolute_write_per_pixel() { + fn merges_latest_timestamp_per_pixel() { let mut canvas = Canvas::new(2); - // Pixel 0: A at 1000 + (10 << 5) = 1320; B at 5000 + (1 << 5) = 5032 -> B wins. - // Pixel 1: A at 1000 + (100 << 5) = 4200; B has coarse 0 -> skipped -> A stays. - let a = frame(1_000, &[([0xaa, 0, 0], 10), ([0xaa, 0, 1], 100)]); - let b = frame(5_000, &[([0, 0, 0xbb], 1), ([0, 0, 0xbc], 0)]); + // Pixel 0: A timestamp 1320, B timestamp 5032 -> B wins. + // Pixel 1: A timestamp 4200, B timestamp 0 (never written) -> A stays. + canvas.merge(&frame(&[(0xaa_0000, 1_320), (0xaa_0001, 4_200)])); + canvas.merge(&frame(&[(0x00_00bb, 5_032), (0x00_00bc, 0)])); - canvas.merge(&a); - canvas.merge(&b); - - assert_eq!(canvas.pixels[0].rgb, [0, 0, 0xbb]); - assert_eq!(canvas.pixels[0].written_ns_since_unix_epoch, 5_032); - assert_eq!(canvas.pixels[1].rgb, [0xaa, 0, 1]); - assert_eq!(canvas.pixels[1].written_ns_since_unix_epoch, 4_200); + assert_eq!(canvas.pixels[0].rgb(), 0x00_00bb); + assert_eq!(canvas.pixels[0].timestamp(), 5_032); + assert_eq!(canvas.pixels[1].rgb(), 0xaa_0001); + assert_eq!(canvas.pixels[1].timestamp(), 4_200); } #[test] - fn blank_or_stale_frame_never_overwrites_live_content() { + fn blank_frame_never_overwrites_live_content() { let mut canvas = Canvas::new(1); - canvas.merge(&frame(1_000, &[([0x12, 0x34, 0x56], 50)])); + canvas.merge(&frame(&[(0x12_3456, 50)])); - // A frame with coarse 0 carries no recent-write info — even with a far-later base it must - // not clobber the live pixel (this is the restarted-worker / blank-canvas case). - canvas.merge(&frame(9_000_000_000, &[([0, 0, 0], 0)])); + // A never-written pixel has timestamp 0 (oldest possible), so it can't clobber live content + // (the restarted-worker / blank-canvas case). + canvas.merge(&frame(&[(0, 0)])); - assert_eq!(canvas.pixels[0].rgb, [0x12, 0x34, 0x56]); - assert_eq!( - canvas.pixels[0].written_ns_since_unix_epoch, - 1_000 + (50 << 5) - ); + assert_eq!(canvas.pixels[0].rgb(), 0x12_3456); + assert_eq!(canvas.pixels[0].timestamp(), 50); } #[test] fn older_write_does_not_replace_newer() { let mut canvas = Canvas::new(1); // Merge the newer write first, then an older one — order must not matter. - canvas.merge(&frame(5_000, &[([0, 0, 0xbb], 1)])); // 5032 - canvas.merge(&frame(1_000, &[([0xaa, 0, 0], 10)])); // 1320, older + canvas.merge(&frame(&[(0x00_00bb, 5_032)])); + canvas.merge(&frame(&[(0xaa_0000, 1_320)])); - assert_eq!(canvas.pixels[0].rgb, [0, 0, 0xbb]); - assert_eq!(canvas.pixels[0].written_ns_since_unix_epoch, 5_032); + assert_eq!(canvas.pixels[0].rgb(), 0x00_00bb); + assert_eq!(canvas.pixels[0].timestamp(), 5_032); } } diff --git a/deich/src/sync.rs b/deich/src/sync.rs index 1e36e7d..1c678df 100644 --- a/deich/src/sync.rs +++ b/deich/src/sync.rs @@ -10,7 +10,7 @@ //! blob is deliberately *not* serialized: a [`WorkerMessage::Frame`] header is immediately followed //! on the wire by `frame_size` raw bytes, keeping serialization off the hot payload. -use std::{io, sync::Arc, time::Duration}; +use std::{io, time::Duration}; use breakwater_parser::{ FrameBuffer, TimeTrackingFrameBuffer, TimeTrackingPixel, get_current_ns_since_unix_epoch, @@ -20,7 +20,6 @@ use tokio::{ io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, net::TcpStream, }; -use tracing::{info, warn}; use uuid::Uuid; /// Identifies the deich sync protocol on the wire ("deic" in ASCII). The collector and workers are @@ -32,9 +31,6 @@ const MAGIC: u32 = 0x6465_6963; /// Control messages are tiny; the big framebuffer blob is sent raw, outside this path. const MAX_MESSAGE_SIZE: usize = 1024 * 1024; -/// How long a worker waits between attempts to (re)connect to the collector. -const RECONNECT_BACKOFF: Duration = Duration::from_secs(1); - /// Messages the collector sends to a worker. #[derive(Debug, Clone, Serialize, Deserialize)] pub enum CollectorMessage { @@ -48,23 +44,13 @@ pub enum WorkerMessage { /// First message after the handshake: identifies which worker this connection belongs to. Hello { worker_id: Uuid }, - /// A framebuffer frame. On the wire this header is immediately followed by - /// [`WorkerConfig::frame_size`] raw framebuffer bytes (not part of this serialized message). - Frame(FrameHeader), -} - -/// Metadata sent ahead of each raw framebuffer blob. -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -pub struct FrameHeader { - /// Which scheduled slot this frame is for (see [`FrameSchedule`]). Because every node derives - /// it from the same wall-clock schedule, the collector can tell when all workers' frames for a - /// slot have arrived, and discard frames for slots it has already rendered. - pub frame_number: u64, - - /// The base the frame's per-pixel `coarse_ns_since_base` values are relative to. This is the - /// worker's *actual* re-base time, deliberately not `frame_number * period`: a temporarily - /// overloaded worker may re-base a slot late, and the timestamps must stay correct regardless. - pub base_ns_since_unix_epoch: u64, + /// A framebuffer frame. On the wire this is immediately followed by + /// [`WorkerConfig::frame_size_bytes`] raw framebuffer bytes (not part of this serialized + /// message). `frame_number` is the scheduled slot this frame is for (see [`FrameSchedule`]); + /// every node derives it from the same wall-clock schedule, so the collector knows which slot a + /// frame belongs to. The per-pixel write timestamps live in the blob itself (absolute, set by + /// the framebuffer), so no base needs to travel with the frame. + Frame { frame_number: u64 }, } /// The wall-clock frame schedule shared by every worker and the collector. Frames are numbered by @@ -108,6 +94,11 @@ pub struct WorkerConfig { pub width: u32, pub height: u32, pub sync_fps: u32, + + /// The collector's startup time (ns since the UNIX epoch). Workers use it as the zero point for + /// their per-pixel timestamps, so every worker's timestamps are mutually comparable at the + /// collector. A 40-bit µs offset from here lasts ~12.7 days of collector uptime. + pub epoch_ns_since_unix_epoch: u64, } impl WorkerConfig { @@ -144,20 +135,12 @@ pub async fn accept_worker(reader: &mut R) -> io::Result( - writer: &mut W, - config: WorkerConfig, -) -> io::Result<()> { - write_message(writer, &CollectorMessage::Config(config)).await -} - -/// Collector side: read a frame's [`FrameHeader`]. The caller must then read exactly -/// [`WorkerConfig::frame_size_bytes`] bytes with [`receive_frame_body`] before the next message, -/// or discard them — splitting the two lets the caller decide whether to keep the blob. -pub async fn receive_frame_header(reader: &mut R) -> io::Result { +/// Collector side: read a frame's `frame_number`. The caller must then read exactly +/// [`WorkerConfig::frame_size_bytes`] bytes with [`receive_frame_body`] before the next message, or +/// discard them — splitting the two lets the caller decide whether to keep the blob. +pub async fn receive_frame_number(reader: &mut R) -> io::Result { match read_message(reader).await? { - WorkerMessage::Frame(header) => Ok(header), + WorkerMessage::Frame { frame_number } => Ok(frame_number), WorkerMessage::Hello { worker_id } => Err(io::Error::new( io::ErrorKind::InvalidData, format!("expected a frame but got another hello from {worker_id}"), @@ -165,6 +148,14 @@ pub async fn receive_frame_header(reader: &mut R) -> io::R } } +/// Collector side: send the worker its config in reply to the hello. +pub async fn send_config( + writer: &mut W, + config: WorkerConfig, +) -> io::Result<()> { + write_message(writer, &CollectorMessage::Config(config)).await +} + /// Collector side: read a frame's raw blob into `pixels` (which must be /// [`WorkerConfig::frame_size_bytes`] long), following a [`receive_frame_header`]. pub async fn receive_frame_body( @@ -181,90 +172,36 @@ pub async fn receive_frame_body( /// (re)connect the collector re-sends the config; we honour the frame rate, but since the /// framebuffer is already allocated we only warn if the geometry changed (live resize is not /// supported yet). -pub async fn sync_framebuffer( - fb: Arc, - collector_address: String, - worker_id: Uuid, - mut stream: TcpStream, - mut config: WorkerConfig, -) { - loop { - let schedule = FrameSchedule::new(config.sync_fps); - if let Err(error) = stream_frames(&fb, &mut stream, schedule).await { - warn!(%error, "Framebuffer sync stream failed, reconnecting"); - } - - (stream, config) = reconnect(&fb, &collector_address, worker_id).await; - } -} - -/// Streams frames aligned to the shared [`FrameSchedule`] until a write fails (returns the error). +/// Streams frames aligned to the shared [`FrameSchedule`] until the connection fails, returning the +/// error. The worker treats that as a signal to tear down and start a fresh session (reconnect, get +/// new config, rebuild the framebuffer), so there is no reconnection logic here. /// -/// Each iteration sleeps until the current slot ends, sends the frame accumulated during it, then -/// re-bases the framebuffer for the next slot. The slot the worker enters is recomputed from the -/// clock each time, so a worker that falls behind (e.g. temporarily overloaded) skips the slots it -/// missed instead of sending a backlog of stale frames. -async fn stream_frames( +/// Each iteration sleeps until the current slot ends and sends the framebuffer, tagged with the +/// just-completed slot number. Per-pixel timestamps are absolute (set by the framebuffer at write +/// time), so there's nothing to re-base — the worker only has to pick the right slot number, which +/// it recomputes from the clock so a worker that falls behind simply skips missed slots. +pub async fn sync_framebuffer( fb: &TimeTrackingFrameBuffer, stream: &mut TcpStream, - schedule: FrameSchedule, + config: WorkerConfig, ) -> io::Result<()> { - // Start accumulating the current slot, basing pixels at the actual time we started. - fb.set_base_ns_since_unix_epoch(get_current_ns_since_unix_epoch()); + let schedule = FrameSchedule::new(config.sync_fps); let mut frame_number = schedule.frame_number_at(get_current_ns_since_unix_epoch()); loop { - // Sleep until the slot we're accumulating ends. + // Sleep until the current slot ends. let slot_end = schedule.frame_start_ns(frame_number + 1); let now = get_current_ns_since_unix_epoch(); tokio::time::sleep(Duration::from_nanos(slot_end.saturating_sub(now))).await; - // Send the just-completed slot's frame. `base_ns` is the framebuffer's *actual* base (set - // below for this slot), so the collector interprets `coarse_ns` correctly even if we - // re-based late; `frame_number` is the schedule slot, for the collector's barrier. - let header = FrameHeader { - frame_number, - base_ns_since_unix_epoch: fb.base_ns_since_unix_epoch(), - }; - write_message(stream, &WorkerMessage::Frame(header)).await?; + write_message(stream, &WorkerMessage::Frame { frame_number }).await?; stream.write_all(fb.as_bytes()).await?; stream.flush().await?; - // Re-base for the next slot at the actual current time, and advance to whichever slot that - // is — skipping any we fell behind on, while always moving forward by at least one. - let now = get_current_ns_since_unix_epoch(); - fb.set_base_ns_since_unix_epoch(now); - frame_number = schedule.frame_number_at(now).max(frame_number + 1); - } -} - -/// Reconnects to the collector (retrying with a backoff), returning the new stream and config. -async fn reconnect( - fb: &TimeTrackingFrameBuffer, - collector_address: &str, - worker_id: Uuid, -) -> (TcpStream, WorkerConfig) { - loop { - match connect(collector_address, worker_id).await { - Ok((stream, config)) => { - if config.width as usize != fb.get_width() - || config.height as usize != fb.get_height() - { - warn!( - ?config, - fb_width = fb.get_width(), - fb_height = fb.get_height(), - "Collector changed canvas geometry; live resize is not supported, keeping the current framebuffer" - ); - } - info!(collector_address, "Reconnected to collector"); - return (stream, config); - } - Err(error) => { - warn!(collector_address, %error, "Failed to reconnect to collector, retrying"); - tokio::time::sleep(RECONNECT_BACKOFF).await; - } - } + // Advance to the current slot, skipping any we fell behind on, always moving forward. + frame_number = schedule + .frame_number_at(get_current_ns_since_unix_epoch()) + .max(frame_number + 1); } } diff --git a/deich/src/worker.rs b/deich/src/worker.rs index ccbcdd5..3d40962 100644 --- a/deich/src/worker.rs +++ b/deich/src/worker.rs @@ -1,18 +1,18 @@ -//! The worker role: runs a Pixelflut server into a time-tracking framebuffer and spawns the -//! background task that syncs that framebuffer to the collector. +//! The worker role: runs a Pixelflut server into a time-tracking framebuffer and syncs that +//! framebuffer to the collector. //! -//! The collector owns the canvas geometry and frame rate, so the worker fetches its config from -//! the collector before it can allocate the framebuffer and start serving. +//! The collector owns the canvas geometry, frame rate and timestamp epoch, so the worker fetches +//! its config from the collector before it can allocate the framebuffer and start serving. Each +//! [`run_session`] runs until the collector connection drops; the worker then tears everything down +//! and starts a fresh session, which transparently picks up a changed config (geometry, fps, or a +//! new epoch after a collector restart). use std::{fs, io, path::Path, sync::Arc, time::Duration}; -use breakwater::{handle_ctrl_c, server::Server, statistics::StatisticsEvent}; -use breakwater_parser::{TimeTrackingFrameBuffer, get_current_ns_since_unix_epoch}; +use breakwater::{server::Server, statistics::StatisticsEvent}; +use breakwater_parser::TimeTrackingFrameBuffer; use color_eyre::eyre::{self, Context}; -use tokio::{ - net::TcpStream, - sync::{broadcast, mpsc}, -}; +use tokio::{net::TcpStream, sync::mpsc}; use tracing::{info, warn}; use uuid::Uuid; @@ -21,26 +21,53 @@ use crate::{ sync::{self, WorkerConfig}, }; -/// Backoff between attempts to reach the collector during startup. -const COLLECTOR_CONNECT_BACKOFF: Duration = Duration::from_secs(1); +/// Backoff between worker sessions (also used while waiting for the collector to become reachable). +const SESSION_BACKOFF: Duration = Duration::from_secs(1); -/// Runs the worker until Ctrl-C: a Pixelflut server plus the background framebuffer sync. +/// Runs the worker until Ctrl-C, restarting the session whenever the collector connection drops. pub async fn run(args: WorkerArgs) -> eyre::Result<()> { let worker_id = load_or_create_worker_id(&args.worker_id_file)?; info!(%worker_id, "Starting worker"); - // Fetch the canvas config from the collector (retrying until it's reachable) before we can - // allocate anything. - let (stream, config) = connect_to_collector(&args.collector_address, worker_id).await; + tokio::select! { + // `session_loop` never returns on its own — it just keeps (re)starting sessions. + result = session_loop(&args, worker_id) => result, + result = tokio::signal::ctrl_c() => { + result.context("failed to wait for ctrl + c")?; + info!("Received Ctrl-C, shutting down"); + Ok(()) + } + } +} + +/// Runs sessions back-to-back forever; a session ending (connection lost, config change, error) is +/// just logged and followed by a fresh one after a short backoff. +async fn session_loop(args: &WorkerArgs, worker_id: Uuid) -> eyre::Result<()> { + loop { + match run_session(args, worker_id).await { + Ok(()) => info!("Collector connection ended; restarting worker session"), + Err(error) => warn!(%error, "Worker session failed; restarting"), + } + tokio::time::sleep(SESSION_BACKOFF).await; + } +} + +/// One worker session: connect to the collector, build the framebuffer from its config, serve +/// Pixelflut into it, and sync it — until the collector connection drops (or the server stops). +/// +/// The server, stats drain and sync all run as `select!` arms (not detached tasks), so when the +/// session ends — here, or because the whole worker is cancelled on Ctrl-C — they're all dropped +/// together. No teardown bookkeeping, no leaked tasks. +async fn run_session(args: &WorkerArgs, worker_id: Uuid) -> eyre::Result<()> { + let (mut stream, config) = connect_to_collector(&args.collector_address, worker_id).await; info!(?config, "Received configuration from collector"); let fb = Arc::new(TimeTrackingFrameBuffer::new( config.width as usize, config.height as usize, - get_current_ns_since_unix_epoch(), + config.epoch_ns_since_unix_epoch, )); let (statistics_tx, statistics_rx) = mpsc::channel::(100); - let (terminate_signal_tx, _terminate_signal_rx) = broadcast::channel::<()>(1); let network_buffer_size = args .network_buffer_size @@ -51,27 +78,20 @@ pub async fn run(args: WorkerArgs) -> eyre::Result<()> { let mut server = Server::new( &args.listen_addresses, fb.clone(), - statistics_tx.clone(), + statistics_tx, network_buffer_size, None, ) .await .context("failed to start pixelflut server")?; - let server_listener_thread = tokio::spawn(async move { server.start().await }); - let stats_drain_thread = tokio::spawn(async move { drain_stats(statistics_rx).await }); - let sync_thread = { - let fb = fb.clone(); - let collector_address = args.collector_address; - tokio::spawn(async move { - sync::sync_framebuffer(fb, collector_address, worker_id, stream, config).await; - }) - }; - - handle_ctrl_c(terminate_signal_tx).await?; - server_listener_thread.abort(); - stats_drain_thread.abort(); - sync_thread.abort(); + tokio::select! { + result = server.start() => result.context("pixelflut server stopped")?, + () = drain_stats(statistics_rx) => {} + result = sync::sync_framebuffer(&fb, &mut stream, config) => { + result.context("framebuffer sync to the collector stopped")?; + } + } Ok(()) } @@ -90,7 +110,7 @@ async fn connect_to_collector( %error, "Waiting for the collector to become reachable" ); - tokio::time::sleep(COLLECTOR_CONNECT_BACKOFF).await; + tokio::time::sleep(SESSION_BACKOFF).await; } } } From f620ebef906a8d3e8e892f9cc38f5ae01ed5ab47 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Sat, 13 Jun 2026 15:25:31 +0200 Subject: [PATCH 5/7] Move the sinks new() functions out of the DisplaySink trait --- breakwater/src/cli_args.rs | 33 -------------- breakwater/src/main.rs | 9 +++- breakwater/src/sinks/egui/mod.rs | 13 +++--- breakwater/src/sinks/ffmpeg.rs | 8 ++-- breakwater/src/sinks/mod.rs | 22 ++------- breakwater/src/sinks/native_display.rs | 8 ++-- breakwater/src/sinks/vnc.rs | 63 +++++++++++++++++++------- 7 files changed, 73 insertions(+), 83 deletions(-) diff --git a/breakwater/src/cli_args.rs b/breakwater/src/cli_args.rs index 99fc1c7..1451866 100644 --- a/breakwater/src/cli_args.rs +++ b/breakwater/src/cli_args.rs @@ -1,10 +1,6 @@ use std::net::SocketAddr; -#[cfg(feature = "vnc")] -use std::net::{SocketAddrV4, SocketAddrV6}; use clap::Parser; -#[cfg(feature = "vnc")] -use color_eyre::eyre::{self, ensure}; use const_format::formatcp; pub const DEFAULT_NETWORK_BUFFER_SIZE: usize = 256 * 1024; @@ -119,32 +115,3 @@ pub struct CliArgs { pub shared_memory_name: Option, } -impl CliArgs { - /// Checks that at most one IP per version (v4/v6) is configured. - /// Returns the (optional) v4 address and (optional) v6 address. - #[cfg(feature = "vnc")] - pub fn get_vnc_listen_addresses( - &self, - ) -> eyre::Result<(Option<&SocketAddrV4>, Option<&SocketAddrV6>)> { - self.vnc_listen_addresses - .iter() - .try_fold((None, None), |(v4, v6), addr| match addr { - SocketAddr::V4(new) => { - // Fail if an IPv4 was already encountered - ensure!( - v4.is_none(), - "You can only specify one IPv4 VNC listen address" - ); - Ok((Some(new), v6)) - } - SocketAddr::V6(new) => { - // Fail if an IPv6 was already encountered - ensure!( - v6.is_none(), - "You can only specify one IPv6 VNC listen address" - ); - Ok((v4, Some(new))) - } - }) - } -} diff --git a/breakwater/src/main.rs b/breakwater/src/main.rs index 1ca3def..a2ebaee 100644 --- a/breakwater/src/main.rs +++ b/breakwater/src/main.rs @@ -31,7 +31,9 @@ async fn main() -> eyre::Result<()> { // Give the user quick feedback on invalid VNC arguments #[cfg(feature = "vnc")] - if let Err(e) = args.get_vnc_listen_addresses() { + if let Err(e) = + breakwater::sinks::vnc::vnc_listen_addresses_v4_v6(&args.vnc_listen_addresses) + { use clap::CommandFactory; let mut cmd = ::command(); // Displays error in the standard 'clap' format and exits @@ -116,7 +118,10 @@ async fn main() -> eyre::Result<()> { if let Some(vnc_sink) = VncSink::new( fb.clone(), - &args, + &args.vnc_listen_addresses, + &args.font, + args.fps, + &args.text, statistics_tx.clone(), statistics_information_rx.resubscribe(), terminate_signal_rx.resubscribe(), diff --git a/breakwater/src/sinks/egui/mod.rs b/breakwater/src/sinks/egui/mod.rs index 0bbed4a..ed6f4b0 100644 --- a/breakwater/src/sinks/egui/mod.rs +++ b/breakwater/src/sinks/egui/mod.rs @@ -67,20 +67,16 @@ pub struct EguiSink { ui_overlay: Arc, } -#[async_trait] -impl DisplaySink for EguiSink { +impl EguiSink { /// This function can return [`None`] in case this sink is not configured (by looking at the `cli_args`). #[instrument(skip_all, err)] - async fn new( + pub async fn new( fb: Arc, cli_args: &crate::cli_args::CliArgs, _statistics_tx: mpsc::Sender, statistics_information_rx: broadcast::Receiver, terminate_signal_rx: broadcast::Receiver<()>, - ) -> eyre::Result> - where - Self: Sized, - { + ) -> eyre::Result> { let viewports = match ( cli_args.viewport.as_slice(), cli_args.native_display, @@ -137,7 +133,10 @@ impl DisplaySink for EguiSink { ui_overlay, })) } +} +#[async_trait] +impl DisplaySink for EguiSink { /// This should only run on the main thread #[instrument(skip(self), err)] async fn run(&mut self) -> eyre::Result<()> { diff --git a/breakwater/src/sinks/ffmpeg.rs b/breakwater/src/sinks/ffmpeg.rs index 5507d2e..d161cd2 100644 --- a/breakwater/src/sinks/ffmpeg.rs +++ b/breakwater/src/sinks/ffmpeg.rs @@ -23,10 +23,9 @@ pub struct FfmpegSink { fps: u32, } -#[async_trait] -impl DisplaySink for FfmpegSink { +impl FfmpegSink { #[instrument(skip_all, err)] - async fn new( + pub async fn new( fb: Arc, cli_args: &crate::cli_args::CliArgs, _statistics_tx: mpsc::Sender, @@ -45,7 +44,10 @@ impl DisplaySink for FfmpegSink { Ok(None) } } +} +#[async_trait] +impl DisplaySink for FfmpegSink { #[instrument(skip(self), err)] async fn run(&mut self) -> eyre::Result<()> { let mut ffmpeg_args: Vec = self diff --git a/breakwater/src/sinks/mod.rs b/breakwater/src/sinks/mod.rs index 4ee53f7..f5b6560 100644 --- a/breakwater/src/sinks/mod.rs +++ b/breakwater/src/sinks/mod.rs @@ -1,13 +1,5 @@ -use std::sync::Arc; - use async_trait::async_trait; use color_eyre::eyre; -use tokio::sync::{broadcast, mpsc}; - -use crate::{ - cli_args::CliArgs, - statistics::{StatisticsEvent, StatisticsInformationEvent}, -}; #[cfg(feature = "egui")] pub mod egui; @@ -19,18 +11,10 @@ pub mod vnc; // The stabilization of async functions in traits in Rust 1.75 did not include support for using traits containing async // functions as dyn Trait, so we still need to use async_trait here. +// +// Each sink has its own inherent `new(..)` constructor (their arguments differ), returning +// `Ok(None)` when the sink isn't configured. This trait only carries the shared run loop. #[async_trait] pub trait DisplaySink { - /// This function can return [`None`] in case this sink is not configured (by looking at the `cli_args`). - async fn new( - fb: Arc, - cli_args: &CliArgs, - statistics_tx: mpsc::Sender, - statistics_information_rx: broadcast::Receiver, - terminate_signal_rx: broadcast::Receiver<()>, - ) -> eyre::Result> - where - Self: Sized; - async fn run(&mut self) -> eyre::Result<()>; } diff --git a/breakwater/src/sinks/native_display.rs b/breakwater/src/sinks/native_display.rs index aa74cc8..29d8dc9 100644 --- a/breakwater/src/sinks/native_display.rs +++ b/breakwater/src/sinks/native_display.rs @@ -30,10 +30,9 @@ pub struct NativeDisplaySink { surface: Option, Arc>>, } -#[async_trait] -impl DisplaySink for NativeDisplaySink { +impl NativeDisplaySink { #[instrument(skip_all, err)] - async fn new( + pub async fn new( fb: Arc, cli_args: &CliArgs, _statistics_tx: mpsc::Sender, @@ -50,7 +49,10 @@ impl DisplaySink for NativeDisplayS surface: None, })) } +} +#[async_trait] +impl DisplaySink for NativeDisplaySink { #[instrument(skip(self), err)] async fn run(&mut self) -> eyre::Result<()> { let fb_clone = self.fb.clone(); diff --git a/breakwater/src/sinks/vnc.rs b/breakwater/src/sinks/vnc.rs index d527443..520aead 100644 --- a/breakwater/src/sinks/vnc.rs +++ b/breakwater/src/sinks/vnc.rs @@ -1,9 +1,15 @@ use core::slice; -use std::{ffi::CString, str::FromStr, sync::Arc, time::Duration}; +use std::{ + ffi::CString, + net::{SocketAddr, SocketAddrV4, SocketAddrV6}, + str::FromStr, + sync::Arc, + time::Duration, +}; use async_trait::async_trait; use breakwater_parser::{FB_BYTES_PER_PIXEL, FrameBuffer}; -use color_eyre::eyre::{self, Context, ContextCompat}; +use color_eyre::eyre::{self, Context, ContextCompat, ensure}; use number_prefix::NumberPrefix; use rusttype::{Font, Scale, point}; use tokio::{ @@ -16,7 +22,6 @@ use vncserver::{ }; use crate::{ - cli_args::CliArgs, sinks::DisplaySink, statistics::{ STATISTICS_INFO_RECV_ERR, STATISTICS_SEND_ERR, StatisticsEvent, StatisticsInformationEvent, @@ -40,29 +45,34 @@ pub struct VncSink<'a, FB: FrameBuffer> { font: Font<'a>, } -#[async_trait] -impl DisplaySink for VncSink<'_, FB> { - async fn new( +impl VncSink<'_, FB> { + /// Returns `Ok(None)` if no VNC listen addresses are configured (the sink is then disabled). + #[tracing::instrument(skip_all, err)] + #[expect(clippy::too_many_arguments, reason = "the sink genuinely needs all of these")] + pub async fn new( fb: Arc, - cli_args: &CliArgs, + vnc_listen_addresses: &[SocketAddr], + font: &str, + target_fps: u32, + text: &str, statistics_tx: mpsc::Sender, statistics_information_rx: broadcast::Receiver, terminate_signal_rx: broadcast::Receiver<()>, ) -> eyre::Result> { - if cli_args.vnc_listen_addresses.is_empty() { + if vnc_listen_addresses.is_empty() { tracing::debug!("VNC sink not enabled as no vnc addresses are specified"); return Ok(None); } // We ship our own copy of Arial.ttf, so that users don't need to download and provide it - let font = if cli_args.font.as_str() == "Arial.ttf" { + let font = if font == "Arial.ttf" { let font_bytes = include_bytes!("../../../Arial.ttf"); Font::try_from_bytes(font_bytes).context("failed to load default font")? } else { - let font_bytes = std::fs::read(&cli_args.font) - .with_context(|| format!("failed to read font from file {}", cli_args.font))?; + let font_bytes = std::fs::read(font) + .with_context(|| format!("failed to read font from file {font}"))?; Font::try_from_vec(font_bytes) - .with_context(|| format!("failed to construct font from file {}", cli_args.font))? + .with_context(|| format!("failed to construct font from file {font}"))? }; let screen = rfb_get_screen(fb.get_width() as i32, fb.get_height() as i32, 8, 3, 4); @@ -77,7 +87,7 @@ impl DisplaySink for VncSink<'_, FB> { (*screen).ipv6port = -1; } - let (v4, v6) = cli_args.get_vnc_listen_addresses()?; + let (v4, v6) = vnc_listen_addresses_v4_v6(vnc_listen_addresses)?; if let Some(v4) = v4 { unsafe { @@ -105,19 +115,21 @@ impl DisplaySink for VncSink<'_, FB> { } } - // FIXME: Only return Some in case VNC is enabled Ok(Some(Self { fb, statistics_tx, statistics_information_rx, terminate_signal_rx, screen, - target_fps: cli_args.fps, - text: cli_args.text.clone(), + target_fps, + text: text.to_owned(), font, })) } +} +#[async_trait] +impl DisplaySink for VncSink<'_, FB> { async fn run(&mut self) -> eyre::Result<()> { let vnc_fb_slice: &mut [u8] = unsafe { slice::from_raw_parts_mut( @@ -251,6 +263,25 @@ impl VncSink<'_, FB> { } } +/// Splits VNC listen addresses into at most one IPv4 and one IPv6 address (libvncserver listens on +/// one of each). Errors if more than one of either version is given. +pub fn vnc_listen_addresses_v4_v6( + addresses: &[SocketAddr], +) -> eyre::Result<(Option<&SocketAddrV4>, Option<&SocketAddrV6>)> { + addresses + .iter() + .try_fold((None, None), |(v4, v6), addr| match addr { + SocketAddr::V4(addr) => { + ensure!(v4.is_none(), "You can only specify one IPv4 VNC listen address"); + Ok((Some(addr), v6)) + } + SocketAddr::V6(addr) => { + ensure!(v6.is_none(), "You can only specify one IPv6 VNC listen address"); + Ok((v4, Some(addr))) + } + }) +} + fn format_per_s(value: f64) -> String { match NumberPrefix::decimal(value) { NumberPrefix::Prefixed(prefix, n) => format!("{n:.1}{prefix}"), From 4dcd98aada7e57d00c617424c54fc9279dae613f Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Sun, 14 Jun 2026 16:31:12 +0200 Subject: [PATCH 6/7] feat: Add different kind of sinks to deich collector --- .../src/framebuffer/time_tracking.rs | 3 +- breakwater/src/cli_args.rs | 55 ++---- breakwater/src/main.rs | 28 ++- breakwater/src/sinks/egui/mod.rs | 52 ++++-- breakwater/src/sinks/ffmpeg.rs | 36 ++-- breakwater/src/sinks/native_display.rs | 3 - breakwater/src/sinks/vnc.rs | 37 +++- deich/Cargo.toml | 5 + deich/src/cli_args.rs | 21 ++- deich/src/collector.rs | 168 ++++++++++++++++-- 10 files changed, 291 insertions(+), 117 deletions(-) diff --git a/breakwater-parser/src/framebuffer/time_tracking.rs b/breakwater-parser/src/framebuffer/time_tracking.rs index 89f788d..cca3daf 100644 --- a/breakwater-parser/src/framebuffer/time_tracking.rs +++ b/breakwater-parser/src/framebuffer/time_tracking.rs @@ -103,7 +103,8 @@ impl FrameBuffer for TimeTrackingFrameBuffer { fn pixel_timestamp(&self, ns_since_unix_epoch: u64) -> u64 { // The per-parse-call timestamp, computed once. `saturating_sub` guards against a worker // whose clock trails the collector's epoch slightly; `.min` clamps the ~12.7 day range. - (ns_since_unix_epoch.saturating_sub(self.epoch_ns_since_unix_epoch) / 1_000).min(TIMESTAMP_MAX) + (ns_since_unix_epoch.saturating_sub(self.epoch_ns_since_unix_epoch) / 1_000) + .min(TIMESTAMP_MAX) } fn set_with_pixel_timestamp(&self, x: usize, y: usize, rgba: u32, pixel_timestamp: u64) { diff --git a/breakwater/src/cli_args.rs b/breakwater/src/cli_args.rs index 1451866..ef38f68 100644 --- a/breakwater/src/cli_args.rs +++ b/breakwater/src/cli_args.rs @@ -39,12 +39,6 @@ pub struct CliArgs { #[clap(short, long, default_value = "Pixelflut server (breakwater)")] pub text: String, - /// The font used to render the text on the screen. - /// Should be a ttf file. - /// If you use the default value a copy that ships with breakwater will be used - no need to download and provide the font. - #[clap(long, default_value = "Arial.ttf")] - pub font: String, - /// Listen address the prometheus exporter should listen on. #[clap(short, long, default_value = "[::]:9100")] pub prometheus_listen_address: String, @@ -63,23 +57,15 @@ pub struct CliArgs { #[clap(long)] pub disable_statistics_save_file: bool, - /// Enable rtmp streaming to configured address, e.g. `rtmp://127.0.0.1:1935/live/test` - #[clap(long)] - pub rtmp_address: Option, - - /// Enable dump of video stream into file. File location will be `/pixelflut_dump_{timestamp}.mp4` - #[clap(long)] - pub video_save_folder: Option, - /// Allow only a certain number of connections per ip address #[clap(short, long)] pub connections_per_ip: Option, - /// VNC server listen address to bind to (multiple can be specified). - /// Only one address of each IP version can be specified - #[cfg(feature = "vnc")] - #[clap(long = "vnc-listen-address")] - pub vnc_listen_addresses: Vec, + /// Create (or use an existing) shared memory region for the framebuffer. + /// This enables other applications to read and write Pixel values to the framebuffer or can be + /// used to persist the canvas across restarts. + #[clap(long)] + pub shared_memory_name: Option, /// Enable native display output. This requires some form of graphical system (so will probably not work on your /// server). @@ -87,31 +73,14 @@ pub struct CliArgs { #[clap(long)] pub native_display: bool, - /// Specify a view port to display the canvas or a certain part of it. Format: `x,x`. - /// Might be specified multiple times for more than one viewport. Useful for multi-projector setups. - /// Defaults to display the entire canvas. - /// Implies --native-display. - #[cfg(feature = "egui")] - #[clap(long)] - pub viewport: Vec, - - /// Specify one or more pixelflut endpoints to display. - #[cfg(feature = "egui")] - #[clap(long)] - pub advertised_endpoints: Vec, + #[clap(flatten)] + pub ffmpeg_sink: crate::sinks::ffmpeg::FfmpegSinkCliArgs, - /// Provide a path to a dylib containing a custom egui overlay. - /// Implies --native-display. - // - // Qualifying import here to avoid feature-specific imports #[cfg(feature = "egui")] - #[clap(long)] - pub ui: Option, + #[clap(flatten)] + pub egui_sink: crate::sinks::egui::EguiSinkCliArgs, - /// Create (or use an existing) shared memory region for the framebuffer. - /// This enables other applications to read and write Pixel values to the framebuffer or can be - /// used to persist the canvas across restarts. - #[clap(long)] - pub shared_memory_name: Option, + #[cfg(feature = "vnc")] + #[clap(flatten)] + pub vnc_sink: crate::sinks::vnc::VncSinkCliArgs, } - diff --git a/breakwater/src/main.rs b/breakwater/src/main.rs index a2ebaee..202ab4c 100644 --- a/breakwater/src/main.rs +++ b/breakwater/src/main.rs @@ -32,7 +32,7 @@ async fn main() -> eyre::Result<()> { // Give the user quick feedback on invalid VNC arguments #[cfg(feature = "vnc")] if let Err(e) = - breakwater::sinks::vnc::vnc_listen_addresses_v4_v6(&args.vnc_listen_addresses) + breakwater::sinks::vnc::vnc_listen_addresses_v4_v6(&args.vnc_sink.listen_addresses) { use clap::CommandFactory; let mut cmd = ::command(); @@ -98,15 +98,10 @@ async fn main() -> eyre::Result<()> { { use breakwater::sinks::native_display::NativeDisplaySink; - if let Some(native_display_sink) = NativeDisplaySink::new( - fb.clone(), - &args, - statistics_tx.clone(), - statistics_information_rx.resubscribe(), - terminate_signal_rx.resubscribe(), - ) - .await - .context("failed to create native display sink")? + if let Some(native_display_sink) = + NativeDisplaySink::new(fb.clone(), terminate_signal_rx.resubscribe()) + .await + .context("failed to create native display sink")? { display_sinks.push(Box::new(native_display_sink)); } @@ -118,8 +113,7 @@ async fn main() -> eyre::Result<()> { if let Some(vnc_sink) = VncSink::new( fb.clone(), - &args.vnc_listen_addresses, - &args.font, + &args.vnc_sink, args.fps, &args.text, statistics_tx.clone(), @@ -136,9 +130,8 @@ async fn main() -> eyre::Result<()> { let mut ffmpeg_thread_present = false; if let Some(ffmpeg_sink) = FfmpegSink::new( fb.clone(), - &args, - statistics_tx.clone(), - statistics_information_rx.resubscribe(), + &args.ffmpeg_sink, + args.fps, terminate_signal_rx.resubscribe(), ) .await @@ -162,8 +155,9 @@ async fn main() -> eyre::Result<()> { match EguiSink::new( fb.clone(), - &args, - statistics_tx.clone(), + &args.egui_sink, + &args.listen_addresses, + args.native_display, statistics_information_rx.resubscribe(), terminate_signal_rx.resubscribe(), ) diff --git a/breakwater/src/sinks/egui/mod.rs b/breakwater/src/sinks/egui/mod.rs index ed6f4b0..bd02d28 100644 --- a/breakwater/src/sinks/egui/mod.rs +++ b/breakwater/src/sinks/egui/mod.rs @@ -1,11 +1,11 @@ -use std::{fmt::Display, str::FromStr, sync::Arc}; +use std::{fmt::Display, net::SocketAddr, str::FromStr, sync::Arc}; use async_trait::async_trait; use breakwater_parser::FrameBuffer; use color_eyre::eyre::{self, Context}; use dynamic_overlay::UiOverlay; use serde::{Deserialize, Serialize}; -use tokio::sync::{broadcast, mpsc}; +use tokio::sync::broadcast; use tracing::instrument; use super::DisplaySink; @@ -67,21 +67,43 @@ pub struct EguiSink { ui_overlay: Arc, } +#[derive(clap::Parser, Debug)] +pub struct EguiSinkCliArgs { + /// Specify a view port to display the canvas or a certain part of it. Format: `x,x`. + /// Might be specified multiple times for more than one viewport. Useful for multi-projector setups. + /// Defaults to display the entire canvas. + /// Implies --native-display. + #[clap(long)] + pub viewport: Vec, + + /// Specify one or more pixelflut endpoints to display. + #[clap(long)] + pub advertised_endpoints: Vec, + + /// Provide a path to a dylib containing a custom egui overlay. + /// Implies --native-display. + // + // Qualifying import here to avoid feature-specific imports + #[clap(long)] + pub ui: Option, +} + impl EguiSink { /// This function can return [`None`] in case this sink is not configured (by looking at the `cli_args`). #[instrument(skip_all, err)] pub async fn new( fb: Arc, - cli_args: &crate::cli_args::CliArgs, - _statistics_tx: mpsc::Sender, + EguiSinkCliArgs { + viewport, + advertised_endpoints, + ui, + }: &EguiSinkCliArgs, + listen_addresses: &[SocketAddr], + native_display: bool, statistics_information_rx: broadcast::Receiver, terminate_signal_rx: broadcast::Receiver<()>, ) -> eyre::Result> { - let viewports = match ( - cli_args.viewport.as_slice(), - cli_args.native_display, - cli_args.ui.as_ref(), - ) { + let viewports = match (viewport.as_slice(), native_display, ui.as_ref()) { (vp, _, _) if !vp.is_empty() => Vec::from(vp), ([], _, Some(_)) | ([], true, _) => vec![ViewportConfig { x: 0, @@ -93,17 +115,17 @@ impl EguiSink { }; let ui_overlay = Arc::new({ - if let Some(ui) = cli_args.ui.as_ref() { + if let Some(ui) = ui.as_ref() { dynamic_overlay::load_and_check(ui).context("failed to load dynamic overlay")? } else { UiOverlay::BuiltIn } }); - let advertised_endpoints = if cli_args.advertised_endpoints.is_empty() { + let advertised_endpoints = if advertised_endpoints.is_empty() { // In case no advertised endpoints to display are given, we calculate the most likely // endpoint(s) to display. - match &cli_args.listen_addresses[..] { + match listen_addresses[..] { // No listeners given, so also no endpoints to advertise [] => vec![], // In case of a single listener we get the local IPs (v4 + v6) and concat them with @@ -118,10 +140,12 @@ impl EguiSink { .collect() } // If multiple listeners are used it's complicated, so we just print them - multiple_listeners => multiple_listeners.iter().map(ToString::to_string).collect(), + ref multiple_listeners => { + multiple_listeners.iter().map(ToString::to_string).collect() + } } } else { - cli_args.advertised_endpoints.clone() + advertised_endpoints.clone() }; Ok(Some(Self { diff --git a/breakwater/src/sinks/ffmpeg.rs b/breakwater/src/sinks/ffmpeg.rs index d161cd2..c7baa14 100644 --- a/breakwater/src/sinks/ffmpeg.rs +++ b/breakwater/src/sinks/ffmpeg.rs @@ -4,15 +4,10 @@ use async_trait::async_trait; use breakwater_parser::FrameBuffer; use chrono::Local; use color_eyre::eyre::{self, Context}; -use tokio::{ - io::AsyncWriteExt, - process::Command, - sync::{broadcast, mpsc}, - time, -}; +use tokio::{io::AsyncWriteExt, process::Command, sync::broadcast, time}; use tracing::instrument; -use crate::{sinks::DisplaySink, statistics::StatisticsInformationEvent}; +use crate::sinks::DisplaySink; pub struct FfmpegSink { fb: Arc, @@ -23,22 +18,35 @@ pub struct FfmpegSink { fps: u32, } +#[derive(clap::Parser, Debug)] +pub struct FfmpegSinkCliArgs { + /// Enable rtmp streaming to configured address, e.g. `rtmp://127.0.0.1:1935/live/test` + #[clap(long)] + pub rtmp_address: Option, + + /// Enable dump of video stream into file. File location will be `/pixelflut_dump_{timestamp}.mp4` + #[clap(long)] + pub video_save_folder: Option, +} + impl FfmpegSink { #[instrument(skip_all, err)] pub async fn new( fb: Arc, - cli_args: &crate::cli_args::CliArgs, - _statistics_tx: mpsc::Sender, - _statistics_information_rx: broadcast::Receiver, + FfmpegSinkCliArgs { + rtmp_address, + video_save_folder, + }: &FfmpegSinkCliArgs, + fps: u32, terminate_signal_rx: broadcast::Receiver<()>, ) -> eyre::Result> { - if cli_args.rtmp_address.is_some() || cli_args.video_save_folder.is_some() { + if rtmp_address.is_some() || video_save_folder.is_some() { Ok(Some(Self { fb, terminate_signal_rx, - rtmp_address: cli_args.rtmp_address.clone(), - video_save_folder: cli_args.video_save_folder.clone(), - fps: cli_args.fps, + rtmp_address: rtmp_address.clone(), + video_save_folder: video_save_folder.clone(), + fps, })) } else { Ok(None) diff --git a/breakwater/src/sinks/native_display.rs b/breakwater/src/sinks/native_display.rs index 29d8dc9..e981b4b 100644 --- a/breakwater/src/sinks/native_display.rs +++ b/breakwater/src/sinks/native_display.rs @@ -34,9 +34,6 @@ impl NativeDisplaySink { #[instrument(skip_all, err)] pub async fn new( fb: Arc, - cli_args: &CliArgs, - _statistics_tx: mpsc::Sender, - _statistics_information_rx: broadcast::Receiver, terminate_signal_rx: broadcast::Receiver<()>, ) -> eyre::Result> { if !cli_args.native_display { diff --git a/breakwater/src/sinks/vnc.rs b/breakwater/src/sinks/vnc.rs index 520aead..f622660 100644 --- a/breakwater/src/sinks/vnc.rs +++ b/breakwater/src/sinks/vnc.rs @@ -45,21 +45,36 @@ pub struct VncSink<'a, FB: FrameBuffer> { font: Font<'a>, } +#[derive(clap::Parser, Debug)] +pub struct VncSinkCliArgs { + /// VNC server listen address to bind to (multiple can be specified). + /// Only one address of each IP version can be specified + #[clap(long = "vnc-listen-address")] + pub listen_addresses: Vec, + + /// The font used to render the text on the screen. + /// Should be a ttf file. + /// If you use the default value a copy that ships with breakwater will be used - no need to download and provide the font. + #[clap(long, default_value = "Arial.ttf")] + pub font: String, +} + impl VncSink<'_, FB> { /// Returns `Ok(None)` if no VNC listen addresses are configured (the sink is then disabled). - #[tracing::instrument(skip_all, err)] - #[expect(clippy::too_many_arguments, reason = "the sink genuinely needs all of these")] + #[expect(clippy::unused_async)] pub async fn new( fb: Arc, - vnc_listen_addresses: &[SocketAddr], - font: &str, + VncSinkCliArgs { + listen_addresses, + font, + }: &VncSinkCliArgs, target_fps: u32, text: &str, statistics_tx: mpsc::Sender, statistics_information_rx: broadcast::Receiver, terminate_signal_rx: broadcast::Receiver<()>, ) -> eyre::Result> { - if vnc_listen_addresses.is_empty() { + if listen_addresses.is_empty() { tracing::debug!("VNC sink not enabled as no vnc addresses are specified"); return Ok(None); } @@ -87,7 +102,7 @@ impl VncSink<'_, FB> { (*screen).ipv6port = -1; } - let (v4, v6) = vnc_listen_addresses_v4_v6(vnc_listen_addresses)?; + let (v4, v6) = vnc_listen_addresses_v4_v6(listen_addresses)?; if let Some(v4) = v4 { unsafe { @@ -272,11 +287,17 @@ pub fn vnc_listen_addresses_v4_v6( .iter() .try_fold((None, None), |(v4, v6), addr| match addr { SocketAddr::V4(addr) => { - ensure!(v4.is_none(), "You can only specify one IPv4 VNC listen address"); + ensure!( + v4.is_none(), + "You can only specify one IPv4 VNC listen address" + ); Ok((Some(addr), v6)) } SocketAddr::V6(addr) => { - ensure!(v6.is_none(), "You can only specify one IPv6 VNC listen address"); + ensure!( + v6.is_none(), + "You can only specify one IPv6 VNC listen address" + ); Ok((v4, Some(addr))) } }) diff --git a/deich/Cargo.toml b/deich/Cargo.toml index 8d0eae9..a4c75fa 100644 --- a/deich/Cargo.toml +++ b/deich/Cargo.toml @@ -11,6 +11,11 @@ repository.workspace = true name = "deich" path = "src/main.rs" +[features] +egui = ["breakwater/egui"] +native-display = ["breakwater/native-display"] +vnc = ["breakwater/vnc"] + [dependencies] breakwater-parser = { workspace = true, features = ["time-tracking"] } breakwater.workspace = true diff --git a/deich/src/cli_args.rs b/deich/src/cli_args.rs index e0b7a63..9903e15 100644 --- a/deich/src/cli_args.rs +++ b/deich/src/cli_args.rs @@ -39,7 +39,7 @@ pub struct WorkerArgs { pub network_buffer_size: i64, /// Address of the collector to fetch the config from and stream the framebuffer to. - #[clap(long, default_value = "127.0.0.1:9999")] + #[clap(long, default_value = "127.0.0.1:1235")] pub collector_address: String, /// File storing this worker's persistent UUID, created on first run. The UUID identifies the @@ -51,7 +51,7 @@ pub struct WorkerArgs { #[derive(Args, Debug)] pub struct CollectorArgs { /// Listen address the workers connect to. - #[clap(short, long, default_value = "[::]:9999")] + #[clap(short, long, default_value = "[::]:1235")] pub listen_address: SocketAddr, /// Width of the canvas. Sent to every worker as part of its config. @@ -66,4 +66,21 @@ pub struct CollectorArgs { /// Must stay well below the ~536 ms window the per-pixel timestamp can represent. #[clap(long, default_value_t = 30, value_parser = clap::value_parser!(u32).range(1..=60))] pub fps: u32, + + #[clap(flatten)] + pub ffmpeg_sink: breakwater::sinks::ffmpeg::FfmpegSinkCliArgs, + + /// Enable native display output. This requires some form of graphical system (so will probably not work on your + /// server). + #[cfg(any(feature = "native-display", feature = "egui"))] + #[clap(long)] + pub native_display: bool, + + #[cfg(feature = "egui")] + #[clap(flatten)] + pub egui_sink: breakwater::sinks::egui::EguiSinkCliArgs, + + #[cfg(feature = "vnc")] + #[clap(flatten)] + pub vnc_sink: breakwater::sinks::vnc::VncSinkCliArgs, } diff --git a/deich/src/collector.rs b/deich/src/collector.rs index ad9c564..12be424 100644 --- a/deich/src/collector.rs +++ b/deich/src/collector.rs @@ -19,9 +19,19 @@ use std::{ time::Duration, }; -use breakwater_parser::{TimeTrackingPixel, get_current_ns_since_unix_epoch, pixels_as_bytes_mut}; +use breakwater::{ + sinks::{DisplaySink, ffmpeg::FfmpegSink}, + statistics::{StatisticsEvent, StatisticsInformationEvent}, +}; +use breakwater_parser::{ + FrameBuffer, SimpleFrameBuffer, TimeTrackingPixel, get_current_ns_since_unix_epoch, + pixels_as_bytes_mut, +}; use color_eyre::eyre::{self, Context}; -use tokio::net::{TcpListener, TcpStream}; +use tokio::{ + net::{TcpListener, TcpStream}, + sync::{broadcast, mpsc}, +}; use tracing::{debug, info, warn}; use uuid::Uuid; @@ -30,9 +40,9 @@ use crate::{ sync::{self, FrameSchedule, WorkerConfig}, }; -/// How long we assume a worker needs to stream a frame to us. The master keeps the last -/// `⌈budget / frame_period⌉` frames "interesting" and renders the oldest of them, so it only -/// renders a frame once every worker has had this long to deliver it. +/// How long we assume a worker needs to stream a frame to us (on top of the slot the worker spends +/// finishing it). Together with that slot, this sets how far behind "now" the master renders, so a +/// frame has had time to arrive — see the `margin` computation in [`run_master`]. const FRAME_STREAM_BUDGET: Duration = Duration::from_millis(50); /// The workers currently connected, keyed by their UUID (value: the connection's peer address, for @@ -54,9 +64,9 @@ struct Canvas { } impl Canvas { - fn new(pixel_count: usize) -> Self { + fn new(width: usize, height: usize) -> Self { Self { - pixels: vec![TimeTrackingPixel::default(); pixel_count], + pixels: vec![TimeTrackingPixel::default(); width * height], } } @@ -70,6 +80,15 @@ impl Canvas { } } } + + fn draw_to_framebuffer(&self, fb: &Arc) { + let pixels = self + .pixels + .iter() + .flat_map(|pixel| pixel.rgb().to_le_bytes()) + .collect::>(); + fb.set_multi_from_start_index(0, &pixels); + } } /// Shared between the master task and the worker-connection tasks. @@ -132,6 +151,7 @@ impl FrameStore { /// Runs the collector until Ctrl-C: accepts worker connections, configures them, stores frames, and /// runs the master render-scheduling task. +#[expect(clippy::too_many_lines)] pub async fn run(args: CollectorArgs) -> eyre::Result<()> { let config = WorkerConfig { width: args.width, @@ -154,12 +174,124 @@ pub async fn run(args: CollectorArgs) -> eyre::Result<()> { let connected_workers: ConnectedWorkers = Arc::new(RwLock::new(HashMap::new())); let frame_store = Arc::new(FrameStore::new()); + // Most of the time we want to render the screen to *something*, otherwise the game is boring + // As the breakwater sinks expect the memory layout of the [`SimpleFrameBuffer`] we need to + // create and maintain one. + let render_fb = Arc::new(SimpleFrameBuffer::new( + args.width as usize, + args.height as usize, + )); + + // If we make the channel to big, stats will start to lag behind + // TODO: Check performance impact in real-world scenario. Maybe the statistics thread blocks the other threads + #[allow(unused_variables)] + let (statistics_tx, mut statistics_rx) = mpsc::channel::(100); + #[allow(unused_variables)] + let (_statistics_information_tx, statistics_information_rx) = + broadcast::channel::(2); + let (_terminate_signal_tx, terminate_signal_rx) = broadcast::channel::<()>(1); + + // FIXME: For now we need to drain the statistics + tokio::spawn(async move { while statistics_rx.recv().await.is_some() {} }); + + let mut display_sinks = Vec:: + Send>>::new(); + + #[cfg(feature = "vnc")] + { + use breakwater::sinks::vnc::VncSink; + + if let Some(vnc_sink) = VncSink::new( + render_fb.clone(), + &args.vnc_sink, + args.fps, + "deich", // FIXME + statistics_tx.clone(), + statistics_information_rx.resubscribe(), + terminate_signal_rx.resubscribe(), + ) + .await + .context("failed to create vnc sink")? + { + display_sinks.push(Box::new(vnc_sink)); + } + } + + if let Some(ffmpeg_sink) = FfmpegSink::new( + render_fb.clone(), + &args.ffmpeg_sink, + args.fps, + terminate_signal_rx.resubscribe(), + ) + .await + .context("failed to create ffmpeg sink")? + { + display_sinks.push(Box::new(ffmpeg_sink)); + } + + let mut sink_threads = Vec::new(); + for mut sink in display_sinks { + sink_threads.push(tokio::spawn(async move { + sink.run().await?; + eyre::Result::<()>::Ok(()) + })); + } + { let frame_store = frame_store.clone(); + let render_fb = render_fb.clone(); let connected_workers = connected_workers.clone(); - tokio::spawn(async move { run_master(&frame_store, &connected_workers, config).await }); + tokio::spawn(async move { + run_master(&frame_store, &connected_workers, config, render_fb).await; + }); + } + + let accept_thread = tokio::spawn(accept_workers( + listener, + connected_workers, + frame_store, + config, + )); + + #[cfg(feature = "egui")] + { + use breakwater::sinks::egui::EguiSink; + + if let Some(mut egui_sink) = EguiSink::new( + render_fb, + &args.egui_sink, + // There are no listen addresses we know about (the traffic comes in via a complicated + // path - virtual IP and stuff) + &[], + args.native_display, + statistics_information_rx.resubscribe(), + terminate_signal_rx.resubscribe(), + ) + .await + .context("failed to create egui sink")? + { + // Some platforms require opening windows from the main thread. + // The tokio::main macro uses Runtime::block_on(future) which runs the future on + // the current thread, which should be the main thread right now. Workers keep being + // accepted by the background task above while egui owns this thread. + egui_sink.run().await.context("failed to run egui sink")?; + return Ok(()); + } } + // No main-thread sink is running, so block on the accept task to keep the collector alive. + accept_thread + .await + .context("the worker-accept task panicked")? +} + +/// Accepts worker connections forever, spawning a handler per connection. Only returns if accepting +/// itself fails. +async fn accept_workers( + listener: TcpListener, + connected_workers: ConnectedWorkers, + frame_store: Arc, + config: WorkerConfig, +) -> eyre::Result<()> { loop { let (stream, peer) = listener .accept() @@ -192,12 +324,17 @@ async fn run_master( frame_store: &FrameStore, connected_workers: &ConnectedWorkers, config: WorkerConfig, + render_fb: Arc, ) { let schedule = FrameSchedule::new(config.sync_fps); - // Render the oldest frame that has had `FRAME_STREAM_BUDGET` to arrive. - let margin = schedule.frames_spanning(FRAME_STREAM_BUDGET); + // How many slots behind "now" the master renders. A worker only sends frame N once slot N has + // *ended* — one period after it began — so a frame is already a full slot old before streaming + // even starts. The `1 +` covers that slot-completion delay; `frames_spanning(..)` adds the + // streaming budget on top. (Without the `1 +`, `--fps 1` rendered a slot whose frame was still + // in flight, so nothing ever merged.) + let margin = 1 + schedule.frames_spanning(FRAME_STREAM_BUDGET); - let mut canvas = Canvas::new(config.width as usize * config.height as usize); + let mut canvas = Canvas::new(config.width as usize, config.height as usize); let mut current_frame = schedule.frame_number_at(get_current_ns_since_unix_epoch()); loop { @@ -218,7 +355,8 @@ async fn run_master( canvas.merge(frame); } - // TODO: render `canvas` to the screen / output sink. + canvas.draw_to_framebuffer(&render_fb); + info!( render_frame, frames_merged = frames.len(), @@ -342,7 +480,7 @@ mod tests { #[test] fn merges_latest_timestamp_per_pixel() { - let mut canvas = Canvas::new(2); + let mut canvas = Canvas::new(1, 2); // Pixel 0: A timestamp 1320, B timestamp 5032 -> B wins. // Pixel 1: A timestamp 4200, B timestamp 0 (never written) -> A stays. @@ -357,7 +495,7 @@ mod tests { #[test] fn blank_frame_never_overwrites_live_content() { - let mut canvas = Canvas::new(1); + let mut canvas = Canvas::new(1, 1); canvas.merge(&frame(&[(0x12_3456, 50)])); // A never-written pixel has timestamp 0 (oldest possible), so it can't clobber live content @@ -370,7 +508,7 @@ mod tests { #[test] fn older_write_does_not_replace_newer() { - let mut canvas = Canvas::new(1); + let mut canvas = Canvas::new(1, 1); // Merge the newer write first, then an older one — order must not matter. canvas.merge(&frame(&[(0x00_00bb, 5_032)])); canvas.merge(&frame(&[(0xaa_0000, 1_320)])); From 9502e372dd78a6f2f964e156eb5d3e93f96372b8 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Sun, 14 Jun 2026 18:33:33 +0200 Subject: [PATCH 7/7] Display better stats for delayed frames --- breakwater-parser/src/original.rs | 6 +- deich/src/collector.rs | 118 +++++++++++++++++++++++++----- deich/src/sync.rs | 5 ++ 3 files changed, 108 insertions(+), 21 deletions(-) diff --git a/breakwater-parser/src/original.rs b/breakwater-parser/src/original.rs index be9aafe..e3d648e 100644 --- a/breakwater-parser/src/original.rs +++ b/breakwater-parser/src/original.rs @@ -58,9 +58,9 @@ impl Parser for OriginalParser { // Encode the timestamp exactly once here, not per pixel: it's constant for the whole parse // call, so computing it per write would just waste throughput on the hot path. #[cfg(feature = "time-tracking")] - let pixel_timestamp = self.fb.pixel_timestamp( - crate::framebuffer::time_tracking::get_current_ns_since_unix_epoch(), - ); + let pixel_timestamp = self + .fb + .pixel_timestamp(crate::framebuffer::time_tracking::get_current_ns_since_unix_epoch()); #[cfg(feature = "binary-sync-pixels")] if let Some(remaining) = &self.remaining_pixel_sync { diff --git a/deich/src/collector.rs b/deich/src/collector.rs index 12be424..565ba90 100644 --- a/deich/src/collector.rs +++ b/deich/src/collector.rs @@ -91,6 +91,17 @@ impl Canvas { } } +/// How an incoming frame relates to the window of frames the master currently wants. +enum FrameInterest { + /// In the window — store it. + Wanted, + /// Outside the window. Carries the window so the caller can tell whether the frame is too old + /// (arrived late) or from the future, and by how much. + OutsideWindow { window: RangeInclusive }, + /// The master hasn't published a window yet (just started up). + NoWindowYet, +} + /// Shared between the master task and the worker-connection tasks. struct FrameStore { /// Frame numbers the master currently wants: the inclusive window `[render_frame ..= current]`. @@ -111,13 +122,20 @@ impl FrameStore { } } - /// Whether the master currently wants frame `frame_number`. - fn is_interested(&self, frame_number: u64) -> bool { - self.interesting + /// Classifies an incoming frame against the window the master currently wants. + fn classify(&self, frame_number: u64) -> FrameInterest { + match self + .interesting .read() .expect("interesting-window lock poisoned") .as_ref() - .is_some_and(|window| window.contains(&frame_number)) + { + None => FrameInterest::NoWindowYet, + Some(window) if window.contains(&frame_number) => FrameInterest::Wanted, + Some(window) => FrameInterest::OutsideWindow { + window: window.clone(), + }, + } } /// Stores a worker's frame (overwriting any previous frame from the same worker for that slot). @@ -408,6 +426,7 @@ async fn serve_frames( ) -> io::Result<()> { sync::send_config(stream, config).await?; + let schedule = FrameSchedule::new(config.sync_fps); let pixel_count = config.width as usize * config.height as usize; // Reused only for discarding frames the master doesn't want. let mut discard = vec![0u8; config.frame_size_bytes()]; @@ -415,20 +434,62 @@ async fn serve_frames( loop { let frame_number = sync::receive_frame_number(stream).await?; - if frame_store.is_interested(frame_number) { - // Read straight into a fresh pixel buffer (outside any lock) so we can hand ownership to - // the store; `read_exact` writes directly into its bytes, no extra copy. - let mut buffer = vec![TimeTrackingPixel::default(); pixel_count]; - sync::receive_frame_body(stream, pixels_as_bytes_mut(&mut buffer)).await?; - frame_store.store(frame_number, worker_id, buffer); - } else { - // Still consume the blob so the stream stays aligned for the next message. - sync::receive_frame_body(stream, &mut discard).await?; - warn!( - %peer, - frame_number, - "Master is not interested in this frame (outside the window); dropping it" - ); + match frame_store.classify(frame_number) { + FrameInterest::Wanted => { + // Read straight into a fresh pixel buffer (outside any lock) so we can hand + // ownership to the store; `read_exact` writes directly into its bytes, no extra copy. + let mut buffer = vec![TimeTrackingPixel::default(); pixel_count]; + sync::receive_frame_body(stream, pixels_as_bytes_mut(&mut buffer)).await?; + frame_store.store(frame_number, worker_id, buffer); + } + other => { + // Still consume the blob so the stream stays aligned for the next message. + sync::receive_frame_body(stream, &mut discard).await?; + log_dropped_frame(peer, frame_number, &other, schedule); + } + } + } +} + +/// Logs a frame the master didn't want, explaining whether it was outdated or from the future and +/// by how much, alongside the tolerance (the size of the window the master accepts). +fn log_dropped_frame( + peer: SocketAddr, + frame_number: u64, + interest: &FrameInterest, + schedule: FrameSchedule, +) { + match interest { + // Can't reach here for `Wanted`, but keep the match exhaustive and cheap. + FrameInterest::Wanted => {} + FrameInterest::NoWindowYet => { + debug!(%peer, frame_number, "Dropping frame: the master hasn't started rendering yet"); + } + FrameInterest::OutsideWindow { window } => { + // The window is `[oldest ..= newest]`; `newest` is the master's current slot and the + // span is how far back it still accepts frames. + let tolerance = schedule.duration_of_frames(window.end() - window.start()); + if frame_number < *window.start() { + // Older than the oldest slot we still accept: it arrived too late. + let delayed_by = schedule.duration_of_frames(window.end() - frame_number); + warn!( + %peer, + frame_number, + ?delayed_by, + ?tolerance, + "Dropping outdated frame: it arrived later than the collector tolerates" + ); + } else { + // Newer than the master's current slot: the worker's clock is ahead of ours. + let ahead_by = schedule.duration_of_frames(frame_number - window.end()); + warn!( + %peer, + frame_number, + ?ahead_by, + ?tolerance, + "Dropping frame from the future (is the worker's clock ahead of the collector's?)" + ); + } } } } @@ -478,6 +539,27 @@ mod tests { .collect() } + #[test] + fn classify_distinguishes_wanted_old_and_new() { + let store = FrameStore::new(); + + // Before the master ticks, there's no window yet. + assert!(matches!(store.classify(42), FrameInterest::NoWindowYet)); + + *store.interesting.write().unwrap() = Some(40..=42); + + assert!(matches!(store.classify(41), FrameInterest::Wanted)); + // Older than the window -> outdated; newer than the window -> from the future. + assert!(matches!( + store.classify(38), + FrameInterest::OutsideWindow { .. } + )); + assert!(matches!( + store.classify(50), + FrameInterest::OutsideWindow { .. } + )); + } + #[test] fn merges_latest_timestamp_per_pixel() { let mut canvas = Canvas::new(1, 2); diff --git a/deich/src/sync.rs b/deich/src/sync.rs index 1c678df..2bda4dc 100644 --- a/deich/src/sync.rs +++ b/deich/src/sync.rs @@ -84,6 +84,11 @@ impl FrameSchedule { let ns = u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX); ns.div_ceil(self.frame_period_ns) } + + /// Wall-clock duration of `frames` frame slots. + pub fn duration_of_frames(&self, frames: u64) -> Duration { + Duration::from_nanos(frames.saturating_mul(self.frame_period_ns)) + } } /// Configuration the collector hands to each worker. The collector is the single source of truth