From 7f31bed57e6f772b2e2a5d1c1ef8824a3145bb7c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 15:57:57 +0000 Subject: [PATCH 1/2] fix(web): play video frames without a jitter buffer WebRTC buffers received video by default to smooth playback, which on a LAN only puts the host's cursor further behind the user's hand. Set jitterBufferTarget (standard) and playoutDelayHint (Chrome) to 0 on the video receiver; audio keeps its buffer. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GRSn3XsjYLc494XwPez2mF --- crates/rkvm-web/src/net/peer.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/rkvm-web/src/net/peer.rs b/crates/rkvm-web/src/net/peer.rs index 88b4ada..331b6dd 100644 --- a/crates/rkvm-web/src/net/peer.rs +++ b/crates/rkvm-web/src/net/peer.rs @@ -84,6 +84,28 @@ pub async fn watch(peer: &Peer, cancelled: impl Fn() -> bool) -> Option } } +/// Asks the browser to play frames as soon as they decode. By default WebRTC +/// holds video in a jitter buffer (often 50-150 ms) to smooth playback, which +/// on a LAN only delays the host's cursor behind the user's hand. +/// `jitterBufferTarget` is the standard knob; `playoutDelayHint` is Chrome's +/// older one. Browsers without either simply ignore the extra property. +/// Audio keeps its buffer: without one it crackles on every late packet. +fn play_out_immediately(ev: &RtcTrackEvent) { + let kind = js_sys::Reflect::get(ev, &"track".into()) + .and_then(|t| js_sys::Reflect::get(&t, &"kind".into())) + .ok() + .and_then(|k| k.as_string()); + if kind.as_deref() != Some("video") { + return; + } + let Ok(receiver) = js_sys::Reflect::get(ev, &"receiver".into()) else { + return; + }; + for key in ["jitterBufferTarget", "playoutDelayHint"] { + let _ = js_sys::Reflect::set(&receiver, &key.into(), &0.into()); + } +} + pub async fn connect(video: &HtmlVideoElement) -> Result { let pc = RtcPeerConnection::new().map_err(js_err("RtcPeerConnection::new"))?; @@ -95,6 +117,7 @@ pub async fn connect(video: &HtmlVideoElement) -> Result { { let video = video.clone(); let on_track = Closure::::new(move |ev: RtcTrackEvent| { + play_out_immediately(&ev); if let Some(stream) = ev.streams().get(0).dyn_ref::() { video.set_src_object(Some(stream)); } From c4dff070cde5909e090a50f5703dc84a3d0f69f3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 16:40:19 +0000 Subject: [PATCH 2/2] fix(video): send a periodic IDR every 10 s and rate-limit forced keyframes With no GOP configured the encoder emitted an IDR every second. At 4K a keyframe is ~350 KB and takes ~28 ms to send, so the 2-3 frames behind it arrive late: a hitch every second that makes the cursor feel like it lags and catches up. Measured on the T6 with a RTP-level client over 60 s of a static desktop: 1 s GOP gave frame-gap p99 46.3 ms and 23 gaps over 50 ms; 10 s GOP gave p99 16.6 ms and none. The dynamic GOP is now 10 s of frames. Viewers still get an IDR at once when they join or report loss (PLI/FIR), but PLI-driven keyframes are spaced at least 500 ms apart: a misbehaving client was able to make the encoder emit ~2,600 IDRs a minute. Resuming encoding for a viewer bypasses the limit, since it needs an IDR to start decoding. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GRSn3XsjYLc494XwPez2mF --- app/src/pipeline.rs | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/app/src/pipeline.rs b/app/src/pipeline.rs index 78a470c..e17487c 100644 --- a/app/src/pipeline.rs +++ b/app/src/pipeline.rs @@ -116,6 +116,17 @@ impl AudioConfig { } } +/// Seconds between periodic IDRs when no GOP is configured. A 4K IDR is +/// several hundred KB and takes ~25 ms to send, so the frames behind it arrive +/// late; at one IDR per second that is a visible hitch every second (measured +/// on the T6: frame-gap p99 46 ms at 1 s, 16.6 ms at 10 s). Viewers that lose +/// a frame ask for an IDR with a PLI, so a long GOP does not delay recovery. +const GOP_SECONDS: i32 = 10; + +/// Minimum spacing of keyframes forced by viewer requests (PLI/FIR), so one +/// misbehaving client cannot make the encoder emit nothing but IDRs. +const KEYFRAME_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_millis(500); + pub struct VideoPipeline { pipeline: gst::Pipeline, appsink: gst_app::AppSink, @@ -127,6 +138,8 @@ pub struct VideoPipeline { /// When the oldest unanswered keyframe request was made; the encoder /// output probe logs how long it took to produce the keyframe. keyframe_requested_at: Arc>>, + /// When a keyframe was last forced, to rate-limit PLI-driven requests. + keyframe_forced_at: parking_lot::Mutex>, capture_src: gst::Element, /// Raw frames the capture source has pushed, paused encoding included. raw_frames: Arc, @@ -350,7 +363,7 @@ impl VideoPipeline { && let Ok(info) = gst_video::VideoInfo::from_caps(&caps) { let fps = info.fps(); - let gop = fps.numer() / fps.denom().max(1); + let gop = (fps.numer() / fps.denom().max(1)).max(1) * GOP_SECONDS; encoder_clone.set_property("gop", gop); info!( "Set dynamic GOP to {} based on source {:.1}fps", @@ -373,6 +386,7 @@ impl VideoPipeline { raw_frame_waiters, raw_frame_requested, keyframe_requested_at, + keyframe_forced_at: parking_lot::Mutex::new(None), capture_src: v4l2src, raw_frames, }) @@ -934,12 +948,22 @@ impl PipelineManager { return; } } + if self + .video + .keyframe_forced_at + .lock() + .is_some_and(|at| at.elapsed() < KEYFRAME_MIN_INTERVAL) + { + debug!("keyframe forced recently, skipping request"); + return; + } self.force_keyframe_now(); } /// Like [`Self::force_keyframe`] but never coalesced: a new viewer needs /// an IDR encoded after its connection is up, even if one is in flight. pub fn force_keyframe_now(&self) { + *self.video.keyframe_forced_at.lock() = Some(std::time::Instant::now()); self.video.keyframe_requested_at.lock().get_or_insert_with(std::time::Instant::now); let mut builder = gst_video::UpstreamForceKeyUnitEvent::builder().all_headers(true); if let Some(running_time) = self.video.encoder.current_running_time() { @@ -956,7 +980,7 @@ impl PipelineManager { /// or pauses it when nobody is watching. pub fn set_encoding_active(&self, active: bool) { if self.video.set_encoding_active(active) && active { - self.force_keyframe(); + self.force_keyframe_now(); } } }