From 6601da0339726339bb78b8e8c42ffa4308fb3ca0 Mon Sep 17 00:00:00 2001 From: Ryan Bateman Date: Fri, 13 Nov 2026 09:52:19 -0800 Subject: [PATCH 1/3] feat(encoder): mark the frame that recovers from an invalidation --- CHANGELOG.md | 10 ++++++++++ src/encoder/api.rs | 11 ++++++++--- src/encoder/codec.rs | 18 ++++++++++++++++-- src/encoder/mod.rs | 7 +++++++ src/encoder/pipeline.rs | 2 ++ tests/b_frames.rs | 4 ++-- tests/rfi.rs | 17 ++++++++++++++++- 7 files changed, 61 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f803c3..63e8764 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Add `EncodedPacket::is_recovery_frame`, set on the first frame shown after an invalidation that recovers without a keyframe, so a streaming host can tell the client where to start decoding again. + +### Changed + +- **Breaking:** `Encoder::invalidate_reference_frames` returns whether a reference survived (`false`: the next frame is an IDR). + ## [0.12.1] - 15-11-2026 ### Fixed diff --git a/src/encoder/api.rs b/src/encoder/api.rs index 5c3183e..346ae4f 100644 --- a/src/encoder/api.rs +++ b/src/encoder/api.rs @@ -27,7 +27,7 @@ pub(crate) trait EncoderApi: Send { fn encode_frame_now(&mut self, input: FrameInput, sync: FrameSync) -> Result; fn flush(&mut self) -> Result<()>; fn request_idr(&mut self); - fn invalidate_reference_frames(&mut self, first_lost_display_order: u64); + fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) -> bool; fn set_color(&mut self, color: StreamColor) -> Result<()>; fn set_rate_control(&mut self, rate_control: RateControl); } @@ -51,7 +51,7 @@ impl EncoderApi for CodecEncoder { fn request_idr(&mut self) { CodecEncoder::request_idr(self) } - fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) { + fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) -> bool { CodecEncoder::invalidate_reference_frames(self, first_lost_display_order) } fn set_color(&mut self, color: StreamColor) -> Result<()> { @@ -497,7 +497,12 @@ impl Encoder { /// their group are encoded later from the references that survive. If /// none survives, the oldest frame that waits is encoded at once as the /// IDR. - pub fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) { + /// + /// Returns `true` when a reference survives: the next frame shown + /// recovers from it and its packet has + /// [`EncodedPacket::is_recovery_frame`](super::EncodedPacket::is_recovery_frame) + /// set. `false` means the next frame is an IDR. + pub fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) -> bool { let local = first_lost_display_order.saturating_sub(self.pts_offset); self.inner.invalidate_reference_frames(local) } diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 2173c58..3fdaf69 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -72,6 +72,9 @@ pub(crate) struct EncoderCommon { pub input_frame_num: u64, /// Monotonic encode-order counter (decode order / DTS; `0` => first frame). pub encode_frame_num: u64, + /// Set when an invalidation left a reference to predict from: the next + /// frame shown is the recovery frame, see [`crate::EncodedPacket::is_recovery_frame`]. + pub recovery_pending: bool, pub dpb_images: Vec, pub dpb_image_memories: Vec, @@ -935,6 +938,7 @@ impl CodecEncoder { self.common.pipeline.submit_synthetic(EncodedPacket { data: hidden.show_packet, frame_type: hidden.frame_type, + is_recovery_frame: false, pts: hidden.display_order, dts: hidden.display_order as i64, stats: EncodedPacketStats { @@ -960,8 +964,14 @@ impl CodecEncoder { self.common.begin_encode(frame, delivery); let setup = self.codec.begin_picture(&mut self.common, plan)?; + // The first frame shown after a recovering invalidation; an IDR in + // between (one was asked for) recovers by itself. + let is_recovery_frame = !plan.hidden + && std::mem::take(&mut self.common.recovery_pending) + && plan.frame_type() != FrameType::Idr; self.common.set_pending_metadata(SlotPacketMetadata { frame_type: plan.frame_type(), + is_recovery_frame, pts: plan.display_order, // With hidden frames the packets are in display order. dts: if C::HIDDEN_FRAMES { @@ -1017,18 +1027,21 @@ impl CodecEncoder { /// /// For the IDR, the oldest frame that waits for its group is encoded as /// one at once, so no frame is encoded from the lost references. - pub fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) { + pub fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) -> bool { if self .codec .invalidate_references(&mut self.common, first_lost_display_order) { - return; + self.common.recovery_pending = true; + return true; } + self.common.recovery_pending = false; let ready = self.common.gop.restart(); if let Err(e) = self.encode_ready(ready) { // Its future resolves with the error, and the next frame is an IDR. tracing::warn!("Encoding the IDR after lost references failed: {e}"); } + false } /// Rebuild session parameters with a new stream colour; the next frame is @@ -1509,6 +1522,7 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result gop, input_frame_num: 0, encode_frame_num: 0, + recovery_pending: false, dpb_images, dpb_image_memories, dpb_image_views, diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index 00314e3..d650db1 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -704,6 +704,12 @@ pub struct EncodedPacket { pub data: Vec, /// Frame type. pub frame_type: FrameType, + /// Whether this is the first frame shown after + /// [`Encoder::invalidate_reference_frames`](crate::Encoder::invalidate_reference_frames) + /// returned `true`: it predicts only from frames before the loss, so a + /// client that stopped decoding at the loss can start again here. + /// Moonlight's protocol marks it as frame type 5. Never set on a key frame. + pub is_recovery_frame: bool, /// Presentation timestamp: the number of the frame in the order it was /// submitted. pub pts: u64, @@ -1048,6 +1054,7 @@ mod tests { let packet = EncodedPacket { data: vec![0x00, 0x00, 0x00, 0x01, 0x67], frame_type: FrameType::Idr, + is_recovery_frame: false, pts: 0, dts: 0, stats: EncodedPacketStats::default(), diff --git a/src/encoder/pipeline.rs b/src/encoder/pipeline.rs index 99fd63f..1db6c34 100644 --- a/src/encoder/pipeline.rs +++ b/src/encoder/pipeline.rs @@ -127,6 +127,7 @@ pub(crate) const ENCODE_PIPELINE_DEPTH: usize = 2; /// when the completion thread reads the slot back. pub(crate) struct SlotPacketMetadata { pub frame_type: FrameType, + pub is_recovery_frame: bool, pub pts: u64, pub dts: i64, /// Codec header (SPS/PPS, VPS/SPS/PPS, or AV1 sequence header) to prepend. @@ -787,6 +788,7 @@ fn run_completion_thread( let packet = result.map(|()| EncodedPacket { data, frame_type: work.metadata.frame_type, + is_recovery_frame: work.metadata.is_recovery_frame, pts: work.metadata.pts, dts: work.metadata.dts, stats, diff --git a/tests/b_frames.rs b/tests/b_frames.rs index c74e907..96ddfa8 100644 --- a/tests/b_frames.rs +++ b/tests/b_frames.rs @@ -163,10 +163,10 @@ fn control_calls_close_the_group() -> Result<(), Box> { submitted += 1; } // The references from before the loss survive: no IDR. - 30 => encoder.invalidate_reference_frames(submitted - 2), + 30 => assert!(encoder.invalidate_reference_frames(submitted - 2)), // Every reference is lost. 33 => { - encoder.invalidate_reference_frames(0); + assert!(!encoder.invalidate_reference_frames(0)); restart = submitted - u64::from(b_frames)..=submitted; } // Everything submitted is encoded, and the stream goes on. diff --git a/tests/rfi.rs b/tests/rfi.rs index 6d8be77..632a72e 100644 --- a/tests/rfi.rs +++ b/tests/rfi.rs @@ -147,6 +147,7 @@ fn run_codec( let mut pending: VecDeque = VecDeque::new(); let mut recovery_is_key: Option = None; let mut later_key_frames = 0; + let mut recovery_frames = Vec::new(); let mut drain_one = |pending: &mut VecDeque, output_file: &mut File, @@ -159,6 +160,9 @@ fn run_codec( if packet.pts > 0 && packet.is_key_frame() { later_key_frames += 1; } + if packet.is_recovery_frame { + recovery_frames.push(packet.pts); + } output_file.write_all(&packet.data)?; Ok(()) }; @@ -174,7 +178,9 @@ fn run_codec( // just before encoding frame `INVALIDATE_AT`. if i == INVALIDATE_AT { let first_lost = INVALIDATE_AT - INVALIDATE_COUNT; - encoder.invalidate_reference_frames(first_lost); + if !encoder.invalidate_reference_frames(first_lost) { + return Err("invalidation left no reference to recover from".into()); + } } let slot = encoder.input_slot()?; @@ -192,6 +198,15 @@ fn run_codec( drop(output_file); drop(drain_one); + // One packet is marked as the recovery frame; without B-frames, the + // frame submitted right after the invalidation. + match (recovery_frames.as_slice(), b_frames) { + ([_], Some(_)) | ([INVALIDATE_AT], None) => {} + (frames, _) => { + return Err(format!("frames {frames:?} were marked as the recovery frame").into()); + } + } + if later_key_frames > 0 { return Err( format!("{later_key_frames} keyframes after the first (RFI did not engage)").into(), From 17f7f6ca34c3472dec20f9d49f6959bad87e4182 Mon Sep 17 00:00:00 2001 From: Ryan Bateman Date: Fri, 13 Nov 2026 09:54:23 -0800 Subject: [PATCH 2/3] feat(encoder): cap the references a frame predicts from --- CHANGELOG.md | 1 + src/encoder/codec.rs | 7 ++++++- src/encoder/mod.rs | 13 +++++++++++++ tests/rfi.rs | 30 ++++++++++++++++++++++-------- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63e8764..a235d7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add `EncodeConfig::with_active_reference_frames`, to predict from fewer references than the device allows while keeping the whole window for invalidation. On RADV, H.264 takes about 75% longer to encode a frame with two references than with one. - Add `EncodedPacket::is_recovery_frame`, set on the first frame shown after an invalidation that recovers without a keyframe, so a streaming host can tell the client where to start decoding again. ### Changed diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index 3fdaf69..b648444 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -1395,7 +1395,12 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result } let mut max_active_reference_pictures = reference_window .min(max_active_supported) - .min(req.max_frame_references); + .min(req.max_frame_references) + .min( + config + .active_reference_frames + .map_or(usize::MAX, |n| n as usize), + ); // `PIXELFORGE_MAX_ACTIVE_REFERENCES=1` predicts from fewer references than // the device allows, to reproduce a device with tighter limits. if let Some(cap) = std::env::var("PIXELFORGE_MAX_ACTIVE_REFERENCES") diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index d650db1..c18aba7 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -443,6 +443,11 @@ pub struct EncodeConfig { /// as the device allows, which can be fewer. The device's DPB size can /// also lower it. pub max_reference_frames: u32, + /// The most references one frame predicts from, below the device's limit; + /// `None` predicts from as many as the device allows. Fewer cost less to + /// encode, and the rest of the window stays for invalidation. B-frames + /// still predict from one reference on each side. + pub active_reference_frames: Option, /// How many B-frames go between two reference frames; see /// [`Self::with_b_frames`]. pub b_frames: u32, @@ -499,6 +504,7 @@ impl EncodeConfig { }, gop_size: DEFAULT_GOP_SIZE, max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES, + active_reference_frames: None, b_frames: 0, b_frame_references: BFrameReferences::Flat, virtual_buffer_size_ms: 1000, @@ -568,6 +574,13 @@ impl EncodeConfig { self } + /// Set the most references a frame predicts from (see + /// [`active_reference_frames`](Self::active_reference_frames)). + pub fn with_active_reference_frames(mut self, count: u32) -> Self { + self.active_reference_frames = Some(count); + self + } + /// Encode `count` B-frames between two reference frames, at most /// [`MAX_B_FRAMES`]. 0, the default, encodes none and adds no latency. /// diff --git a/tests/rfi.rs b/tests/rfi.rs index 632a72e..40613bf 100644 --- a/tests/rfi.rs +++ b/tests/rfi.rs @@ -81,13 +81,18 @@ fn rfi() -> Result<(), Box> { BFrameSupport::Flat => Some(BFrameReferences::Flat), BFrameSupport::Pyramid => Some(BFrameReferences::Pyramid), }; - let runs = std::iter::once(None).chain(b_frames.map(Some)); - for references in runs { - let name = match references { - None => format!("{codec:?}"), - Some(references) => format!("{codec:?} B-frames {references:?}"), + // Without B-frames, also predicting from one reference, as a streaming + // host does to save encode time: the rest of the window recovers. + let runs = [(None, None), (None, Some(1))] + .into_iter() + .chain(b_frames.map(|references| (Some(references), None))); + for (references, active) in runs { + let name = match (references, active) { + (None, None) => format!("{codec:?}"), + (None, Some(active)) => format!("{codec:?} predicting from {active}"), + (Some(references), _) => format!("{codec:?} B-frames {references:?}"), }; - if let Err(e) = run_codec(&context, &dir, codec, references, &input_path) { + if let Err(e) = run_codec(&context, &dir, codec, references, active, &input_path) { println!("{name}: FAIL: {e}"); failures.push(format!("{name}: {e}")); } @@ -105,10 +110,15 @@ fn run_codec( dir: &std::path::Path, codec: Codec, b_frames: Option, + active_references: Option, input_path: &str, ) -> Result<(), Box> { let ext = if codec == Codec::AV1 { "obu" } else { "bin" }; - let suffix = if b_frames.is_some() { "_b" } else { "" }; + let suffix = match (b_frames, active_references) { + (Some(_), _) => "_b".to_string(), + (None, Some(active)) => format!("_active{active}"), + (None, None) => String::new(), + }; let output_filename = dir .join(format!("output_rfi_{codec:?}{suffix}.{ext}")) .to_string_lossy() @@ -132,6 +142,10 @@ fn run_codec( Some(references) => config.with_b_frames(3).with_b_frame_references(references), None => config, }; + let config = match active_references { + Some(active) => config.with_active_reference_frames(active), + None => config, + }; // At most `b_frames` frames wait for their group, so with more futures // pending than that the oldest is ready (this also checks that claim). let held = config.b_frames.max(2) as usize; @@ -250,7 +264,7 @@ fn run_codec( } println!( - "{codec:?} {b_frames:?}: PASS — recovered with a P-frame, full-stream PSNR {psnr:.2} dB" + "{codec:?} {b_frames:?} {active_references:?}: PASS — recovered with a P-frame, full-stream PSNR {psnr:.2} dB" ); Ok(()) } From d365e056424a1a59d523fc37d418592da179a65f Mon Sep 17 00:00:00 2001 From: Ryan Bateman Date: Wed, 18 Nov 2026 04:12:51 +0000 Subject: [PATCH 3/3] docs(encoder): clarify recovery-frame comments --- src/encoder/codec.rs | 4 ++-- src/encoder/mod.rs | 2 +- tests/rfi.rs | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/encoder/codec.rs b/src/encoder/codec.rs index b648444..c33aa70 100644 --- a/src/encoder/codec.rs +++ b/src/encoder/codec.rs @@ -964,8 +964,8 @@ impl CodecEncoder { self.common.begin_encode(frame, delivery); let setup = self.codec.begin_picture(&mut self.common, plan)?; - // The first frame shown after a recovering invalidation; an IDR in - // between (one was asked for) recovers by itself. + // The first frame shown after an invalidation that kept a reference. An + // IDR requested in between recovers by itself and isn't marked. let is_recovery_frame = !plan.hidden && std::mem::take(&mut self.common.recovery_pending) && plan.frame_type() != FrameType::Idr; diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index c18aba7..16f80e6 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -443,7 +443,7 @@ pub struct EncodeConfig { /// as the device allows, which can be fewer. The device's DPB size can /// also lower it. pub max_reference_frames: u32, - /// The most references one frame predicts from, below the device's limit; + /// The most references one frame predicts from, capped at the device's limit; /// `None` predicts from as many as the device allows. Fewer cost less to /// encode, and the rest of the window stays for invalidation. B-frames /// still predict from one reference on each side. diff --git a/tests/rfi.rs b/tests/rfi.rs index 40613bf..9cca91d 100644 --- a/tests/rfi.rs +++ b/tests/rfi.rs @@ -81,8 +81,9 @@ fn rfi() -> Result<(), Box> { BFrameSupport::Flat => Some(BFrameReferences::Flat), BFrameSupport::Pyramid => Some(BFrameReferences::Pyramid), }; - // Without B-frames, also predicting from one reference, as a streaming - // host does to save encode time: the rest of the window recovers. + // Without B-frames, also run predicting from one reference (as a streaming + // host may, to save encode time) and check that the rest of the window + // still recovers. let runs = [(None, None), (None, Some(1))] .into_iter() .chain(b_frames.map(|references| (Some(references), None)));