Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]

### Added

- Add `EncodeConfig::with_active_reference_frames`, to predict from fewer references than the device allows while keeping the whole window for invalidation. On RADV, H.264 takes about 75% longer to encode a frame with two references than with one.
- Add `EncodedPacket::is_recovery_frame`, set on the first frame shown after an invalidation that recovers without a keyframe, so a streaming host can tell the client where to start decoding again.

### Changed

- **Breaking:** `Encoder::invalidate_reference_frames` returns whether a reference survived (`false`: the next frame is an IDR).

## [0.12.1] - 15-11-2026

### Fixed
Expand Down
11 changes: 8 additions & 3 deletions src/encoder/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ pub(crate) trait EncoderApi: Send {
fn encode_frame_now(&mut self, input: FrameInput, sync: FrameSync) -> Result<EncodeFuture>;
fn flush(&mut self) -> Result<()>;
fn request_idr(&mut self);
fn invalidate_reference_frames(&mut self, first_lost_display_order: u64);
fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) -> bool;
fn set_color(&mut self, color: StreamColor) -> Result<()>;
fn set_rate_control(&mut self, rate_control: RateControl);
}
Expand All @@ -51,7 +51,7 @@ impl<C: VideoCodec> EncoderApi for CodecEncoder<C> {
fn request_idr(&mut self) {
CodecEncoder::request_idr(self)
}
fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) {
fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) -> bool {
CodecEncoder::invalidate_reference_frames(self, first_lost_display_order)
}
fn set_color(&mut self, color: StreamColor) -> Result<()> {
Expand Down Expand Up @@ -497,7 +497,12 @@ impl Encoder {
/// their group are encoded later from the references that survive. If
/// none survives, the oldest frame that waits is encoded at once as the
/// IDR.
pub fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) {
///
/// Returns `true` when a reference survives: the next frame shown
/// recovers from it and its packet has
/// [`EncodedPacket::is_recovery_frame`](super::EncodedPacket::is_recovery_frame)
/// set. `false` means the next frame is an IDR.
pub fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) -> bool {
let local = first_lost_display_order.saturating_sub(self.pts_offset);
self.inner.invalidate_reference_frames(local)
}
Expand Down
25 changes: 22 additions & 3 deletions src/encoder/codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ pub(crate) struct EncoderCommon {
pub input_frame_num: u64,
/// Monotonic encode-order counter (decode order / DTS; `0` => first frame).
pub encode_frame_num: u64,
/// Set when an invalidation left a reference to predict from: the next
/// frame shown is the recovery frame, see [`crate::EncodedPacket::is_recovery_frame`].
pub recovery_pending: bool,

pub dpb_images: Vec<vk::Image>,
pub dpb_image_memories: Vec<vk::DeviceMemory>,
Expand Down Expand Up @@ -935,6 +938,7 @@ impl<C: VideoCodec> CodecEncoder<C> {
self.common.pipeline.submit_synthetic(EncodedPacket {
data: hidden.show_packet,
frame_type: hidden.frame_type,
is_recovery_frame: false,
pts: hidden.display_order,
dts: hidden.display_order as i64,
stats: EncodedPacketStats {
Expand All @@ -960,8 +964,14 @@ impl<C: VideoCodec> CodecEncoder<C> {
self.common.begin_encode(frame, delivery);

let setup = self.codec.begin_picture(&mut self.common, plan)?;
// The first frame shown after an invalidation that kept a reference. An
// IDR requested in between recovers by itself and isn't marked.
let is_recovery_frame = !plan.hidden
&& std::mem::take(&mut self.common.recovery_pending)
&& plan.frame_type() != FrameType::Idr;
self.common.set_pending_metadata(SlotPacketMetadata {
frame_type: plan.frame_type(),
is_recovery_frame,
pts: plan.display_order,
// With hidden frames the packets are in display order.
dts: if C::HIDDEN_FRAMES {
Expand Down Expand Up @@ -1017,18 +1027,21 @@ impl<C: VideoCodec> CodecEncoder<C> {
///
/// For the IDR, the oldest frame that waits for its group is encoded as
/// one at once, so no frame is encoded from the lost references.
pub fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) {
pub fn invalidate_reference_frames(&mut self, first_lost_display_order: u64) -> bool {
if self
.codec
.invalidate_references(&mut self.common, first_lost_display_order)
{
return;
self.common.recovery_pending = true;
return true;
}
self.common.recovery_pending = false;
let ready = self.common.gop.restart();
if let Err(e) = self.encode_ready(ready) {
// Its future resolves with the error, and the next frame is an IDR.
tracing::warn!("Encoding the IDR after lost references failed: {e}");
}
false
}

/// Rebuild session parameters with a new stream colour; the next frame is
Expand Down Expand Up @@ -1382,7 +1395,12 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result<CommonInit>
}
let mut max_active_reference_pictures = reference_window
.min(max_active_supported)
.min(req.max_frame_references);
.min(req.max_frame_references)
.min(
config
.active_reference_frames
.map_or(usize::MAX, |n| n as usize),
);
// `PIXELFORGE_MAX_ACTIVE_REFERENCES=1` predicts from fewer references than
// the device allows, to reproduce a device with tighter limits.
if let Some(cap) = std::env::var("PIXELFORGE_MAX_ACTIVE_REFERENCES")
Expand Down Expand Up @@ -1509,6 +1527,7 @@ pub(crate) fn build_encoder_common(req: CommonInitRequest) -> Result<CommonInit>
gop,
input_frame_num: 0,
encode_frame_num: 0,
recovery_pending: false,
dpb_images,
dpb_image_memories,
dpb_image_views,
Expand Down
20 changes: 20 additions & 0 deletions src/encoder/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,11 @@ pub struct EncodeConfig {
/// as the device allows, which can be fewer. The device's DPB size can
/// also lower it.
pub max_reference_frames: u32,
/// The most references one frame predicts from, capped at the device's limit;
/// `None` predicts from as many as the device allows. Fewer cost less to
/// encode, and the rest of the window stays for invalidation. B-frames
/// still predict from one reference on each side.
pub active_reference_frames: Option<u32>,
/// How many B-frames go between two reference frames; see
/// [`Self::with_b_frames`].
pub b_frames: u32,
Expand Down Expand Up @@ -499,6 +504,7 @@ impl EncodeConfig {
},
gop_size: DEFAULT_GOP_SIZE,
max_reference_frames: DEFAULT_MAX_REFERENCE_FRAMES,
active_reference_frames: None,
b_frames: 0,
b_frame_references: BFrameReferences::Flat,
virtual_buffer_size_ms: 1000,
Expand Down Expand Up @@ -568,6 +574,13 @@ impl EncodeConfig {
self
}

/// Set the most references a frame predicts from (see
/// [`active_reference_frames`](Self::active_reference_frames)).
pub fn with_active_reference_frames(mut self, count: u32) -> Self {
self.active_reference_frames = Some(count);
self
}

/// Encode `count` B-frames between two reference frames, at most
/// [`MAX_B_FRAMES`]. 0, the default, encodes none and adds no latency.
///
Expand Down Expand Up @@ -704,6 +717,12 @@ pub struct EncodedPacket {
pub data: Vec<u8>,
/// Frame type.
pub frame_type: FrameType,
/// Whether this is the first frame shown after
/// [`Encoder::invalidate_reference_frames`](crate::Encoder::invalidate_reference_frames)
/// returned `true`: it predicts only from frames before the loss, so a
/// client that stopped decoding at the loss can start again here.
/// Moonlight's protocol marks it as frame type 5. Never set on a key frame.
pub is_recovery_frame: bool,
/// Presentation timestamp: the number of the frame in the order it was
/// submitted.
pub pts: u64,
Expand Down Expand Up @@ -1048,6 +1067,7 @@ mod tests {
let packet = EncodedPacket {
data: vec![0x00, 0x00, 0x00, 0x01, 0x67],
frame_type: FrameType::Idr,
is_recovery_frame: false,
pts: 0,
dts: 0,
stats: EncodedPacketStats::default(),
Expand Down
2 changes: 2 additions & 0 deletions src/encoder/pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ pub(crate) const ENCODE_PIPELINE_DEPTH: usize = 2;
/// when the completion thread reads the slot back.
pub(crate) struct SlotPacketMetadata {
pub frame_type: FrameType,
pub is_recovery_frame: bool,
pub pts: u64,
pub dts: i64,
/// Codec header (SPS/PPS, VPS/SPS/PPS, or AV1 sequence header) to prepend.
Expand Down Expand Up @@ -787,6 +788,7 @@ fn run_completion_thread(
let packet = result.map(|()| EncodedPacket {
data,
frame_type: work.metadata.frame_type,
is_recovery_frame: work.metadata.is_recovery_frame,
pts: work.metadata.pts,
dts: work.metadata.dts,
stats,
Expand Down
4 changes: 2 additions & 2 deletions tests/b_frames.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,10 +163,10 @@ fn control_calls_close_the_group() -> Result<(), Box<dyn std::error::Error>> {
submitted += 1;
}
// The references from before the loss survive: no IDR.
30 => encoder.invalidate_reference_frames(submitted - 2),
30 => assert!(encoder.invalidate_reference_frames(submitted - 2)),
// Every reference is lost.
33 => {
encoder.invalidate_reference_frames(0);
assert!(!encoder.invalidate_reference_frames(0));
restart = submitted - u64::from(b_frames)..=submitted;
}
// Everything submitted is encoded, and the stream goes on.
Expand Down
48 changes: 39 additions & 9 deletions tests/rfi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,13 +81,19 @@ fn rfi() -> Result<(), Box<dyn std::error::Error>> {
BFrameSupport::Flat => Some(BFrameReferences::Flat),
BFrameSupport::Pyramid => Some(BFrameReferences::Pyramid),
};
let runs = std::iter::once(None).chain(b_frames.map(Some));
for references in runs {
let name = match references {
None => format!("{codec:?}"),
Some(references) => format!("{codec:?} B-frames {references:?}"),
// Without B-frames, also run predicting from one reference (as a streaming
// host may, to save encode time) and check that the rest of the window
// still recovers.
let runs = [(None, None), (None, Some(1))]
.into_iter()
.chain(b_frames.map(|references| (Some(references), None)));
for (references, active) in runs {
let name = match (references, active) {
(None, None) => format!("{codec:?}"),
(None, Some(active)) => format!("{codec:?} predicting from {active}"),
(Some(references), _) => format!("{codec:?} B-frames {references:?}"),
};
if let Err(e) = run_codec(&context, &dir, codec, references, &input_path) {
if let Err(e) = run_codec(&context, &dir, codec, references, active, &input_path) {
println!("{name}: FAIL: {e}");
failures.push(format!("{name}: {e}"));
}
Expand All @@ -105,10 +111,15 @@ fn run_codec(
dir: &std::path::Path,
codec: Codec,
b_frames: Option<BFrameReferences>,
active_references: Option<u32>,
input_path: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let ext = if codec == Codec::AV1 { "obu" } else { "bin" };
let suffix = if b_frames.is_some() { "_b" } else { "" };
let suffix = match (b_frames, active_references) {
(Some(_), _) => "_b".to_string(),
(None, Some(active)) => format!("_active{active}"),
(None, None) => String::new(),
};
let output_filename = dir
.join(format!("output_rfi_{codec:?}{suffix}.{ext}"))
.to_string_lossy()
Expand All @@ -132,6 +143,10 @@ fn run_codec(
Some(references) => config.with_b_frames(3).with_b_frame_references(references),
None => config,
};
let config = match active_references {
Some(active) => config.with_active_reference_frames(active),
None => config,
};
// At most `b_frames` frames wait for their group, so with more futures
// pending than that the oldest is ready (this also checks that claim).
let held = config.b_frames.max(2) as usize;
Expand All @@ -147,6 +162,7 @@ fn run_codec(
let mut pending: VecDeque<pixelforge::EncodeFuture> = VecDeque::new();
let mut recovery_is_key: Option<bool> = None;
let mut later_key_frames = 0;
let mut recovery_frames = Vec::new();

let mut drain_one = |pending: &mut VecDeque<pixelforge::EncodeFuture>,
output_file: &mut File,
Expand All @@ -159,6 +175,9 @@ fn run_codec(
if packet.pts > 0 && packet.is_key_frame() {
later_key_frames += 1;
}
if packet.is_recovery_frame {
recovery_frames.push(packet.pts);
}
output_file.write_all(&packet.data)?;
Ok(())
};
Expand All @@ -174,7 +193,9 @@ fn run_codec(
// just before encoding frame `INVALIDATE_AT`.
if i == INVALIDATE_AT {
let first_lost = INVALIDATE_AT - INVALIDATE_COUNT;
encoder.invalidate_reference_frames(first_lost);
if !encoder.invalidate_reference_frames(first_lost) {
return Err("invalidation left no reference to recover from".into());
}
}

let slot = encoder.input_slot()?;
Expand All @@ -192,6 +213,15 @@ fn run_codec(
drop(output_file);
drop(drain_one);

// One packet is marked as the recovery frame; without B-frames, the
// frame submitted right after the invalidation.
match (recovery_frames.as_slice(), b_frames) {
([_], Some(_)) | ([INVALIDATE_AT], None) => {}
(frames, _) => {
return Err(format!("frames {frames:?} were marked as the recovery frame").into());
}
}

if later_key_frames > 0 {
return Err(
format!("{later_key_frames} keyframes after the first (RFI did not engage)").into(),
Expand Down Expand Up @@ -235,7 +265,7 @@ fn run_codec(
}

println!(
"{codec:?} {b_frames:?}: PASS — recovered with a P-frame, full-stream PSNR {psnr:.2} dB"
"{codec:?} {b_frames:?} {active_references:?}: PASS — recovered with a P-frame, full-stream PSNR {psnr:.2} dB"
);
Ok(())
}
Expand Down
Loading