From 9c9fa921b9de1bace66976a3dabdb899ecd240c3 Mon Sep 17 00:00:00 2001 From: Tiisetso Mosala <65363519+trmosala@users.noreply.github.com> Date: Wed, 18 Nov 2026 06:56:57 +0200 Subject: [PATCH 1/2] feat(decoder): reset at stream discontinuities --- README.md | 6 ++ src/decoder/av1/parser.rs | 8 ++ src/decoder/av1/session.rs | 15 ++++ src/decoder/av1/stream_tests.rs | 33 +++++++ src/decoder/common.rs | 27 +++--- src/decoder/driver.rs | 30 ++++++- src/decoder/frames.rs | 9 ++ src/decoder/h264/dpb.rs | 33 +++++++ src/decoder/h264/session.rs | 6 ++ src/decoder/h265/dpb.rs | 4 + src/decoder/h265/dpb_tests.rs | 40 +++++++++ src/decoder/h265/session.rs | 7 ++ src/decoder/mod.rs | 25 ++++++ src/decoder/pipeline.rs | 9 ++ src/lib.rs | 6 ++ tests/decode_streams.rs | 147 ++++++++++++++++++++++++++++++++ 16 files changed, 390 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 25f1c627..5956dc98 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,12 @@ pictures out instead of handing over its own. To get a frame's samples on the CPU (for a file, a test or software processing), copy it with [`YuvDownloader`]. +To seek, call [`Decoder::reset`] and feed again from a keyframe. The +decoder keeps the session for unchanged stream parameters and discards +buffered input and pictures still waiting for reordering. Frames already +sent to the source can still be read. Use the `pts` supplied with each +call to distinguish frames from before and after the seek. + ### RGB input An encoder takes RGB frames once it is told what they are, with a diff --git a/src/decoder/av1/parser.rs b/src/decoder/av1/parser.rs index a2a8e77b..60f9ec36 100644 --- a/src/decoder/av1/parser.rs +++ b/src/decoder/av1/parser.rs @@ -99,6 +99,14 @@ impl Av1Parser { Self::default() } + /// Clear reference and partial-frame state while retaining the sequence. + pub fn reset(&mut self) { + *self = Self { + sequence: self.sequence.take(), + ..Self::default() + }; + } + /// The sequence header in effect, once one has been seen. pub fn sequence_header(&self) -> Option<&SequenceHeader> { self.sequence.as_ref() diff --git a/src/decoder/av1/session.rs b/src/decoder/av1/session.rs index 5a5e271c..64679eb6 100644 --- a/src/decoder/av1/session.rs +++ b/src/decoder/av1/session.rs @@ -224,6 +224,15 @@ impl Av1Decoder { }); } Some(Av1Event::Frame(frame)) => { + // An intra-only picture can parse without references, + // but only a key frame can start decoding. + if self.common.session.as_ref().is_none_or(|s| s.needs_reset) + && frame.header.frame_type != FrameType::Key + { + self.parser.discard_frame(); + missing.get_or_insert_with(|| "no key frame decoded yet".to_string()); + continue; + } // The frame shown by the previous one has already been // handed over; close off its pipeline slot first. if let Decoded::Shown(_) = outcome { @@ -459,6 +468,12 @@ impl CodecDecoder for Av1Decoder { &mut self.common } + fn reset(&mut self) { + self.parser.reset(); + self.grain_applied.fill(false); + self.shown_since_key = 0; + } + fn complete_prefix(buffer: &[u8]) -> usize { // An error here is input that is not low-overhead OBUs; decoding the // buffer reports it. diff --git a/src/decoder/av1/stream_tests.rs b/src/decoder/av1/stream_tests.rs index 64e75f48..886815a4 100644 --- a/src/decoder/av1/stream_tests.rs +++ b/src/decoder/av1/stream_tests.rs @@ -622,3 +622,36 @@ fn corrupt_streams_do_not_panic() { } } } + +#[test] +fn reset_keeps_sequence_header_and_forgets_references() { + let data = fixture("aom_altref"); + let starts = temporal_unit_starts(data).unwrap(); + let mut parser = Av1Parser::new(); + let mut keyframe = None; + for obu in iter_obus(&data[..starts[1]]) { + let obu = obu.unwrap(); + if let Some(Av1Event::Frame(_)) = parser.parse_obu(&obu).unwrap() { + keyframe = Some(obu); + parser.commit_frame(Some(0)).unwrap(); + } + } + let keyframe = keyframe.unwrap(); + assert!(parser.refs().dpb_slot_in_use(0)); + + parser.reset(); + assert!(parser.sequence_header().is_some()); + assert_eq!(*parser.refs(), Default::default()); + assert!(iter_obus(&data[starts[1]..starts[2]]).any(|obu| { + matches!( + parser.parse_obu(&obu.unwrap()).unwrap(), + Some(Av1Event::MissingReferences(_)) + ) + })); + assert!(matches!( + parser.parse_obu(&keyframe).unwrap(), + Some(Av1Event::Frame(_)) + )); + parser.commit_frame(Some(1)).unwrap(); + assert!(parser.refs().dpb_slot_in_use(1)); +} diff --git a/src/decoder/common.rs b/src/decoder/common.rs index 6dc4b340..07d68deb 100644 --- a/src/decoder/common.rs +++ b/src/decoder/common.rs @@ -98,7 +98,10 @@ pub(crate) struct DecodeSession { pub dpb: Vec>, /// Whether each slot has been written at least once (governs the /// UNDEFINED-vs-DPB old layout in the pre-decode barrier). - pub dpb_slot_active: Vec, + dpb_slot_written: Vec, + /// Initialize the Vulkan DPB before the next decode. Image layouts survive + /// the coding reset, so they are tracked separately. + pub needs_reset: bool, pub use_layered_dpb: bool, /// True when the driver decodes into the DPB image directly; false when a @@ -465,7 +468,8 @@ impl DecoderCommon { chroma_format: plan.chroma_format, dpb, slot_pins: Arc::new(SlotPins::default()), - dpb_slot_active: vec![false; plan.slot_count], + dpb_slot_written: vec![false; plan.slot_count], + needs_reset: true, use_layered_dpb: plan.use_layered_dpb, coincide: plan.coincide, spare_slots: plan.spare_slots, @@ -590,7 +594,7 @@ impl DecoderCommon { // afterwards. let picture_layout = self.picture_layout(); let (dst_image, dst_layer) = session.dpb_image_for_slot(slot); - let old_layout = if session.dpb_slot_active[slot as usize] { + let old_layout = if session.dpb_slot_written[slot as usize] { picture_layout } else { vk::ImageLayout::UNDEFINED @@ -710,24 +714,21 @@ impl DecoderCommon { /// decodes in decode order despite the CPU running ahead. pub fn submit_decode(&mut self) -> Result<()> { let queue = self.decode_queue; - self.pipeline.submit_decode(&self.context, queue) + self.pipeline.submit_decode(&self.context, queue)?; + self.session.as_mut().expect("active session").needs_reset = false; + Ok(()) } /// Record `vkCmdBeginVideoCoding`, `vkCmdDecodeVideo` and /// `vkCmdEndVideoCoding` from a linked `record`, resetting the session - /// first if this is its first decode. + /// first if this decode starts a new sequence after creation or reset. pub fn record_coding(&self, record: &DecodeRecord) { let command_buffer = self.decode_command_buffer(); - // A session's first decode sees no active slot. - let first_use = self - .session - .as_ref() - .is_none_or(|s| !s.dpb_slot_active.iter().any(|&a| a)); + let needs_reset = self.session.as_ref().expect("active session").needs_reset; unsafe { self.video_queue_fn .cmd_begin_video_coding(command_buffer, record.begin_info()); - // On the first use of the session, all DPB slots must be reset. - if first_use { + if needs_reset { let control = vk::VideoCodingControlInfoKHR::default() .flags(vk::VideoCodingControlFlagsKHR::RESET); self.video_queue_fn @@ -757,7 +758,7 @@ impl DecoderCommon { let output_layout = self.output_layout(); let generation = self.generation; let session = self.session.as_mut().expect("a picture was just decoded"); - session.dpb_slot_active[slot as usize] = true; + session.dpb_slot_written[slot as usize] = true; let (image, image_view, layout, array_layer) = if session.coincide { let (image, layer) = session.dpb_image_for_slot(slot); (image, session.dpb_entry(slot).2, picture_layout, layer) diff --git a/src/decoder/driver.rs b/src/decoder/driver.rs index fbd48081..e3e782d1 100644 --- a/src/decoder/driver.rs +++ b/src/decoder/driver.rs @@ -12,7 +12,7 @@ use tracing::debug; use crate::decoder::common::DecoderCommon; use crate::decoder::frames::{DecodedPicture, ReorderBuffer}; use crate::decoder::{DecodeConfig, DecodeStatus, DecoderApi, FrameReceiver, Framing}; -use crate::error::Result; +use crate::error::{PixelForgeError, Result}; /// What a codec does; everything else is [`StreamDecoder`]'s. pub(crate) trait CodecDecoder: Send { @@ -26,6 +26,9 @@ pub(crate) trait CodecDecoder: Send { fn common(&self) -> &DecoderCommon; fn common_mut(&mut self) -> &mut DecoderCommon; + /// Forget picture and reference state, keeping stream parameters. + fn reset(&mut self); + /// How many leading bytes of `buffer` hold coded pictures known to be /// complete. /// @@ -94,6 +97,7 @@ pub(crate) struct StreamDecoder { /// that does not exist yet, so they are skipped and the caller is told to /// request a keyframe. This is the normal state when joining a stream. awaiting_keyframe: bool, + finished: bool, } impl StreamDecoder { @@ -105,6 +109,7 @@ impl StreamDecoder { carry: Vec::new(), carry_pts: 0, awaiting_keyframe: true, + finished: false, } } @@ -124,7 +129,8 @@ impl StreamDecoder { let split = C::complete_prefix(&buffer); let result = if split > 0 { - self.decode_frames(&buffer[..split], self.carry_pts) + let pts = if carried == 0 { pts } else { self.carry_pts }; + self.decode_frames(&buffer[..split], pts) } else { // Not enough bytes to complete a picture yet. Ok(DecodeStatus::Buffered) @@ -215,6 +221,25 @@ impl DecoderApi for StreamDecoder { } } + fn reset(&mut self) -> Result<()> { + if self.finished { + return Err(PixelForgeError::InvalidInput( + "decoder has already finished".to_string(), + )); + } + let common = self.codec.common_mut(); + common.pipeline.drain_submitted(); + self.reorder.discard(); + self.carry.clear(); + self.carry_pts = 0; + self.awaiting_keyframe = true; + if let Some(session) = &mut common.session { + session.needs_reset = true; + } + self.codec.reset(); + Ok(()) + } + /// Decode what is still buffered and emit the pictures held for /// reordering. Call at end of stream. fn finish(&mut self) -> Result<()> { @@ -227,6 +252,7 @@ impl DecoderApi for StreamDecoder { } let frames = self.reorder.flush(); self.codec.common_mut().finish_stream(frames); + self.finished = true; Ok(()) } diff --git a/src/decoder/frames.rs b/src/decoder/frames.rs index efb5f75d..907dc537 100644 --- a/src/decoder/frames.rs +++ b/src/decoder/frames.rs @@ -480,6 +480,15 @@ impl ReorderBuffer { Ok(Retained::Pool { index }) } + /// Release buffered pictures once their GPU work has completed. + pub fn discard(&mut self) { + for entry in self.buffered.drain(..) { + if let Retained::Pool { index } = entry.retained { + self.pool.images[index].state = PoolState::Free; + } + } + } + /// Emit every buffered picture in display order. Call at end of stream. pub fn flush(&mut self) -> Vec { self.drain_all() diff --git a/src/decoder/h264/dpb.rs b/src/decoder/h264/dpb.rs index e5c933c3..be05f5ca 100644 --- a/src/decoder/h264/dpb.rs +++ b/src/decoder/h264/dpb.rs @@ -105,6 +105,10 @@ impl DecodeDpb { } } + pub fn reset(&mut self) { + *self = Self::new(self.slot_count, self.pins.clone()); + } + /// Current reference pictures, in the order they were decoded. pub fn references(&self) -> &[RefPicture] { &self.refs @@ -837,6 +841,35 @@ mod tests { assert_eq!(dpb.try_allocate_slot().unwrap(), slot); } + #[test] + fn reset_restores_initial_dpb_and_keeps_pins() { + let sps = sps(1); + let pins = Arc::new(SlotPins::default()); + let mut dpb = DecodeDpb::new(4, pins.clone()); + let slot = decode_ref(&mut dpb, &sps, &header(15, 0)); + pins.pin(slot); + decode_ref( + &mut dpb, + &sps, + &header_with_mmco( + 1, + 0, + vec![Mmco::MaxLongTermIdx { + max_long_term_frame_idx_plus1: 2, + }], + ), + ); + assert_eq!(dpb.prev_frame_num_offset, 16); + assert_eq!(dpb.max_long_term_frame_idx, Some(1)); + + dpb.reset(); + assert!(dpb.references().is_empty()); + assert_eq!(dpb.prev_frame_num_offset, 0); + assert_eq!(dpb.prev_frame_num, 0); + assert_eq!(dpb.max_long_term_frame_idx, None); + assert_eq!(dpb.try_allocate_slot(), Some(1)); + } + #[test] fn test_idr_flushes_dpb() { let sps = sps(0); diff --git a/src/decoder/h264/session.rs b/src/decoder/h264/session.rs index 6e3fd882..4ae99a5b 100644 --- a/src/decoder/h264/session.rs +++ b/src/decoder/h264/session.rs @@ -554,6 +554,12 @@ impl CodecDecoder for H264Decoder { &mut self.common } + fn reset(&mut self) { + if let Some(dpb) = &mut self.dpb { + dpb.reset(); + } + } + fn complete_prefix(buffer: &[u8]) -> usize { super::complete_prefix(buffer) } diff --git a/src/decoder/h265/dpb.rs b/src/decoder/h265/dpb.rs index c5440965..7e069128 100644 --- a/src/decoder/h265/dpb.rs +++ b/src/decoder/h265/dpb.rs @@ -190,6 +190,10 @@ impl DecodeDpb { } } + pub fn reset(&mut self) { + *self = Self::new(self.slot_count, self.pins.clone()); + } + /// Current reference pictures, in the order they were decoded. /// /// After [`Self::begin_picture`] these are exactly the pictures in the diff --git a/src/decoder/h265/dpb_tests.rs b/src/decoder/h265/dpb_tests.rs index f330e665..3291950e 100644 --- a/src/decoder/h265/dpb_tests.rs +++ b/src/decoder/h265/dpb_tests.rs @@ -663,3 +663,43 @@ fn rejects_bad_input() { .is_err() ); } + +#[test] +fn reset_restores_cra_start_state_and_keeps_pins() { + let sps = sps(1); + let pins = Arc::new(SlotPins::default()); + let mut dpb = DecodeDpb::new(6, pins.clone()); + let (_, slot) = decode(&mut dpb, nal(NalType::IdrNLp, 0), &idr(), &sps); + pins.pin(slot); + for lsb in [6, 12, 2] { + decode( + &mut dpb, + nal(NalType::TrailR, 0), + &slice(lsb, &[(-6, true)], &[]), + &sps, + ); + } + assert_eq!(dpb.prev_tid0_poc, 18); + + dpb.reset(); + assert!(dpb.references().is_empty()); + assert_eq!(dpb.prev_tid0_poc, 0); + assert_eq!( + skip_reason(&mut dpb, nal(NalType::TrailR, 0), &slice(4, &[], &[]), &sps), + SkipReason::AwaitingIrap + ); + + let (cra, new_slot) = decode( + &mut dpb, + nal(NalType::Cra, 0), + &slice(4, &[(-2, false)], &[]), + &sps, + ); + assert_ne!(new_slot, slot); + assert!(cra.is_keyframe && cra.no_rasl_output_flag); + assert_eq!(cra.poc, 4); + assert_eq!( + skip_reason(&mut dpb, nal(NalType::RaslR, 0), &slice(3, &[], &[]), &sps), + SkipReason::RaslWithoutReferences + ); +} diff --git a/src/decoder/h265/session.rs b/src/decoder/h265/session.rs index db29a547..75d40d27 100644 --- a/src/decoder/h265/session.rs +++ b/src/decoder/h265/session.rs @@ -472,6 +472,13 @@ impl CodecDecoder for H265Decoder { &mut self.common } + fn reset(&mut self) { + if let Some(dpb) = &mut self.dpb { + dpb.reset(); + } + self.pending_eos = false; + } + fn complete_prefix(buffer: &[u8]) -> usize { super::complete_prefix(buffer) } diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 4dac751d..048376d1 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -432,6 +432,7 @@ pub(crate) type FrameReceiver = futures_channel::mpsc::UnboundedReceiver Result; + fn reset(&mut self) -> Result<()>; fn finish(&mut self) -> Result<()>; fn take_frame_receiver(&mut self) -> Option; fn picture_format(&self) -> Option; @@ -540,6 +541,11 @@ impl Decoder { self.sink.decode(data, pts) } + /// Declare a discontinuity. See [`DecodeSink::reset`]. + pub fn reset(&mut self) -> Result<()> { + self.sink.reset() + } + /// End the stream. See [`DecodeSink::finish`]. pub fn finish(&mut self) -> Result<()> { self.sink.finish() @@ -589,6 +595,25 @@ impl DecodeSink { self.inner.decode(data, pts) } + /// Declare a discontinuity, as when seeking to a new position in a file. + /// + /// Discards buffered input and frames held back for reordering. Start + /// feeding again at a keyframe; earlier pictures yield + /// [`DecodeStatus::NeedsKeyframe`]. Parameter sets and sequence headers + /// already seen are kept, and unchanged parameters reuse the session. + /// + /// Waits for submitted GPU work to complete. Frames already handed to + /// the completion thread are on the source when this returns, unless + /// already taken. Use distinct `pts` values to tell old and new frames + /// apart. Frames already taken remain valid. + /// + /// Returns [`PixelForgeError::InvalidInput`] after a successful + /// [`finish`](Self::finish). Errors from GPU work already submitted are + /// reported by the source. + pub fn reset(&mut self) -> Result<()> { + self.inner.reset() + } + /// End the stream: decode whatever is still buffered, emit the frames held /// back for reordering, and close the source. /// diff --git a/src/decoder/pipeline.rs b/src/decoder/pipeline.rs index 4cc78b78..33024d4c 100644 --- a/src/decoder/pipeline.rs +++ b/src/decoder/pipeline.rs @@ -201,6 +201,15 @@ impl DecodePipeline { self.slot_sync.wait_all_free(); } + /// Wait for submitted work and for the completion thread to send its frames. + pub(crate) fn drain_submitted(&mut self) { + // An error can leave a submission in the current slot before hand-off. + if self.last_fence.is_some() { + self.end_picture(Vec::new()); + } + self.wait_all_free(); + } + /// Ensure the current slot's staging buffer holds at least `size` bytes. /// /// Growing destroys the old buffer, so the slot must already be free (the diff --git a/src/lib.rs b/src/lib.rs index 4d4d3814..8a0fe679 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -180,6 +180,12 @@ //! the CPU (for a file, a test or software processing), copy it with //! [`YuvDownloader`]. //! +//! To seek, call [`Decoder::reset`] and feed again from a keyframe. The +//! decoder keeps the session for unchanged stream parameters and discards +//! buffered input and pictures still waiting for reordering. Frames already +//! sent to the source can still be read. Use the `pts` supplied with each +//! call to distinguish frames from before and after the seek. +//! //! ## RGB input //! //! An encoder takes RGB frames once it is told what they are, with a diff --git a/tests/decode_streams.rs b/tests/decode_streams.rs index b21078ad..c66354f9 100644 --- a/tests/decode_streams.rs +++ b/tests/decode_streams.rs @@ -207,3 +207,150 @@ fn h264_zerolatency_has_no_delay() -> Result<(), Box> { fn h264_novui_poc2_has_no_delay() -> Result<(), Box> { check_no_latency("novui_poc2.264") } + +/// Compare replay after a discontinuity with a decoder started at the seek +/// point. Priming and replay use different PTS so an old frame cannot pass. +fn check_reset( + codec: Codec, + priming: &[u8], + seek: &[u8], + expected_count: Option, +) -> Result<(), Box> { + common::init_logging(); + let context = VideoContextBuilder::new() + .app_name("pixelforge-decode-reset") + .require_decode(codec) + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .build()?; + let config = DecodeConfig::new(codec).with_byte_stream(); + let mut readback = pixelforge::YuvDownloader::new(&context)?; + let mut fresh = Decoder::new(context.clone(), config.clone())?; + let (mut sink, mut source) = Decoder::new(context, config)?.split(); + + const BEFORE: u64 = 10; + const AFTER: u64 = 20; + sink.decode(priming, BEFORE)?; + sink.reset()?; + let FramePoll::Frame(old) = source.try_next_frame()? else { + panic!("reset must leave submitted frames on the source"); + }; + let generation = old.generation(); + let old_pixels = readback.download(&old)?; + assert_eq!(old.pts(), BEFORE); + while let FramePoll::Frame(frame) = source.try_next_frame()? { + assert_eq!(frame.pts(), BEFORE); + } + + sink.decode(seek, AFTER)?; + fresh.decode(seek, AFTER)?; + sink.finish()?; + fresh.finish()?; + let mut count = 0; + while let Some(frame) = pollster::block_on(source.next_frame())? { + let expected = pollster::block_on(fresh.next_frame())?.expect("extra frame after reset"); + assert_eq!(frame.pts(), AFTER); + assert_eq!(frame.display_order(), expected.display_order()); + assert_eq!(frame.generation(), generation); + assert_eq!(frame.is_keyframe(), expected.is_keyframe()); + let actual = readback.download(&frame)?; + let expected = readback.download(&expected)?; + assert!( + actual.y == expected.y && actual.uv == expected.uv, + "frame {count}" + ); + count += 1; + } + assert!(pollster::block_on(fresh.next_frame())?.is_none()); + assert!(count > 0); + if let Some(expected) = expected_count { + assert_eq!(count, expected); + } + let still_held = readback.download(&old)?; + assert!(old_pixels.y == still_held.y && old_pixels.uv == still_held.uv); + assert!(matches!( + sink.reset(), + Err(pixelforge::PixelForgeError::InvalidInput(_)) + )); + Ok(()) +} + +macro_rules! reset_tests { + ($($test:ident: $codec:expr, $prime:literal, $seek:literal, $count:expr;)*) => { + $( + #[test] + #[ignore = "requires a Vulkan Video device"] + fn $test() -> Result<(), Box> { + check_reset( + $codec, + include_bytes!($prime), + include_bytes!($seek), + $count, + ) + } + )* + }; +} + +reset_tests! { + reset_h264_bframes: Codec::H264, "data/bframes.264", "data/bframes.264", None; + reset_h265_closedgop: Codec::H265, "data/closedgop.265", "data/closedgop.265", None; + reset_h265_cra: Codec::H265, "data/opengop.265", "data/cra_start.265", Some(6); + reset_h265_main10: Codec::H265, "data/main10.265", "data/main10.265", None; + reset_av1_altref: Codec::AV1, "data/av1/aom_altref.obu", "data/av1/aom_altref.obu", None; +} + +#[test] +#[ignore = "requires a Vulkan Video device"] +fn reset_h264_drops_reorder_and_keeps_parameter_sets() -> Result<(), Box> { + use pixelforge::decoder::DecodeStatus::*; + + common::init_logging(); + let units = h264_access_units(include_bytes!("data/bframes.264")); + // Skip SPS, PPS and SEI to feed just the IDR slice after the first decode. + let key = units[0] + .windows(4) + .position(|w| w[..3] == [0, 0, 1] && w[3] & 0x1f == 5) + .map(|i| &units[0][i..]) + .expect("first access unit has an IDR slice"); + for force_copy in [false, true] { + let mut builder = VideoContextBuilder::new() + .app_name("pixelforge-decode-reset") + .require_decode(Codec::H264) + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()); + if force_copy { + builder = builder.without_unified_image_layouts(); + } + let context = builder.build()?; + let mut decoder = Decoder::new(context, DecodeConfig::new(Codec::H264))?; + decoder.reset()?; + assert_eq!(decoder.decode(units[0], 0)?, Decoded); + decoder.reset()?; + assert!(matches!(decoder.try_next_frame()?, FramePoll::Pending)); + assert_eq!(decoder.decode(units[1], 0)?, NeedsKeyframe); + + // The fixture holds two frames for reordering. Vary the number of + // decodes before each reset, without waiting for the last submission. + let mut copied_images = std::collections::HashSet::new(); + for seek in 1..=300u64 { + let following = seek as usize % 8; + assert_eq!(decoder.decode(key, seek)?, Decoded); + for unit in &units[1..=following] { + assert_eq!(decoder.decode(unit, seek)?, Decoded); + } + decoder.reset()?; + let mut sent = 0; + while let FramePoll::Frame(frame) = decoder.try_next_frame()? { + assert_eq!(frame.pts(), seek); + assert_eq!(frame.generation(), 1); + if force_copy { + copied_images.insert(frame.image()); + } + sent += 1; + } + assert_eq!(sent, (1 + following).saturating_sub(2), "seek {seek}"); + } + // Eight pictures is the most any iteration can put in the pool. + assert!(copied_images.len() <= 8, "copy pool grew across resets"); + } + Ok(()) +} From a43b041cbb60f2f15400db33169abc9463894362 Mon Sep 17 00:00:00 2001 From: Tiisetso Mosala <65363519+trmosala@users.noreply.github.com> Date: Wed, 18 Nov 2026 06:58:36 +0200 Subject: [PATCH 2/2] feat(decoder): flush without closing the source --- README.md | 4 ++- src/decoder/driver.rs | 62 +++++++++++++++++++++++++++-------------- src/decoder/mod.rs | 21 ++++++++++++++ src/decoder/pipeline.rs | 25 ++++++++++++++++- src/lib.rs | 4 ++- tests/decode_streams.rs | 59 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 151 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 5956dc98..a4daa13d 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,9 @@ To seek, call [`Decoder::reset`] and feed again from a keyframe. The decoder keeps the session for unchanged stream parameters and discards buffered input and pictures still waiting for reordering. Frames already sent to the source can still be read. Use the `pts` supplied with each -call to distinguish frames from before and after the seek. +call to distinguish frames from before and after the seek. At the end of +a sequence, [`Decoder::flush`] delivers the remaining frames and keeps the +decoder ready for another keyframe. ### RGB input diff --git a/src/decoder/driver.rs b/src/decoder/driver.rs index e3e782d1..34b10c6c 100644 --- a/src/decoder/driver.rs +++ b/src/decoder/driver.rs @@ -209,6 +209,34 @@ impl StreamDecoder { } Ok(DecodeStatus::Buffered) } + + fn check_open(&self) -> Result<()> { + if self.finished { + return Err(PixelForgeError::InvalidInput( + "decoder has already finished".to_string(), + )); + } + Ok(()) + } + + fn reset_input(&mut self) { + self.carry.clear(); + self.carry_pts = 0; + self.awaiting_keyframe = true; + if let Some(session) = &mut self.codec.common_mut().session { + session.needs_reset = true; + } + self.codec.reset(); + } + + fn decode_tail(&mut self) -> Result<()> { + // End of stream is also a picture boundary in byte-stream framing. + if !self.carry.is_empty() { + let tail = std::mem::take(&mut self.carry); + self.decode_frames(&tail, self.carry_pts)?; + } + Ok(()) + } } impl DecoderApi for StreamDecoder { @@ -222,34 +250,26 @@ impl DecoderApi for StreamDecoder { } fn reset(&mut self) -> Result<()> { - if self.finished { - return Err(PixelForgeError::InvalidInput( - "decoder has already finished".to_string(), - )); - } - let common = self.codec.common_mut(); - common.pipeline.drain_submitted(); + self.check_open()?; + self.codec.common_mut().pipeline.drain_submitted(); self.reorder.discard(); - self.carry.clear(); - self.carry_pts = 0; - self.awaiting_keyframe = true; - if let Some(session) = &mut common.session { - session.needs_reset = true; - } - self.codec.reset(); + self.reset_input(); + Ok(()) + } + + fn flush(&mut self) -> Result<()> { + self.check_open()?; + self.decode_tail()?; + let frames = self.reorder.flush(); + self.codec.common_mut().pipeline.flush_stream(frames)?; + self.reset_input(); Ok(()) } /// Decode what is still buffered and emit the pictures held for /// reordering. Call at end of stream. fn finish(&mut self) -> Result<()> { - // In byte-stream framing the tail is a picture that was never seen to - // end because nothing followed it. End of stream is what ends it. - if !self.carry.is_empty() { - let tail = std::mem::take(&mut self.carry); - let pts = self.carry_pts; - self.decode_frames(&tail, pts)?; - } + self.decode_tail()?; let frames = self.reorder.flush(); self.codec.common_mut().finish_stream(frames); self.finished = true; diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 048376d1..fd1bbe4c 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -433,6 +433,7 @@ pub(crate) type FrameReceiver = futures_channel::mpsc::UnboundedReceiver Result; fn reset(&mut self) -> Result<()>; + fn flush(&mut self) -> Result<()>; fn finish(&mut self) -> Result<()>; fn take_frame_receiver(&mut self) -> Option; fn picture_format(&self) -> Option; @@ -546,6 +547,11 @@ impl Decoder { self.sink.reset() } + /// Deliver remaining frames. See [`DecodeSink::flush`]. + pub fn flush(&mut self) -> Result<()> { + self.sink.flush() + } + /// End the stream. See [`DecodeSink::finish`]. pub fn finish(&mut self) -> Result<()> { self.sink.finish() @@ -614,6 +620,21 @@ impl DecodeSink { self.inner.reset() } + /// End the current sequence and keep the source open for another one. + /// + /// Decodes the remaining input as at [`finish`](Self::finish), then emits + /// the pictures still held for reordering in display order. On success, + /// all frames have been sent to the source, so + /// [`DecodeSource::try_next_frame`] can drain them until + /// [`FramePoll::Pending`]. + /// + /// Feeding can resume at a keyframe, with the same retained parameters as + /// after [`reset`](Self::reset). Like that call, this fails after a + /// successful `finish`. It can also fail while decoding the buffered tail. + pub fn flush(&mut self) -> Result<()> { + self.inner.flush() + } + /// End the stream: decode whatever is still buffered, emit the frames held /// back for reordering, and close the source. /// diff --git a/src/decoder/pipeline.rs b/src/decoder/pipeline.rs index 33024d4c..35de2c1a 100644 --- a/src/decoder/pipeline.rs +++ b/src/decoder/pipeline.rs @@ -40,7 +40,7 @@ //! instead: past it, pictures are copied out rather than pinned. use std::sync::Arc; -use std::sync::mpsc::{Receiver, Sender}; +use std::sync::mpsc::{self, Receiver, Sender}; use std::thread::JoinHandle; use ash::vk::{self, Handle}; @@ -96,6 +96,8 @@ struct WorkItem { frames: Vec, /// End of stream: after this item's frames, the consumer sees `None`. end_of_stream: bool, + /// Acknowledge delivery to a caller waiting for a flush. + delivered: Option>, } /// Rotating set of [`DecodeSlot`]s, the timelines that order their submissions, @@ -381,10 +383,27 @@ impl DecodePipeline { fence, frames, end_of_stream: false, + delivered: None, }); self.current_slot = (self.current_slot + 1) % self.slots.len(); } + /// Send `frames` after earlier submissions and wait until they are sent. + pub(crate) fn flush_stream(&mut self, frames: Vec) -> Result<()> { + self.drain_submitted(); + let (delivered, completion) = mpsc::channel(); + self.send(WorkItem { + slot_index: None, + fence: None, + frames, + end_of_stream: false, + delivered: Some(delivered), + }); + completion.recv().map_err(|_| { + PixelForgeError::Driver("decode completion thread stopped during flush".to_string()) + }) + } + /// End the stream: deliver `frames`, then close the channel so the consumer /// sees the end after everything already in flight. pub(crate) fn finish_stream(&mut self, frames: Vec) { @@ -393,6 +412,7 @@ impl DecodePipeline { fence: None, frames, end_of_stream: true, + delivered: None, }); } @@ -479,5 +499,8 @@ fn run_completion_thread( if let Some(index) = work.slot_index { slot_sync.set_free(index); } + if let Some(delivered) = work.delivered { + let _ = delivered.send(()); + } } } diff --git a/src/lib.rs b/src/lib.rs index 8a0fe679..915a0549 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -184,7 +184,9 @@ //! decoder keeps the session for unchanged stream parameters and discards //! buffered input and pictures still waiting for reordering. Frames already //! sent to the source can still be read. Use the `pts` supplied with each -//! call to distinguish frames from before and after the seek. +//! call to distinguish frames from before and after the seek. At the end of +//! a sequence, [`Decoder::flush`] delivers the remaining frames and keeps the +//! decoder ready for another keyframe. //! //! ## RGB input //! diff --git a/tests/decode_streams.rs b/tests/decode_streams.rs index c66354f9..c79d6572 100644 --- a/tests/decode_streams.rs +++ b/tests/decode_streams.rs @@ -354,3 +354,62 @@ fn reset_h264_drops_reorder_and_keeps_parameter_sets() -> Result<(), Box Result<(), Box> { + use pixelforge::decoder::DecodeStatus; + + common::init_logging(); + let context = VideoContextBuilder::new() + .app_name("pixelforge-decode-flush") + .require_decode(Codec::H264) + .enable_validation(std::env::var("PIXELFORGE_VALIDATION").is_ok()) + .build()?; + let config = DecodeConfig::new(Codec::H264).with_byte_stream(); + let mut readback = pixelforge::YuvDownloader::new(&context)?; + let mut fresh = Decoder::new(context.clone(), config.clone())?; + let mut decoder = Decoder::new(context, config)?; + let stream = include_bytes!("data/bframes.264"); + let units = h264_access_units(stream); + fresh.decode(stream, 0)?; + fresh.finish()?; + let mut reference = Vec::new(); + while let Some(frame) = pollster::block_on(fresh.next_frame())? { + reference.push((frame.display_order(), readback.download(&frame)?)); + } + assert_eq!(reference.len(), units.len()); + + decoder.flush()?; + assert!(matches!(decoder.try_next_frame()?, FramePoll::Pending)); + for pts in [10, 20] { + if pts == 20 { + assert_eq!( + decoder.decode(&stream[units[0].len()..], 1)?, + DecodeStatus::NeedsKeyframe + ); + decoder.flush()?; + assert!(matches!(decoder.try_next_frame()?, FramePoll::Pending)); + } + decoder.decode(stream, pts)?; + decoder.flush()?; + for (order, expected) in &reference { + let FramePoll::Frame(frame) = decoder.try_next_frame()? else { + panic!("flushed frame {order} must be ready when flush returns"); + }; + assert_eq!(frame.pts(), pts); + assert_eq!(frame.display_order(), *order); + assert_eq!(frame.generation(), 1); + let actual = readback.download(&frame)?; + assert!(actual.y == expected.y && actual.uv == expected.uv); + } + assert!(matches!(decoder.try_next_frame()?, FramePoll::Pending)); + } + decoder.finish()?; + assert!(matches!( + decoder.flush(), + Err(pixelforge::PixelForgeError::InvalidInput(_)) + )); + assert!(pollster::block_on(decoder.next_frame())?.is_none()); + Ok(()) +}