From e1390b8a8694e2900470dc628bdb2297298b5201 Mon Sep 17 00:00:00 2001 From: Tiisetso Mosala <65363519+trmosala@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:17:21 +0200 Subject: [PATCH] fix(decoder): keep copied frame images alive with their frames --- src/decoder/common.rs | 19 +++- src/decoder/driver.rs | 10 +-- src/decoder/frames.rs | 204 ++++++++++++++---------------------------- 3 files changed, 87 insertions(+), 146 deletions(-) diff --git a/src/decoder/common.rs b/src/decoder/common.rs index 07d68de..b1624ea 100644 --- a/src/decoder/common.rs +++ b/src/decoder/common.rs @@ -35,7 +35,7 @@ use crate::video::{ }; use crate::vulkan::VideoContext; -/// One DPB image and everything that lives and dies with it. +/// One decoder image and everything that lives and dies with it. /// /// Held behind an `Arc` shared with every frame decoded into it, so a session /// rebuild frees only the images nothing is still holding. The rest outlive the @@ -44,13 +44,28 @@ use crate::vulkan::VideoContext; /// full stop, with no exception for a stream that changes resolution. pub(crate) struct DpbImage { context: VideoContext, - image: vk::Image, + pub(super) image: vk::Image, memory: vk::DeviceMemory, /// One view per slot backed by this image: all of them for a layered DPB, /// exactly one when each slot has an image of its own. views: Vec, } +impl DpbImage { + pub(super) fn without_views( + context: VideoContext, + image: vk::Image, + memory: vk::DeviceMemory, + ) -> Self { + Self { + context, + image, + memory, + views: Vec::new(), + } + } +} + impl std::fmt::Debug for DpbImage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // `VideoContext` is not Debug, and the handles are what matter anyway. diff --git a/src/decoder/driver.rs b/src/decoder/driver.rs index 34b10c6..898d01e 100644 --- a/src/decoder/driver.rs +++ b/src/decoder/driver.rs @@ -74,6 +74,7 @@ pub(crate) enum Decoded { /// Drives a [`CodecDecoder`]. pub(crate) struct StreamDecoder { + // Dropped first to drain in-flight copies before the reorder pool releases its images. codec: C, /// Display-order reordering, which holds pictures by pinning their DPB @@ -288,12 +289,3 @@ impl DecoderApi for StreamDecoder { .map(|s| s.picture_format) } } - -impl Drop for StreamDecoder { - fn drop(&mut self) { - // The reorder pool's images must be freed while the codec's - // `DecoderCommon` (and its device) is still alive. Fields drop after - // this, so the codec goes afterwards. - self.reorder.destroy(self.codec.common()); - } -} diff --git a/src/decoder/frames.rs b/src/decoder/frames.rs index 907dc53..4ea2e9a 100644 --- a/src/decoder/frames.rs +++ b/src/decoder/frames.rs @@ -46,10 +46,6 @@ pub(crate) struct DecodedPicture { pub film_grain: bool, } -/// An image in the frame pool, by index. -#[derive(Debug, Clone, Copy)] -pub(crate) struct PoolIndex(usize); - /// DPB slots reserved by frames the caller is still holding. /// /// A pinned slot is one the driver decoded into and the caller is now reading, @@ -123,36 +119,14 @@ impl SlotPins { } } -/// Pins released by frames the caller has dropped, waiting to be reclaimed. -/// -/// A [`DecodedFrame`] can be dropped on any thread, including while the decoder -/// is submitting the next picture, so releases land here and the decoder folds -/// them in the next time it needs storage. -#[derive(Debug, Default)] -pub(crate) struct ReleaseQueue { - released: Mutex>, -} - -impl ReleaseQueue { - fn push(&self, index: PoolIndex) { - self.released.lock().unwrap().push(index); - } - - fn take(&self) -> Vec { - std::mem::take(&mut *self.released.lock().unwrap()) - } -} - /// Keeps a [`DecodedFrame`]'s storage reserved for as long as the frame lives. /// /// Dropping the frame drops this, which returns the storage to the decoder. #[derive(Debug)] pub(crate) enum FramePin { - /// A copy in the frame pool. Released lazily: the decoder never waits on - /// pool images, it just allocates another one. Pool { - index: PoolIndex, - releases: Arc, + #[allow(dead_code)] + image: Arc, }, /// The DPB slot the picture was decoded into, handed out without a copy. /// Releases the slot on drop, and holds the image itself so that a session @@ -167,36 +141,22 @@ pub(crate) enum FramePin { impl Drop for FramePin { fn drop(&mut self) { - match self { - FramePin::Pool { index, releases } => releases.push(*index), - FramePin::DpbSlot { slot, pins, .. } => pins.release(*slot), + if let FramePin::DpbSlot { slot, pins, .. } = self { + pins.release(*slot); } } } -/// State of one image in the frame pool. -#[derive(Clone, Copy, PartialEq, Eq)] -enum PoolState { - /// Reusable. - Free, - /// Holds a decoded picture awaiting its turn in display order. - Buffered, - /// Handed to the caller; reserved until their [`DecodedFrame`] is dropped. - HandedOut, -} - /// One pooled image, sized to the picture it currently holds. /// /// No image view: pixelforge only ever writes to a pool image, and a consumer /// reading one needs a view built for how they intend to read it (a sampled /// multi-planar view needs a ycbcr conversion, a copy needs no view at all). struct PoolImage { - image: vk::Image, - memory: vk::DeviceMemory, + storage: Arc, coded_width: u32, coded_height: u32, format: vk::Format, - state: PoolState, } /// Images that decoded pictures are copied into so they outlive their DPB slot. @@ -211,29 +171,20 @@ struct PoolImage { /// are ordinary allocations rather than a hardware-bounded resource. struct FramePool { images: Vec, - releases: Arc, } impl FramePool { fn new() -> Self { - Self { - images: Vec::new(), - releases: Arc::new(ReleaseQueue::default()), - } - } - - /// Fold in the images released by dropped frames. - fn reclaim(&mut self) { - for PoolIndex(i) in self.releases.take() { - self.images[i].state = PoolState::Free; - } + Self { images: Vec::new() } } /// A free image matching `picture`'s geometry, creating or resizing one as /// needed. Resolution changes recreate a mismatched image. - fn acquire(&mut self, common: &DecoderCommon, picture: &DecodedPicture) -> Result { - self.reclaim(); - + fn acquire( + &mut self, + common: &DecoderCommon, + picture: &DecodedPicture, + ) -> Result> { let format = common .session .as_ref() @@ -241,13 +192,13 @@ impl FramePool { .expect("session active while decoding"); let matching = self.images.iter().position(|p| { - p.state == PoolState::Free + Arc::strong_count(&p.storage) == 1 && p.coded_width == picture.coded_width && p.coded_height == picture.coded_height && p.format == format }); if let Some(i) = matching { - return Ok(i); + return Ok(self.images[i].storage.clone()); } let (image, memory) = create_pool_image( @@ -257,61 +208,30 @@ impl FramePool { format, &common.picture_sharing_families(), )?; - let slot = PoolImage { + let storage = Arc::new(DpbImage::without_views( + common.context.clone(), image, memory, + )); + let slot = PoolImage { + storage: storage.clone(), coded_width: picture.coded_width, coded_height: picture.coded_height, format, - state: PoolState::Free, }; // Reuse a free-but-mismatched image if one exists, else grow the pool. - if let Some(i) = self.images.iter().position(|p| p.state == PoolState::Free) { - self.destroy_image(common, i); + if let Some(i) = self + .images + .iter() + .position(|p| Arc::strong_count(&p.storage) == 1) + { + common.pipeline.wait_all_free(); self.images[i] = slot; - Ok(i) } else { self.images.push(slot); - Ok(self.images.len() - 1) } - } - - /// Mark `index` as handed to the caller and mint its pin. - fn hand_out(&mut self, index: usize) -> FramePin { - self.images[index].state = PoolState::HandedOut; - FramePin::Pool { - index: PoolIndex(index), - releases: self.releases.clone(), - } - } - - fn destroy_image(&mut self, common: &DecoderCommon, i: usize) { - let p = &self.images[i]; - if p.image == vk::Image::null() { - return; - } - unsafe { - // The pool image may be a copy target of a submission in flight. - common.pipeline.wait_all_free(); - common.context.device().destroy_image(p.image, None); - common.context.device().free_memory(p.memory, None); - } - } - - /// Whether any frame handed to the caller is still alive. - fn has_live_frames(&mut self) -> bool { - self.reclaim(); - self.images.iter().any(|p| p.state == PoolState::HandedOut) - } - - /// Free every pool image. The caller must be done with handed-out frames. - fn destroy(&mut self, common: &DecoderCommon) { - for i in 0..self.images.len() { - self.destroy_image(common, i); - self.images[i].image = vk::Image::null(); - } - self.images.clear(); + Ok(storage) } } @@ -333,7 +253,7 @@ enum Retained { array_layer: u32, }, /// A private copy in the frame pool. - Pool { index: usize }, + Pool { image: Arc }, } /// A buffered picture, awaiting its turn in display order. @@ -473,20 +393,15 @@ impl ReorderBuffer { }); } - let index = self.pool.acquire(common, picture)?; - common.record_picture_copy(picture, self.pool.images[index].image)?; + let image = self.pool.acquire(common, picture)?; + common.record_picture_copy(picture, image.image)?; common.submit_copy()?; - self.pool.images[index].state = PoolState::Buffered; - Ok(Retained::Pool { index }) + Ok(Retained::Pool { image }) } /// Release buffered pictures once their GPU work has completed. pub fn discard(&mut self) { - for entry in self.buffered.drain(..) { - if let Retained::Pool { index } = entry.retained { - self.pool.images[index].state = PoolState::Free; - } - } + self.buffered.clear(); } /// Emit every buffered picture in display order. Call at end of stream. @@ -524,8 +439,8 @@ impl ReorderBuffer { layout, array_layer, } => (image, image_view, layout, array_layer, pin), - Retained::Pool { index } => ( - self.pool.images[index].image, + Retained::Pool { image } => ( + image.image, // Pool images carry no view (see PoolImage); a caller needing // one for GPU work creates it over `image` with the usage it // wants. @@ -533,7 +448,7 @@ impl ReorderBuffer { // copy_picture_to_image leaves the pool image in TRANSFER_DST. vk::ImageLayout::TRANSFER_DST_OPTIMAL, 0, - self.pool.hand_out(index), + FramePin::Pool { image }, ), }; @@ -557,23 +472,6 @@ impl ReorderBuffer { pin: Some(pin), } } - - /// Release everything this buffer holds. - /// - /// Drops the buffered pictures first, which releases the DPB slots they - /// pinned, then frees the pool images. Frames the caller still holds point - /// at these images, so this warns rather than silently leaving dangling - /// handles behind. - pub fn destroy(&mut self, common: &DecoderCommon) { - self.buffered.clear(); - if self.pool.has_live_frames() { - tracing::warn!( - "decoder dropped while decoded frames are still alive; their images \ - are now invalid. Drop every DecodedFrame before the Decoder." - ); - } - self.pool.destroy(common); - } } /// A plain device-local image for the reorder pool: a copy target, readback @@ -652,6 +550,42 @@ fn create_pool_image( mod tests { use super::*; + #[test] + #[ignore = "requires a Vulkan Video device"] + fn pooled_frame_outlives_decoder() -> Result<()> { + use crate::{Codec, DecodeConfig, Decoder, FramePoll, VideoContextBuilder, YuvDownloader}; + + let _ = tracing_subscriber::fmt().with_test_writer().try_init(); + let context = VideoContextBuilder::new() + .require_decode(Codec::H264) + .enable_validation(std::env::var_os("PIXELFORGE_VALIDATION").is_some()) + .build()?; + let mut downloader = YuvDownloader::new(&context)?; + let mut decoder = Decoder::new(context, DecodeConfig::new(Codec::H264).with_byte_stream())?; + decoder.decode(include_bytes!("../../tests/data/bframes.264"), 0)?; + decoder.flush()?; + let frame = loop { + let FramePoll::Frame(frame) = decoder.try_next_frame()? else { + panic!("expected a copied frame") + }; + if frame.layout() == vk::ImageLayout::TRANSFER_DST_OPTIMAL { + break frame; + } + }; + let Some(FramePin::Pool { image }) = &frame.pin else { + panic!("missing pool image owner") + }; + let allocation = Arc::downgrade(image); + let before = downloader.download(&frame)?; + drop(decoder); + assert_eq!(allocation.strong_count(), 1); + let after = downloader.download(&frame)?; + assert!(before.y == after.y && before.uv == after.uv); + drop(frame); + assert_eq!(allocation.strong_count(), 0); + Ok(()) + } + #[test] fn a_slot_pinned_twice_stays_pinned_until_both_release() { let pins = SlotPins::default();