Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,14 @@ pictures out instead of handing over its own. To get a frame's samples on
the CPU (for a file, a test or software processing), copy it with
[`YuvDownloader`].

To seek, call [`Decoder::reset`] and feed again from a keyframe. The
decoder keeps the session for unchanged stream parameters and discards
buffered input and pictures still waiting for reordering. Frames already
sent to the source can still be read. Use the `pts` supplied with each
call to distinguish frames from before and after the seek. At the end of
a sequence, [`Decoder::flush`] delivers the remaining frames and keeps the
decoder ready for another keyframe.

### RGB input

An encoder takes RGB frames once it is told what they are, with a
Expand Down
8 changes: 8 additions & 0 deletions src/decoder/av1/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,14 @@ impl Av1Parser {
Self::default()
}

/// Clear reference and partial-frame state while retaining the sequence.
pub fn reset(&mut self) {
*self = Self {
sequence: self.sequence.take(),
..Self::default()
};
}

/// The sequence header in effect, once one has been seen.
pub fn sequence_header(&self) -> Option<&SequenceHeader> {
self.sequence.as_ref()
Expand Down
15 changes: 15 additions & 0 deletions src/decoder/av1/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,15 @@ impl Av1Decoder {
});
}
Some(Av1Event::Frame(frame)) => {
// An intra-only picture can parse without references,
// but only a key frame can start decoding.
if self.common.session.as_ref().is_none_or(|s| s.needs_reset)
&& frame.header.frame_type != FrameType::Key
{
self.parser.discard_frame();
missing.get_or_insert_with(|| "no key frame decoded yet".to_string());
continue;
}
// The frame shown by the previous one has already been
// handed over; close off its pipeline slot first.
if let Decoded::Shown(_) = outcome {
Expand Down Expand Up @@ -459,6 +468,12 @@ impl CodecDecoder for Av1Decoder {
&mut self.common
}

fn reset(&mut self) {
self.parser.reset();
self.grain_applied.fill(false);
self.shown_since_key = 0;
}

fn complete_prefix(buffer: &[u8]) -> usize {
// An error here is input that is not low-overhead OBUs; decoding the
// buffer reports it.
Expand Down
33 changes: 33 additions & 0 deletions src/decoder/av1/stream_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -622,3 +622,36 @@ fn corrupt_streams_do_not_panic() {
}
}
}

#[test]
fn reset_keeps_sequence_header_and_forgets_references() {
let data = fixture("aom_altref");
let starts = temporal_unit_starts(data).unwrap();
let mut parser = Av1Parser::new();
let mut keyframe = None;
for obu in iter_obus(&data[..starts[1]]) {
let obu = obu.unwrap();
if let Some(Av1Event::Frame(_)) = parser.parse_obu(&obu).unwrap() {
keyframe = Some(obu);
parser.commit_frame(Some(0)).unwrap();
}
}
let keyframe = keyframe.unwrap();
assert!(parser.refs().dpb_slot_in_use(0));

parser.reset();
assert!(parser.sequence_header().is_some());
assert_eq!(*parser.refs(), Default::default());
assert!(iter_obus(&data[starts[1]..starts[2]]).any(|obu| {
matches!(
parser.parse_obu(&obu.unwrap()).unwrap(),
Some(Av1Event::MissingReferences(_))
)
}));
assert!(matches!(
parser.parse_obu(&keyframe).unwrap(),
Some(Av1Event::Frame(_))
));
parser.commit_frame(Some(1)).unwrap();
assert!(parser.refs().dpb_slot_in_use(1));
}
27 changes: 14 additions & 13 deletions src/decoder/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,10 @@ pub(crate) struct DecodeSession {
pub dpb: Vec<Arc<DpbImage>>,
/// Whether each slot has been written at least once (governs the
/// UNDEFINED-vs-DPB old layout in the pre-decode barrier).
pub dpb_slot_active: Vec<bool>,
dpb_slot_written: Vec<bool>,
/// Initialize the Vulkan DPB before the next decode. Image layouts survive
/// the coding reset, so they are tracked separately.
pub needs_reset: bool,
pub use_layered_dpb: bool,

/// True when the driver decodes into the DPB image directly; false when a
Expand Down Expand Up @@ -465,7 +468,8 @@ impl DecoderCommon {
chroma_format: plan.chroma_format,
dpb,
slot_pins: Arc::new(SlotPins::default()),
dpb_slot_active: vec![false; plan.slot_count],
dpb_slot_written: vec![false; plan.slot_count],
needs_reset: true,
use_layered_dpb: plan.use_layered_dpb,
coincide: plan.coincide,
spare_slots: plan.spare_slots,
Expand Down Expand Up @@ -590,7 +594,7 @@ impl DecoderCommon {
// afterwards.
let picture_layout = self.picture_layout();
let (dst_image, dst_layer) = session.dpb_image_for_slot(slot);
let old_layout = if session.dpb_slot_active[slot as usize] {
let old_layout = if session.dpb_slot_written[slot as usize] {
picture_layout
} else {
vk::ImageLayout::UNDEFINED
Expand Down Expand Up @@ -710,24 +714,21 @@ impl DecoderCommon {
/// decodes in decode order despite the CPU running ahead.
pub fn submit_decode(&mut self) -> Result<()> {
let queue = self.decode_queue;
self.pipeline.submit_decode(&self.context, queue)
self.pipeline.submit_decode(&self.context, queue)?;
self.session.as_mut().expect("active session").needs_reset = false;
Ok(())
}

/// Record `vkCmdBeginVideoCoding`, `vkCmdDecodeVideo` and
/// `vkCmdEndVideoCoding` from a linked `record`, resetting the session
/// first if this is its first decode.
/// first if this decode starts a new sequence after creation or reset.
pub fn record_coding<C: RecordCodec>(&self, record: &DecodeRecord<C>) {
let command_buffer = self.decode_command_buffer();
// A session's first decode sees no active slot.
let first_use = self
.session
.as_ref()
.is_none_or(|s| !s.dpb_slot_active.iter().any(|&a| a));
let needs_reset = self.session.as_ref().expect("active session").needs_reset;
unsafe {
self.video_queue_fn
.cmd_begin_video_coding(command_buffer, record.begin_info());
// On the first use of the session, all DPB slots must be reset.
if first_use {
if needs_reset {
let control = vk::VideoCodingControlInfoKHR::default()
.flags(vk::VideoCodingControlFlagsKHR::RESET);
self.video_queue_fn
Expand Down Expand Up @@ -757,7 +758,7 @@ impl DecoderCommon {
let output_layout = self.output_layout();
let generation = self.generation;
let session = self.session.as_mut().expect("a picture was just decoded");
session.dpb_slot_active[slot as usize] = true;
session.dpb_slot_written[slot as usize] = true;
let (image, image_view, layout, array_layer) = if session.coincide {
let (image, layer) = session.dpb_image_for_slot(slot);
(image, session.dpb_entry(slot).2, picture_layout, layer)
Expand Down
64 changes: 55 additions & 9 deletions src/decoder/driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use tracing::debug;
use crate::decoder::common::DecoderCommon;
use crate::decoder::frames::{DecodedPicture, ReorderBuffer};
use crate::decoder::{DecodeConfig, DecodeStatus, DecoderApi, FrameReceiver, Framing};
use crate::error::Result;
use crate::error::{PixelForgeError, Result};

/// What a codec does; everything else is [`StreamDecoder`]'s.
pub(crate) trait CodecDecoder: Send {
Expand All @@ -26,6 +26,9 @@ pub(crate) trait CodecDecoder: Send {
fn common(&self) -> &DecoderCommon;
fn common_mut(&mut self) -> &mut DecoderCommon;

/// Forget picture and reference state, keeping stream parameters.
fn reset(&mut self);

/// How many leading bytes of `buffer` hold coded pictures known to be
/// complete.
///
Expand Down Expand Up @@ -94,6 +97,7 @@ pub(crate) struct StreamDecoder<C: CodecDecoder> {
/// that does not exist yet, so they are skipped and the caller is told to
/// request a keyframe. This is the normal state when joining a stream.
awaiting_keyframe: bool,
finished: bool,
}

impl<C: CodecDecoder> StreamDecoder<C> {
Expand All @@ -105,6 +109,7 @@ impl<C: CodecDecoder> StreamDecoder<C> {
carry: Vec::new(),
carry_pts: 0,
awaiting_keyframe: true,
finished: false,
}
}

Expand All @@ -124,7 +129,8 @@ impl<C: CodecDecoder> StreamDecoder<C> {
let split = C::complete_prefix(&buffer);

let result = if split > 0 {
self.decode_frames(&buffer[..split], self.carry_pts)
let pts = if carried == 0 { pts } else { self.carry_pts };
self.decode_frames(&buffer[..split], pts)
} else {
// Not enough bytes to complete a picture yet.
Ok(DecodeStatus::Buffered)
Expand Down Expand Up @@ -203,6 +209,34 @@ impl<C: CodecDecoder> StreamDecoder<C> {
}
Ok(DecodeStatus::Buffered)
}

fn check_open(&self) -> Result<()> {
if self.finished {
return Err(PixelForgeError::InvalidInput(
"decoder has already finished".to_string(),
));
}
Ok(())
}

fn reset_input(&mut self) {
self.carry.clear();
self.carry_pts = 0;
self.awaiting_keyframe = true;
if let Some(session) = &mut self.codec.common_mut().session {
session.needs_reset = true;
}
self.codec.reset();
}

fn decode_tail(&mut self) -> Result<()> {
// End of stream is also a picture boundary in byte-stream framing.
if !self.carry.is_empty() {
let tail = std::mem::take(&mut self.carry);
self.decode_frames(&tail, self.carry_pts)?;
}
Ok(())
}
}

impl<C: CodecDecoder> DecoderApi for StreamDecoder<C> {
Expand All @@ -215,18 +249,30 @@ impl<C: CodecDecoder> DecoderApi for StreamDecoder<C> {
}
}

fn reset(&mut self) -> Result<()> {
self.check_open()?;
self.codec.common_mut().pipeline.drain_submitted();
self.reorder.discard();
self.reset_input();
Ok(())
}

fn flush(&mut self) -> Result<()> {
self.check_open()?;
self.decode_tail()?;
let frames = self.reorder.flush();
self.codec.common_mut().pipeline.flush_stream(frames)?;
self.reset_input();
Ok(())
}

/// Decode what is still buffered and emit the pictures held for
/// reordering. Call at end of stream.
fn finish(&mut self) -> Result<()> {
// In byte-stream framing the tail is a picture that was never seen to
// end because nothing followed it. End of stream is what ends it.
if !self.carry.is_empty() {
let tail = std::mem::take(&mut self.carry);
let pts = self.carry_pts;
self.decode_frames(&tail, pts)?;
}
self.decode_tail()?;
let frames = self.reorder.flush();
self.codec.common_mut().finish_stream(frames);
self.finished = true;
Ok(())
}

Expand Down
9 changes: 9 additions & 0 deletions src/decoder/frames.rs
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,15 @@ impl ReorderBuffer {
Ok(Retained::Pool { index })
}

/// Release buffered pictures once their GPU work has completed.
pub fn discard(&mut self) {
for entry in self.buffered.drain(..) {
if let Retained::Pool { index } = entry.retained {
self.pool.images[index].state = PoolState::Free;
}
}
}

/// Emit every buffered picture in display order. Call at end of stream.
pub fn flush(&mut self) -> Vec<DecodedFrame> {
self.drain_all()
Expand Down
33 changes: 33 additions & 0 deletions src/decoder/h264/dpb.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,10 @@ impl DecodeDpb {
}
}

pub fn reset(&mut self) {
*self = Self::new(self.slot_count, self.pins.clone());
}

/// Current reference pictures, in the order they were decoded.
pub fn references(&self) -> &[RefPicture] {
&self.refs
Expand Down Expand Up @@ -837,6 +841,35 @@ mod tests {
assert_eq!(dpb.try_allocate_slot().unwrap(), slot);
}

#[test]
fn reset_restores_initial_dpb_and_keeps_pins() {
let sps = sps(1);
let pins = Arc::new(SlotPins::default());
let mut dpb = DecodeDpb::new(4, pins.clone());
let slot = decode_ref(&mut dpb, &sps, &header(15, 0));
pins.pin(slot);
decode_ref(
&mut dpb,
&sps,
&header_with_mmco(
1,
0,
vec![Mmco::MaxLongTermIdx {
max_long_term_frame_idx_plus1: 2,
}],
),
);
assert_eq!(dpb.prev_frame_num_offset, 16);
assert_eq!(dpb.max_long_term_frame_idx, Some(1));

dpb.reset();
assert!(dpb.references().is_empty());
assert_eq!(dpb.prev_frame_num_offset, 0);
assert_eq!(dpb.prev_frame_num, 0);
assert_eq!(dpb.max_long_term_frame_idx, None);
assert_eq!(dpb.try_allocate_slot(), Some(1));
}

#[test]
fn test_idr_flushes_dpb() {
let sps = sps(0);
Expand Down
6 changes: 6 additions & 0 deletions src/decoder/h264/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,12 @@ impl CodecDecoder for H264Decoder {
&mut self.common
}

fn reset(&mut self) {
if let Some(dpb) = &mut self.dpb {
dpb.reset();
}
}

fn complete_prefix(buffer: &[u8]) -> usize {
super::complete_prefix(buffer)
}
Expand Down
4 changes: 4 additions & 0 deletions src/decoder/h265/dpb.rs
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,10 @@ impl DecodeDpb {
}
}

pub fn reset(&mut self) {
*self = Self::new(self.slot_count, self.pins.clone());
}

/// Current reference pictures, in the order they were decoded.
///
/// After [`Self::begin_picture`] these are exactly the pictures in the
Expand Down
Loading
Loading