diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e061349d..4f9a048e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,6 +131,33 @@ jobs: ERIKA_NATIVE_TARGET="${{ matrix.target }}" cargo clippy --locked -p erika --all-targets --all-features --no-deps + - name: Build wgpu_decode_png example + if: matrix.arch == 'arm64' + run: | + # Link against libclang_rt.osx to resolve __isPlatformVersionAtLeast, + # a symbol the sanitizer runtime needs but that may be weakly referenced. + # See commits acae10f and 3498b8e for context. + RT_DIR="$(dirname "$(find "$(dirname "$(xcrun -f clang)")/../lib/clang" -name libclang_rt.osx.a | head -1)")" + echo "clang_rt dir: $RT_DIR" + ERIKA_NATIVE_PROFILE="$PROFILE" \ + ERIKA_NATIVE_TARGET="${{ matrix.target }}" \ + RUSTFLAGS="-C link-arg=-L$RT_DIR -C link-arg=-lclang_rt.osx" \ + cargo build --locked -p wgpu_decode_png --release --target "${{ matrix.target }}" + - name: Package erika-capi bundle + if: matrix.arch == 'arm64' + run: >- + bash packaging/bundle.sh erika-capi-macos-arm64 + dist/erika-capi-macos-arm64.zip + "target/${{ matrix.target }}/release/liberika_capi.a" + - name: Upload test artifacts + if: matrix.arch == 'arm64' + uses: actions/upload-artifact@v4 + with: + name: dv-test-macos-arm64 + path: | + target/aarch64-apple-darwin/release/wgpu_decode_png + dist/erika-capi-macos-arm64.zip + if-no-files-found: error ios: name: iOS device staticlib diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b828ce1..cad77013 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,77 @@ ## Unreleased +### Renderer + +#### Dolby Vision and HDR tone mapping + +- Dolby Vision RPU mapping via libplacebo-style piecewise polynomial/MMR + reshaping, followed by the RPU nonlinear matrix, PQ linearization, and + LMS→RGB composite. Profile 5/8 VUI tags are forced to BT.2020/PQ. +- Profile 5 falls back to software decode on mobile backends (MediaCodec / + generic AvCodec) that cannot attach RPU side data to hardware frames; + VideoToolbox and D3D11VA keep hardware decode. +- Profile 7 FEL/MEL RPUs are no longer rejected outright. An RPU whose + `disable_residual_flag` asks for an enhancement layer still applies its + base-layer reshaping curves, color matrices, and L1 trims — dropping the + whole RPU threw those away and left the frame as plain HDR10. The + un-composable residual is reported through the new `Frame::dovi_el_status()` + and one throttled `dovi_el_not_composed` diagnostic per stream, matching + libplacebo's split between `nlq_active` and "consumers that have not bound an + enhancement layer must not look at these fields". +- Per-frame Dolby Vision L1 brightness (`min_pq`/`max_pq`/`avg_pq`) drives the + tone-map source peak and scene-average pivot; the static mastering-display + (L0) peak still decides output-mode negotiation and whether tone mapping / + the perceptual gamut LUT are enabled. +- Tone mapping switched to a libplacebo-style IPT-domain color map with + **BT.2390 EETF as the default** operator, plus spline, BT.2446 method A, + SMPTE ST 2094-10, Mobius, Reinhard, and clip. Black-point compensation uses + `target black = target peak / contrast_ratio` (auto 1000:1 for SDR). +- Perceptual gamut mapping via a CPU-generated 48×32×256 IPT 3D LUT + (following libplacebo's lattice and index mapping) when a tone-mapped HDR + source is compressed into a smaller gamut; generated on a background thread + so the first wide-gamut frame does not stall the render thread. Chroma + rolloff is currently a simplified dead-zone blend rather than libplacebo's + full per-hue boundary search. +- Scene-adaptive HDR10: software-decoded PQ frames without DoVi L1 get a + CPU-measured scene-average luminance (`luma_stats`) that drives the + tone-map pivot, mirroring mpv's `--hdr-compute-peak`. +- Metal EDR negotiation reads the presenting screen rather than + `NSScreen.mainScreen`, and Auto HDR is decided from that display's + capability. Callers must set `CAMetalLayer.delegate` (examples updated). +- Metal: the perceptual gamut LUT is only marked active when the texture + resolved for the frame actually matches its (source, target, target-peak) + key. A headroom or primaries change while a new LUT generates used to bind + the 1x1x1 placeholder with `gamut_lut_enabled` still set, rendering black + frames until generation finished. D3D11 now masks the uniform the same way + while its LUT is still generating. +- ST 2094-10 tone mapping picks its knee in the PQ domain, matching + libplacebo's internal rescale, and `ToneMapConfig::curve_param` now tunes + its knee adaptation (default 0.70, libplacebo's `param_def`). Previously the + parameter had no effect and the knee was selected in linear nits. +- The tone map, its black-point compensation, and the perceptual gamut LUT are + enabled from the static mastering-display (L0) peak rather than the + per-frame Dolby Vision L1 peak, so dark scenes no longer drop them for a + frame and then restore them. +- Metal EDR / extended-linear output linearizes subtitle and danmaku colors + before compositing into the linear drawable; previously they were written + gamma-encoded, making mid-tones and colored text too bright. +- HDR10 scene-luma measurement honors the frame's color range: limited (TV) + range luma planes are expanded over the legal code span like the shaders' + `expand_ycbcr_range`, instead of being measured as full range. The + measurement also distinguishes `yuv420p10le` (10-bit code in bits `[9:0]`) + from P010 (left-aligned in `[15:6]`); treating both as P010 measured + software Main10 HDR10 as near-black. +- The perceptual gamut LUT's I axis now spans the target's `[black, peak]` in + PQ codes (libplacebo's `gamut.min_luma`/`max_luma`) instead of + `[0, peak]`; generation and all three backends' samplers agree, and the + cache key includes the target black, so a `contrast_ratio` change + regenerates the LUT. SDR targets shift their gamut mapping by up to ~3.5% + of the I axis; HDR/EDR targets are unchanged (black = 0). +- A degenerate ST 2094-10 anchor set (duplicated/non-finite points) falls back + to the identity curve (Clip) instead of zero coefficients, which the shader + would have rendered as a black frame. + ## 0.1.8 - 2026-09-07 ### Compatibility diff --git a/crates/erika/src/core.rs b/crates/erika/src/core.rs index c8a26fe3..42a289bf 100644 --- a/crates/erika/src/core.rs +++ b/crates/erika/src/core.rs @@ -463,6 +463,10 @@ pub struct PlayerVideoFrame { pub media_time: Duration, pub late_by: Option, pub generation: u64, + /// Measured scene-average luminance (nits) attached by the presenter for + /// HDR10 software frames without Dolby Vision L1 metadata. `None` for + /// every other source; renderers fold it into the tone-map pivot. + pub scene_avg_nits: Option, } impl PlayerVideoFrame { @@ -481,6 +485,7 @@ impl PlayerVideoFrame { media_time, late_by, generation, + scene_avg_nits: None, }) } } diff --git a/crates/erika/src/ffmpeg.rs b/crates/erika/src/ffmpeg.rs index beaea6fa..7772e481 100644 --- a/crates/erika/src/ffmpeg.rs +++ b/crates/erika/src/ffmpeg.rs @@ -12,9 +12,10 @@ use std::time::Duration; use crate::core::{ColorPrimaries, FrameRate, TrackInfo, TrackKind, TransferFunction, VideoParams}; use crate::renderer::pipeline::{ - Chromaticity, ColorRange, ContentLightMetadata, HdrMetadata, MasteringDisplayMetadata, - MatrixCoefficients, + Chromaticity, ColorRange, ContentLightMetadata, DoviComponentCurve, DoviFramePq, + DoviSourceMetadata, HdrMetadata, MasteringDisplayMetadata, MatrixCoefficients, RgbMatrix, }; +use crate::renderer::pipeline::{DOVI_MAX_MMR_ORDER, DOVI_MAX_PIECES}; use crate::source::{ByteRange, MediaSource}; use crate::subtitle::{ AssTrackResources, DecodedSubtitleFrame, SubtitleBitmapPlane, SubtitleFontAttachment, @@ -544,6 +545,13 @@ impl CodecParameters<'_> { pub fn kind(self) -> Option { unsafe { track_kind((*self.ptr).codec_type) } } + + /// Container-signalled Dolby Vision configuration profile (`dvcC`/`dvvC` + /// carried as `AV_PKT_DATA_DOVI_CONF` codec parameters side data), used to + /// steer decode backend selection. + pub fn dolby_vision_profile(self) -> Option { + unsafe { codec_parameters_dolby_vision_profile(self.ptr) } + } } pub struct OwnedCodecParameters { @@ -555,6 +563,13 @@ pub struct OwnedCodecParameters { unsafe impl Send for OwnedCodecParameters {} impl OwnedCodecParameters { + /// Container-signalled Dolby Vision configuration profile (`dvcC`/`dvvC` + /// carried as `AV_PKT_DATA_DOVI_CONF` codec parameters side data), used to + /// steer decode backend selection. + pub fn dolby_vision_profile(&self) -> Option { + unsafe { codec_parameters_dolby_vision_profile(self.ptr) } + } + fn copy_from(parameters: CodecParameters<'_>) -> Result { let ptr = unsafe { sys::avcodec_parameters_alloc() }; if ptr.is_null() { @@ -2149,6 +2164,53 @@ impl Frame { } } + /// Zero-copy view of a software frame's luma plane. + /// + /// `yuv420p10le` and `p010le` both store two bytes per sample but pack + /// the 10-bit code differently: software Main10 (`yuv420p10le`) keeps the + /// code in bits `[9:0]`, while P010 left-aligns it in bits `[15:6]`. + /// Callers must respect [`LumaPlaneView::sample_layout`] — treating both + /// as P010 silently mis-measures software HDR10. + pub fn luma_plane_view(&self) -> Option> { + let width = self.width() as usize; + let height = self.height() as usize; + if width == 0 || height == 0 { + return None; + } + let format = self.raw_pixel_format(); + let (bytes_per_sample, layout) = if format == sys::AVPixelFormat_AV_PIX_FMT_YUV420P + || format == sys::AVPixelFormat_AV_PIX_FMT_NV12 + { + (1, LumaSampleLayout::U8) + } else if format == sys::AVPixelFormat_AV_PIX_FMT_YUV420P10LE { + (2, LumaSampleLayout::Packed10) + } else if format == sys::AVPixelFormat_AV_PIX_FMT_P010LE { + (2, LumaSampleLayout::P010) + } else { + return None; + }; + unsafe { + let frame = &*self.ptr; + let data = frame.data[0] as *const u8; + if data.is_null() { + return None; + } + let stride = usize::try_from(frame.linesize[0]).ok()?; + if stride < width.checked_mul(bytes_per_sample)? { + return None; + } + Some(LumaPlaneView { + data, + stride, + width, + height, + bytes_per_sample, + layout, + _frame: std::marker::PhantomData, + }) + } + } + pub fn is_videotoolbox(&self) -> bool { self.raw_pixel_format() == sys::AVPixelFormat_AV_PIX_FMT_VIDEOTOOLBOX } @@ -2386,6 +2448,61 @@ impl Frame { unsafe { frame_hdr_metadata(self.ptr) } } + /// Per-frame Dolby Vision RPU metadata parsed by the HEVC decoder + /// (`AV_FRAME_DATA_DOVI_METADATA`). Present on RPU-carrying streams such + /// as profiles 5, 7, and 8, on both software and hardware decoded frames + /// (FFmpeg parses the RPU on the CPU and attaches it regardless). + pub fn dovi_metadata(&self) -> Option { + unsafe { + frame_dovi_metadata_result(self.ptr) + .ok() + .map(|(metadata, _)| metadata) + } + } + + /// Reports the enhancement-layer residual an RPU asks for but this renderer + /// cannot compose (Profile 7 FEL/MEL). The base-layer mapping still applies + /// to such frames, so this is diagnostic signal rather than a rejection. + /// Returns `None` when the frame carries no usable RPU at all. + pub fn dovi_el_status(&self) -> Option { + unsafe { frame_dovi_metadata_result(self.ptr).ok().map(|(_, el)| el) } + } + + /// Returns `None` when this frame's Dolby Vision metadata can feed the + /// mapping path. Otherwise reports why it cannot. Frames without any RPU + /// side data report [`DoviRejectReason::MissingSideData`], which callers + /// must interpret against the stream's Dolby Vision profile: absent RPUs + /// are normal on Profile 8, but Profile 5 has no HDR10-compatible base + /// layer, so a missing RPU leaves the frame unmappable. + pub fn dovi_unavailable_reason(&self) -> Option { + match self.dovi_mapping_probe() { + Ok(_) => None, + Err(reason) => Some(reason), + } + } + + /// One-pass probe of this frame's RPU: `Ok` carries the enhancement-layer + /// status when the mapping path can consume the RPU, `Err` reports why it + /// cannot. Prefer this over calling [`Frame::dovi_unavailable_reason`] + /// and [`Frame::dovi_el_status`] back-to-back, which re-parses the side + /// data. + pub fn dovi_mapping_probe(&self) -> std::result::Result { + if self.ptr.is_null() { + return Err(DoviRejectReason::MissingSideData); + } + let has_side_data = unsafe { + !sys::av_frame_get_side_data( + self.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + ) + .is_null() + }; + if !has_side_data { + return Err(DoviRejectReason::MissingSideData); + } + unsafe { frame_dovi_metadata_result(self.ptr).map(|(_, el)| el) } + } + pub fn transfer_to_system_memory(&self) -> Result { let frame = Frame::alloc(self.time_base)?; check( @@ -2636,6 +2753,169 @@ pub struct Nv12Frame { pub chroma: Vec, } +/// How a luma plane stores each sample. 10-bit frames share two bytes per +/// sample but not the same bit packing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LumaSampleLayout { + /// 8-bit `yuv420p` / `nv12`. + U8, + /// 10-bit packed LE (`yuv420p10le`): code lives in bits `[9:0]`. + Packed10, + /// 10-bit P010 LE: code left-aligned in bits `[15:6]`. + P010, +} + +impl LumaSampleLayout { + /// Decode one little-endian sample into a 10-bit code in `[0, 1023]`. + /// Returns `None` for the 8-bit layout. + pub fn decode_10bit(self, sample: u16) -> Option { + match self { + Self::U8 => None, + Self::Packed10 => Some(sample & 0x03FF), + Self::P010 => Some(sample >> 6), + } + } +} + +/// Strided, zero-copy view of a software frame's luma plane (see +/// [`Frame::luma_plane_view`]). +#[derive(Clone, Copy)] +pub struct LumaPlaneView<'a> { + data: *const u8, + stride: usize, + width: usize, + height: usize, + bytes_per_sample: usize, + layout: LumaSampleLayout, + _frame: std::marker::PhantomData<&'a Frame>, +} + +impl<'a> LumaPlaneView<'a> { + pub fn width(&self) -> u32 { + self.width as u32 + } + + pub fn height(&self) -> u32 { + self.height as u32 + } + + /// 10-bit samples (two bytes per sample)? + pub fn is_10bit(&self) -> bool { + self.bytes_per_sample == 2 + } + + pub fn sample_layout(&self) -> LumaSampleLayout { + self.layout + } + + /// Test-only view over a packed luma buffer: `stride == width * + /// bytes_per_sample`. Tests that care about stride padding lay the rows + /// out manually and point `data` at row 0. + #[cfg(test)] + pub(crate) fn from_packed( + data: &'a [u8], + width: u32, + height: u32, + is_10bit: bool, + ) -> Option { + let layout = if is_10bit { + LumaSampleLayout::P010 + } else { + LumaSampleLayout::U8 + }; + Self::from_packed_with_layout(data, width, height, layout) + } + + /// Test-only view with an explicit sample layout. + #[cfg(test)] + pub(crate) fn from_packed_with_layout( + data: &'a [u8], + width: u32, + height: u32, + layout: LumaSampleLayout, + ) -> Option { + let width = width as usize; + let height = height as usize; + let bytes_per_sample = match layout { + LumaSampleLayout::U8 => 1, + LumaSampleLayout::Packed10 | LumaSampleLayout::P010 => 2, + }; + if width == 0 || height == 0 || data.len() < width * height * bytes_per_sample { + return None; + } + let stride = width * bytes_per_sample; + Some(Self::from_strided_with_layout( + data, + stride, + width as u32, + height as u32, + layout, + )) + } + + /// Test-only view over rows of `width` samples separated by `stride` + /// bytes (the decoder-alignment layout `Frame::luma_plane_view` exposes). + #[cfg(test)] + pub(crate) fn from_strided( + data: &'a [u8], + stride: usize, + width: u32, + height: u32, + is_10bit: bool, + ) -> Self { + let layout = if is_10bit { + LumaSampleLayout::P010 + } else { + LumaSampleLayout::U8 + }; + Self::from_strided_with_layout(data, stride, width, height, layout) + } + + /// Test-only view with an explicit sample layout. + #[cfg(test)] + pub(crate) fn from_strided_with_layout( + data: &'a [u8], + stride: usize, + width: u32, + height: u32, + layout: LumaSampleLayout, + ) -> Self { + assert!(data.len() >= stride * height as usize); + Self { + data: data.as_ptr(), + stride, + width: width as usize, + height: height as usize, + bytes_per_sample: match layout { + LumaSampleLayout::U8 => 1, + LumaSampleLayout::Packed10 | LumaSampleLayout::P010 => 2, + }, + layout, + _frame: std::marker::PhantomData, + } + } + + /// Row `index` of the luma plane: `width` samples of 1 or 2 bytes each. + /// `None` when the index is out of bounds. + pub fn row(&self, index: usize) -> Option<&'a [u8]> { + if index >= self.height { + return None; + } + let row_bytes = self.width.checked_mul(self.bytes_per_sample)?; + let offset = index.checked_mul(self.stride)?.checked_add(row_bytes)?; + let total = self.stride.checked_mul(self.height)?; + if offset > total { + return None; + } + // SAFETY: `data` points at a valid luma plane whose rows hold at + // least `row_bytes` bytes every `stride` bytes (validated when the + // view was built), and the frame outlives `'a`. + Some(unsafe { std::slice::from_raw_parts(self.data.add(index * self.stride), row_bytes) }) + } +} + +unsafe impl Send for LumaPlaneView<'_> {} + /// GPU upload format for a repacked planar frame. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PlanarPixelFormat { @@ -4316,6 +4596,401 @@ unsafe fn frame_hdr_metadata(frame: *const sys::AVFrame) -> Option Some(HdrMetadata::new(mastering_display, content_light)) } +/// Reads the container's Dolby Vision configuration record from codec +/// parameters side data (`AV_PKT_DATA_DOVI_CONF` in ffmpeg 8), returning the +/// Dolby Vision profile number. +unsafe fn codec_parameters_dolby_vision_profile( + parameters: *const sys::AVCodecParameters, +) -> Option { + if parameters.is_null() { + return None; + } + let side_data = unsafe { + sys::av_packet_side_data_get( + (*parameters).coded_side_data, + (*parameters).nb_coded_side_data, + sys::AVPacketSideDataType_AV_PKT_DATA_DOVI_CONF, + ) + }; + if side_data.is_null() { + return None; + } + let data = unsafe { (*side_data).data }; + if data.is_null() { + return None; + } + let size = unsafe { (*side_data).size }; + if usize::try_from(size).ok()? < mem::size_of::() { + return None; + } + let record = data as *const sys::AVDOVIDecoderConfigurationRecord; + Some(unsafe { (*record).dv_profile }) +} + +/// Why a decoded frame cannot feed the Dolby Vision mapping path. +/// +/// Reportable through [`Frame::dovi_unavailable_reason`] so a stream whose +/// RPUs are missing or rejected stays diagnosable instead of silently falling +/// back to the base layer — Profile 5 in particular has no HDR10-compatible +/// base layer, and an unmapped Profile 8 frame with a rejected RPU displays +/// with wrong colors. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DoviRejectReason { + /// The frame carries no RPU side data at all. Normal for profiles whose + /// base layer is HDR10-compatible (e.g. Profile 8); always wrong for + /// Profile 5. + MissingSideData, + /// Side data exists but failed structural validation (size, alignment, + /// sub-structure offsets). + MalformedSideData, + /// Base-layer bit depth outside FFmpeg's 8..=16 range. + UnsupportedBitDepth, + /// `coef_log2_denom` beyond FFmpeg's fixed-point range (above 32). + UnsupportedCoefDenom, + /// Reshaping curves invalid or absent (pivots, orders, unknown methods). + InvalidCurves, + /// Color matrices/offsets invalid or the RGB→LMS matrix is singular. + InvalidColorMetadata, +} + +impl DoviRejectReason { + pub fn label(self) -> &'static str { + match self { + Self::MissingSideData => "rpu side data missing", + Self::MalformedSideData => "rpu side data malformed", + Self::UnsupportedBitDepth => "base layer bit depth outside 8..=16", + Self::UnsupportedCoefDenom => "coef_log2_denom above 32", + Self::InvalidCurves => "invalid reshaping curves", + Self::InvalidColorMetadata => "invalid color metadata", + } + } +} + +/// Enhancement-layer information an RPU carries that this renderer does not +/// compose. Profile 7 keeps its enhancement layer in a second HEVC layer that +/// this renderer never decodes, so the NLQ residual these fields describe has +/// nothing to be added to. libplacebo draws the same line: it exposes +/// `nlq_active` and documents that "consumers that have not bound an +/// enhancement layer must not look at these fields", while still applying the +/// base-layer reshaping, color matrices, and trims. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct DoviElStatus { + /// `disable_residual_flag == 0`: the RPU expects its residual to be + /// composed with an enhancement layer (Profile 7 FEL/MEL). + pub residual_requested: bool, + /// The RPU carries a non-trivial NLQ definition for that residual. + pub nlq_nontrivial: bool, +} + +impl DoviElStatus { + /// Whether the RPU asked for anything this renderer cannot compose. + pub fn is_active(self) -> bool { + self.residual_requested || self.nlq_nontrivial + } +} + +/// Reads the decoder's parsed Dolby Vision RPU side data and converts it into +/// shader-ready floats, mirroring libplacebo's `pl_map_dovi_metadata`: pivots +/// are normalized by the base-layer bit depth and curve coefficients by +/// `2^-coef_log2_denom`. The second value reports any enhancement-layer +/// residual the RPU asked for but this renderer cannot compose; it never +/// rejects an otherwise valid RPU. +unsafe fn frame_dovi_metadata_result( + frame: *const sys::AVFrame, +) -> std::result::Result<(DoviSourceMetadata, DoviElStatus), DoviRejectReason> { + if frame.is_null() { + return Err(DoviRejectReason::MissingSideData); + } + let side_data = unsafe { + sys::av_frame_get_side_data(frame, sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA) + }; + if side_data.is_null() { + return Err(DoviRejectReason::MissingSideData); + } + let data = unsafe { (*side_data).data }; + if data.is_null() { + return Err(DoviRejectReason::MalformedSideData); + } + let size = usize::try_from(unsafe { (*side_data).size }) + .ok() + .ok_or(DoviRejectReason::MalformedSideData)?; + if size < mem::size_of::() { + return Err(DoviRejectReason::MalformedSideData); + } + if (data as usize) % mem::align_of::() != 0 { + return Err(DoviRejectReason::MalformedSideData); + } + // The sub-structures live behind byte offsets inside this side data buffer + // (`av_dovi_get_header` and friends are C inline helpers bindgen does not + // emit), so resolve them by the same pointer arithmetic here with bounds checks. + let metadata = unsafe { *data.cast::() }; + let data_address = data as usize; + let aligned = |offset: usize, alignment: usize| { + data_address + .checked_add(offset) + .is_some_and(|address| address % alignment == 0) + }; + let in_bounds = |offset: usize, structure: usize| { + offset.checked_add(structure).is_some_and(|end| end <= size) + }; + if !in_bounds( + metadata.header_offset, + mem::size_of::(), + ) || !in_bounds( + metadata.mapping_offset, + mem::size_of::(), + ) || !in_bounds( + metadata.color_offset, + mem::size_of::(), + ) || !aligned( + metadata.header_offset, + mem::align_of::(), + ) || !aligned( + metadata.mapping_offset, + mem::align_of::(), + ) || !aligned( + metadata.color_offset, + mem::align_of::(), + ) { + return Err(DoviRejectReason::MalformedSideData); + } + let header = unsafe { + &*data + .add(metadata.header_offset) + .cast::() + }; + let mapping = unsafe { + &*data + .add(metadata.mapping_offset) + .cast::() + }; + let color = unsafe { + &*data + .add(metadata.color_offset) + .cast::() + }; + + let bl_bit_depth = usize::from(header.bl_bit_depth); + let coef_denom = u32::from(header.coef_log2_denom); + // Validate bit depth and coefficient denominator before using them in + // shifts below; malformed side data must be rejected, never panic. + // FFmpeg uses coef_log2_denom up to 32 for float RPU coefficients. + if !(8..=16).contains(&bl_bit_depth) { + return Err(DoviRejectReason::UnsupportedBitDepth); + } + if coef_denom > 32 { + return Err(DoviRejectReason::UnsupportedCoefDenom); + } + // This renderer has no enhancement-layer input, so the NLQ residual below + // is never composed. That does not make the RPU unusable: the base-layer + // reshaping curves, color matrices, and L1 trims still apply, and dropping + // the whole RPU would throw them away for every Profile 7 FEL frame. + // libplacebo keeps the same separation (it exposes `nlq_active` but only + // composes it when an enhancement layer is bound). + // + // FFmpeg marks an absent NLQ with the AV_DOVI_NLQ_NONE sentinel (-1); + // method 0 with all-neutral parameters is likewise an identity mapping. + let nlq_nontrivial = match mapping.nlq_method_idc as i32 { + -1 => false, + 0 => mapping.nlq.iter().any(|params| { + params.nlq_offset != 0 + || params.linear_deadzone_slope != 0 + || params.linear_deadzone_threshold != 0 + || (params.vdr_in_max != 0 + && params.vdr_in_max != (1_u64 << u32::from(header.coef_log2_denom))) + }), + _ => true, + }; + let el = DoviElStatus { + residual_requested: header.disable_residual_flag == 0, + nlq_nontrivial, + }; + + let pivot_scale = 1.0_f32 / (((1_usize << bl_bit_depth) - 1) as f32); + let coefficient_scale = 2.0_f32.powi(-(coef_denom as i32)); + + let mut reshaping = [DoviComponentCurve::default(); 3]; + let mut has_curve = false; + for (component, curve) in reshaping.iter_mut().enumerate() { + let source = &mapping.curves[component]; + let num_pivots = usize::from(source.num_pivots); + if num_pivots == 0 { + continue; + } + has_curve = true; + if !(2..=DOVI_MAX_PIECES + 1).contains(&num_pivots) { + return Err(DoviRejectReason::InvalidCurves); + } + let max_pivot = ((1_usize << bl_bit_depth) - 1) as u16; + let mut previous = None; + for &pivot in source.pivots[..num_pivots].iter() { + if pivot > max_pivot || previous.is_some_and(|previous| pivot <= previous) { + return Err(DoviRejectReason::InvalidCurves); + } + previous = Some(pivot); + } + curve.num_pivots = num_pivots as u8; + for (slot, &pivot) in curve.pivots[..num_pivots] + .iter_mut() + .zip(source.pivots[..num_pivots].iter()) + { + *slot = pivot_scale * f32::from(pivot); + } + for segment in 0..num_pivots - 1 { + match source.mapping_idc[segment] { + sys::AVDOVIMappingMethod_AV_DOVI_MAPPING_MMR => { + let order = usize::from(source.mmr_order[segment]); + if order == 0 || order > DOVI_MAX_MMR_ORDER { + return Err(DoviRejectReason::InvalidCurves); + } + curve.mmr_orders[segment] = order as u8; + curve.mmr_constants[segment] = + coefficient_scale * source.mmr_constant[segment] as f32; + let destination = &mut curve.mmr_coeffs[segment][..order]; + for (order_index, coefficients) in destination.iter_mut().enumerate() { + for (index, coefficient) in coefficients.iter_mut().enumerate() { + *coefficient = coefficient_scale + * source.mmr_coef[segment][order_index][index] as f32; + } + } + } + sys::AVDOVIMappingMethod_AV_DOVI_MAPPING_POLYNOMIAL => { + let poly_order = usize::from(source.poly_order[segment]); + if !(1..=2).contains(&poly_order) { + return Err(DoviRejectReason::InvalidCurves); + } + let coefficients = &source.poly_coef[segment]; + for (order_index, slot) in curve.poly_coeffs[segment].iter_mut().enumerate() { + *slot = if order_index <= poly_order { + coefficient_scale * coefficients[order_index] as f32 + } else { + 0.0 + }; + } + } + _ => return Err(DoviRejectReason::InvalidCurves), + } + } + } + if !has_curve { + return Err(DoviRejectReason::InvalidCurves); + } + + let rational_matrix = |values: &[sys::AVRational; 9]| -> Option<[[f32; 3]; 3]> { + let mut matrix = [[0.0_f32; 3]; 3]; + for row in 0..3 { + for col in 0..3 { + let value = &values[row * 3 + col]; + if value.den == 0 { + return None; + } + let value = value.num as f32 / value.den as f32; + if !value.is_finite() { + return None; + } + matrix[row][col] = value; + } + } + Some(matrix) + }; + + let mut nonlinear_offset = [0.0_f32; 3]; + for (index, slot) in nonlinear_offset.iter_mut().enumerate() { + let value = &color.ycc_to_rgb_offset[index]; + if value.den == 0 { + return Err(DoviRejectReason::InvalidColorMetadata); + } + *slot = value.num as f32 / value.den as f32; + if !slot.is_finite() { + return Err(DoviRejectReason::InvalidColorMetadata); + } + } + + if color.source_min_pq > 4095 + || color.source_max_pq > 4095 + || (color.source_max_pq != 0 && color.source_min_pq > color.source_max_pq) + { + return Err(DoviRejectReason::InvalidColorMetadata); + } + + let nonlinear_matrix = + rational_matrix(&color.ycc_to_rgb_matrix).ok_or(DoviRejectReason::InvalidColorMetadata)?; + let rgb_to_lms = + rational_matrix(&color.rgb_to_lms_matrix).ok_or(DoviRejectReason::InvalidColorMetadata)?; + let determinant = rgb_to_lms[0][0] + * (rgb_to_lms[1][1] * rgb_to_lms[2][2] - rgb_to_lms[1][2] * rgb_to_lms[2][1]) + - rgb_to_lms[0][1] + * (rgb_to_lms[1][0] * rgb_to_lms[2][2] - rgb_to_lms[1][2] * rgb_to_lms[2][0]) + + rgb_to_lms[0][2] + * (rgb_to_lms[1][0] * rgb_to_lms[2][1] - rgb_to_lms[1][1] * rgb_to_lms[2][0]); + if !determinant.is_finite() || determinant == 0.0 { + return Err(DoviRejectReason::InvalidColorMetadata); + } + + // Level 1 per-frame brightness metadata lives in the DM extension blocks + // that the RPU decoder appends right after the color structure. `av_dovi_find_level` + // is exported by FFmpeg but performs an unchecked pointer walk, so validate + // the block region here like the sub-structures above. L1 is optional + // signal quality metadata: an invalid or absent block never rejects the RPU. + let l1 = unsafe { frame_dovi_level1(data, &metadata, size) }; + + Ok(( + DoviSourceMetadata { + reshaping, + nonlinear_matrix: RgbMatrix::new(nonlinear_matrix), + nonlinear_offset, + rgb_to_lms: RgbMatrix::new(rgb_to_lms), + source_min_pq: color.source_min_pq, + source_max_pq: color.source_max_pq, + l1, + }, + el, + )) +} + +/// Reads the dynamic DM level 1 block (per-frame min/max/avg luminance in +/// 12-bit PQ codes) from the validated ext-block region, matching +/// `av_dovi_get_ext`'s pointer arithmetic. +unsafe fn frame_dovi_level1( + data: *const u8, + metadata: &sys::AVDOVIMetadata, + size: usize, +) -> Option { + let count = usize::try_from(metadata.num_ext_blocks).ok()?; + if count == 0 { + return None; + } + let block_size = metadata.ext_block_size; + if block_size < mem::size_of::() { + return None; + } + let total = block_size.checked_mul(count)?; + let end = metadata.ext_block_offset.checked_add(total)?; + if end > size { + return None; + } + if (metadata.ext_block_offset as usize) % mem::align_of::() != 0 { + return None; + } + let ext = unsafe { data.add(metadata.ext_block_offset) }; + for index in 0..count { + let block = unsafe { &*ext.add(block_size * index).cast::() }; + if block.level != 1 { + continue; + } + let l1 = unsafe { block.__bindgen_anon_1.l1 }; + if l1.max_pq == 0 || (l1.min_pq != 0 && l1.min_pq > l1.max_pq) { + return None; + } + return Some(DoviFramePq { + min_pq: l1.min_pq, + max_pq: l1.max_pq, + avg_pq: l1.avg_pq, + }); + } + None +} + unsafe fn mastering_display_metadata( frame: *const sys::AVFrame, ) -> Option { @@ -5071,6 +5746,59 @@ mod tests { ); } + #[test] + fn playback_fixture_has_no_dolby_vision_profile() { + let path = std::env::var_os("ERIKA_PLAYBACK_FIXTURE") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| { + Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/playback/playback-fixture.mkv") + }); + let demuxer = Demuxer::open_path(&path).unwrap(); + let video = demuxer + .probe() + .tracks + .iter() + .find(|track| track.kind == TrackKind::Video) + .unwrap(); + + let parameters = demuxer.owned_codec_parameters(video.id as i32).unwrap(); + assert_eq!(parameters.dolby_vision_profile(), None); + } + + #[test] + fn dv_sample_reports_dolby_vision_profile() { + let Some(path) = std::env::var_os("ERIKA_DV_SAMPLE") else { + return; + }; + let demuxer = Demuxer::open_path(&path).unwrap(); + let video = demuxer + .probe() + .tracks + .iter() + .find(|track| track.kind == TrackKind::Video) + .expect("DV sample must contain a video stream"); + + let parameters = demuxer.owned_codec_parameters(video.id as i32).unwrap(); + assert_eq!(parameters.dolby_vision_profile(), Some(5)); + } + + #[test] + fn dv_profile_8_sample_reports_profile() { + let Some(path) = std::env::var_os("ERIKA_DV_PROFILE_8_SAMPLE") else { + return; + }; + let demuxer = Demuxer::open_path(&path).unwrap(); + let video = demuxer + .probe() + .tracks + .iter() + .find(|track| track.kind == TrackKind::Video) + .expect("DV Profile 8 sample must contain a video stream"); + + let parameters = demuxer.owned_codec_parameters(video.id as i32).unwrap(); + assert_eq!(parameters.dolby_vision_profile(), Some(8)); + } + #[test] fn real_ass_container_preserves_header_fonts_and_matroska_chunk_when_env_is_set() { let Ok(path) = std::env::var("ERIKA_ASS_SAMPLE") else { @@ -5308,6 +6036,469 @@ mod tests { sys::AVRational { num, den } } + #[test] + fn frame_reads_dovi_side_data() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { + let mut size = 0_usize; + let metadata = sys::av_dovi_metadata_alloc(&mut size); + assert!(!metadata.is_null()); + assert!(size >= mem::size_of::()); + let header = (metadata as *mut u8) + .add((*metadata).header_offset) + .cast::(); + let mapping = (metadata as *mut u8) + .add((*metadata).mapping_offset) + .cast::(); + let color = (metadata as *mut u8) + .add((*metadata).color_offset) + .cast::(); + + (*header).bl_bit_depth = 10; + (*header).el_bit_depth = 10; + (*header).coef_log2_denom = 13; + (*header).disable_residual_flag = 1; + + let curve = &mut (*mapping).curves[0]; + curve.num_pivots = 3; + curve.pivots[0] = 0; + curve.pivots[1] = 256; + curve.pivots[2] = 1023; + curve.mapping_idc[0] = sys::AVDOVIMappingMethod_AV_DOVI_MAPPING_POLYNOMIAL; + curve.poly_order[0] = 1; + curve.poly_coef[0][0] = 0; + curve.poly_coef[0][1] = 1 << 13; + curve.poly_coef[0][2] = 0; + curve.mapping_idc[1] = sys::AVDOVIMappingMethod_AV_DOVI_MAPPING_MMR; + curve.mmr_order[1] = 1; + curve.mmr_constant[1] = 1024; + curve.mmr_coef[1][0][0] = 4096; + curve.mmr_coef[1][0][6] = -8192; + + (*color).ycc_to_rgb_matrix = [ + rational(9575, 8192), + rational(0, 8192), + rational(14742, 8192), + rational(9575, 8192), + rational(1754, 8192), + rational(4383, 8192), + rational(9575, 8192), + rational(17372, 8192), + rational(0, 8192), + ]; + (*color).ycc_to_rgb_offset = [rational(1, 4), rational(2, 1), rational(2, 1)]; + (*color).rgb_to_lms_matrix = [ + rational(5845, 16384), + rational(9702, 16384), + rational(837, 16384), + rational(2568, 16384), + rational(12256, 16384), + rational(1561, 16384), + rational(0, 16384), + rational(679, 16384), + rational(15705, 16384), + ]; + (*color).source_min_pq = 62; + (*color).source_max_pq = 3079; + + let side_data = sys::av_frame_new_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + size, + ); + assert!(!side_data.is_null()); + ptr::copy_nonoverlapping(metadata.cast::(), (*side_data).data, size); + sys::av_free(metadata.cast()); + } + + let dovi = frame.dovi_metadata().unwrap(); + let luma = &dovi.reshaping[0]; + assert_eq!(luma.num_pivots, 3); + assert_close(luma.pivots[1], 256.0 / 1023.0); + assert_close(luma.pivots[2], 1.0); + assert_close(luma.poly_coeffs[0][1], 1.0); + assert_eq!(luma.mmr_orders[1], 1); + assert_close(luma.mmr_constants[1], 0.125); + assert_close(luma.mmr_coeffs[1][0][0], 0.5); + assert_close(luma.mmr_coeffs[1][0][6], -1.0); + // Chroma and luma curves without pivots stay empty. + assert_eq!(dovi.reshaping[1].num_pivots, 0); + assert_close(dovi.nonlinear_matrix.rows()[0][0], 9575.0 / 8192.0); + assert_close(dovi.nonlinear_offset[0], 0.25); + assert_close(dovi.rgb_to_lms.rows()[2][2], 15705.0 / 16384.0); + assert_eq!(dovi.source_min_pq, 62); + assert_eq!(dovi.source_max_pq, 3079); + + unsafe { + let side_data = sys::av_frame_get_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + ); + assert!(!side_data.is_null()); + let metadata = *(*side_data).data.cast::(); + let header = &mut *((*side_data) + .data + .add(metadata.header_offset) + .cast::()); + let mapping = &mut *((*side_data) + .data + .add(metadata.mapping_offset) + .cast::()); + mapping.curves[0].mmr_order[1] = 0; + assert_eq!(frame.dovi_metadata(), None); + header.disable_residual_flag = 0; + assert_eq!(frame.dovi_metadata(), None); + } + } + + #[test] + fn frame_rejects_truncated_or_invalid_dovi_side_data() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { + let side_data = sys::av_frame_new_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + 4, + ); + assert!(!side_data.is_null()); + } + assert_eq!(frame.dovi_metadata(), None); + } + + /// Attaches a known-good RPU: 10-bit base layer, `coef_log2_denom = 32` + /// with the identity polynomial scaled to match, no residual, identity + /// matrices, valid PQ range. + unsafe fn attach_valid_dovi_rpu(frame: &Frame) { + unsafe { + let mut size = 0_usize; + let metadata = sys::av_dovi_metadata_alloc(&mut size); + assert!(!metadata.is_null()); + let header = (metadata as *mut u8) + .add((*metadata).header_offset) + .cast::(); + let mapping = (metadata as *mut u8) + .add((*metadata).mapping_offset) + .cast::(); + let color = (metadata as *mut u8) + .add((*metadata).color_offset) + .cast::(); + + (*header).bl_bit_depth = 10; + (*header).el_bit_depth = 10; + (*header).coef_log2_denom = 32; + (*header).disable_residual_flag = 1; + + let curve = &mut (*mapping).curves[0]; + curve.num_pivots = 2; + curve.pivots[0] = 0; + curve.pivots[1] = 1023; + curve.mapping_idc[0] = sys::AVDOVIMappingMethod_AV_DOVI_MAPPING_POLYNOMIAL; + curve.poly_order[0] = 1; + curve.poly_coef[0][0] = 0; + curve.poly_coef[0][1] = 1_i64 << 32; + curve.poly_coef[0][2] = 0; + + (*color).ycc_to_rgb_matrix = [ + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + ]; + (*color).ycc_to_rgb_offset = [rational(0, 1), rational(0, 1), rational(0, 1)]; + (*color).rgb_to_lms_matrix = [ + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + ]; + (*color).source_min_pq = 0; + (*color).source_max_pq = 3079; + + let side_data = sys::av_frame_new_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + size, + ); + assert!(!side_data.is_null()); + ptr::copy_nonoverlapping(metadata.cast::(), (*side_data).data, size); + sys::av_free(metadata.cast()); + } + } + + unsafe fn mutate_dovi_rpu( + frame: &Frame, + mutate: impl FnOnce( + &mut sys::AVDOVIRpuDataHeader, + &mut sys::AVDOVIDataMapping, + &mut sys::AVDOVIColorMetadata, + ), + ) { + unsafe { + let side_data = sys::av_frame_get_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + ); + assert!(!side_data.is_null()); + let metadata = *(*side_data).data.cast::(); + let header = &mut *((*side_data) + .data + .add(metadata.header_offset) + .cast::()); + let mapping = &mut *((*side_data) + .data + .add(metadata.mapping_offset) + .cast::()); + let color = &mut *((*side_data) + .data + .add(metadata.color_offset) + .cast::()); + mutate(header, mapping, color); + } + } + + /// Writes a single dynamic DM level 1 ext block into the side data. + /// `av_dovi_metadata_alloc` reserves the full ext block array inside the + /// same allocation, so the first block lands at `ext_block_offset`. + unsafe fn with_dovi_l1(frame: &Frame, min_pq: u16, max_pq: u16, avg_pq: u16) { + unsafe { + let side_data = sys::av_frame_get_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + ); + assert!(!side_data.is_null()); + let metadata = &mut *(*side_data).data.cast::(); + let block = &mut *((*side_data) + .data + .add(metadata.ext_block_offset) + .cast::()); + block.level = 1; + block.__bindgen_anon_1.l1 = sys::AVDOVIDmLevel1 { + min_pq, + max_pq, + avg_pq, + }; + metadata.num_ext_blocks = 1; + } + } + + #[test] + fn dovi_l1_brightness_metadata_is_parsed() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { attach_valid_dovi_rpu(&frame) }; + assert_eq!(frame.dovi_metadata().unwrap().l1, None); + + unsafe { with_dovi_l1(&frame, 62, 2200, 1500) }; + let l1 = frame + .dovi_metadata() + .unwrap() + .l1 + .expect("level 1 block should be parsed"); + assert_eq!( + l1, + DoviFramePq { + min_pq: 62, + max_pq: 2200, + avg_pq: 1500 + } + ); + + // Inverted min/max and an all-zero block are treated as absent. + unsafe { with_dovi_l1(&frame, 2200, 100, 1500) }; + assert_eq!(frame.dovi_metadata().unwrap().l1, None); + unsafe { with_dovi_l1(&frame, 0, 0, 0) }; + assert_eq!(frame.dovi_metadata().unwrap().l1, None); + } + + #[test] + fn dovi_malformed_ext_region_skips_l1_but_keeps_rpu() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { + attach_valid_dovi_rpu(&frame); + let side_data = sys::av_frame_get_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + ); + assert!(!side_data.is_null()); + let metadata = &mut *(*side_data).data.cast::(); + metadata.ext_block_offset = usize::MAX; + metadata.ext_block_size = mem::size_of::(); + metadata.num_ext_blocks = 5; + } + let dovi = frame + .dovi_metadata() + .expect("RPU must remain usable when the ext region is malformed"); + assert_eq!(dovi.l1, None); + } + + #[test] + fn frame_reads_dovi_side_data_with_32bit_coef_denom() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { attach_valid_dovi_rpu(&frame) }; + + let dovi = frame + .dovi_metadata() + .expect("32-bit coef denominator should be accepted"); + let luma = &dovi.reshaping[0]; + assert_eq!(luma.num_pivots, 2); + assert_close(luma.poly_coeffs[0][1], 1.0); + + // Test with 31-bit denominator + unsafe { + mutate_dovi_rpu(&frame, |header, mapping, _| { + header.coef_log2_denom = 31; + mapping.curves[0].poly_coef[0][1] = 1_i64 << 31; + }); + } + let dovi31 = frame + .dovi_metadata() + .expect("31-bit coef denominator should be accepted"); + assert_close(dovi31.reshaping[0].poly_coeffs[0][1], 1.0); + + // Test with invalid 33-bit denominator (should be rejected) + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.coef_log2_denom = 33); + } + assert_eq!(frame.dovi_metadata(), None); + } + + #[test] + fn dovi_unavailable_reason_classifies_rejections() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + assert_eq!( + frame.dovi_unavailable_reason(), + Some(DoviRejectReason::MissingSideData) + ); + + unsafe { attach_valid_dovi_rpu(&frame) }; + assert_eq!(frame.dovi_unavailable_reason(), None); + + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.coef_log2_denom = 33); + } + assert_eq!( + frame.dovi_unavailable_reason(), + Some(DoviRejectReason::UnsupportedCoefDenom) + ); + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.coef_log2_denom = 32); + } + + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.disable_residual_flag = 0); + } + // A residual request is reported through `dovi_el_status`, not rejected: + // the base-layer mapping still applies (see + // `fel_residual_keeps_the_base_layer_mapping`). + assert_eq!(frame.dovi_unavailable_reason(), None); + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.disable_residual_flag = 1); + } + + unsafe { + mutate_dovi_rpu(&frame, |_, mapping, _| mapping.curves[0].pivots[1] = 0); + } + assert_eq!( + frame.dovi_unavailable_reason(), + Some(DoviRejectReason::InvalidCurves) + ); + unsafe { + mutate_dovi_rpu(&frame, |_, mapping, _| mapping.curves[0].pivots[1] = 1023); + } + + unsafe { + mutate_dovi_rpu(&frame, |_, _, color| { + color.rgb_to_lms_matrix = [ + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + ]; + }); + } + assert_eq!( + frame.dovi_unavailable_reason(), + Some(DoviRejectReason::InvalidColorMetadata) + ); + + // Restoring the matrix makes the RPU usable again, and the reason + // tracks the metadata rather than sticky stream state. + unsafe { + mutate_dovi_rpu(&frame, |_, _, color| { + color.rgb_to_lms_matrix = [ + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + ]; + }); + } + assert_eq!(frame.dovi_unavailable_reason(), None); + assert!(frame.dovi_metadata().is_some()); + } + + #[test] + fn fel_residual_keeps_the_base_layer_mapping() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { attach_valid_dovi_rpu(&frame) }; + + // Profile 7 FEL: the RPU asks for residual composition, which this + // renderer cannot do (no enhancement-layer input). The RPU itself is + // still valid, so the base-layer reshaping must survive; only the + // un-composable part is reported. + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.disable_residual_flag = 0); + } + assert_eq!(frame.dovi_unavailable_reason(), None); + let metadata = frame.dovi_metadata().expect("FEL RPU stays usable"); + assert_eq!(metadata.reshaping[0].num_pivots, 2); + assert_close(metadata.reshaping[0].pivots[1], 1.0); + let status = frame.dovi_el_status().expect("FEL status is reported"); + assert!(status.residual_requested); + assert!(!status.nlq_nontrivial); + assert!(status.is_active()); + + // A non-trivial NLQ definition is likewise reported, never rejected. + unsafe { + mutate_dovi_rpu(&frame, |_, mapping, _| mapping.nlq_method_idc = 1); + } + assert_eq!(frame.dovi_unavailable_reason(), None); + assert!(frame.dovi_metadata().is_some()); + let status = frame.dovi_el_status().expect("NLQ status is reported"); + assert!(status.nlq_nontrivial); + assert!(status.is_active()); + + // Profile 8 style RPUs (no residual, no NLQ) report an inactive status + // so callers do not diagnose them. + unsafe { + mutate_dovi_rpu(&frame, |header, mapping, _| { + header.disable_residual_flag = 1; + mapping.nlq_method_idc = -1; + }); + } + assert_eq!(frame.dovi_el_status(), Some(DoviElStatus::default())); + assert!(!frame.dovi_el_status().unwrap().is_active()); + } + #[test] fn frame_rate_normalizes_positive_rationals() { assert_eq!( diff --git a/crates/erika/src/lib.rs b/crates/erika/src/lib.rs index d37bdf56..4586f974 100644 --- a/crates/erika/src/lib.rs +++ b/crates/erika/src/lib.rs @@ -6,6 +6,7 @@ pub mod core; pub mod danmaku; pub mod debug_hud; pub mod ffmpeg; +pub(crate) mod luma_stats; #[cfg(target_env = "ohos")] pub mod ohos; pub mod overlay; diff --git a/crates/erika/src/luma_stats.rs b/crates/erika/src/luma_stats.rs new file mode 100644 index 00000000..b3f9e50e --- /dev/null +++ b/crates/erika/src/luma_stats.rs @@ -0,0 +1,403 @@ +//! Frame-luminance statistics feeding the tone map's scene-adaptive pivot. +//! +//! mpv/libplacebo measure per-frame average/peak brightness on the GPU +//! (`--hdr-compute-peak`) and feed it to the tone-map curve, which is what +//! makes HDR10 (static-metadata-only) content behave like Dolby Vision's +//! per-frame L1. This module computes the same signals on the CPU for +//! software-decoded frames, sampling a sparse grid of rows/columns so a 4K +//! frame costs well under a millisecond, then smooths them with the same +//! IIR filter libplacebo uses (τ = smoothing_period). +//! +//! Only the luma plane is needed: HDR10 software frames arrive as NV12 +//! (8-bit), `yuv420p10le` (10-bit packed LE, code in bits `[9:0]`), or P010 +//! (10-bit MSB-aligned in 16-bit LE). The view reports which packing it +//! carries; treating `yuv420p10le` as P010 would measure near-black. The +//! frame's color range decides the sample normalization — HDR10 streams are +//! normally limited (TV) range, so measuring them as full range would lift +//! black and compress the highlights before the PQ re-encode. + +/// How many rows/columns of the frame are sampled (stride = dim / SAMPLES). +const SAMPLES: usize = 48; +/// libplacebo `pl_peak_detect_default_params.smoothing_period` (frames). +const SMOOTHING_PERIOD: f32 = 20.0; + +const PQ_M1: f32 = 2610.0 / 4096.0 * 1.0 / 4.0; +const PQ_M2: f32 = 2523.0 / 4096.0 * 128.0; +const PQ_C1: f32 = 3424.0 / 4096.0; +const PQ_C2: f32 = 2413.0 / 4096.0 * 32.0; +const PQ_C3: f32 = 2392.0 / 4096.0 * 32.0; + +/// One frame's measured luminance in PQ code (12-bit-ish precision). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FrameLumaStats { + /// Mean luma in PQ code over the sampled grid. + pub avg_pq: f32, + /// Maximum sampled luma in PQ code. + pub max_pq: f32, +} + +/// Stateful IIR smoothing of the per-frame signals, mirroring libplacebo's +/// `update_peak_buf`: `state += coeff * (measured - state)` with +/// `coeff = 1 - exp(-1/smoothing_period)`. +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct LumaSmoother { + pub avg_pq: f32, + pub max_pq: f32, + /// 0 until the first frame is pushed. + initialized: bool, +} + +impl LumaSmoother { + pub fn new() -> Self { + Self::default() + } + + /// Fold a frame's measurements into the running estimate and return the + /// smoothed (avg, max) in PQ code. + pub fn push(&mut self, measured: FrameLumaStats) -> (f32, f32) { + let coeff = 1.0 - (-1.0 / SMOOTHING_PERIOD).exp(); + if !self.initialized { + self.avg_pq = measured.avg_pq; + self.max_pq = measured.max_pq; + self.initialized = true; + } else { + self.avg_pq += coeff * (measured.avg_pq - self.avg_pq); + self.max_pq += coeff * (measured.max_pq - self.max_pq); + } + (self.avg_pq, self.max_pq) + } + + pub fn reset(&mut self) { + *self = Self::default(); + } +} + +/// Convert a normalized luma sample in [0, 1] (after range expansion and +/// bit-depth normalization) to its PQ code. +fn pq_code_of_luma(luma: f32) -> f32 { + let p = luma.clamp(0.0, 1.0).powf(PQ_M1); + ((PQ_C1 + PQ_C2 * p) / (1.0 + PQ_C3 * p)).powf(PQ_M2) +} + +/// Convert a normalized PQ-encoded luma sample to a PQ code over 10 k nits. +/// The luma plane of an HDR10 frame holds PQ-encoded Y'; decode it to linear +/// nits first (PQ EOTF), then re-encode so the average is perceptual. +fn sample_code(normalized: f32) -> f32 { + let linear_nits = pq_eotf(normalized.clamp(0.0, 1.0)) * 10000.0; + pq_code_of_luma(linear_nits / 10000.0) +} + +/// Measure a strided luma plane in place (see +/// `crate::ffmpeg::Frame::luma_plane_view`): 8-bit samples, or 10-bit samples +/// in either the `yuv420p10le` packed layout or the P010 MSB-aligned layout, +/// sampled on the same sparse grid as the packed variants. `full_range` +/// selects the sample normalization: full-range planes normalize by the code +/// maximum, limited (TV) planes by the legal 16..235 / 64..940 span, exactly +/// like the shaders' `expand_ycbcr_range`. This is the presenter's per-frame +/// path — repacking the whole frame with `to_planar_frame` would copy +/// megabytes to sample a 48x48 grid. +pub fn measure_luma_plane_view( + view: &crate::ffmpeg::LumaPlaneView<'_>, + full_range: bool, +) -> Option { + let w = view.width() as usize; + let h = view.height() as usize; + if w == 0 || h == 0 { + return None; + } + let is_10bit = view.is_10bit(); + let sample_bytes = if is_10bit { 2 } else { 1 }; + let x_step = (w / SAMPLES).max(1); + let y_step = (h / SAMPLES).max(1); + let mut sum = 0.0_f64; + let mut max_code = 0.0_f32; + let mut count = 0_u64; + let mut row_index = 0usize; + while row_index < h { + let Some(row) = view.row(row_index) else { + break; + }; + let sample_count = row.len() / sample_bytes; + let mut x = 0usize; + while x < sample_count { + let sample = if is_10bit { + let raw = u16::from_le_bytes([row[x * 2], row[x * 2 + 1]]); + // yuv420p10le keeps the code in bits [9:0]; p010le left-aligns + // it in [15:6]. Treating both as P010 crushes software Main10. + view.sample_layout() + .decode_10bit(raw) + .expect("10-bit layout decodes a 10-bit code") as f32 + } else { + row[x] as f32 + }; + let normalized = expand_luma_sample(sample, is_10bit, full_range); + let code = sample_code(normalized); + sum += code as f64; + if code > max_code { + max_code = code; + } + count += 1; + x += x_step; + } + row_index += y_step; + } + if count == 0 { + return None; + } + Some(FrameLumaStats { + avg_pq: (sum / count as f64) as f32, + max_pq: max_code, + }) +} + +/// Normalize one luma code to [0, 1] in the encoded domain, applying the +/// limited-range expansion when the plane is TV range. +fn expand_luma_sample(sample: f32, is_10bit: bool, full_range: bool) -> f32 { + let (black, span, peak) = if is_10bit { + (64.0, 876.0, 1023.0) + } else { + (16.0, 219.0, 255.0) + }; + if full_range { + return (sample / peak).clamp(0.0, 1.0); + } + ((sample - black) / span).clamp(0.0, 1.0) +} + +fn pq_eotf(code: f32) -> f32 { + let p = code.clamp(0.0, 1.0).powf(1.0 / PQ_M2); + let num = (p - PQ_C1).max(0.0); + let den = (PQ_C2 - PQ_C3 * p).max(1e-9); + (num / den).powf(1.0 / PQ_M1) +} + +/// Convert a PQ code back to nits (for diagnostics and for feeding the +/// `tone_map_extra.y` scene-average slot, which expects nits). +pub fn pq_code_to_nits(code: f32) -> f32 { + 10000.0 * pq_eotf(code) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ffmpeg::LumaPlaneView; + + fn view(luma: &[u8], w: u32, h: u32, is_10bit: bool) -> LumaPlaneView<'_> { + LumaPlaneView::from_packed(luma, w, h, is_10bit).unwrap() + } + + #[test] + fn nv12_black_frame_measures_near_zero() { + let (w, h) = (1920_u32, 1080_u32); + let luma = vec![0_u8; (w * h) as usize]; + let stats = measure_luma_plane_view(&view(&luma, w, h, false), true).unwrap(); + assert!(pq_code_to_nits(stats.avg_pq) < 0.5, "avg {}", stats.avg_pq); + assert!(pq_code_to_nits(stats.max_pq) < 0.5, "max {}", stats.max_pq); + } + + #[test] + fn nv12_full_white_measures_ten_k_nits() { + let (w, h) = (1920_u32, 1080_u32); + let luma = vec![255_u8; (w * h) as usize]; + let stats = measure_luma_plane_view(&view(&luma, w, h, false), true).unwrap(); + assert!((pq_code_to_nits(stats.avg_pq) - 10000.0).abs() < 20.0); + assert!((pq_code_to_nits(stats.max_pq) - 10000.0).abs() < 20.0); + } + + #[test] + fn p010_white_measures_ten_k_nits() { + let (w, h) = (1920_u32, 1080_u32); + let mut luma = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + luma.extend_from_slice(&(1023_u16 << 6).to_le_bytes()); + } + let stats = measure_luma_plane_view(&view(&luma, w, h, true), true).unwrap(); + assert!((pq_code_to_nits(stats.avg_pq) - 10000.0).abs() < 20.0); + } + + #[test] + fn yuv420p10le_white_measures_ten_k_nits() { + // Software Main10 packs the 10-bit code in bits [9:0], not P010's + // [15:6]. Measuring a full-range white plane as P010 would yield + // sample 15 and a near-black scene average. + let (w, h) = (1920_u32, 1080_u32); + let mut luma = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + luma.extend_from_slice(&1023_u16.to_le_bytes()); + } + let packed = LumaPlaneView::from_packed_with_layout( + &luma, + w, + h, + crate::ffmpeg::LumaSampleLayout::Packed10, + ) + .unwrap(); + let stats = measure_luma_plane_view(&packed, true).unwrap(); + assert!( + (pq_code_to_nits(stats.avg_pq) - 10000.0).abs() < 20.0, + "avg {}", + stats.avg_pq + ); + } + + #[test] + fn yuv420p10le_limited_white_hits_ten_k_nits() { + // Limited-range white (940) must expand to the code peak, not be + // crushed by a mistaken >> 6. + let (w, h) = (64_u32, 64_u32); + let mut luma = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + luma.extend_from_slice(&940_u16.to_le_bytes()); + } + let packed = LumaPlaneView::from_packed_with_layout( + &luma, + w, + h, + crate::ffmpeg::LumaSampleLayout::Packed10, + ) + .unwrap(); + let stats = measure_luma_plane_view(&packed, false).unwrap(); + assert!( + (pq_code_to_nits(stats.avg_pq) - 10000.0).abs() < 20.0, + "avg {}", + stats.avg_pq + ); + } + + #[test] + fn strided_rows_measure_like_packed_rows() { + // A stride larger than the visible row (decoder alignment padding) + // must not change the measurement. + let (w, h) = (64_u32, 48_u32); + let row_bytes = w as usize; + let stride = row_bytes + 64; + let mut padded = vec![255_u8; stride * h as usize]; // padding bright + for y in 0..h as usize { + for x in 0..row_bytes { + padded[y * stride + x] = 16; + } + } + let strided_view = LumaPlaneView::from_strided(&padded, stride, w, h, false); + // Same visible samples, tightly packed. + let packed: Vec = (0..h as usize) + .flat_map(|y| padded[y * stride..y * stride + row_bytes].to_vec()) + .collect(); + let strided = measure_luma_plane_view(&strided_view, true).unwrap(); + let packed_stats = measure_luma_plane_view(&view(&packed, w, h, false), true).unwrap(); + assert!((strided.avg_pq - packed_stats.avg_pq).abs() < 1e-6); + assert!((strided.max_pq - packed_stats.max_pq).abs() < 1e-6); + assert!( + pq_code_to_nits(strided.max_pq) < 100.0, + "padding leaked: {}", + pq_code_to_nits(strided.max_pq) + ); + } + + #[test] + fn bright_highlights_raise_the_peak_but_not_the_mean() { + let (w, h) = (1920_u32, 1080_u32); + // Mostly dim (16/255) with a small bright region (255). + let mut luma = vec![16_u8; (w * h) as usize]; + for y in (h / 4)..(h / 2) { + for x in (w / 4)..(w / 2) { + luma[(y * w + x) as usize] = 255; + } + } + let stats = measure_luma_plane_view(&view(&luma, w, h, false), true).unwrap(); + let avg = pq_code_to_nits(stats.avg_pq); + let max = pq_code_to_nits(stats.max_pq); + assert!(avg < 100.0, "avg {avg}"); + assert!(max > 9000.0, "max {max}"); + } + + #[test] + fn limited_range_black_and_white_hit_the_code_ends() { + // HDR10 streams are normally limited (TV) range: 64 is the 10-bit PQ + // black code and 940 is the 10 000-nit code, so measuring them as full + // range would lift black and clip the highlights before the PQ + // re-encode. + let (w, h) = (1920_u32, 1080_u32); + let mut black = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + black.extend_from_slice(&(64_u16 << 6).to_le_bytes()); + } + let stats = measure_luma_plane_view(&view(&black, w, h, true), false).unwrap(); + assert!( + pq_code_to_nits(stats.avg_pq) < 0.5, + "avg {} -> {} nits", + stats.avg_pq, + pq_code_to_nits(stats.avg_pq) + ); + + let mut white = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + white.extend_from_slice(&(940_u16 << 6).to_le_bytes()); + } + let stats = measure_luma_plane_view(&view(&white, w, h, true), false).unwrap(); + assert!( + (pq_code_to_nits(stats.max_pq) - 10000.0).abs() < 20.0, + "limited-range white should reach 10 k nits: {}", + pq_code_to_nits(stats.max_pq) + ); + } + + #[test] + fn limited_range_mid_gray_matches_the_shader_expansion() { + // 10-bit limited-range 512 expands to (512 - 64) / 876, the same + // normalization `expand_ycbcr_range` applies in the video shaders. + let (w, h) = (16_u32, 16_u32); + let mut luma = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + luma.extend_from_slice(&(512_u16 << 6).to_le_bytes()); + } + let stats = measure_luma_plane_view(&view(&luma, w, h, true), false).unwrap(); + let expected = (512.0 - 64.0) / 876.0; + let code = sample_code(expected); + assert!((stats.avg_pq - code).abs() < 1e-5, "avg {}", stats.avg_pq); + // The full-range reading is visibly different, so the range matters. + let full = measure_luma_plane_view(&view(&luma, w, h, true), true).unwrap(); + assert!((full.avg_pq - stats.avg_pq).abs() > 1e-3); + } + + #[test] + fn smoother_converges_and_tracks_step() { + let mut smoother = LumaSmoother::new(); + let dim = FrameLumaStats { + avg_pq: 0.2, + max_pq: 0.5, + }; + // First sample initializes directly. + let (avg, max) = smoother.push(dim); + assert_eq!(avg, 0.2); + assert_eq!(max, 0.5); + // A sustained new level converges asymptotically (τ=20). + let bright = FrameLumaStats { + avg_pq: 0.6, + max_pq: 0.9, + }; + let mut last = 0.0_f32; + for _ in 0..300 { + let (avg, _) = smoother.push(bright); + last = avg; + } + assert!((last - 0.6).abs() < 0.01, "converged avg {last}"); + // A single bright frame only nudges the estimate a little. + let mut s2 = LumaSmoother::new(); + s2.push(dim); + s2.push(bright); + let (avg, max) = (s2.avg_pq, s2.max_pq); + assert!(avg < 0.25 && avg > 0.2, "avg {avg}"); + assert!(max < 0.55 && max > 0.5, "max {max}"); + } + + #[test] + fn reset_clears_state() { + let mut smoother = LumaSmoother::new(); + smoother.push(FrameLumaStats { + avg_pq: 0.8, + max_pq: 0.9, + }); + smoother.reset(); + assert_eq!(smoother, LumaSmoother::default()); + } +} diff --git a/crates/erika/src/playback.rs b/crates/erika/src/playback.rs index 313c5e06..b6f595ef 100644 --- a/crates/erika/src/playback.rs +++ b/crates/erika/src/playback.rs @@ -15,7 +15,8 @@ use crate::core::{ }; use crate::ffmpeg::{ self, AudioResampler, Decoder, DecoderBackend, DecoderConfig, DecoderOutputFrame, Demuxer, - Frame, OwnedCodecParameters, PcmAudioFrame, PcmFormat, StreamSelection, SubtitleDecoder, + DoviRejectReason, Frame, OwnedCodecParameters, PcmAudioFrame, PcmFormat, StreamSelection, + SubtitleDecoder, }; use crate::source::{self, source_from_uri_with_hint, source_from_uri_with_options}; use crate::subtitle::{ @@ -731,6 +732,8 @@ pub struct PlaybackSession { audio_seek_dropped_packets: usize, mediacodec_surface_disabled: bool, video_decoder_fallbacks: u64, + dolby_vision_profile: Option, + dovi_rpu_diagnostics: DoviRpuDiagnostics, video_decoder_events: VecDeque, queue_limits: PlaybackQueueLimits, buffer_recovery_audio: Duration, @@ -877,25 +880,29 @@ impl PlaybackSession { let mut video_decoder = None; let mut video_decoder_fallbacks = 0u64; let mut video_decoder_events = VecDeque::new(); + let mut dolby_vision_profile = None; let mut selected_streams = Vec::new(); if let Some(stream_index) = selected_video_track { selected_streams.push(stream_index); let parameters = codec_parameters_for(&codec_parameters, stream_index)?; let codec = parameters.codec_name(); - let decoder_config = config.video_decode.decoder_config(); + let requested_config = config.video_decode.decoder_config(); + dolby_vision_profile = parameters.dolby_vision_profile(); + let (decoder_config, decode_fallback_reason) = + dolby_vision_decode_fallback(dolby_vision_profile, requested_config); video_decoder = Some( match open_video_decoder(parameters, decoder_config, &decoder_resources) { Ok(decoder) => { let event = VideoDecoderEvent { stage: video_decoder_open_stage(decoder_config).to_string(), - requested_backend: decoder_config.backend, + requested_backend: requested_config.backend, previous_backend: None, active_backend: decoder.backend(), fallback_count: 0, codec: codec.clone(), pixel_format: None, line_sizes: None, - reason: None, + reason: decode_fallback_reason, }; trace::diagnostic(event.structured_message()); video_decoder_events.push_back(event); @@ -1232,6 +1239,8 @@ impl PlaybackSession { audio_seek_dropped_packets: 0, mediacodec_surface_disabled, video_decoder_fallbacks, + dolby_vision_profile, + dovi_rpu_diagnostics: DoviRpuDiagnostics::default(), video_decoder_events, queue_limits, buffer_recovery_audio: buffer_recovery_audio_for_request(request), @@ -2756,6 +2765,8 @@ impl PlaybackSession { let status = drain_video_frames( self.video_decoder.as_mut().expect("video decoder exists"), &mut self.video_frames, + self.dolby_vision_profile, + &mut self.dovi_rpu_diagnostics, )?; let video_frame_limit = self.active_video_frame_queue_limit(); if demand != PlaybackPumpDemand::BufferingAudio { @@ -5767,12 +5778,20 @@ fn trace_clock_correction( fn drain_video_frames( decoder: &mut Decoder, frames: &mut VecDeque, + dolby_vision_profile: Option, + dovi_rpu_diagnostics: &mut DoviRpuDiagnostics, ) -> Result { let decode_backend = decoder.backend(); let mut status = DecoderDrainStatus::default(); loop { match decoder.receive_frame()? { DecoderOutputFrame::Frame(frame) => { + diagnose_dovi_frame( + dolby_vision_profile, + dovi_rpu_diagnostics, + &frame, + decode_backend, + ); frames.push_back(DecodedVideoFrame { frame, decode_backend, @@ -5788,6 +5807,99 @@ fn drain_video_frames( } } +/// Per-stream accounting for decoded frames whose Dolby Vision RPU cannot +/// feed the mapping path, and for frames whose RPU asks for an enhancement +/// layer this renderer never decodes. +#[derive(Debug, Default)] +struct DoviRpuDiagnostics { + unavailable_frames: u64, + el_frames: u64, +} + +/// Re-report interval after the first diagnostic, so a broken stream stays +/// visible over long playback without logging at frame rate. +const DOVI_RPU_DIAGNOSTIC_INTERVAL: u64 = 1024; + +impl DoviRpuDiagnostics { + fn record_unavailable_frame(&mut self) -> u64 { + self.unavailable_frames = self.unavailable_frames.saturating_add(1); + self.unavailable_frames + } + + fn record_el_frame(&mut self) -> u64 { + self.el_frames = self.el_frames.saturating_add(1); + self.el_frames + } + + fn should_report(count: u64) -> bool { + count == 1 || count % DOVI_RPU_DIAGNOSTIC_INTERVAL == 0 + } +} + +/// Decides whether a frame lacking usable RPU metadata warrants a diagnostic. +/// A missing RPU is only fatal on Profile 5, whose base layer is not +/// HDR10-compatible; other profiles either repeat RPUs across frames or fall +/// back to a base layer that displays correctly. A *present but rejected* +/// RPU always warrants one: the stream intended to carry Dolby Vision mapping +/// and the frame will silently display unmapped. +fn dovi_rpu_diagnostic_warranted(profile: Option, reason: DoviRejectReason) -> bool { + reason != DoviRejectReason::MissingSideData || profile == Some(5) +} + +fn diagnose_dovi_frame( + profile: Option, + diagnostics: &mut DoviRpuDiagnostics, + frame: &Frame, + decode_backend: DecoderBackend, +) { + // One parse: both the reject reason and the EL residual status come from + // the same `frame_dovi_metadata_result` result. + let el_status = match frame.dovi_mapping_probe() { + Ok(status) => status, + Err(reason) => { + if !dovi_rpu_diagnostic_warranted(profile, reason) { + return; + } + let count = diagnostics.record_unavailable_frame(); + if !DoviRpuDiagnostics::should_report(count) { + return; + } + trace::diagnostic( + serde_json::json!({ + "event": "dovi_rpu_unavailable", + "profile": profile, + "decodeBackend": decode_backend.as_str(), + "reason": reason.label(), + "framesAffected": count, + }) + .to_string(), + ); + return; + } + }; + // The RPU is usable, but it may ask for an enhancement layer this renderer + // never decodes (Profile 7 FEL/MEL). The base-layer mapping still applies, + // so report the gap instead of pretending the stream was composed. + if !el_status.is_active() { + return; + } + let count = diagnostics.record_el_frame(); + if !DoviRpuDiagnostics::should_report(count) { + return; + } + trace::diagnostic( + serde_json::json!({ + "event": "dovi_el_not_composed", + "profile": profile, + "decodeBackend": decode_backend.as_str(), + "residualRequested": el_status.residual_requested, + "nlqNontrivial": el_status.nlq_nontrivial, + "framesAffected": count, + }) + .to_string(), + ); +} + fn pop_matching_audio_frame( frames: &mut VecDeque, keep_frame: &mut impl FnMut(&mut PcmAudioFrame) -> bool, @@ -5936,6 +6048,30 @@ fn should_fallback_video_decoder_open_error(backend: DecoderBackend, codec: Opti && codec.is_some_and(|codec| codec.eq_ignore_ascii_case("av1"))) } +/// VideoToolbox and D3D11VA hardware decoders retain the Dolby Vision RPU +/// metadata side data on the hardware AVFrame, allowing GPU textures + RPU +/// reshaping via shaders without software decode. +/// +/// Backends without GPU shader RPU mapping (e.g. Android MediaCodec with +/// direct surface output, or OpenHarmony AVCodec) fall back to software +/// decoding for Profile 5 streams to preserve color mapping. +fn dolby_vision_decode_fallback( + profile: Option, + requested: DecoderConfig, +) -> (DecoderConfig, Option) { + let mobile_hardware_fallback = matches!( + requested.backend, + DecoderBackend::MediaCodec | DecoderBackend::AvCodec + ); + if !mobile_hardware_fallback || profile != Some(5) { + return (requested, None); + } + ( + DecoderConfig::software(), + Some("dolby vision profile 5 on mobile/embedded backend requires software decode for RPU mapping".to_string()), + ) +} + fn video_decoder_open_stage(config: DecoderConfig) -> &'static str { match (config.backend, config.mediacodec_surface) { (DecoderBackend::MediaCodec, true) => "open_surface", @@ -8509,4 +8645,113 @@ mod tests { assert_ne!(first.vsyncs, second.vsyncs); assert!(first.residual_error_nanos.signum() != second.residual_error_nanos.signum()); } + + #[test] + fn dolby_vision_profile_5_stays_on_hardware_for_videotoolbox_and_d3d11va() { + let vt_config = DecoderConfig { + backend: DecoderBackend::VideoToolbox, + mediacodec_surface: false, + }; + let (config, reason) = dolby_vision_decode_fallback(Some(5), vt_config); + assert_eq!(config.backend, DecoderBackend::VideoToolbox); + assert_eq!(reason, None); + + let d3d11_config = DecoderConfig { + backend: DecoderBackend::D3d11va, + mediacodec_surface: false, + }; + let (config, reason) = dolby_vision_decode_fallback(Some(5), d3d11_config); + assert_eq!(config.backend, DecoderBackend::D3d11va); + assert_eq!(reason, None); + } + + #[test] + fn dolby_vision_profile_5_falls_back_to_software_decode_on_mobile_backends() { + let avcodec_config = DecoderConfig { + backend: DecoderBackend::AvCodec, + mediacodec_surface: false, + }; + let (config, reason) = dolby_vision_decode_fallback(Some(5), avcodec_config); + assert_eq!(config.backend, DecoderBackend::Software); + assert!(reason.is_some()); + assert!(reason.unwrap().contains("profile 5")); + + let mediacodec_config = DecoderConfig { + backend: DecoderBackend::MediaCodec, + mediacodec_surface: true, + }; + let (config, reason) = dolby_vision_decode_fallback(Some(5), mediacodec_config); + assert_eq!(config.backend, DecoderBackend::Software); + assert!(reason.is_some()); + assert!(reason.unwrap().contains("profile 5")); + } + + #[test] + fn dolby_vision_profile_8_stays_on_hardware_decode() { + let hardware_config = DecoderConfig { + backend: DecoderBackend::VideoToolbox, + mediacodec_surface: false, + }; + let (config, reason) = dolby_vision_decode_fallback(Some(8), hardware_config); + + assert_eq!(config.backend, DecoderBackend::VideoToolbox); + assert_eq!(reason, None); + } + + #[test] + fn dolby_vision_software_decode_stays_software() { + let software_config = DecoderConfig::software(); + let (config, reason) = dolby_vision_decode_fallback(Some(5), software_config); + + assert_eq!(config.backend, DecoderBackend::Software); + assert_eq!(reason, None); + } + + #[test] + fn dovi_rpu_missing_is_diagnostic_only_on_profile_5() { + assert!(dovi_rpu_diagnostic_warranted( + Some(5), + DoviRejectReason::MissingSideData + )); + assert!(!dovi_rpu_diagnostic_warranted( + Some(8), + DoviRejectReason::MissingSideData + )); + assert!(!dovi_rpu_diagnostic_warranted( + None, + DoviRejectReason::MissingSideData + )); + assert!(dovi_rpu_diagnostic_warranted( + None, + DoviRejectReason::UnsupportedCoefDenom + )); + assert!(dovi_rpu_diagnostic_warranted( + Some(8), + DoviRejectReason::InvalidCurves + )); + } + + #[test] + fn dovi_rpu_diagnostics_report_first_then_spaced() { + assert!(DoviRpuDiagnostics::should_report(1)); + assert!(!DoviRpuDiagnostics::should_report(2)); + assert!(!DoviRpuDiagnostics::should_report( + DOVI_RPU_DIAGNOSTIC_INTERVAL - 1 + )); + assert!(DoviRpuDiagnostics::should_report( + DOVI_RPU_DIAGNOSTIC_INTERVAL + )); + assert!(!DoviRpuDiagnostics::should_report( + DOVI_RPU_DIAGNOSTIC_INTERVAL + 1 + )); + + let mut diagnostics = DoviRpuDiagnostics::default(); + assert_eq!(diagnostics.record_unavailable_frame(), 1); + assert_eq!(diagnostics.record_unavailable_frame(), 2); + // The enhancement-layer counter is independent, so a stream that both + // loses RPUs and carries FEL residual reports each at its own cadence. + assert_eq!(diagnostics.record_el_frame(), 1); + assert_eq!(diagnostics.record_unavailable_frame(), 3); + assert_eq!(diagnostics.record_el_frame(), 2); + } } diff --git a/crates/erika/src/presenter.rs b/crates/erika/src/presenter.rs index 1351ad49..3df6f489 100644 --- a/crates/erika/src/presenter.rs +++ b/crates/erika/src/presenter.rs @@ -42,6 +42,7 @@ use crate::danmaku::{ }; use crate::debug_hud::{DebugHud, DebugHudSnapshot}; use crate::ffmpeg::DecoderBackend; +use crate::luma_stats::LumaSmoother; #[cfg(target_env = "ohos")] use crate::ohos::ohaudio::{OHAudioOutput, OHAudioOutputConfig}; use crate::overlay::{OverlayFrame, OverlayTimeline, OverlayViewport}; @@ -306,6 +307,11 @@ pub struct PresenterRuntime { current_danmaku_prepared: Option, danmaku_plan_replacement_pending: bool, rejected_video_import_route: Option, + /// Rolling scene-average luminance (HDR10 without DoVi L1), injected + /// into each software frame's `scene_avg_nits` to drive the tone-map + /// pivot like Dolby Vision L1 would. + scene_luma_smoother: LumaSmoother, + last_scene_luma_generation: u64, current_media_time: Duration, current_generation: u64, current_surface_metrics: Option, @@ -358,6 +364,33 @@ fn should_reject_video_import( rejected == Some(candidate) } +/// Measure the scene-average luminance of an HDR10 (PQ, no Dolby Vision L1) +/// software-decoded frame and fold it into `smoother`, returning the smoothed +/// average in nits. Hardware frames (no CPU plane) and non-HDR10 sources keep +/// the previous estimate (`None` when nothing is known yet). The luma plane is +/// sampled in place with its native stride — no frame repack happens here. +fn measure_frame_scene_avg(frame: &PlayerVideoFrame, smoother: &mut LumaSmoother) -> Option { + use crate::core::TransferFunction; + use crate::luma_stats::{measure_luma_plane_view, pq_code_to_nits}; + use crate::renderer::pipeline::ColorRange; + // Hardware payloads have no CPU plane to sample. + let decoded = frame.frame.decoded_frame()?; + // Only PQ (HDR10) sources. When the RPU already carries L1 brightness, + // prefer that over measuring the base layer. + if decoded.transfer_function() != TransferFunction::Pq { + return None; + } + if decoded.dovi_metadata().and_then(|dovi| dovi.l1).is_some() { + return None; + } + let full_range = matches!(decoded.color_range(), ColorRange::Full); + let plane = decoded.luma_plane_view()?; + let measured = measure_luma_plane_view(&plane, full_range)?; + let (avg_pq, _max_pq) = smoother.push(measured); + let nits = pq_code_to_nits(avg_pq); + (nits.is_finite() && nits > 0.0).then_some(nits) +} + fn should_report_video_frame_backpressure(drop_count: u64) -> bool { drop_count == 1 || drop_count.is_power_of_two() } @@ -717,6 +750,8 @@ impl PresenterRuntime { current_danmaku_prepared: None, danmaku_plan_replacement_pending: false, rejected_video_import_route: None, + scene_luma_smoother: LumaSmoother::new(), + last_scene_luma_generation: 0, current_media_time: Duration::ZERO, current_generation: 1, current_surface_metrics: None, @@ -1943,10 +1978,21 @@ impl PresenterRuntime { break; } match self.video_frames.try_recv() { - Ok(frame) => { + Ok(mut frame) => { if frame.generation != self.player.playback_generation() { continue; } + if frame.generation != self.last_scene_luma_generation { + // New playback: start the smoothing state fresh so a + // seek cannot carry the old scene's brightness over. + self.scene_luma_smoother.reset(); + self.last_scene_luma_generation = frame.generation; + } + if let Some(scene_avg) = + measure_frame_scene_avg(&frame, &mut self.scene_luma_smoother) + { + frame.scene_avg_nits = Some(scene_avg); + } #[cfg(target_os = "android")] let mediacodec_surface = frame.frame.is_mediacodec(); #[cfg(not(target_os = "android"))] diff --git a/crates/erika/src/renderer.rs b/crates/erika/src/renderer.rs index 16977504..d0b2c4be 100644 --- a/crates/erika/src/renderer.rs +++ b/crates/erika/src/renderer.rs @@ -7,6 +7,8 @@ pub mod d3d11; #[cfg(target_os = "windows")] mod d3d11_artcnn; mod frame; +/// CPU-side perceptual gamut LUT generation (libplacebo-compatible). +pub mod gamut; pub mod metal; #[cfg(all(feature = "wgpu", target_env = "ohos"))] pub(crate) mod ohos_vulkan; diff --git a/crates/erika/src/renderer/d3d11.rs b/crates/erika/src/renderer/d3d11.rs index 166560bb..b51f55c2 100644 --- a/crates/erika/src/renderer/d3d11.rs +++ b/crates/erika/src/renderer/d3d11.rs @@ -19,9 +19,9 @@ use ::windows::Win32::Graphics::Direct3D11::{ D3D11_RENDER_TARGET_BLEND_DESC, D3D11_RESOURCE_MISC_SHARED, D3D11_SAMPLER_DESC, D3D11_SDK_VERSION, D3D11_SHADER_RESOURCE_VIEW_DESC, D3D11_SHADER_RESOURCE_VIEW_DESC_0, D3D11_SUBRESOURCE_DATA, D3D11_TEX2D_ARRAY_SRV, D3D11_TEX2D_SRV, D3D11_TEXTURE2D_DESC, - D3D11_USAGE_DEFAULT, D3D11_VIEWPORT, D3D11CreateDevice, ID3D11BlendState, ID3D11Buffer, - ID3D11Device, ID3D11DeviceContext, ID3D11InputLayout, ID3D11Multithread, ID3D11PixelShader, - ID3D11Query, ID3D11RenderTargetView, ID3D11Resource, ID3D11SamplerState, + D3D11_TEXTURE3D_DESC, D3D11_USAGE_DEFAULT, D3D11_VIEWPORT, D3D11CreateDevice, ID3D11BlendState, + ID3D11Buffer, ID3D11Device, ID3D11DeviceContext, ID3D11InputLayout, ID3D11Multithread, + ID3D11PixelShader, ID3D11Query, ID3D11RenderTargetView, ID3D11Resource, ID3D11SamplerState, ID3D11ShaderResourceView, ID3D11Texture2D, ID3D11VertexShader, }; use ::windows::Win32::Graphics::Dxgi::Common::{ @@ -49,9 +49,12 @@ use crate::core::{ use crate::danmaku::{ DanmakuAtlasUpdate, DanmakuGlyphAtlas, DanmakuGlyphInstance, DanmakuRenderPlan, }; -use crate::ffmpeg::Frame; +use crate::ffmpeg::{Frame, PlanarPixelFormat}; use crate::overlay::OverlayFrame; use crate::renderer::d3d11_artcnn::D3d11ArtCnn; +use crate::renderer::gamut::{ + GamutLut, GamutLutJob, GamutLutParams, LUT_SIZE_C, LUT_SIZE_H, LUT_SIZE_I, pack_rgba16f, +}; use crate::renderer::metal::{MetalRendererConfig, VideoAlphaMode}; use crate::renderer::output::{ ActiveOutputEncoding, OutputFallbackReason, OutputRuntimeStatus, OutputSurfaceFormat, @@ -76,6 +79,17 @@ struct VsOut { float2 texcoord : TEXCOORD0; }; +struct DoviUniforms { + float4 dovi_flags; + float4 dovi_pivots[6]; + float4 dovi_bounds[3]; + float4 dovi_coefficients[24]; + float4 dovi_mmr[144]; + float4 dovi_nonlinear_matrix[3]; + float4 dovi_nonlinear_offset; + float4 dovi_lms_matrix[3]; +}; + cbuffer VideoConstants : register(b0) { uint is_p010; uint full_range; @@ -88,6 +102,14 @@ cbuffer VideoConstants : register(b0) { float4 nits; float4 luma_coefficients; float4 gamut_matrix_rows[3]; + float4 ipt_matrix_rows[9]; + float4 tone_map_extra; + float4 tone_map_coeffs; + uint gamut_lut_enabled; + uint gamut_primaries; + uint gamut_reserved0; + uint gamut_reserved1; + DoviUniforms dovi; // xy scales native/packed luma coordinates; zw scales native chroma. // D3D11VA textures can be allocation-aligned beyond the visible frame. float4 texture_scales; @@ -95,6 +117,7 @@ cbuffer VideoConstants : register(b0) { Texture2D lumaTex : register(t0); Texture2D chromaTex : register(t1); +Texture3D gamutLut : register(t2); SamplerState videoSampler : register(s0); float source_peak_nits() { @@ -184,26 +207,24 @@ float3 source_reference_to_nits(float3 rgb) { return max(rgb, float3(0.0, 0.0, 0.0)) * source_reference_white_nits(); } -float3 tone_map_nits(float3 input_nits) { - float source_peak = source_peak_nits(); - float target_peak = target_peak_nits(); - float3 x = max(input_nits, float3(0.0, 0.0, 0.0)) / target_peak; - float white = max(source_peak / target_peak, 1.0); - if (tone_map == 1u) { - float white2 = white * white; - return target_peak * clamp((x * (float3(1.0, 1.0, 1.0) + x / white2)) / (float3(1.0, 1.0, 1.0) + x), float3(0.0, 0.0, 0.0), float3(1.0, 1.0, 1.0)); - } - if (tone_map == 2u) { - float knee = 0.75; - float denom = max(white - knee, 0.0001); - float3 knee3 = float3(knee, knee, knee); - float3 t = clamp((x - knee3) / denom, float3(0.0, 0.0, 0.0), float3(1.0, 1.0, 1.0)); - float3 shoulder = knee3 + (1.0 - knee) * (float3(1.0, 1.0, 1.0) - pow(float3(1.0, 1.0, 1.0) - t, float3(2.0, 2.0, 2.0))); - return target_peak * lerp(x, shoulder, step(knee3, x)); - } - return target_peak * clamp(x, float3(0.0, 0.0, 0.0), float3(1.0, 1.0, 1.0)); +float pq_code(float nits) { + return pq_inverse_eotf(clamp(nits, 0.0, 10000.0) / 10000.0); +} + +float nits_from_pq(float code) { + return 10000.0 * pq_eotf(clamp(code, 0.0, 1.0)); } +// libplacebo pl_smoothstep with arbitrary edge order (HLSL smoothstep has +// undefined results when edge0 >= edge1, and libplacebo's knee tuning term +// deliberately uses reversed edges). +float sstep(float edge0, float edge1, float x) { + float t = clamp((x - edge0) / (edge1 - edge0), 0.0, 1.0); + return t * t * (3.0 - 2.0 * t); +} + +// Simple primaries conversion (HDR10 output path); the tone-mapped path +// converts primaries inside the IPT roundtrip instead. float3 apply_gamut_map(float3 rgb) { return float3( dot(gamut_matrix_rows[0].xyz, rgb), @@ -212,8 +233,270 @@ float3 apply_gamut_map(float3 rgb) { ); } +// libplacebo st2094_pick_knee evaluated on absolute PQ codes. The source +// pivot follows the scene average luminance when known and stays within +// [10%, 80%] of the range; the destination pivot rescales it into the output +// range and then adapts towards the 1:1 line (knee_adaptation 0.4). +float2 st2094_pick_knee(float src_min, float src_max, float src_avg, float dst_min, float dst_max) { + const float knee_adaptation = 0.4; + const float min_knee = 0.1; + const float max_knee = 0.8; + const float def_knee = 0.4; + float src_knee_min = lerp(src_min, src_max, min_knee); + float src_knee_max = lerp(src_min, src_max, max_knee); + float dst_knee_min = lerp(dst_min, dst_max, min_knee); + float dst_knee_max = lerp(dst_min, dst_max, max_knee); + float fallback = lerp(src_min, src_max, def_knee); + float src_knee = clamp(src_avg > 0.0 ? src_avg : fallback, src_knee_min, src_knee_max); + float target = (src_knee - src_min) / max(src_max - src_min, 0.000001); + float adapted = lerp(dst_min, dst_max, target); + float tuning = 1.0 - sstep(max_knee, def_knee, target) * sstep(min_knee, def_knee, target); + float adaptation = lerp(knee_adaptation, 1.0, tuning); + float dst_knee = clamp(lerp(src_knee, adapted, adaptation), dst_knee_min, dst_knee_max); + return float2(src_knee, dst_knee); +} + +// The tone-map curve evaluated on the IPT intensity axis (PQ codes), +// mirroring libplacebo's tone-map functions. `param` is the per-operator +// curve parameter from ToneMapConfig::curve_param (0 = operator default). +float tone_map_curve_pq(float x_in, float param) { + float src_peak = source_peak_nits(); + float dst_peak = target_peak_nits(); + float src_avg = tone_map_extra.y; + float dst_black = tone_map_extra.z; + float in_min = 0.0; + float in_max = max(pq_code(src_peak), 0.000001); + float out_min = pq_code(dst_black); + float out_max = max(pq_code(dst_peak), 0.000001); + float out_range = max(out_max - out_min, 0.000001); + float x = clamp(x_in, in_min, in_max); + if (tone_map == 0u) { + // Clip: values within the source range pass through untouched. + return x; + } + if (tone_map == 1u) { + // Reinhard (output-relative, libplacebo pl_tone_map_reinhard). + // + // libplacebo evaluates this curve in the linear PL_HDR_NORM domain + // and only the PQ-domain operators (BT.2390, spline, ST 2094-10) + // work on encoded values. Running it on PQ codes bends the curve's + // meaning: against a 1000->203 nit target mid-tones darken badly, and + // the Mobius knee lands far below its intended position. Rescale to + // linear nits here and encode back at the end. + float in_max_nits = max(src_peak, 0.000001); + float out_min_nits = dst_black; + float out_range_nits = max(dst_peak - dst_black, 0.000001); + float peak = in_max_nits / out_range_nits; + float contrast = param > 0.0 ? param : 0.5; + float offset = (1.0 - contrast) / max(contrast, 0.000001); + float scale = (peak + offset) / peak; + float t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; + float mapped = t / (t + offset) * scale; + return pq_code(mapped * out_range_nits + out_min_nits); + } + if (tone_map == 2u) { + // Mobius: Mobius transform with a 1:1 linear region below the knee, + // also evaluated in linear nits (see the Reinhard note above). The + // knee j is relative to the output range, so the linear region ends + // at dst_black + j * (dst_peak - dst_black). + float in_max_nits = max(src_peak, 0.000001); + float out_min_nits = dst_black; + float out_range_nits = max(dst_peak - dst_black, 0.000001); + float peak = in_max_nits / out_range_nits; + float j = param > 0.0 ? param : 0.3; + float a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + float b = (j * j - 2.0 * j * peak + peak) / max(peak - 1.0, 0.000001); + float scale = (b * b + 2.0 * b * j + j * j) / (b - a); + float t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; + float mapped = t > j ? scale * (t + a) / (t + b) : t; + return pq_code(mapped * out_range_nits + out_min_nits); + } + if (tone_map == 3u) { + // ITU-R BT.2390 EETF with black-point compensation (the libplacebo + // version also compensates target black; the earlier port skipped it). + float knee_offset = param > 0.0 ? param : 1.0; + float max_lum = clamp(out_max / in_max, 0.0, 1.0); + float min_lum = out_min / in_max; + float ks = (1.0 + knee_offset) * max_lum - knee_offset; + float bp = min(max(1.0 / max(min_lum, 0.000001), 0.0), 4.0); + float u = x / in_max; + if (ks < 1.0 && u > ks) { + float tb = (u - ks) / (1.0 - ks); + float tb2 = tb * tb; + float tb3 = tb2 * tb; + u = (2.0 * tb3 - 3.0 * tb2 + 1.0) * ks + + (tb3 - 2.0 * tb2 + tb) * (1.0 - ks) + + (-2.0 * tb3 + 3.0 * tb2) * max_lum; + } + if (u < 1.0) { + u = u + min_lum * pow(1.0 - u, bp); + float gain = max_lum < 1.0 + ? 1.0 / (1.0 + min_lum / max_lum * pow(1.0 - max_lum, bp)) + : 1.0; + u = gain * (u - min_lum) + min_lum; + } + return u * in_max; + } + if (tone_map == 4u) { + // Spline: perceptually linear single-pivot polynomial, the default + // tone map of libplacebo and mpv's gpu-next renderer. + float contrast = param > 0.0 ? param : 0.3; + float fallback_avg = clamp(0.4 * src_peak, 100.0, 400.0); + float effective_src_avg = src_avg > 0.0 ? src_avg : fallback_avg; + float2 knee = st2094_pick_knee( + in_min, + in_max, + pq_code(effective_src_avg), + out_min, + out_max + ); + float src_pivot = knee.x; + float dst_pivot = knee.y; + float slope0 = (dst_pivot - out_min) / max(src_pivot - in_min, 0.000001); + float ratio = clamp(1.5 * (in_max / out_max - 1.0), 0.2, 1.2); + float slope = pow(slope0, (1.0 - contrast) * ratio); + float in_min0 = in_min - src_pivot; + float in_max0 = in_max - src_pivot; + float out_min0 = out_min - dst_pivot; + float out_max0 = out_max - dst_pivot; + float pa = (out_min0 - slope * in_min0) / (in_min0 * in_min0); + float qa = (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0 * in_max0); + float qb = -3.0 * (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0); + float xr = x - src_pivot; + float mapped = xr > 0.0 + ? ((qa * xr + qb) * xr + slope) * xr + : (pa * xr + slope) * xr; + return mapped + dst_pivot; + } + if (tone_map == 5u) { + // ITU-R BT.2446 method A: Weber-law log compression from the source + // peak envelope and a standardized S-curve (mpv's recommended curve + // for well-mastered content). + float phdr = 1.0 + 32.0 * pow(src_peak / 10000.0, 1.0 / 2.4); + float psdr = 1.0 + 32.0 * pow(dst_peak / 10000.0, 1.0 / 2.4); + float t = pow(nits_from_pq(x) / max(src_peak, 0.000001), 1.0 / 2.4); + t = log(1.0 + (phdr - 1.0) * t) / log(phdr); + if (t <= 0.7399) { + t = 1.0770 * t; + } else if (t < 0.9909) { + t = (-1.1510 * t + 2.7811) * t - 0.6302; + } else { + t = 0.5 * t + 0.5; + } + t = (pow(psdr, t) - 1.0) / (psdr - 1.0); + // BT.1886 EOTF from the target black point and peak. + float lb = pow(max(dst_black, 0.0), 1.0 / 2.4); + float lw = pow(max(dst_peak, 0.0), 1.0 / 2.4); + return pq_code(pow((lw - lb) * t + lb, 2.4)); + } + // SMPTE ST 2094-10 (DolbyVision's dynamic-metadata curve): rational + // Mobius interpolation in absolute nits; coefficients are solved per + // frame on the CPU from the same scene pivot. + float c1 = tone_map_coeffs.x; + float c2 = tone_map_coeffs.y; + float c3 = tone_map_coeffs.z; + float x_nits = nits_from_pq(x); + float y_nits = (c1 + c2 * x_nits) / max(1.0 + c3 * x_nits, 0.000001); + return pq_code(clamp(y_nits, 0.0, 10000.0)); +} + +float3 tone_map_nits(float3 input_nits) { + if (target_transfer == 3u) { + // HDR10 output: convert primaries by the gamut matrix and clamp to + // the PQ range (no tone mapping; the display does the HDR mapping). + return clamp(apply_gamut_map(max(input_nits, float3(0.0, 0.0, 0.0)) / source_reference_white_nits()) + * source_reference_white_nits(), float3(0.0, 0.0, 0.0), float3(10000.0, 10000.0, 10000.0)); + } + // libplacebo color map: RGB in source primaries (absolute nits) to + // HPE-LMS, PQ-encode, IPT, map the intensity axis and apply the + // hue-preserving chroma rule, then decode back to RGB in the target + // primaries. The primaries conversion happens inside this roundtrip. + float3 rgb = max(input_nits, float3(0.0, 0.0, 0.0)); + float3 lms = float3( + dot(ipt_matrix_rows[0].xyz, rgb), + dot(ipt_matrix_rows[1].xyz, rgb), + dot(ipt_matrix_rows[2].xyz, rgb) + ); + float3 lmspq = float3(pq_code(lms.r), pq_code(lms.g), pq_code(lms.b)); + float3 ipt = float3( + dot(float3(0.4, 0.4, 0.2), lmspq), + dot(float3(4.455, -4.851, 0.396), lmspq), + dot(float3(0.8056, 0.3572, -1.1628), lmspq) + ); + float i_orig = ipt.x; + ipt.x = tone_map_curve_pq(ipt.x, tone_map_extra.x); + // Libplacebo's chroma rule: clamp the saturation boost when brightening + // and desaturate (by the cubic hull term) when the mapping darkens. + float2 hull = float2(i_orig, ipt.x); + float2 hull_c = ((hull - float2(6.0, 6.0)) * hull + float2(9.0, 9.0)) * hull; + float ratio = min(i_orig / max(ipt.x, 0.000001), hull_c.y / max(hull_c.x, 0.000001)); + ipt.yz = ipt.yz * ratio; + if (gamut_lut_enabled != 0u) { + // I axis spans the target's [black, peak] in PQ codes, matching + // libplacebo's gamut.min_luma/max_luma (tone_map_extra.z is the + // target black in nits, the same value the LUT was generated for). + float lut_min = pq_code(tone_map_extra.z); + float lut_max = max(pq_code(target_peak_nits()), 0.000001); + float lut_range = max(lut_max - lut_min, 0.000001); + float3 pos = float3( + clamp((ipt.x - lut_min) / lut_range, 0.0, 1.0), + clamp(2.0 * length(ipt.yz), 0.0, 1.0), + 0.5 + 0.5 * atan2(ipt.z, ipt.y) / 3.14159265 + ); + // libplacebo's texel_scale: the lattice position must be remapped to + // the texel-center coordinate, otherwise the low end of the chroma + // axis (whose first texel stores zero chroma) leaks in and crushes + // saturation. + float3 idx = float3( + pos.x * (47.0 / 48.0) + 0.5 / 48.0, + pos.y * (31.0 / 32.0) + 0.5 / 32.0, + pos.z * (255.0 / 256.0) + 0.5 / 256.0 + ); + float3 sampled = gamutLut.Sample(videoSampler, idx).xyz; + ipt = float3(sampled.x, sampled.y - 0.5, sampled.z - 0.5); + } + float3 lmspq_out = float3( + dot(float3(1.0, 0.0975689, 0.205226), ipt), + dot(float3(1.0, -0.113876, 0.133217), ipt), + dot(float3(1.0, 0.0326151, -0.676887), ipt) + ); + float3 lms_out = float3( + nits_from_pq(lmspq_out.r), + nits_from_pq(lmspq_out.g), + nits_from_pq(lmspq_out.b) + ); + return float3( + dot(ipt_matrix_rows[6].xyz, lms_out), + dot(ipt_matrix_rows[7].xyz, lms_out), + dot(ipt_matrix_rows[8].xyz, lms_out) + ); +} + +// Hue-preserving gamut mapping: the linear gamut matrix can push highly +// saturated wide-gamut colors outside the target gamut (negative +// components). Blending those towards luma shifts hue - BT.2020 primary +// red picks up blue and turns pink. Instead blend towards the naive clip +// by an out-of-gamut smoothstep factor: slightly-out colors stay nearly +// intact, strongly-out primaries land on the pure target primary with +// their hue intact, matching mpv's perceptual gamut handling. Mirrors the +// WGSL/MSL gamut_compress and the Rust reference in pipeline.rs tests. +// Brightness overshoot (> 1) is left for the tone map. +float3 gamut_compress(float3 rgb) { + float lo = min(rgb.r, min(rgb.g, rgb.b)); + float outness = max(-lo, 0.0); + float k = smoothstep(0.0, 1.0, outness); + return lerp(rgb, clamp(rgb, 0.0, 1.0), k); +} + float3 target_nits_to_reference_linear(float3 input_nits) { - return max(input_nits, float3(0.0, 0.0, 0.0)) / target_reference_white_nits(); + // libplacebo's encode maps [target black, target peak] onto [0, 1] where + // 1.0 is the target reference white, so the tone-map black-point + // compensation lands back on true black instead of lifting it. + float black = tone_map_extra.z; + float peak = target_peak_nits(); + float range = max(peak - black, 0.0001); + return max(input_nits - float3(black, black, black), float3(0.0, 0.0, 0.0)) / range + * (range / target_reference_white_nits()); } float3 target_reference_linear_to_output(float3 rgb) { @@ -261,6 +544,12 @@ float4 final_output(float3 rgb, float alpha) { } void expand_ycbcr_range(float y_in, float2 cbcr_in, out float y, out float2 cbcr) { + if (is_p010 != 0u) { + // P010 stores 10-bit codes as code << 6 in a 16-bit UNORM texture. + const float p010_scale = 65535.0 / 65472.0; + y_in *= p010_scale; + cbcr_in *= p010_scale; + } if (full_range != 0u) { y = y_in; cbcr = cbcr_in - float2(0.5, 0.5); @@ -275,6 +564,83 @@ void expand_ycbcr_range(float y_in, float2 cbcr_in, out float y, out float2 cbcr cbcr = (cbcr_in - float2(128.0 / 255.0, 128.0 / 255.0)) * (255.0 / 224.0); } +// Dolby Vision RPU reshaping, ported from libplacebo's `pl_shader_dovi_reshape` +// (the renderer behind mpv's Dolby Vision mapping). The base-layer signal is +// reshaped per component through piecewise polynomial/MMR curves selected by +// pivot comparison, where MMR coefficients mix all three raw components. +float3 dovi_reshaped_signal(float3 sig_in) { + float3 sig = clamp(sig_in, 0.0, 1.0); + float result[3] = { sig.r, sig.g, sig.b }; + float4 flags = dovi.dovi_flags; + for (uint c = 0; c < 3u; c++) { + uint segments = uint(flags[1u + c]); + if (segments == 0u) { + continue; + } + float s = result[c]; + uint index = 0u; + for (uint i = 0u; i < 7u; i++) { + float4 pivot_row = dovi.dovi_pivots[2u * c + i / 4u]; + float pivot = pivot_row[i % 4u]; + if (s >= pivot) { + index = index + 1u; + } + } + float4 coeff = dovi.dovi_coefficients[8u * c + index]; + if (coeff.w < 0.5) { + s = (coeff.z * s + coeff.y) * s + coeff.x; + } else { + uint base = 48u * c + uint(coeff.y); + uint order = uint(coeff.w); + float4 sig_x = float4( + sig.x * sig.y, + sig.x * sig.z, + sig.y * sig.z, + sig.x * sig.y * sig.z + ); + s = coeff.x; + s = s + dot(dovi.dovi_mmr[base].xyz, sig); + s = s + dot(dovi.dovi_mmr[base + 1u], sig_x); + if (order >= 2u) { + float3 sig2 = sig * sig; + float4 sig_x2 = sig_x * sig_x; + s = s + dot(dovi.dovi_mmr[base + 2u].xyz, sig2); + s = s + dot(dovi.dovi_mmr[base + 3u], sig_x2); + if (order >= 3u) { + s = s + dot(dovi.dovi_mmr[base + 4u].xyz, sig2 * sig); + s = s + dot(dovi.dovi_mmr[base + 5u], sig_x2 * sig_x); + } + } + } + float4 bounds = dovi.dovi_bounds[c]; + result[c] = clamp(s, bounds.x, bounds.y); + } + return float3(result[0], result[1], result[2]); +} + +// Reshaped nonlinear signal to PQ-encoded IPT via the RPU's ycc_to_rgb matrix +// and signal offsets. Applying the RPU offsets keeps integer offset codes +// exactly on sample codes (2^bits/(2^bits-1) folded in on the CPU). +float3 dovi_signal_to_pq_rgb(float3 sig) { + float3 reshaped = dovi_reshaped_signal(sig) - dovi.dovi_nonlinear_offset.xyz; + return float3( + dot(dovi.dovi_nonlinear_matrix[0].xyz, reshaped), + dot(dovi.dovi_nonlinear_matrix[1].xyz, reshaped), + dot(dovi.dovi_nonlinear_matrix[2].xyz, reshaped) + ); +} + +// Linearized BT.2020-referred HPE LMS back to linear RGB, using the composite +// of the fixed HPE inverse with the RPU's rgb_to_lms matrix (premultiplied on +// the CPU, matching libplacebo's dovi_lms2rgb). +float3 dovi_lms_to_rgb(float3 lin) { + return float3( + dot(dovi.dovi_lms_matrix[0].xyz, lin), + dot(dovi.dovi_lms_matrix[1].xyz, lin), + dot(dovi.dovi_lms_matrix[2].xyz, lin) + ); +} + VsOut vs_main(VsIn input) { VsOut output; output.position = float4(input.position, 0.0, 1.0); @@ -335,22 +701,38 @@ float4 ps_main(VsOut input) : SV_Target { ? sample_packed_luma(luma_coord) : lumaTex.Sample(videoSampler, luma_coord).r; float2 cbcr_sample = chromaTex.Sample(videoSampler, chroma_coord).rg; - float y; - float2 cbcr; - expand_ycbcr_range(y_sample, cbcr_sample, y, cbcr); - - float kr = luma_coefficients.x; - float kg = max(luma_coefficients.y, 0.000001); - float kb = luma_coefficients.z; + bool dovi_enabled = dovi.dovi_flags.x != 0.0; + bool dovi_ycbcr_input = dovi_enabled && (base_input_mode == 0u || base_input_mode == 2u); float3 rgb; - rgb.r = y + 2.0 * (1.0 - kr) * cbcr.y; - rgb.b = y + 2.0 * (1.0 - kb) * cbcr.x; - rgb.g = (y - kr * rgb.r - kb * rgb.b) / kg; + if (dovi_ycbcr_input) { + // The base layer carries the raw 12-bit DV signal (10-bit container, + // full range); range expansion and the YCbCr matrix are replaced by + // the RPU reshaping + ycc_to_rgb path. + float3 sig = float3(y_sample, cbcr_sample.x, cbcr_sample.y); + if (is_p010 != 0u) { + sig *= 65535.0 / 65472.0; + } + rgb = dovi_signal_to_pq_rgb(sig); + } else { + float y; + float2 cbcr; + expand_ycbcr_range(y_sample, cbcr_sample, y, cbcr); + + float kr = luma_coefficients.x; + float kg = max(luma_coefficients.y, 0.000001); + float kb = luma_coefficients.z; + rgb.r = y + 2.0 * (1.0 - kr) * cbcr.y; + rgb.b = y + 2.0 * (1.0 - kb) * cbcr.x; + rgb.g = (y - kr * rgb.r - kb * rgb.b) / kg; + } rgb = transfer_to_source_reference_linear(rgb); - rgb = apply_gamut_map(rgb); + if (dovi_ycbcr_input) { + rgb = dovi_lms_to_rgb(rgb); + } rgb = source_reference_to_nits(rgb); rgb = tone_map_nits(rgb); rgb = target_nits_to_reference_linear(rgb); + rgb = gamut_compress(rgb); rgb = target_reference_linear_to_output(rgb); float alpha = 1.0; if (packed_alpha) { @@ -560,6 +942,7 @@ pub struct D3d11RendererStats { pub surface_width: u32, pub surface_height: u32, pub rendered_frames: u64, + pub software_video_frames: u64, pub hardware_video_frames: u64, pub zero_copy_video_frames: u64, pub direct_zero_copy_video_frames: u64, @@ -630,6 +1013,7 @@ impl AttachedSurface { struct ImportedVideoFrame { _frame: Frame, _texture: ID3D11Texture2D, + _chroma_texture: Option, luma: ID3D11ShaderResourceView, chroma: ID3D11ShaderResourceView, width: u32, @@ -794,8 +1178,12 @@ impl D3d11OutputMode { } fn target_color_for_source(self, source: SourceColorState) -> TargetColorState { - let _ = source; - self.target_color() + match self { + Self::Sdr if source.is_hdr() => { + TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709) + } + _ => self.target_color(), + } } } @@ -840,6 +1228,12 @@ pub struct D3d11Renderer { upscaler: D3d11ArtCnn, next_frame_token: u64, hdr10_output_unavailable: bool, + /// Cached perceptual gamut LUT (3D RGBA16F SRV), keyed the same way as + /// the wgpu/Metal caches: (source, target, target black PQ, target peak PQ). + gamut_lut: Option<(u32, u32, u32, u32, ID3D11ShaderResourceView)>, + /// Background generation for a cache miss; the fast `gamut_compress` + /// path renders until the LUT lands. + gamut_lut_job: Option, stats: D3d11RendererStats, } @@ -861,6 +1255,8 @@ impl D3d11Renderer { upscaler: D3d11ArtCnn::default(), next_frame_token: 0, hdr10_output_unavailable: false, + gamut_lut: None, + gamut_lut_job: None, stats: D3d11RendererStats::default(), }) } @@ -955,6 +1351,11 @@ impl D3d11Renderer { surface.output_texture = None; } } + // The cached LUT SRV belongs to the previous device, and D3D11 cannot + // bind a resource across devices. Drop it even when the key is + // unchanged: the CPU-side `gamut_lut_job` only holds parameters, so a + // later frame regenerates the texture on the new device and reuses it. + self.gamut_lut = None; self.state = Some(state); self.recreate_surface_targets()?; Ok(()) @@ -1198,6 +1599,7 @@ impl D3d11Renderer { let imported = ImportedVideoFrame { _frame: retained_frame, _texture: texture, + _chroma_texture: None, luma, chroma, width: visible_width, @@ -1573,6 +1975,95 @@ impl D3d11Renderer { Ok(()) } + /// Return a cached (or freshly generated) perceptual gamut LUT SRV for + /// the given uniforms, or `None` while the fast path is active or the + /// background generation is still pending. Callers must then mask + /// `gamut_lut_enabled` off so the shader keeps the fast path. + fn gamut_lut_view( + &mut self, + uniforms: &VideoUniforms, + ) -> Result> { + if uniforms.gamut_lut_enabled == 0 { + return Ok(None); + } + let packed = uniforms._gamut_primaries; + let source = packed >> 8; + let target = packed & 0xff; + // The LUT's I axis spans [target black, target peak] in PQ codes; + // either endpoint changing invalidates the cached LUT. + let black_pq = d3d_quantize_luma_pq(uniforms.tone_map_extra[2]); + let peak_pq = d3d_quantize_luma_pq(uniforms.nits[1]); + if let Some((s, t, b, p, srv)) = &self.gamut_lut { + if *s == source && *t == target && *b == black_pq && *p == peak_pq { + return Ok(Some(srv.clone())); + } + } + let params = GamutLutParams { + source: d3d_code_to_primaries(source), + target: d3d_code_to_primaries(target), + // Same target black the shader derives from tone_map_extra.z. + min_luma: d3d_pq_code_for_lut(uniforms.tone_map_extra[2]), + max_luma: d3d_pq_code_for_lut(uniforms.nits[1]), + }; + let job_params = self + .gamut_lut_job + .as_ref() + .map(GamutLutJob::params) + .filter(|job_params| *job_params == params); + if job_params.is_none() { + // First request (or the key changed): spawn generation and keep + // the fast path for this frame. + self.gamut_lut_job = Some(GamutLutJob::spawn(params)); + return Ok(None); + } + let Some(lut) = self.gamut_lut_job.as_ref().and_then(GamutLutJob::poll) else { + return Ok(None); + }; + self.gamut_lut_job = None; + let state = self.state.as_ref().expect("device ensured for gamut lut"); + // Pack (I, P+0.5, T+0.5) into RGBA16F bytes laid out I x C x H. + let texels = pack_rgba16f(&lut.texels, 1.0); + let initial = [D3D11_SUBRESOURCE_DATA { + pSysMem: texels.as_ptr() as *const c_void, + SysMemPitch: (LUT_SIZE_I * 4 * 2) as u32, + SysMemSlicePitch: (LUT_SIZE_I * LUT_SIZE_C * 4 * 2) as u32, + }]; + let desc = D3D11_TEXTURE3D_DESC { + Width: LUT_SIZE_I as u32, + Height: LUT_SIZE_C as u32, + Depth: LUT_SIZE_H as u32, + MipLevels: 1, + Format: DXGI_FORMAT_R16G16B16A16_FLOAT, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let mut texture = None; + unsafe { + state + .device + .CreateTexture3D(&desc, Some(initial.as_ptr()), Some(&mut texture)) + .map_err(|error| d3d_error("ID3D11Device::CreateTexture3D(gamut lut)", error))?; + } + let resource = texture + .expect("gamut lut texture created") + .cast::() + .map_err(|error| d3d_error("cast to ID3D11Resource", error))?; + let mut srv = None; + unsafe { + state + .device + .CreateShaderResourceView(&resource, None, Some(&mut srv)) + .map_err(|error| { + d3d_error("ID3D11Device::CreateShaderResourceView(gamut lut)", error) + })?; + } + let srv = srv.expect("gamut lut srv created"); + self.gamut_lut = Some((source, target, black_pq, peak_pq, srv.clone())); + Ok(Some(srv)) + } + fn render_video(&mut self, context: RenderFrameContext<'_>) -> Result { if self.current_video.is_none() { return Ok(false); @@ -1649,6 +2140,16 @@ impl D3d11Renderer { } else { None }; + let mut video_constants = { + let video = self.current_video.as_ref().expect("video checked"); + video.constants + }; + let gamut_lut = self.gamut_lut_view(&video_constants)?; + if gamut_lut.is_none() { + // Background generation pending: keep the fast gamut_compress + // path until the LUT lands. + video_constants.gamut_lut_enabled = 0; + } let video = self.current_video.as_ref().expect("video checked"); let state = self.state.as_ref().expect("device ensured"); let surface = self.surface.as_ref().expect("surface ensured"); @@ -1684,7 +2185,14 @@ impl D3d11Renderer { ], ); } - state.draw_video(video, upscaled_luma.as_ref(), scene_rtv, target_rect)?; + state.draw_video( + video, + video_constants, + upscaled_luma.as_ref(), + gamut_lut.as_ref(), + scene_rtv, + target_rect, + )?; if !overlay_draws.is_empty() { // Subtitle coordinates are produced in the video-frame viewport. // Composite them through the same aspect-fit viewport as the video @@ -1902,10 +2410,104 @@ impl RendererBackend for D3d11Renderer { "d3d11: hardware frame is not importable as D3D11VA".to_string(), )); } - self.stats.cpu_video_frame_fallbacks += 1; - Err(PlayerError::Renderer( - "d3d11: software frames require WgpuFallback or a CPU upload path".to_string(), - )) + let decoded = frame.frame.decoded_frame().ok_or_else(|| { + PlayerError::Renderer("d3d11: video payload has no CPU-readable frame".to_string()) + })?; + let planar = decoded.to_planar_frame().ok_or_else(|| { + PlayerError::Renderer(format!( + "d3d11: unsupported software video frame format {}", + decoded + .pixel_format() + .unwrap_or_else(|| "unknown".to_string()) + )) + })?; + self.ensure_default_device()?; + let source = source_color_for_frame(frame); + let output_mode = self.select_output_mode_for_source(source)?; + let target = output_mode.target_color_for_source(source); + let (texture_format, bytes_per_sample) = match planar.format { + PlanarPixelFormat::Nv12 => (D3d11VideoTextureFormat::Nv12, 1_u32), + PlanarPixelFormat::P010 => (D3d11VideoTextureFormat::P010, 2_u32), + }; + let width = planar.width.max(1); + let height = planar.height.max(1); + let chroma_width = width.div_ceil(2); + let chroma_height = height.div_ceil(2); + let luma_pitch = width + .checked_mul(bytes_per_sample) + .ok_or_else(|| PlayerError::Renderer("d3d11: luma row pitch overflowed".to_string()))?; + let chroma_pitch = chroma_width + .checked_mul(2) + .and_then(|pitch| pitch.checked_mul(bytes_per_sample)) + .ok_or_else(|| { + PlayerError::Renderer("d3d11: chroma row pitch overflowed".to_string()) + })?; + let sample_bytes = bytes_per_sample as usize; + let expected_luma = (width as usize) + .checked_mul(height as usize) + .and_then(|samples| samples.checked_mul(sample_bytes)) + .ok_or_else(|| { + PlayerError::Renderer("d3d11: luma plane size overflowed".to_string()) + })?; + let expected_chroma = (chroma_width as usize) + .checked_mul(chroma_height as usize) + .and_then(|samples| samples.checked_mul(2)) + .and_then(|bytes| bytes.checked_mul(sample_bytes)) + .ok_or_else(|| { + PlayerError::Renderer("d3d11: chroma plane size overflowed".to_string()) + })?; + if planar.luma.len() != expected_luma || planar.chroma.len() != expected_chroma { + return Err(PlayerError::Renderer(format!( + "d3d11: invalid {:?} plane sizes (luma {}, expected {}; chroma {}, expected {})", + planar.format, + planar.luma.len(), + expected_luma, + planar.chroma.len(), + expected_chroma, + ))); + } + let (luma, chroma) = { + let state = self.state.as_ref().expect("device ensured"); + ( + create_overlay_texture( + state, + width, + height, + texture_format.luma_srv(), + &planar.luma, + luma_pitch, + )?, + create_overlay_texture( + state, + chroma_width, + chroma_height, + texture_format.chroma_srv(), + &planar.chroma, + chroma_pitch, + )?, + ) + }; + let retained_frame = decoded.try_clone_ref().map_err(|error| { + PlayerError::Renderer(format!("d3d11: av_frame_ref failed: {error}")) + })?; + self.stats.software_video_frames += 1; + let frame_token = self.next_frame_token; + self.next_frame_token = self.next_frame_token.wrapping_add(1); + self.current_video = Some(ImportedVideoFrame { + _frame: retained_frame, + _texture: luma._texture, + _chroma_texture: Some(chroma._texture), + luma: luma.view, + chroma: chroma.view, + width, + height, + tex_rect: D3d11TexRect::FULL, + _array_index: 0, + frame_token, + constants: constants_for_frame(source, texture_format, target) + .packed_alpha_right(self.video_alpha_mode.has_alpha()), + }); + Ok(()) } fn clear_current_frame(&mut self) -> Result<()> { @@ -1937,7 +2539,7 @@ impl RendererBackend for D3d11Renderer { danmaku_draw_items: self.stats.danmaku_items, overlay_alpha_atlas_uploads: self.stats.overlay_alpha_atlas_uploads, overlay_alpha_atlas_reuses: self.stats.overlay_alpha_atlas_reuses, - software_video_frames: 0, + software_video_frames: self.stats.software_video_frames, hardware_video_frames: self.stats.hardware_video_frames, zero_copy_video_frames: self.stats.zero_copy_video_frames, direct_zero_copy_video_frames: self.stats.direct_zero_copy_video_frames, @@ -2113,10 +2715,17 @@ impl D3d11DeviceState { }) } + /// Draw the imported video frame. `constants` is the per-frame uniform + /// payload the caller already resolved (including any + /// `gamut_lut_enabled` mask for a still-generating LUT); the stored + /// `ImportedVideoFrame.constants` is left untouched so a later present + /// can re-enable the LUT once it lands. fn draw_video( &self, video: &ImportedVideoFrame, + constants: VideoUniforms, upscaled_luma: Option<&ID3D11ShaderResourceView>, + gamut_lut: Option<&ID3D11ShaderResourceView>, render_target: &ID3D11RenderTargetView, target: D3d11DrawRect, ) -> Result<()> { @@ -2131,7 +2740,7 @@ impl D3d11DeviceState { let stride = mem::size_of::() as u32; let offset = 0u32; let vertices = video_vertices(D3d11TexRect::FULL); - let mut common = video.constants; + let mut common = constants; if upscaled_luma.is_some() { common = common.packed_d2s_luma_input(); } @@ -2184,20 +2793,20 @@ impl D3d11DeviceState { self.context.PSSetShader(&self.pixel_shader, None); self.context .PSSetConstantBuffers(0, Some(&[Some(self.constants.clone())])); - self.context.PSSetShaderResources( - 0, - Some(&[ - Some(upscaled_luma.cloned().unwrap_or_else(|| video.luma.clone())), - Some(video.chroma.clone()), - ]), - ); + let srvs = [ + Some(upscaled_luma.cloned().unwrap_or_else(|| video.luma.clone())), + Some(video.chroma.clone()), + gamut_lut.cloned(), + ]; + self.context.PSSetShaderResources(0, Some(&srvs)); self.context .PSSetSamplers(0, Some(&[Some(self.sampler.clone())])); self.context .OMSetRenderTargets(Some(&[Some(render_target.clone())]), None); self.context.OMSetBlendState(None, None, u32::MAX); self.context.Draw(6, 0); - self.context.PSSetShaderResources(0, Some(&[None, None])); + self.context + .PSSetShaderResources(0, Some(&[None, None, None])); } Ok(()) } @@ -2395,6 +3004,31 @@ fn aspect_fit_rect( } } +fn d3d_pq_code_for_lut(nits: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = (nits / 10000.0).clamp(0.0, 1.0).powf(m1); + ((c1 + c2 * p) / (1.0 + c3 * p).max(0.000_001)).powf(m2) +} + +/// Quantize a luminance (nits) to its PQ code for the LUT cache key, matching +/// the wgpu/Metal quantizers. 16-bit PQ resolution keeps the sub-1-nit target +/// blacks of SDR targets distinct. +fn d3d_quantize_luma_pq(nits: f32) -> u32 { + (d3d_pq_code_for_lut(nits) * 65535.0) as u32 +} + +fn d3d_code_to_primaries(code: u32) -> ColorPrimaries { + match code { + 1 => ColorPrimaries::Bt2020, + 2 => ColorPrimaries::DisplayP3, + _ => ColorPrimaries::Bt709, + } +} + fn create_default_device() -> Result<(ID3D11Device, ID3D11DeviceContext)> { trace("create_default_device: D3D11CreateDevice"); let feature_levels = [ @@ -3094,6 +3728,8 @@ fn source_color_for_frame(frame: &PlayerVideoFrame) -> SourceColorState { .range(frame.frame.color_range()) .matrix(frame.frame.matrix_coefficients()) .hdr_metadata(frame.frame.hdr_metadata()) + .dovi(frame.frame.dovi_metadata()) + .measured_scene_avg_nits(frame.scene_avg_nits) } fn constants_for_frame( @@ -3732,6 +4368,48 @@ mod tests { assert_eq!(sdr.scene_linear, 0); } + #[test] + fn hlsl_evaluates_reinhard_and_mobius_in_linear_nits() { + // libplacebo runs these two curves in the linear PL_HDR_NORM domain + // and only the PQ-domain operators work on encoded values; evaluating + // them on PQ codes moved the Mobius knee far below its intended + // position. The HLSL is compiled by the D3D11 runtime, not by the Rust + // build, so this pins the source shape rather than executing it. + let source = std::str::from_utf8(SHADER_SOURCE).unwrap(); + for branch in ["if (tone_map == 1u)", "if (tone_map == 2u)"] { + let start = source.find(branch).expect("branch"); + let rest = &source[start..]; + let end = rest.find("\n }").expect("branch end"); + let body = &rest[..end]; + assert!( + body.contains("float out_range_nits = max(dst_peak - dst_black, 0.000001);"), + "{branch} must scale in linear nits" + ); + assert!( + body.contains("nits_from_pq(x)"), + "{branch} must decode the PQ code before the curve" + ); + assert!( + body.contains("return pq_code("), + "{branch} must re-encode the result" + ); + } + } + + #[test] + fn hlsl_gamut_lut_samples_the_target_black_to_peak_axis() { + // libplacebo's LUT I axis is [target black, target peak]; a backend + // that samples `ipt.x / peak` again would diverge from the generated + // LUT. The lattice position must also be remapped to the texel-center + // coordinate (libplacebo's `texel_scale`), or the zero-chroma texel at + // the low end of the C axis crushes saturation. + let source = std::str::from_utf8(SHADER_SOURCE).unwrap(); + assert!(source.contains("float lut_min = pq_code(tone_map_extra.z);")); + assert!(source.contains("clamp((ipt.x - lut_min) / lut_range, 0.0, 1.0)")); + assert!(source.contains("pos.y * (31.0 / 32.0) + 0.5 / 32.0")); + assert!(source.contains("pos.x * (47.0 / 48.0) + 0.5 / 48.0")); + } + #[test] fn overlay_uniforms_enable_scene_linear_only_for_hdr10() { let hdr10 = diff --git a/crates/erika/src/renderer/frame.rs b/crates/erika/src/renderer/frame.rs index 402e92bc..3b99fd65 100644 --- a/crates/erika/src/renderer/frame.rs +++ b/crates/erika/src/renderer/frame.rs @@ -1,6 +1,6 @@ use crate::core::{ColorPrimaries, TransferFunction}; use crate::ffmpeg::{D3d11vaTexture, Frame, PlanarFrame, Result as FfmpegResult}; -use crate::renderer::pipeline::{ColorRange, HdrMetadata, MatrixCoefficients}; +use crate::renderer::pipeline::{ColorRange, DoviSourceMetadata, HdrMetadata, MatrixCoefficients}; #[cfg(any(target_os = "android", target_env = "ohos"))] use std::sync::Arc; @@ -26,6 +26,7 @@ pub struct VideoFrameDescriptor { pub range: ColorRange, pub matrix: MatrixCoefficients, pub hdr_metadata: Option, + pub dovi_metadata: Option, } impl VideoFrameDescriptor { @@ -41,6 +42,7 @@ impl VideoFrameDescriptor { range: frame.color_range(), matrix: frame.matrix_coefficients(), hdr_metadata: frame.hdr_metadata(), + dovi_metadata: frame.dovi_metadata(), } } } @@ -214,6 +216,16 @@ impl VideoFramePayload { } } + pub fn dovi_metadata(&self) -> Option { + match self { + Self::Decoded(frame) => frame.dovi_metadata(), + #[cfg(target_os = "android")] + Self::AndroidHardwareBuffer(frame) => frame.descriptor.dovi_metadata, + #[cfg(target_env = "ohos")] + Self::OhosNativeBuffer(frame) => frame.descriptor.dovi_metadata, + } + } + pub fn has_hw_frames_context(&self) -> bool { match self { Self::Decoded(frame) => frame.has_hw_frames_context(), diff --git a/crates/erika/src/renderer/gamut.rs b/crates/erika/src/renderer/gamut.rs new file mode 100644 index 00000000..413eead1 --- /dev/null +++ b/crates/erika/src/renderer/gamut.rs @@ -0,0 +1,751 @@ +//! Perceptual gamut mapping via a precomputed IPT-space 3D LUT, following +//! libplacebo's `pl_gamut_map_perceptual` LUT layout (the default gamut map +//! of libplacebo/mpv). The mapping converts a color from the source primaries +//! into the destination primaries *inside* IPT and then rolls off +//! out-of-gamut chroma towards the destination gamut, protecting in-gamut +//! colors (dead zone) with a Möbius soft clip. +//! +//! The chroma rolloff is currently a simplified dead-zone blend plus +//! softclip rather than libplacebo's full per-hue boundary search, so highly +//! saturated BT.2020 colors can diverge slightly from mpv. +//! +//! Because the boundary search is expensive, libplacebo precomputes it once +//! into a 3D LUT over the IPT intensity I, the IPT chroma magnitude C and the +//! IPT hue angle h; the renderers sample it per pixel. This module generates +//! the same LUT lattice on the CPU (48 x 32 x 256 = 393216 texels), and the +//! shaders do the lookup with the same index mapping. +//! +//! Note: libplacebo currently *samples the perceptual LUT in ICh space* with +//! the I channel *in absolute PQ units* over the target display range +//! `[min_luma, max_luma]`. We reproduce that layout exactly so the WGSL/MSL +//! and HLSL samplers match the reference pixels. + +use crate::core::ColorPrimaries; +use crate::renderer::pipeline::{RgbMatrix, ipt_rgb2lms_matrix}; + +/// LUT dimensions, matching libplacebo's `pl_color_map_default_params` +/// `lut3d_size = {48, 32, 256}` (I, C, h). +pub const LUT_SIZE_I: usize = 48; +pub const LUT_SIZE_C: usize = 32; +pub const LUT_SIZE_H: usize = 256; +/// Per-texel component count (RGB with a padded w). +pub const LUT_COMPONENTS: usize = 4; + +const PQ_M1: f32 = 2610.0 / 4096.0 * 1.0 / 4.0; +const PQ_M2: f32 = 2523.0 / 4096.0 * 128.0; +const PQ_C1: f32 = 3424.0 / 4096.0; +const PQ_C2: f32 = 2413.0 / 4096.0 * 32.0; +const PQ_C3: f32 = 2392.0 / 4096.0 * 32.0; + +#[allow(dead_code)] +/// 4% crosstalk HPE matrix shared with the tone map. +fn hpe_crosstalk() -> RgbMatrix { + let c = 0.04_f32; + RgbMatrix::new([ + [1.0 - 2.0 * c, c, c], + [c, 1.0 - 2.0 * c, c], + [c, c, 1.0 - 2.0 * c], + ]) +} + +fn rgb2lms(primaries: ColorPrimaries) -> RgbMatrix { + ipt_rgb2lms_matrix(primaries) +} + +#[allow(dead_code)] +fn lms2rgb(primaries: ColorPrimaries) -> RgbMatrix { + rgb2lms(primaries).inverse() +} + +/// Perceptual gamut mapping parameters that change how the LUT is computed. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct GamutLutParams { + /// Source primaries of the tone-mapped (still source-referred) signal. + pub source: ColorPrimaries, + /// Destination (display) primaries. + pub target: ColorPrimaries, + /// Minimum display luminance in PQ code (absolute), i.e. libplacebo's + /// `gamut.min_luma`: the target's black point, which is also the tone + /// map's output minimum (0 for HDR/EDR targets, `peak / contrast` for SDR + /// ones). + pub min_luma: f32, + /// Maximum display luminance in PQ code (libplacebo's `gamut.max_luma`, + /// the tone map's output peak). + pub max_luma: f32, +} + +/// A generated 3D perceptual gamut LUT. Texels are (I, P, T) triples with a +/// padded component, normalized to [0, 1] like libplacebo's uint16 upload +/// (`I`, `P + 0.5`, `T + 0.5` scaled). +#[derive(Debug, Clone, PartialEq)] +pub struct GamutLut { + /// size = LUT_SIZE_I * LUT_SIZE_C * LUT_SIZE_H, stored as float triples + /// (R=IPT.I, G=IPT.P + 0.5, B=IPT.T + 0.5) in [0, 1]. + pub texels: Vec<[f32; 3]>, + /// `min_luma`/`max_luma` in PQ that the LUT's I axis spans. + pub params: GamutLutParams, +} + +impl GamutLut { + /// Generate the perceptual LUT, mirroring libplacebo's + /// `pl_gamut_map_perceptual` evaluated over the lattice texels. + /// + /// libplacebo caches the per-hue boundary peak (`saturate`); we instead + /// precompute the source/destination peaks for each of the `LUT_SIZE_H` + /// hue slices once, which the texel loop then reuses — same math, no + /// repeated golden-section searches. + pub fn generate(params: GamutLutParams) -> Self { + let src = GamutState::new(params.source, params); + let dst = GamutState::new(params.target, params); + let mut texels = Vec::with_capacity(LUT_SIZE_I * LUT_SIZE_C * LUT_SIZE_H); + // Lattice order must match the 3D texture addressing used by the + // shaders: width = I (48, stride 1), height = C (32, stride width), + // depth = h (256, stride width * height). The loop runs h outer, + // C middle, I inner so that width (I) is contiguous in memory, and + // the expensive per-hue `saturate` search runs 256 times instead of + // 393,216 times. + for hx in 0..LUT_SIZE_H { + let h = -std::f32::consts::PI + + 2.0 * std::f32::consts::PI * hx as f32 / (LUT_SIZE_H - 1) as f32; + let src_peak = saturate(h, &src); + let dst_peak = saturate(h, &dst); + let max_c = src_peak[1].max(dst_peak[1]).max(1e-9); + let soft = Softclip { + knee: 0.70, + _desat: 0.35, + }; + for cx in 0..LUT_SIZE_C { + let c = 0.5 * cx as f32 / (LUT_SIZE_C - 1) as f32; + for ix in 0..LUT_SIZE_I { + let i = params.min_luma + + (params.max_luma - params.min_luma) * ix as f32 / (LUT_SIZE_I - 1) as f32; + let mapped = perceptual_map_at( + [i, c, h], + &src, + &dst, + src_peak, + dst_peak, + max_c, + &soft, + params, + ); + texels.push([mapped[0], mapped[1] + 0.5, mapped[2] + 0.5]); + } + } + } + Self { texels, params } + } +} + +/// A `GamutLut::generate` running on a background thread, shared by the +/// Metal/wgpu/D3D11 renderers. Generation costs hundreds of milliseconds +/// (393k texels of PQ roundtrips plus 256 hue-boundary searches); running it +/// on the render thread stalls the first HDR wide-gamut frame. The renderer +/// polls `poll()` each frame and keeps the fast `gamut_compress` path until +/// the LUT lands. +pub struct GamutLutJob { + params: GamutLutParams, + result: std::sync::mpsc::Receiver, +} + +impl GamutLutJob { + /// Spawn generation for `params`. The thread is detached: it writes into + /// the channel and exits, so dropping the job simply discards an unused + /// result. + pub fn spawn(params: GamutLutParams) -> Self { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + std::thread::Builder::new() + .name("erika-gamut-lut".to_string()) + .spawn(move || { + let lut = GamutLut::generate(params); + let _ = sender.send(lut); + }) + .expect("spawn gamut LUT generation thread"); + Self { + params, + result: receiver, + } + } + + /// The parameters this job is generating for. + pub fn params(&self) -> GamutLutParams { + self.params + } + + /// Take the finished LUT when generation has completed. + pub fn poll(&self) -> Option { + self.result.try_recv().ok() + } +} + +#[cfg(test)] +mod job_tests { + use super::*; + + #[test] + fn job_produces_the_same_lut_as_inline_generation() { + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 0.75, + }; + let job = GamutLutJob::spawn(params); + let lut = loop { + if let Some(lut) = job.poll() { + break lut; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + }; + assert_eq!(lut.params, params); + assert_eq!(lut.texels.len(), LUT_SIZE_I * LUT_SIZE_C * LUT_SIZE_H); + // Polling after completion yields nothing further. + assert!(job.poll().is_none()); + } +} + +fn pq_oetf(x: f32) -> f32 { + let x = x.max(0.0).powf(PQ_M1); + ((PQ_C1 + PQ_C2 * x) / (1.0 + PQ_C3 * x)).powf(PQ_M2) +} + +fn pq_eotf(x: f32) -> f32 { + let x = x.max(0.0).powf(1.0 / PQ_M2); + let num = (x - PQ_C1).max(0.0); + let den = (PQ_C2 - PQ_C3 * x).max(1e-9); + (num / den).powf(1.0 / PQ_M1) +} + +fn rgb2ipt(rgb: [f32; 3], gamut: &GamutState) -> [f32; 3] { + let lms = gamut.rgb2lms.mul_vec(rgb); + let lp = pq_oetf(lms[0]); + let mp = pq_oetf(lms[1]); + let sp = pq_oetf(lms[2]); + [ + 0.4000 * lp + 0.4000 * mp + 0.2000 * sp, + 4.4550 * lp - 4.8510 * mp + 0.3960 * sp, + 0.8056 * lp + 0.3572 * mp - 1.1628 * sp, + ] +} + +fn ipt2rgb(ipt: [f32; 3], gamut: &GamutState) -> [f32; 3] { + let lp = ipt[0] + 0.0975689 * ipt[1] + 0.205226 * ipt[2]; + let mp = ipt[0] - 0.1138760 * ipt[1] + 0.133217 * ipt[2]; + let sp = ipt[0] + 0.0326151 * ipt[1] - 0.676887 * ipt[2]; + let l = pq_eotf(lp); + let m = pq_eotf(mp); + let s = pq_eotf(sp); + gamut.lms2rgb.mul_vec([l, m, s]) +} + +/// Perceptual map of one IPT color given as (I in PQ code, chroma, hue), +/// mirroring libplacebo's `perceptual()` body with the per-hue peaks already +/// computed. `src`/`dst` are the gamut states, `src_peak`/`dst_peak` the +/// maximally-saturated boundary colors at this hue, `max_c` their chroma +/// max (the dead-zone denominator). +fn perceptual_map_at( + ich: [f32; 3], + src: &GamutState, + dst: &GamutState, + _src_peak: [f32; 3], + _dst_peak: [f32; 3], + max_c: f32, + soft: &Softclip, + _params: GamutLutParams, +) -> [f32; 3] { + let ipt_in = ich2ipt(ich); + let mapped = rgb2ipt(ipt2rgb(ipt_in, src), dst); + + // Protect in-gamut region: blend only colors whose chroma exceeds the + // perceptual dead zone (30% of the peak), scaling to full strength. + let deadzone = 0.30_f32; + let strength = 0.80_f32; + let k = pl_smoothstep(deadzone, 1.0, ich[1] / max_c) * strength; + let ipt = [ + ipt_in[0] + (mapped[0] - ipt_in[0]) * k, + ipt_in[1] + (mapped[1] - ipt_in[1]) * k, + ipt_in[2] + (mapped[2] - ipt_in[2]) * k, + ]; + + let rgb = ipt2rgb(ipt, dst); + let max_rgb = rgb[0].max(rgb[1]).max(rgb[2]); + let out = [ + softclip(rgb[0], max_rgb, dst.max_rgb, soft).max(dst.min_rgb), + softclip(rgb[1], max_rgb, dst.max_rgb, soft).max(dst.min_rgb), + softclip(rgb[2], max_rgb, dst.max_rgb, soft).max(dst.min_rgb), + ]; + rgb2ipt(out, dst) +} + +fn softclip(value: f32, source: f32, target: f32, c: &Softclip) -> f32 { + if target == 0.0 { + return 0.0; + } + let peak = source / target; + let x = (value / target).min(peak); + if x <= c.knee || peak <= 1.0 { + return value; + } + let j = c.knee; + let a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + let b = (j * j - 2.0 * j * peak + peak) / (peak - 1.0).max(1e-6); + let scale = (b * b + 2.0 * b * j + j * j) / (b - a); + scale * (x + a) / (x + b) * target +} + +struct Softclip { + knee: f32, + _desat: f32, +} + +fn pl_smoothstep(edge0: f32, edge1: f32, x: f32) -> f32 { + let t = ((x - edge0) / (edge1 - edge0)).clamp(0.0, 1.0); + t * t * (3.0 - 2.0 * t) +} + +struct GamutState { + rgb2lms: RgbMatrix, + lms2rgb: RgbMatrix, + min_rgb: f32, + max_rgb: f32, + min_luma: f32, + max_luma: f32, +} + +impl GamutState { + fn new(primaries: ColorPrimaries, params: GamutLutParams) -> Self { + let m = rgb2lms(primaries); + Self { + lms2rgb: m.inverse(), + rgb2lms: m, + min_rgb: pq_eotf(params.min_luma) - 1e-6, + max_rgb: pq_eotf(params.max_luma) + 1e-6, + min_luma: params.min_luma, + max_luma: params.max_luma, + } + } +} + +#[allow(dead_code)] +fn ipt2ich(ipt: [f32; 3]) -> [f32; 3] { + [ + ipt[0], + (ipt[1] * ipt[1] + ipt[2] * ipt[2]).sqrt(), + ipt[2].atan2(ipt[1]), + ] +} + +fn ich2ipt(ich: [f32; 3]) -> [f32; 3] { + [ich[0], ich[1] * ich[2].cos(), ich[1] * ich[2].sin()] +} + +/// Returns the maximally saturated in-gamut color of `gamut` at `hue`, +/// using a golden-section search over I with a bounded binary search for the +/// C boundary (mirrors libplacebo's `saturate`). +fn saturate(hue: f32, gamut: &GamutState) -> [f32; 3] { + let inv_phi = 0.618_033_988_749_894_8_f32; + let inv_phi2 = 0.381_966_011_250_105_15_f32; + + // Golden-section bracket over I, keeping the full (I, C, h) points so + // each iteration re-bounds the boundary search like the C version. + let (mut lo_i, mut lo_c) = (gamut.min_luma, 0.0_f32); + let (hi_i, mut hi_c) = (gamut.max_luma, 0.0_f32); + let mut de = hi_i - lo_i; + let mut a = [lo_i + inv_phi2 * de, 0.0, hue]; + let mut b = [lo_i + inv_phi * de, 0.0, hue]; + a[1] = desat_bounded(a[0], hue, 0.0, 0.5, gamut)[1]; + b[1] = desat_bounded(b[0], hue, 0.0, 0.5, gamut)[1]; + + while de > 5e-5 { + de *= inv_phi; + if a[1] > b[1] { + hi_c = b[1]; + b = a; + a[0] = lo_i + inv_phi2 * de; + a[1] = desat_bounded(a[0], hue, lo_c - 5e-5, 0.5, gamut)[1]; + } else { + lo_i = a[0]; + lo_c = a[1]; + a = b; + b[0] = lo_i + inv_phi * de; + b[1] = desat_bounded(b[0], hue, hi_c - 5e-5, 0.5, gamut)[1]; + } + } + + if a[1] > b[1] { + [a[0], a[1], hue] + } else { + [b[0], b[1], hue] + } +} + +/// Find the gamut boundary at luminance `i` and hue `h` within `[cmin, cmax]`. +fn desat_bounded(i: f32, h: f32, cmin: f32, cmax: f32, gamut: &GamutState) -> [f32; 3] { + if i <= gamut.min_luma { + return [gamut.min_luma, 0.0, h]; + } + if i >= gamut.max_luma { + return [gamut.max_luma, 0.0, h]; + } + let max_di = i * 5e-5; + let mut lo = cmin; + let mut hi = cmax; + loop { + let c = (lo + hi) / 2.0; + if ingamut(ich2ipt([i, c, h]), gamut) { + lo = c; + } else { + hi = c; + } + if hi - lo <= max_di { + return [i, (lo + hi) / 2.0, h]; + } + } +} + +fn ingamut(ipt: [f32; 3], gamut: &GamutState) -> bool { + let rgb = ipt2rgb(ipt, gamut); + rgb[0] >= gamut.min_rgb + && rgb[0] <= gamut.max_rgb + && rgb[1] >= gamut.min_rgb + && rgb[1] <= gamut.max_rgb + && rgb[2] >= gamut.min_rgb + && rgb[2] <= gamut.max_rgb +} + +/// Convert an f32 to IEEE-754 binary16 (round-to-nearest-even) for packing +/// RGBA16F textures. The gamut LUT texels carry values in [-0.5, 1.0], so a +/// half-float texture is exact enough and is filterable on every backend +/// (Rgba32Float is not filterable on wgpu). +pub fn f32_to_f16(value: f32) -> u16 { + let bits = value.to_bits(); + let sign = (bits >> 16) & 0x8000; + let exp = ((bits >> 23) & 0xff) as i32; + let mant = bits & 0x7fffff; + if exp == 0xff { + return (sign | 0x7c00 | u32::from(mant != 0)) as u16; + } + let e = exp - 127 + 15; + if e >= 0x1f { + return (sign | 0x7c00) as u16; // overflow -> inf + } + if e <= 0 { + return if e <= -10 { + sign as u16 // underflow -> signed zero + } else { + let m = mant | 0x800000; + let shift = (14 - e) as u32; + let rounded = (m >> shift) + u32::from((m >> (shift - 1)) & 1 == 1); + (sign | rounded) as u16 + }; + } + let rounded = mant + 0x1000 + ((mant >> 13) & 1); + let mut e = e as u32; + if rounded & 0x800000 != 0 { + e += 1; + } + (sign | (e << 10) | ((rounded >> 13) & 0x3ff)) as u16 +} + +/// Pack the f32 texel triples into interleaved RGBA16F little-endian bytes, +/// matching the `Rgba16Float` textures of the Metal/wgpu/D3D11 backends. +pub fn pack_rgba16f(texels: &[[f32; 3]], alpha: f32) -> Vec { + let mut bytes = Vec::with_capacity(texels.len() * 8); + for texel in texels { + for channel in texel.iter().copied().chain(std::iter::once(alpha)) { + bytes.extend_from_slice(&f32_to_f16(channel).to_le_bytes()); + } + } + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn f16_packing_roundtrips_and_matches_half_layout() { + // Known IEEE-754 binary16 bit patterns. + assert_eq!(f32_to_f16(0.0), 0x0000); + assert_eq!(f32_to_f16(1.0), 0x3c00); + assert_eq!(f32_to_f16(0.5), 0x3800); + assert_eq!(f32_to_f16(-0.5), 0xb800); + assert_eq!(f32_to_f16(10000.0), 0x70e2); + assert_eq!(f32_to_f16(-1.0), 0xbc00); + // Padded alpha keeps texel pitch at 8 bytes. + let packed = pack_rgba16f(&[[0.0, 0.5, 1.0]], 1.0); + assert_eq!(packed.len(), 8); + assert_eq!(&packed[0..2], &0x0000_u16.to_le_bytes()); + assert_eq!(&packed[2..4], &0x3800_u16.to_le_bytes()); + assert_eq!(&packed[4..6], &0x3c00_u16.to_le_bytes()); + assert_eq!(&packed[6..8], &0x3c00_u16.to_le_bytes()); + } + + #[test] + fn lut_generation_is_deterministic_and_bounded() { + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 1.0, + }; + let a = GamutLut::generate(params); + let b = GamutLut::generate(params); + assert_eq!(a.texels.len(), LUT_SIZE_I * LUT_SIZE_C * LUT_SIZE_H); + assert_eq!(a, b); + for texel in &a.texels { + assert!( + texel[0] >= 0.0 && texel[0] <= 1.0, + "I out of range: {texel:?}" + ); + assert!( + texel[1] >= 0.0 && texel[1] <= 1.0, + "P out of range: {texel:?}" + ); + assert!( + texel[2] >= 0.0 && texel[2] <= 1.0, + "T out of range: {texel:?}" + ); + } + } + + #[test] + fn in_gamut_colors_stay_near_identity_in_dead_zone() { + // A low-chroma, mid-intensity color (inside both gamuts) maps close + // to itself: the dead-zone blend keeps k small. + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 1.0, + }; + // I=0.5 (mid), C=0.02, h=0 + let src = GamutState::new(params.source, params); + let dst = GamutState::new(params.target, params); + let h = 0.0_f32; + let src_peak = saturate(h, &src); + let dst_peak = saturate(h, &dst); + let soft = Softclip { + knee: 0.70, + _desat: 0.35, + }; + let max_c = src_peak[1].max(dst_peak[1]).max(1e-9); + let out = perceptual_map_at( + [0.5, 0.02, h], + &src, + &dst, + src_peak, + dst_peak, + max_c, + &soft, + params, + ); + assert!( + (out[0] - 0.5).abs() < 0.05 && (out[1] - 0.02).abs() < 0.1, + "out={out:?}" + ); + } + + #[test] + fn saturated_wide_gamut_color_is_brought_in_gamut() { + // BT.2020 primary green is far outside BT.709; the map must reduce + // its chroma without flipping the hue wildly. + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.5, + max_luma: 0.9, + }; + // I at mid-display, high chroma, hue of ~primary green in IPT. + let src = GamutState::new(params.source, params); + let dst = GamutState::new(params.target, params); + let h = 0.9_f32; + let src_peak = saturate(h, &src); + let dst_peak = saturate(h, &dst); + let soft = Softclip { + knee: 0.70, + _desat: 0.35, + }; + let max_c = src_peak[1].max(dst_peak[1]).max(1e-9); + let out = perceptual_map_at( + [0.7, 0.4, h], + &src, + &dst, + src_peak, + dst_peak, + max_c, + &soft, + params, + ); + let rgb = ipt2rgb([out[0], out[1], out[2]], &dst); + assert!( + rgb.iter().all(|v| *v >= -1e-4 && *v <= 1.0 + 1e-4), + "still out of gamut: {rgb:?}" + ); + } +} + +#[cfg(test)] +mod anchor_audit { + use super::*; + use crate::core::ColorPrimaries; + + #[test] + fn grayscale_anchors_survive_perceptual_map() { + // Black/mid-gray/white (chroma ≈ 0) must keep chroma near zero after + // the perceptual map; a LUT numeric bug shows up here first. + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 1.0, + }; + let src = GamutState::new(params.source, params); + let dst = GamutState::new(params.target, params); + // IPT mid-gray: any I, P = T = 0 + for i in [0.0, 0.1, 0.5, 0.9] { + let mapped = perceptual_map_at( + [i, 0.0, 0.0], + &src, + &dst, + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + 1.0, + &Softclip { + knee: 0.7, + _desat: 0.35, + }, + params, + ); + assert!( + mapped[1].abs() < 1e-3 && mapped[2].abs() < 1e-3, + "chroma leak on gray: {mapped:?}" + ); + } + } + + #[test] + fn lut_gray_column_keeps_chroma_near_zero_and_rises_with_i() { + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 1.0, + }; + let lut = GamutLut::generate(params); + // C = 0 (chroma axis 0), mid hue: chroma should stay near zero and + // I should increase along the lattice axis. + let mut previous_i = None; + for ix in [0, 8, 24, 47] { + let idx = 128 * LUT_SIZE_C * LUT_SIZE_I + ix; + let t = lut.texels[idx]; + // Texels pack (I, P + 0.5, T + 0.5); zero chroma is 0.5/0.5. + assert!( + (t[1] - 0.5).abs() < 1e-3 && (t[2] - 0.5).abs() < 1e-3, + "chroma leak at I index {ix}: {t:?}" + ); + if let Some(previous) = previous_i { + let current = t[0]; + assert!( + current >= previous, + "I not monotone at index {ix}: {current} < {previous}" + ); + } + previous_i = Some(t[0]); + } + } +} + +#[cfg(test)] +mod layout_audit { + use super::*; + use crate::core::ColorPrimaries; + + #[test] + fn texel_layout_is_i_major_within_h_slice() { + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 1.0, + }; + let lut = GamutLut::generate(params); + // Lattice order is h-outer / C-mid / I-inner: walking one I step must + // stay inside the same (h, C) cell, so the mid-h gray column rises in + // I and stays chroma-free, while the C=1 row starts after LUT_SIZE_I. + let mid_h_start = 128 * LUT_SIZE_C * LUT_SIZE_I; + let first_i = lut.texels[mid_h_start]; + let last_i = lut.texels[mid_h_start + LUT_SIZE_I - 1]; + assert!( + last_i[0] >= first_i[0], + "I axis must increase along the inner lattice: {first_i:?} -> {last_i:?}" + ); + // Next C row begins exactly after one full I axis. + let next_c = lut.texels[mid_h_start + LUT_SIZE_I]; + assert!( + (next_c[1] - 0.5).abs() > (first_i[1] - 0.5).abs(), + "C=0 and C=1 rows should differ in chroma packing: {first_i:?} vs {next_c:?}" + ); + } +} + +#[cfg(test)] +mod dim_audit { + use super::*; + use crate::core::ColorPrimaries; + + #[test] + fn map_keeps_mid_brightness() { + // Tone-mapped mid-grays must not collapse to black: I stays near the + // input band even with a little chroma. + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: pq_code_for_t(203.0), + }; + let src = GamutState::new(params.source, params); + let dst = GamutState::new(params.target, params); + for (i, c) in [ + (0.3, 0.0), + (0.3, 0.02), + (0.3, 0.05), + (0.3, 0.1), + (0.5, 0.05), + ] { + let h = 1.0_f32; + let sp = saturate(h, &src); + let dp = saturate(h, &dst); + let maxc = sp[1].max(dp[1]).max(1e-9); + let out = perceptual_map_at( + [i, c, h], + &src, + &dst, + sp, + dp, + maxc, + &Softclip { + knee: 0.7, + _desat: 0.35, + }, + params, + ); + assert!( + out[0] > i * 0.5, + "mid-gray I collapsed: in I={i} C={c} -> out {:?}", + out + ); + assert!(out[0].is_finite() && out[1].is_finite() && out[2].is_finite()); + } + } + + fn pq_code_for_t(n: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = (n / 10000.0).clamp(0.0, 1.0).powf(m1); + ((c1 + c2 * p) / (1.0 + c3 * p).max(1e-6)).powf(m2) + } +} diff --git a/crates/erika/src/renderer/metal/apple.rs b/crates/erika/src/renderer/metal/apple.rs index b881a15f..ebe12394 100644 --- a/crates/erika/src/renderer/metal/apple.rs +++ b/crates/erika/src/renderer/metal/apple.rs @@ -32,7 +32,7 @@ use objc2_foundation::NSString; use objc2_metal::{ MTLBlendFactor, MTLBlendOperation, MTLClearColor, MTLCreateSystemDefaultDevice, MTLLoadAction, MTLOrigin, MTLPixelFormat, MTLRegion, MTLResourceOptions, MTLSize, MTLStorageMode, - MTLStoreAction, MTLTextureDescriptor, MTLTextureUsage, + MTLStoreAction, MTLTextureDescriptor, MTLTextureType, MTLTextureUsage, }; use objc2_metal::{ MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLCommandBufferStatus, MTLCommandEncoder, @@ -49,6 +49,9 @@ use objc2_quartz_core::{kCAContentsFormatRGBA8Uint, kCAContentsFormatRGBA16Float use crate::core::{ColorPrimaries, RendererResourceStats, SurfaceMetrics, TransferFunction}; use crate::danmaku::{DanmakuAtlasUpdate, DanmakuGlyphAtlas, DanmakuRenderPlan}; +use crate::renderer::gamut::{ + GamutLut, GamutLutJob, GamutLutParams, LUT_SIZE_C, LUT_SIZE_H, LUT_SIZE_I, pack_rgba16f, +}; use crate::renderer::metal::upscaler::LumaUpscaler; use crate::renderer::metal::{ ClearColor, DanmakuRenderFrame, ImportedVideoFormat, ImportedVideoFrameInfo, @@ -57,8 +60,9 @@ use crate::renderer::metal::{ VideoFrameTextureSource, VideoRenderFrame, fourcc_string, metal_drawable_pixel_format, metal_target_color, }; -use crate::renderer::pipeline::{ColorRange, LumaUpscalerMode, ToneMapOperator}; -use crate::renderer::pipeline::{SourceColorState, TargetColorState}; +use crate::renderer::output::negotiate_output_mode; +use crate::renderer::pipeline::{ColorRange, DoviUniforms, LumaUpscalerMode}; +use crate::renderer::pipeline::{SourceColorState, TargetColorState, VideoRenderPipeline}; use crate::renderer::presentation::PresentationLayout as VideoPresentationLayout; use crate::subtitle::{AssColor, SubtitleAlphaBitmap}; use crate::trace; @@ -114,6 +118,49 @@ pub struct ImportedVideoFrameResult { pub textures: ImportedVideoFrameTextures, } +/// Identity of a perceptual gamut LUT: it is only valid for one +/// (source, target, target-black, target-peak) combination. The black/peak +/// pair sets the LUT's I range. A cached LUT whose key does not match the +/// current frame must never be bound — the shader would sample a LUT built for +/// a different display/gamut. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct GamutLutKey { + source: u32, + target: u32, + target_black_pq: u32, + target_peak_pq: u32, +} + +impl GamutLutKey { + /// The key for a frame's color pipeline. It deliberately reads only the + /// static target/primaries state: per-frame content brightness (Dolby + /// Vision L1, measured scene average) must not force LUT regeneration. + fn for_pipeline(pipeline: &VideoRenderPipeline) -> Self { + let packed = pipeline.gamut_primaries_code(); + let target_black_nits = pipeline.tone_map_extra()[2]; + Self { + source: packed >> 8, + target: packed & 0xff, + target_black_pq: quantize_luma_pq(target_black_nits), + target_peak_pq: quantize_luma_pq(pipeline.target.peak_nits), + } + } +} + +/// Quantize a luminance (nits) to its PQ code for the LUT cache key. The key +/// only has to change when the LUT's I axis changes, so 16-bit PQ resolution +/// is ample (and resolves the sub-1-nit target blacks of SDR targets, which a +/// linear nits quantization would collapse together). +fn quantize_luma_pq(nits: f32) -> u32 { + (pq_code_for_lut(nits) * 65535.0) as u32 +} + +/// Cache of the generated perceptual gamut LUT for [`GamutLutKey`]. +struct GamutLutCache { + key: GamutLutKey, + texture: Retained>, +} + pub struct MetalRendererImpl { device: Retained>, queue: Retained>, @@ -139,7 +186,24 @@ pub struct MetalRendererImpl { pending_gpu_timing: Option>>, stats: MetalRendererStats, layer_color_space_label: &'static str, + /// Perceptual gamut LUT (3D RGBA16Float) cached per (source, target, + /// peak) key; `None` when the fast path is in use. + gamut_lut: Option, + /// Background generation for a cache miss; the fast `gamut_compress` + /// path renders until the LUT lands. + gamut_lut_job: Option, + dummy_gamut_lut: Option>>, logged_first_video_frame: bool, + /// EDR headroom of the display the layer is presented on, published by + /// the host through `RendererBackend::set_output_headroom`. + /// + /// Read here instead of querying AppKit: `select_output_mode_for_source` + /// runs on the render thread (CVDisplayLink or the host's render queue), + /// where touching `NSView`/`NSWindow`/`NSApplication` violates AppKit's + /// main-thread requirement. `None` means no host has reported a value + /// yet, which negotiates as SDR — the same fallback this used when AppKit + /// could not answer. + reported_display_headroom: Option, } fn hdr_debug_enabled() -> bool { @@ -154,6 +218,24 @@ fn hdr_debug_enabled() -> bool { .unwrap_or(false) } +fn pq_code_for_lut(nits: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = (nits / 10000.0).clamp(0.0, 1.0).powf(m1); + ((c1 + c2 * p) / (1.0 + c3 * p).max(0.000_001)).powf(m2) +} + +fn code_to_primaries(code: u32) -> ColorPrimaries { + match code { + 1 => ColorPrimaries::Bt2020, + 2 => ColorPrimaries::DisplayP3, + _ => ColorPrimaries::Bt709, + } +} + impl MetalRendererImpl { pub fn new(config: MetalRendererConfig) -> Result { let device = MTLCreateSystemDefaultDevice().ok_or_else(|| { @@ -192,7 +274,11 @@ impl MetalRendererImpl { pending_gpu_timing: None, stats: MetalRendererStats::default(), layer_color_space_label: "unconfigured", + gamut_lut: None, + gamut_lut_job: None, + dummy_gamut_lut: None, logged_first_video_frame: false, + reported_display_headroom: None, }) } @@ -367,6 +453,39 @@ impl MetalRendererImpl { self.output_mode } + pub fn is_hdr10_pq(&self) -> bool { + self.output_mode.is_edr() + && matches!( + self.layer_color_space_label, + "itur-2100-pq" | "display-p3-pq" + ) + } + + /// EDR headroom of the display the player window is presented on, as + /// published by the host through `RendererBackend::set_output_headroom`. + /// + /// The host resolves this on the main thread (it owns the `NSView` and + /// knows which screen the window is on) and pushes it here; querying + /// AppKit from this renderer would run on the render thread and violate + /// AppKit's main-thread requirement. No report yet negotiates as SDR. + fn display_edr_headroom(&self) -> f32 { + self.reported_display_headroom.unwrap_or(1.0) + } + + /// Records a host-reported display headroom. `known == false` clears the + /// cache so negotiation falls back to SDR until a real value arrives. + pub fn set_output_headroom(&mut self, headroom: f32, known: bool) { + let resolved = if known && headroom.is_finite() && headroom > 0.0 { + Some(headroom) + } else { + None + }; + if resolved != self.reported_display_headroom { + self.reported_display_headroom = resolved; + self.stats.headroom_updates = self.stats.headroom_updates.saturating_add(1); + } + } + fn select_output_mode_for_source(&mut self, source: SourceColorState) { let source_is_hdr = source.is_hdr(); if source_is_hdr { @@ -378,7 +497,18 @@ impl MetalRendererImpl { let selected = if self.flutter_texture_attached { MetalOutputMode::Sdr } else { - self.requested_output_mode.resolve_for_source(source_is_hdr) + #[cfg(target_os = "macos")] + { + negotiate_output_mode( + self.requested_output_mode, + source_is_hdr, + self.display_edr_headroom(), + ) + } + #[cfg(not(target_os = "macos"))] + { + self.requested_output_mode.resolve_for_source(source_is_hdr) + } }; if selected != self.output_mode { self.set_output_mode(selected); @@ -586,6 +716,159 @@ impl MetalRendererImpl { Ok(()) } + /// Return a cached (or freshly generated) perceptual gamut LUT texture + /// for the frame's color pipeline, or `None` when the fast path is used + /// or the background generation is still pending. `Some` is returned only + /// when the texture matches the frame's key; callers must mask + /// `gamut_lut_enabled` off on `None` so the shader keeps the fast path + /// instead of sampling the 1x1x1 placeholder. + fn gamut_lut_texture( + &mut self, + frame: &VideoRenderFrame<'_>, + ) -> Result>>> { + if !frame.pipeline.gamut_lut_active() { + return Ok(None); + } + let key = GamutLutKey::for_pipeline(&frame.pipeline); + if let Some(cached) = &self.gamut_lut { + if cached.key == key { + return Ok(Some(cached.texture.clone())); + } + } + let params = GamutLutParams { + source: code_to_primaries(key.source), + target: code_to_primaries(key.target), + // Same target black the shader derives from tone_map_extra.z. + min_luma: pq_code_for_lut(frame.pipeline.tone_map_extra()[2]), + max_luma: pq_code_for_lut(frame.pipeline.target.peak_nits), + }; + let job_params = self + .gamut_lut_job + .as_ref() + .map(GamutLutJob::params) + .filter(|job_params| *job_params == params); + if job_params.is_none() { + // First request (or the key changed): spawn generation and keep + // the fast path for this frame. + self.gamut_lut_job = Some(GamutLutJob::spawn(params)); + return Ok(None); + } + let Some(lut) = self.gamut_lut_job.as_ref().and_then(GamutLutJob::poll) else { + return Ok(None); + }; + self.gamut_lut_job = None; + let texture = self.upload_gamut_lut(&lut)?; + self.gamut_lut = Some(GamutLutCache { + key, + texture: texture.clone(), + }); + Ok(Some(texture)) + } + + /// Pack (I, P+0.5, T+0.5) into a fresh 3D RGBA16Float texture. + fn upload_gamut_lut( + &mut self, + lut: &GamutLut, + ) -> Result>> { + // Metal lacks a 3D convenience constructor in this binding; build the + // descriptor from the 2D factory and switch the type/depth. + let descriptor = unsafe { + MTLTextureDescriptor::texture2DDescriptorWithPixelFormat_width_height_mipmapped( + MTLPixelFormat::RGBA16Float, + LUT_SIZE_I, + LUT_SIZE_C, + false, + ) + }; + descriptor.setTextureType(MTLTextureType::Type3D); + unsafe { + descriptor.setDepth(LUT_SIZE_H); + } + descriptor.setUsage(MTLTextureUsage::ShaderRead); + descriptor.setResourceOptions(MTLResourceOptions::StorageModeShared); + let texture = self + .device + .newTextureWithDescriptor(&descriptor) + .ok_or_else(|| { + PlayerError::Renderer( + "newTextureWithDescriptor (gamut LUT) returned nil".to_string(), + ) + })?; + let rgba16 = pack_rgba16f(&lut.texels, 1.0); + let region = MTLRegion { + origin: objc2_metal::MTLOrigin { x: 0, y: 0, z: 0 }, + size: objc2_metal::MTLSize { + width: LUT_SIZE_I, + height: LUT_SIZE_C, + depth: LUT_SIZE_H, + }, + }; + let bytes_per_row = LUT_SIZE_I * 4 * 2; // RGBA16F = 8 bytes/texel + unsafe { + texture.replaceRegion_mipmapLevel_slice_withBytes_bytesPerRow_bytesPerImage( + region, + 0, + 0, + NonNull::new(rgba16.as_ptr().cast::().cast_mut()) + .expect("gamut lut pointer is non-null"), + bytes_per_row, + LUT_SIZE_C * bytes_per_row, // one z-slice = width * bytes_per_row + ); + } + Ok(texture) + } + + fn dummy_gamut_lut_texture(&mut self) -> Result>> { + if let Some(dummy) = &self.dummy_gamut_lut { + return Ok(dummy.clone()); + } + let descriptor = unsafe { + MTLTextureDescriptor::texture2DDescriptorWithPixelFormat_width_height_mipmapped( + MTLPixelFormat::RGBA16Float, + 1, + 1, + false, + ) + }; + descriptor.setTextureType(MTLTextureType::Type3D); + unsafe { + descriptor.setDepth(1); + } + descriptor.setUsage(MTLTextureUsage::ShaderRead); + descriptor.setResourceOptions(MTLResourceOptions::StorageModeShared); + let texture = self + .device + .newTextureWithDescriptor(&descriptor) + .ok_or_else(|| { + PlayerError::Renderer( + "newTextureWithDescriptor (dummy gamut LUT) returned nil".to_string(), + ) + })?; + // 1 texel: (I=0, P=0, T=0) stored with P+0.5, T+0.5 -> [0.0, 0.5, 0.5, 1.0] + let dummy_data: [u16; 4] = [0x0000, 0x3800, 0x3800, 0x3c00]; + let region = MTLRegion { + origin: objc2_metal::MTLOrigin { x: 0, y: 0, z: 0 }, + size: objc2_metal::MTLSize { + width: 1, + height: 1, + depth: 1, + }, + }; + unsafe { + texture.replaceRegion_mipmapLevel_slice_withBytes_bytesPerRow_bytesPerImage( + region, + 0, + 0, + NonNull::new(dummy_data.as_ptr().cast::().cast_mut()) + .expect("dummy lut pointer is non-null"), + 8, + 8, + ); + } + self.dummy_gamut_lut = Some(texture.clone()); + Ok(texture) + } + pub fn render_video_frame(&mut self, frame: VideoRenderFrame<'_>) -> Result<()> { self.render_video_frame_inner(frame, None, None) } @@ -855,12 +1138,23 @@ impl MetalRendererImpl { "renderCommandEncoderWithDescriptor returned nil".to_string(), )); }; + // Resolve the LUT before building the uniforms: while a new key's + // background generation is pending, the shader keeps the fast + // gamut_compress path (the dummy texture holds texture 2). + // Readiness comes from the returned texture, not from the cache: + // a stale cache entry for another key must not enable the LUT. + let cached_gamut_lut = self.gamut_lut_texture(&frame)?; + let gamut_lut_ready = cached_gamut_lut.is_some(); + let gamut_lut = match cached_gamut_lut { + Some(lut) => lut, + None => self.dummy_gamut_lut_texture()?, + }; let uniforms = VideoUniforms { is_p010: matches!(frame.frame.info.format, ImportedVideoFormat::P010) as u32, full_range: matches!(frame.pipeline.source.range, ColorRange::Full) as u32, source_transfer: transfer_code(frame.pipeline.source.transfer), target_transfer: transfer_code(frame.pipeline.target.transfer), - tone_map: tone_map_code(frame.pipeline.tone_map.operator), + tone_map: frame.pipeline.tone_map_uniform_code(), edr_output: self.output_mode.is_edr() as u32, _reserved0: self.video_alpha_mode as u32, _reserved1: 0, @@ -874,10 +1168,26 @@ impl MetalRendererImpl { ], luma_coefficients: luma_coefficients(frame.pipeline.luma_coefficients()), gamut_matrix_rows: frame.pipeline.gamut_matrix().row4s(), + ipt_matrix_rows: frame.pipeline.ipt_matrix_rows(), + tone_map_extra: frame.pipeline.tone_map_extra(), + tone_map_coeffs: frame.pipeline.tone_map_coeffs(), + gamut_lut_enabled: if gamut_lut_ready { + frame.pipeline.gamut_lut_active() as u32 + } else { + 0 + }, + gamut_primaries: frame.pipeline.gamut_primaries_code(), + gamut_reserved0: 0, + gamut_reserved1: 0, + dovi: DoviUniforms::of_for_representation( + &frame.pipeline.source, + matches!(frame.frame.info.format, ImportedVideoFormat::P010), + ), }; encoder.setRenderPipelineState(&pipeline); encoder.setFragmentTexture_atIndex(Some(luma), 0); encoder.setFragmentTexture_atIndex(Some(chroma), 1); + encoder.setFragmentTexture_atIndex(Some(&gamut_lut), 2); encoder.setFragmentSamplerState_atIndex(Some(&sampler), 0); encoder.setVertexBytes_length_atIndex( NonNull::new( @@ -1052,12 +1362,20 @@ impl MetalRendererImpl { )); }; + // Resolve the LUT before building the uniforms (same gating as + // the live path above). + let cached_gamut_lut = self.gamut_lut_texture(&frame)?; + let gamut_lut_ready = cached_gamut_lut.is_some(); + let gamut_lut = match cached_gamut_lut { + Some(lut) => lut, + None => self.dummy_gamut_lut_texture()?, + }; let uniforms = VideoUniforms { is_p010: matches!(frame.frame.info.format, ImportedVideoFormat::P010) as u32, full_range: matches!(frame.pipeline.source.range, ColorRange::Full) as u32, source_transfer: transfer_code(frame.pipeline.source.transfer), target_transfer: transfer_code(frame.pipeline.target.transfer), - tone_map: tone_map_code(frame.pipeline.tone_map.operator), + tone_map: frame.pipeline.tone_map_uniform_code(), edr_output: self.output_mode.is_edr() as u32, _reserved0: 0, _reserved1: 0, @@ -1071,10 +1389,26 @@ impl MetalRendererImpl { ], luma_coefficients: luma_coefficients(frame.pipeline.luma_coefficients()), gamut_matrix_rows: frame.pipeline.gamut_matrix().row4s(), + ipt_matrix_rows: frame.pipeline.ipt_matrix_rows(), + tone_map_extra: frame.pipeline.tone_map_extra(), + tone_map_coeffs: frame.pipeline.tone_map_coeffs(), + gamut_lut_enabled: if gamut_lut_ready { + frame.pipeline.gamut_lut_active() as u32 + } else { + 0 + }, + gamut_primaries: frame.pipeline.gamut_primaries_code(), + gamut_reserved0: 0, + gamut_reserved1: 0, + dovi: DoviUniforms::of_for_representation( + &frame.pipeline.source, + matches!(frame.frame.info.format, ImportedVideoFormat::P010), + ), }; encoder.setRenderPipelineState(&pipeline); encoder.setFragmentTexture_atIndex(Some(luma), 0); encoder.setFragmentTexture_atIndex(Some(chroma), 1); + encoder.setFragmentTexture_atIndex(Some(&gamut_lut), 2); encoder.setFragmentSamplerState_atIndex(Some(&sampler), 0); encoder.setVertexBytes_length_atIndex( NonNull::new( @@ -1208,7 +1542,15 @@ impl MetalRendererImpl { &plane.rgba, )?; let (x, y, width, height) = plane.scaled_rect(viewport_width, viewport_height); - let uniforms = OverlayUniforms::from_plane(x, y, width, height, layout, target); + let uniforms = OverlayUniforms::from_plane( + x, + y, + width, + height, + layout, + target, + self.output_mode.is_edr(), + ); unsafe { encoder.setRenderPipelineState(&pipeline); encoder.setFragmentTexture_atIndex(Some(&*texture), 0); @@ -1248,6 +1590,7 @@ impl MetalRendererImpl { atlas_height, layout, target, + self.output_mode.is_edr(), ); unsafe { encoder.setRenderPipelineState(&pipeline); @@ -1417,7 +1760,7 @@ impl MetalRendererImpl { viewport: layout.overlay_viewport(), target_transfer: transfer_code(target.transfer), _reserved0: 0, - ui_nits: [ui_reference_white_nits(target), 0.0, 0.0, 0.0], + ui_nits: ui_output_nits(target, self.output_mode.is_edr()), }; unsafe { encoder.setRenderPipelineState(pipeline); @@ -2160,6 +2503,14 @@ struct VideoUniforms { nits: [f32; 4], luma_coefficients: [f32; 4], gamut_matrix_rows: [[f32; 4]; 3], + ipt_matrix_rows: [[f32; 4]; 9], + tone_map_extra: [f32; 4], + tone_map_coeffs: [f32; 4], + gamut_lut_enabled: u32, + gamut_primaries: u32, + gamut_reserved0: u32, + gamut_reserved1: u32, + dovi: DoviUniforms, } fn metal_pixel_format(format: MetalDrawablePixelFormat) -> MTLPixelFormat { @@ -2264,14 +2615,28 @@ fn ui_reference_white_nits(target: TargetColorState) -> f32 { } } -fn tone_map_code(operator: ToneMapOperator) -> u32 { - match operator { - ToneMapOperator::Clip => 0, - ToneMapOperator::Reinhard => 1, - ToneMapOperator::Mobius => 2, +/// `ui_nits.y`: non-zero when the drawable holds linear light (Apple EDR / +/// extended-linear output), in which case the UI color must be linearized +/// before compositing — the video pass writes linear values into the same +/// buffer. PQ targets encode the UI inside the shader and SDR targets +/// composite in the output transfer, so both keep 0 here. +fn ui_linear_reference_white_nits(target: TargetColorState, edr_output: bool) -> f32 { + if !edr_output || matches!(target.transfer, TransferFunction::Pq) { + 0.0 + } else { + target.reference_white_nits.max(1.0) } } +fn ui_output_nits(target: TargetColorState, edr_output: bool) -> [f32; 4] { + [ + ui_reference_white_nits(target), + ui_linear_reference_white_nits(target, edr_output), + 0.0, + 0.0, + ] +} + fn luma_coefficients(coeffs: crate::renderer::pipeline::LumaCoefficients) -> [f32; 4] { [coeffs.kr, coeffs.kg, coeffs.kb, 0.0] } @@ -2330,6 +2695,7 @@ impl OverlayUniforms { height: u32, layout: VideoPresentationLayout, target: TargetColorState, + edr_output: bool, ) -> Self { Self { rect: layout.map_source_rect(x as f32, y as f32, width as f32, height as f32), @@ -2338,7 +2704,7 @@ impl OverlayUniforms { overlay_mode: 0, target_transfer: transfer_code(target.transfer), color: [1.0, 1.0, 1.0, 1.0], - ui_nits: [ui_reference_white_nits(target), 0.0, 0.0, 0.0], + ui_nits: ui_output_nits(target, edr_output), } } @@ -2349,6 +2715,7 @@ impl OverlayUniforms { atlas_height: usize, layout: VideoPresentationLayout, target: TargetColorState, + edr_output: bool, ) -> Self { let color = AssColor::from_libass_rgba(bitmap.color_rgba); let atlas_width = atlas_width.max(1) as f32; @@ -2375,7 +2742,7 @@ impl OverlayUniforms { color.blue as f32 / 255.0, color.alpha as f32 / 255.0, ], - ui_nits: [ui_reference_white_nits(target), 0.0, 0.0, 0.0], + ui_nits: ui_output_nits(target, edr_output), } } } @@ -2792,6 +3159,21 @@ struct VideoUniforms { float4 nits; float4 luma_coefficients; float4 gamut_matrix_rows[3]; + float4 ipt_matrix_rows[9]; + float4 tone_map_extra; + float4 tone_map_coeffs; + uint gamut_lut_enabled; + uint gamut_primaries; + uint gamut_reserved0; + uint gamut_reserved1; + float4 dovi_flags; + float4 dovi_pivots[6]; + float4 dovi_bounds[3]; + float4 dovi_coefficients[24]; + float4 dovi_mmr[144]; + float4 dovi_nonlinear_matrix[3]; + float4 dovi_nonlinear_offset; + float4 dovi_lms_matrix[3]; }; float source_peak_nits(constant VideoUniforms& uniforms) { @@ -2881,25 +3263,16 @@ float3 source_reference_to_nits(float3 rgb, constant VideoUniforms& uniforms) { return max(rgb, float3(0.0)) * source_reference_white_nits(uniforms); } -float3 tone_map_nits(float3 nits, constant VideoUniforms& uniforms) { - float source_peak = source_peak_nits(uniforms); - float target_peak = target_peak_nits(uniforms); - float3 x = max(nits, float3(0.0)) / target_peak; - float white = max(source_peak / target_peak, 1.0); - if (uniforms.tone_map == 1) { - float white2 = white * white; - return target_peak * clamp((x * (float3(1.0) + x / white2)) / (float3(1.0) + x), 0.0, 1.0); - } - if (uniforms.tone_map == 2) { - constexpr float knee = 0.75; - float denom = max(white - knee, 0.0001); - float3 t = clamp((x - float3(knee)) / denom, 0.0, 1.0); - float3 shoulder = knee + (1.0 - knee) * (float3(1.0) - pow(float3(1.0) - t, float3(2.0))); - return target_peak * mix(x, shoulder, step(float3(knee), x)); - } - return target_peak * clamp(x, 0.0, 1.0); +float pq_code(float nits) { + return pq_inverse_eotf(clamp(nits, 0.0, 10000.0) / 10000.0); +} + +float nits_from_pq(float code) { + return 10000.0 * pq_eotf(clamp(code, 0.0, 1.0)); } +// Simple primaries conversion (HDR10 output path); the tone-mapped path +// converts primaries inside the IPT roundtrip instead. float3 apply_gamut_map(float3 rgb, constant VideoUniforms& uniforms) { return float3( dot(uniforms.gamut_matrix_rows[0].xyz, rgb), @@ -2908,8 +3281,286 @@ float3 apply_gamut_map(float3 rgb, constant VideoUniforms& uniforms) { ); } +// libplacebo pl_smoothstep with arbitrary edge order (Metal smoothstep has +// undefined results when edge0 >= edge1, and libplacebo's knee tuning term +// deliberately uses reversed edges). +float sstep(float edge0, float edge1, float x) { + float t = clamp((x - edge0) / (edge1 - edge0), 0.0, 1.0); + return t * t * (3.0 - 2.0 * t); +} + +// libplacebo st2094_pick_knee evaluated on absolute PQ codes. The source +// pivot follows the scene average luminance when known and stays within +// [10%, 80%] of the range; the destination pivot rescales it into the output +// range and then adapts towards the 1:1 line (knee_adaptation 0.4). +float2 st2094_pick_knee(float src_min, float src_max, float src_avg, float dst_min, float dst_max) { + constexpr float knee_adaptation = 0.4; + constexpr float min_knee = 0.1; + constexpr float max_knee = 0.8; + constexpr float def_knee = 0.4; + float src_knee_min = mix(src_min, src_max, min_knee); + float src_knee_max = mix(src_min, src_max, max_knee); + float dst_knee_min = mix(dst_min, dst_max, min_knee); + float dst_knee_max = mix(dst_min, dst_max, max_knee); + float fallback = mix(src_min, src_max, def_knee); + float src_knee = clamp(src_avg > 0.0 ? src_avg : fallback, src_knee_min, src_knee_max); + float target = (src_knee - src_min) / max(src_max - src_min, 0.000001); + float adapted = mix(dst_min, dst_max, target); + float tuning = 1.0 - sstep(max_knee, def_knee, target) * sstep(min_knee, def_knee, target); + float adaptation = mix(knee_adaptation, 1.0, tuning); + float dst_knee = clamp(mix(src_knee, adapted, adaptation), dst_knee_min, dst_knee_max); + return float2(src_knee, dst_knee); +} + +// The tone-map curve evaluated on the IPT intensity axis (PQ codes), +// mirroring libplacebo's tone-map functions. `param` is the per-operator +// curve parameter from ToneMapConfig::curve_param (0 = operator default). +float tone_map_curve_pq(float x_in, float param, constant VideoUniforms& uniforms) { + float src_peak = source_peak_nits(uniforms); + float dst_peak = target_peak_nits(uniforms); + float src_avg = uniforms.tone_map_extra.y; + float dst_black = uniforms.tone_map_extra.z; + float in_min = 0.0; + float in_max = max(pq_code(src_peak), 0.000001); + float out_min = pq_code(dst_black); + float out_max = max(pq_code(dst_peak), 0.000001); + float out_range = max(out_max - out_min, 0.000001); + float x = clamp(x_in, in_min, in_max); + if (uniforms.tone_map == 0) { + // Clip: values within the source range pass through untouched. + return x; + } + if (uniforms.tone_map == 1) { + // Reinhard (output-relative, libplacebo pl_tone_map_reinhard). + // + // libplacebo evaluates this curve in the linear PL_HDR_NORM domain + // and only the PQ-domain operators (BT.2390, spline, ST 2094-10) + // work on encoded values. Running it on PQ codes bends the curve's + // meaning: against a 1000->203 nit target mid-tones darken badly, and + // the Mobius knee lands far below its intended position. Rescale to + // linear nits here and encode back at the end. + float in_max_nits = max(src_peak, 0.000001); + float out_min_nits = dst_black; + float out_range_nits = max(dst_peak - dst_black, 0.000001); + float peak = in_max_nits / out_range_nits; + float contrast = param > 0.0 ? param : 0.5; + float offset = (1.0 - contrast) / max(contrast, 0.000001); + float scale = (peak + offset) / peak; + float t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; + float mapped = t / (t + offset) * scale; + return pq_code(mapped * out_range_nits + out_min_nits); + } + if (uniforms.tone_map == 2) { + // Mobius: Mobius transform with a 1:1 linear region below the knee, + // also evaluated in linear nits (see the Reinhard note above). The + // knee j is relative to the output range, so the linear region ends + // at dst_black + j * (dst_peak - dst_black). + float in_max_nits = max(src_peak, 0.000001); + float out_min_nits = dst_black; + float out_range_nits = max(dst_peak - dst_black, 0.000001); + float peak = in_max_nits / out_range_nits; + float j = param > 0.0 ? param : 0.3; + float a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + float b = (j * j - 2.0 * j * peak + peak) / max(peak - 1.0, 0.000001); + float scale = (b * b + 2.0 * b * j + j * j) / (b - a); + float t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; + float mapped = t > j ? scale * (t + a) / (t + b) : t; + return pq_code(mapped * out_range_nits + out_min_nits); + } + if (uniforms.tone_map == 3) { + // ITU-R BT.2390 EETF with black-point compensation (the libplacebo + // version also compensates target black; the earlier port skipped it). + float knee_offset = param > 0.0 ? param : 1.0; + float max_lum = clamp(out_max / in_max, 0.0, 1.0); + float min_lum = out_min / in_max; + float ks = (1.0 + knee_offset) * max_lum - knee_offset; + float bp = min(max(1.0 / max(min_lum, 0.000001), 0.0), 4.0); + float u = x / in_max; + if (ks < 1.0 && u > ks) { + float tb = (u - ks) / (1.0 - ks); + float tb2 = tb * tb; + float tb3 = tb2 * tb; + u = (2.0 * tb3 - 3.0 * tb2 + 1.0) * ks + + (tb3 - 2.0 * tb2 + tb) * (1.0 - ks) + + (-2.0 * tb3 + 3.0 * tb2) * max_lum; + } + if (u < 1.0) { + u = u + min_lum * pow(1.0 - u, bp); + float gain = max_lum < 1.0 + ? 1.0 / (1.0 + min_lum / max_lum * pow(1.0 - max_lum, bp)) + : 1.0; + u = gain * (u - min_lum) + min_lum; + } + return u * in_max; + } + if (uniforms.tone_map == 4) { + // Spline: perceptually linear single-pivot polynomial, the default + // tone map of libplacebo and mpv's gpu-next renderer. + float contrast = param > 0.0 ? param : 0.3; + float fallback_avg = clamp(0.4 * src_peak, 100.0, 400.0); + float effective_src_avg = src_avg > 0.0 ? src_avg : fallback_avg; + float2 knee = st2094_pick_knee( + in_min, + in_max, + pq_code(effective_src_avg), + out_min, + out_max + ); + float src_pivot = knee.x; + float dst_pivot = knee.y; + float slope0 = (dst_pivot - out_min) / max(src_pivot - in_min, 0.000001); + float ratio = clamp(1.5 * (in_max / out_max - 1.0), 0.2, 1.2); + float slope = pow(slope0, (1.0 - contrast) * ratio); + float in_min0 = in_min - src_pivot; + float in_max0 = in_max - src_pivot; + float out_min0 = out_min - dst_pivot; + float out_max0 = out_max - dst_pivot; + float pa = (out_min0 - slope * in_min0) / (in_min0 * in_min0); + float qa = (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0 * in_max0); + float qb = -3.0 * (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0); + float xr = x - src_pivot; + float mapped = xr > 0.0 + ? ((qa * xr + qb) * xr + slope) * xr + : (pa * xr + slope) * xr; + return mapped + dst_pivot; + } + if (uniforms.tone_map == 5) { + // ITU-R BT.2446 method A: Weber-law log compression from the source + // peak envelope and a standardized S-curve (mpv's recommended curve + // for well-mastered content). + float phdr = 1.0 + 32.0 * pow(src_peak / 10000.0, 1.0 / 2.4); + float psdr = 1.0 + 32.0 * pow(dst_peak / 10000.0, 1.0 / 2.4); + float t = pow(nits_from_pq(x) / max(src_peak, 0.000001), 1.0 / 2.4); + t = log(1.0 + (phdr - 1.0) * t) / log(phdr); + if (t <= 0.7399) { + t = 1.0770 * t; + } else if (t < 0.9909) { + t = (-1.1510 * t + 2.7811) * t - 0.6302; + } else { + t = 0.5 * t + 0.5; + } + t = (pow(psdr, t) - 1.0) / (psdr - 1.0); + // BT.1886 EOTF from the target black point and peak. + float lb = pow(max(dst_black, 0.0), 1.0 / 2.4); + float lw = pow(max(dst_peak, 0.0), 1.0 / 2.4); + return pq_code(pow((lw - lb) * t + lb, 2.4)); + } + // SMPTE ST 2094-10 (DolbyVision's dynamic-metadata curve): rational + // Mobius interpolation in absolute nits; coefficients are solved per + // frame on the CPU from the same scene pivot. + float c1 = uniforms.tone_map_coeffs.x; + float c2 = uniforms.tone_map_coeffs.y; + float c3 = uniforms.tone_map_coeffs.z; + float x_nits = nits_from_pq(x); + float y_nits = (c1 + c2 * x_nits) / max(1.0 + c3 * x_nits, 0.000001); + return pq_code(clamp(y_nits, 0.0, 10000.0)); +} + +float3 tone_map_nits( + float3 input_nits, + texture3d gamut_lut, + sampler video_sampler, + constant VideoUniforms& uniforms +) { + if (uniforms.target_transfer == 3) { + // HDR10 output: convert primaries by the gamut matrix and clamp to + // the PQ range (no tone mapping; the display does the HDR mapping). + return clamp(apply_gamut_map(max(input_nits, float3(0.0)) / source_reference_white_nits(uniforms), uniforms) + * source_reference_white_nits(uniforms), float3(0.0), float3(10000.0)); + } + // libplacebo color map: RGB in source primaries (absolute nits) to + // HPE-LMS, PQ-encode, IPT, map the intensity axis and apply the + // hue-preserving chroma rule, optionally sample the 3D gamut LUT in + // IPT space, then decode back to RGB in the target primaries (rows 6-8). + // The primaries conversion and gamut mapping happen in this single IPT pass. + float3 rgb = max(input_nits, float3(0.0)); + float3 lms = float3( + dot(uniforms.ipt_matrix_rows[0].xyz, rgb), + dot(uniforms.ipt_matrix_rows[1].xyz, rgb), + dot(uniforms.ipt_matrix_rows[2].xyz, rgb) + ); + float3 lmspq = float3(pq_code(lms.r), pq_code(lms.g), pq_code(lms.b)); + float3 ipt = float3( + dot(float3(0.4, 0.4, 0.2), lmspq), + dot(float3(4.455, -4.851, 0.396), lmspq), + dot(float3(0.8056, 0.3572, -1.1628), lmspq) + ); + float i_orig = ipt.x; + ipt.x = tone_map_curve_pq(ipt.x, uniforms.tone_map_extra.x, uniforms); + // Libplacebo's chroma rule: clamp the saturation boost when brightening + // and desaturate (by the cubic hull term) when the mapping darkens. + float2 hull = float2(i_orig, ipt.x); + float2 hull_c = ((hull - float2(6.0)) * hull + float2(9.0)) * hull; + float ratio = min(i_orig / max(ipt.x, 0.000001), hull_c.y / max(hull_c.x, 0.000001)); + ipt.yz = ipt.yz * ratio; + + if (uniforms.gamut_lut_enabled != 0) { + // I axis spans the target's [black, peak] in PQ codes, matching + // libplacebo's gamut.min_luma/max_luma (tone_map_extra.z is the + // target black in nits, the same value the LUT was generated for). + float lut_min = pq_code(uniforms.tone_map_extra.z); + float lut_max = max(pq_code(target_peak_nits(uniforms)), 0.000001); + float lut_range = max(lut_max - lut_min, 0.000001); + float3 pos = float3( + clamp((ipt.x - lut_min) / lut_range, 0.0, 1.0), + clamp(2.0 * length(ipt.yz), 0.0, 1.0), + 0.5 + 0.5 * atan2(ipt.z, ipt.y) / 3.14159265 + ); + // libplacebo's texel_scale: the lattice position must be remapped to + // the texel-center coordinate, otherwise the low end of the chroma + // axis (whose first texel stores zero chroma) leaks in and crushes + // saturation. + float3 idx = float3( + pos.x * (47.0 / 48.0) + 0.5 / 48.0, + pos.y * (31.0 / 32.0) + 0.5 / 32.0, + pos.z * (255.0 / 256.0) + 0.5 / 256.0 + ); + float3 sampled = gamut_lut.sample(video_sampler, idx).xyz; + ipt = float3(sampled.x, sampled.y - 0.5, sampled.z - 0.5); + } + + float3 lmspq_out = float3( + dot(float3(1.0, 0.0975689, 0.205226), ipt), + dot(float3(1.0, -0.113876, 0.133217), ipt), + dot(float3(1.0, 0.0326151, -0.676887), ipt) + ); + float3 lms_out = float3( + nits_from_pq(lmspq_out.r), + nits_from_pq(lmspq_out.g), + nits_from_pq(lmspq_out.b) + ); + return float3( + dot(uniforms.ipt_matrix_rows[6].xyz, lms_out), + dot(uniforms.ipt_matrix_rows[7].xyz, lms_out), + dot(uniforms.ipt_matrix_rows[8].xyz, lms_out) + ); +} + +// Hue-preserving gamut mapping: the linear gamut matrix can push highly +// saturated wide-gamut colors outside the target gamut (negative +// components). Blending those towards luma shifts hue — BT.2020 primary +// red picks up blue and turns pink. Instead blend towards the naive clip +// by an out-of-gamut smoothstep factor: slightly-out colors stay nearly +// intact, strongly-out primaries land on the pure target primary with +// their hue intact, matching mpv's perceptual gamut handling. Mirrors the +// WGSL/HLSL `gamut_compress` and the Rust reference in pipeline.rs tests. +// Brightness overshoot (> 1) is left for the tone map. +float3 gamut_compress(float3 rgb) { + float lo = min(rgb.r, min(rgb.g, rgb.b)); + float outness = max(-lo, 0.0); + float k = smoothstep(0.0, 1.0, outness); + return mix(rgb, clamp(rgb, 0.0, 1.0), k); +} + float3 target_nits_to_reference_linear(float3 nits, constant VideoUniforms& uniforms) { - return max(nits, float3(0.0)) / target_reference_white_nits(uniforms); + // libplacebo's encode maps [target black, target peak] onto [0, 1] where + // 1.0 is the target reference white, so the tone-map black-point + // compensation lands back on true black instead of lifting it. + float black = uniforms.tone_map_extra.z; + float peak = target_peak_nits(uniforms); + float range = max(peak - black, 0.0001); + return max(nits - float3(black), float3(0.0)) / range + * (range / target_reference_white_nits(uniforms)); } float3 target_reference_linear_to_output(float3 rgb, constant VideoUniforms& uniforms) { @@ -2949,17 +3600,25 @@ float4 final_output(float3 rgb, float alpha, constant VideoUniforms& uniforms) { return float4(premultiplied, alpha); } -float3 sdr_ui_color_to_target_output(float3 rgb, uint target_transfer, float reference_white_nits) { +// SDR composites the UI in the output transfer. HDR10 (target_transfer == 3) +// PQ-encodes it against ui_nits.x. An Apple EDR / extended-linear drawable +// holds linear light (ui_nits.y != 0), so the sRGB-encoded UI color must be +// linearized and scaled to the same reference white as the video pass. +float3 sdr_ui_color_to_target_output(float3 rgb, uint target_transfer, float4 ui_nits) { if (target_transfer == 3) { constexpr float pq_absolute_peak_nits = 10000.0; float3 linear = pow(max(rgb, float3(0.0)), float3(2.2)); - float3 nits = linear * max(reference_white_nits, 1.0); + float3 nits = linear * max(ui_nits.x, 1.0); return float3( pq_inverse_eotf(nits.r / pq_absolute_peak_nits), pq_inverse_eotf(nits.g / pq_absolute_peak_nits), pq_inverse_eotf(nits.b / pq_absolute_peak_nits) ); } + if (ui_nits.y > 0.0) { + float3 linear_rgb = pow(max(rgb, float3(0.0)), float3(2.2)); + return linear_rgb * (max(ui_nits.x, 1.0) / max(ui_nits.y, 1.0)); + } return rgb; } @@ -2969,6 +3628,12 @@ struct RangeExpandedYCbCr { }; RangeExpandedYCbCr expand_ycbcr_range(float y, float2 cbcr, constant VideoUniforms& uniforms) { + if (uniforms.is_p010 != 0) { + // P010 stores 10-bit codes as code << 6 in a 16-bit UNORM texture. + constexpr float p010_scale = 65535.0 / 65472.0; + y *= p010_scale; + cbcr *= p010_scale; + } if (uniforms.full_range != 0) { return RangeExpandedYCbCr { y, cbcr - float2(0.5) }; } @@ -2984,6 +3649,83 @@ RangeExpandedYCbCr expand_ycbcr_range(float y, float2 cbcr, constant VideoUnifor return RangeExpandedYCbCr { y, cbcr }; } +// Dolby Vision RPU reshaping, ported from libplacebo's `pl_shader_dovi_reshape` +// (the renderer behind mpv's Dolby Vision mapping). The base-layer signal is +// reshaped per component through piecewise polynomial/MMR curves selected by +// pivot comparison, where MMR coefficients mix all three raw components. +float3 dovi_reshaped_signal(float3 sig_in, constant VideoUniforms& uniforms) { + float3 sig = clamp(sig_in, 0.0, 1.0); + float result[3] = { sig.r, sig.g, sig.b }; + float4 flags = uniforms.dovi_flags; + for (uint c = 0u; c < 3u; c = c + 1u) { + uint segments = uint(flags[1u + c]); + if (segments == 0u) { + continue; + } + float s = result[c]; + uint index = 0u; + for (uint i = 0u; i < 7u; i = i + 1u) { + float4 pivot_row = uniforms.dovi_pivots[2u * c + i / 4u]; + float pivot = pivot_row[i % 4u]; + if (s >= pivot) { + index = index + 1u; + } + } + float4 coeff = uniforms.dovi_coefficients[8u * c + index]; + if (coeff.w < 0.5) { + s = (coeff.z * s + coeff.y) * s + coeff.x; + } else { + uint base = 48u * c + uint(coeff.y); + uint order = uint(coeff.w); + float4 sig_x = float4( + sig.x * sig.y, + sig.x * sig.z, + sig.y * sig.z, + sig.x * sig.y * sig.z + ); + s = coeff.x; + s = s + dot(uniforms.dovi_mmr[base].xyz, sig); + s = s + dot(uniforms.dovi_mmr[base + 1u], sig_x); + if (order >= 2u) { + float3 sig2 = sig * sig; + float4 sig_x2 = sig_x * sig_x; + s = s + dot(uniforms.dovi_mmr[base + 2u].xyz, sig2); + s = s + dot(uniforms.dovi_mmr[base + 3u], sig_x2); + if (order >= 3u) { + s = s + dot(uniforms.dovi_mmr[base + 4u].xyz, sig2 * sig); + s = s + dot(uniforms.dovi_mmr[base + 5u], sig_x2 * sig_x); + } + } + } + float4 bounds = uniforms.dovi_bounds[c]; + result[c] = clamp(s, bounds.x, bounds.y); + } + return float3(result[0], result[1], result[2]); +} + +// Reshaped nonlinear signal to PQ-encoded IPT via the RPU's ycc_to_rgb matrix +// and signal offsets. Applying the RPU offsets keeps integer offset codes +// exactly on sample codes (2^bits/(2^bits-1) folded in on the CPU). +float3 dovi_signal_to_pq_rgb(float3 sig, constant VideoUniforms& uniforms) { + float3 reshaped = dovi_reshaped_signal(sig, uniforms) - uniforms.dovi_nonlinear_offset.xyz; + return float3( + dot(uniforms.dovi_nonlinear_matrix[0].xyz, reshaped), + dot(uniforms.dovi_nonlinear_matrix[1].xyz, reshaped), + dot(uniforms.dovi_nonlinear_matrix[2].xyz, reshaped) + ); +} + +// Linearized BT.2020-referred HPE LMS back to linear RGB, using the composite +// of the fixed HPE inverse with the RPU's rgb_to_lms matrix (premultiplied on +// the CPU, matching libplacebo's dovi_lms2rgb). +float3 dovi_lms_to_rgb(float3 linear, constant VideoUniforms& uniforms) { + return float3( + dot(uniforms.dovi_lms_matrix[0].xyz, linear), + dot(uniforms.dovi_lms_matrix[1].xyz, linear), + dot(uniforms.dovi_lms_matrix[2].xyz, linear) + ); +} + vertex VertexOut erika_video_vertex( uint vertex_id [[vertex_id]], constant VideoUniforms& uniforms [[buffer(0)]]) { @@ -3014,6 +3756,7 @@ fragment float4 erika_video_fragment( VertexOut in [[stage_in]], texture2d luma_texture [[texture(0)]], texture2d chroma_texture [[texture(1)]], + texture3d gamut_lut [[texture(2)]], sampler video_sampler [[sampler(0)]], constant VideoUniforms& uniforms [[buffer(0)]]) { bool packed_alpha = uniforms.video_alpha_mode == 1; @@ -3021,24 +3764,39 @@ fragment float4 erika_video_fragment( ? float2(in.tex_coord.x * 0.5, in.tex_coord.y) : in.tex_coord; float2 alpha_coord = float2(0.5 + in.tex_coord.x * 0.5, in.tex_coord.y); - float y = luma_texture.sample(video_sampler, color_coord).r; - float2 cbcr = chroma_texture.sample(video_sampler, color_coord).rg; - RangeExpandedYCbCr expanded = expand_ycbcr_range(y, cbcr, uniforms); - y = expanded.y; - cbcr = expanded.cbcr; - - float kr = uniforms.luma_coefficients.x; - float kg = max(uniforms.luma_coefficients.y, 0.000001); - float kb = uniforms.luma_coefficients.z; + float y_sample = luma_texture.sample(video_sampler, color_coord).r; + float2 cbcr_sample = chroma_texture.sample(video_sampler, color_coord).rg; + bool dovi_enabled = uniforms.dovi_flags.x != 0.0; float3 rgb; - rgb.r = y + 2.0 * (1.0 - kr) * cbcr.y; - rgb.b = y + 2.0 * (1.0 - kb) * cbcr.x; - rgb.g = (y - kr * rgb.r - kb * rgb.b) / kg; + if (dovi_enabled) { + // The base layer carries the raw 12-bit DV signal (10-bit container, + // full range); range expansion and the YCbCr matrix are replaced by + // the RPU reshaping + ycc_to_rgb path. + float3 sig = float3(y_sample, cbcr_sample.x, cbcr_sample.y); + if (uniforms.is_p010 != 0) { + sig *= 65535.0 / 65472.0; + } + rgb = dovi_signal_to_pq_rgb(sig, uniforms); + } else { + RangeExpandedYCbCr expanded = expand_ycbcr_range(y_sample, cbcr_sample, uniforms); + float y = expanded.y; + float2 cbcr = expanded.cbcr; + + float kr = uniforms.luma_coefficients.x; + float kg = max(uniforms.luma_coefficients.y, 0.000001); + float kb = uniforms.luma_coefficients.z; + rgb.r = y + 2.0 * (1.0 - kr) * cbcr.y; + rgb.b = y + 2.0 * (1.0 - kb) * cbcr.x; + rgb.g = (y - kr * rgb.r - kb * rgb.b) / kg; + } rgb = transfer_to_source_reference_linear(rgb, uniforms); - rgb = apply_gamut_map(rgb, uniforms); + if (dovi_enabled) { + rgb = dovi_lms_to_rgb(rgb, uniforms); + } rgb = source_reference_to_nits(rgb, uniforms); - rgb = tone_map_nits(rgb, uniforms); + rgb = tone_map_nits(rgb, gamut_lut, video_sampler, uniforms); rgb = target_nits_to_reference_linear(rgb, uniforms); + rgb = gamut_compress(rgb); rgb = target_reference_linear_to_output(rgb, uniforms); float alpha = 1.0; if (packed_alpha) { @@ -3096,14 +3854,14 @@ fragment float4 erika_overlay_fragment( float3 rgb = sdr_ui_color_to_target_output( uniforms.color.rgb, uniforms.target_transfer, - uniforms.ui_nits.x + uniforms.ui_nits ); return float4(rgb, uniforms.color.a * sampled.r); } sampled.rgb = sdr_ui_color_to_target_output( sampled.rgb, uniforms.target_transfer, - uniforms.ui_nits.x + uniforms.ui_nits ); return sampled; } @@ -3173,7 +3931,7 @@ fragment float4 erika_danmaku_batch_fragment( float3 rgb = sdr_ui_color_to_target_output( in.color.rgb, uniforms.target_transfer, - uniforms.ui_nits.x + uniforms.ui_nits ); return float4(rgb, in.color.a * mask); } @@ -3305,10 +4063,12 @@ fn create_plane_texture( mod tests { use super::{ DANMAKU_FILL_ATLAS_TEXTURE, DANMAKU_OUTLINE_ATLAS_TEXTURE, DanmakuBatchInstance, - VIDEO_SHADER_SOURCE, for_each_ordered_danmaku_instance, metal_pixel_format, + VIDEO_SHADER_SOURCE, for_each_ordered_danmaku_instance, metal_pixel_format, ui_output_nits, }; + use crate::core::{ColorPrimaries, TransferFunction}; use crate::danmaku::DanmakuGlyphInstance; use crate::renderer::metal::MetalDrawablePixelFormat; + use crate::renderer::pipeline::TargetColorState; use objc2_metal::MTLPixelFormat; #[test] @@ -3348,6 +4108,153 @@ mod tests { assert!(VIDEO_SHADER_SOURCE.contains("return final_output(rgb, alpha, uniforms)")); } + #[test] + fn overlay_shader_linearizes_the_ui_for_extended_linear_output() { + // The EDR/extended-linear drawable holds linear light, so the overlay + // and danmaku passes must not composite the sRGB-encoded UI color + // directly (ui_nits.y carries the scene-linear reference white). + assert!(VIDEO_SHADER_SOURCE.contains("if (ui_nits.y > 0.0)")); + assert!( + VIDEO_SHADER_SOURCE + .contains("float3 linear_rgb = pow(max(rgb, float3(0.0)), float3(2.2))") + ); + assert!(VIDEO_SHADER_SOURCE.contains("max(ui_nits.x, 1.0) / max(ui_nits.y, 1.0)")); + } + + #[test] + fn gamut_lut_shader_samples_the_target_black_to_peak_axis() { + // libplacebo's LUT I axis is [target black, target peak]; sampling + // `ipt.x / peak` again would diverge from the generated LUT. The + // lattice position must also be remapped to the texel-center + // coordinate (libplacebo's `texel_scale`), or the zero-chroma texel at + // the low end of the C axis crushes saturation. + assert!(VIDEO_SHADER_SOURCE.contains("float lut_min = pq_code(uniforms.tone_map_extra.z)")); + assert!(VIDEO_SHADER_SOURCE.contains("clamp((ipt.x - lut_min) / lut_range, 0.0, 1.0)")); + assert!(VIDEO_SHADER_SOURCE.contains("pos.y * (31.0 / 32.0) + 0.5 / 32.0")); + assert!(VIDEO_SHADER_SOURCE.contains("pos.x * (47.0 / 48.0) + 0.5 / 48.0")); + } + + #[test] + fn ui_output_nits_flags_only_extended_linear_output() { + let sdr = TargetColorState::sdr(ColorPrimaries::Bt709); + assert_eq!(ui_output_nits(sdr, false), [100.0, 0.0, 0.0, 0.0]); + + let hdr10 = TargetColorState::hdr10(ColorPrimaries::Bt2020); + assert_eq!(ui_output_nits(hdr10, false), [203.0, 0.0, 0.0, 0.0]); + + // `metal_target_color` shapes a non-PQ EDR target with a 100-nit + // reference white; the UI is then linearized against it. + let edr = TargetColorState { + primaries: ColorPrimaries::Bt709, + transfer: TransferFunction::Srgb, + peak_nits: 400.0, + reference_white_nits: 100.0, + edr_headroom: 4.0, + }; + assert_eq!(ui_output_nits(edr, true), [100.0, 100.0, 0.0, 0.0]); + assert_eq!(ui_output_nits(edr, false), [100.0, 0.0, 0.0, 0.0]); + + // An explicit EDR request clamped to headroom 1.0 still renders the + // linear drawable, so the flag must come from the output mode rather + // than from the headroom. + let edr_unit = TargetColorState { + edr_headroom: 1.0, + peak_nits: 100.0, + ..edr + }; + assert_eq!(ui_output_nits(edr_unit, true), [100.0, 100.0, 0.0, 0.0]); + + // A PQ EDR target encodes the UI inside the shader (branch on + // target_transfer), so it must not also set the linear flag. + let edr_pq = TargetColorState { + primaries: ColorPrimaries::Bt2020, + transfer: TransferFunction::Pq, + peak_nits: 10_000.0, + reference_white_nits: 203.0, + edr_headroom: 4.0, + }; + assert_eq!(ui_output_nits(edr_pq, true), [203.0, 0.0, 0.0, 0.0]); + } + + #[test] + fn gamut_lut_key_tracks_only_static_pipeline_state() { + // Regression guard for the placeholder-LUT black frames: the cache may + // only be reused when the key equals the current frame's key, so the + // key must change for anything that changes the LUT (target black and + // peak, source/target primaries) and must not change for per-frame + // content brightness. + use super::{GamutLutKey, quantize_luma_pq}; + use crate::renderer::pipeline::{SourceColorState, ToneMapConfig, VideoRenderPipeline}; + + let source = |peak_nits: f32| { + SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .nominal_peak_nits(peak_nits) + }; + let sdr_target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let key = |source: SourceColorState, target: TargetColorState| { + GamutLutKey::for_pipeline(&VideoRenderPipeline::new(source, target)) + }; + + let base = key(source(1000.0), sdr_target); + // Dolby Vision L1 / measured brightness move `nominal_peak_nits` every + // frame; the LUT must not be regenerated for them. + assert_eq!( + base, + key(source(4000.0), sdr_target), + "a per-frame source peak must not invalidate the LUT" + ); + // A different target peak (EDR headroom) changes the LUT's I axis. + let edr_target = TargetColorState { + primaries: ColorPrimaries::Bt709, + transfer: TransferFunction::Srgb, + peak_nits: 400.0, + reference_white_nits: 100.0, + edr_headroom: 4.0, + }; + assert_ne!( + base, + key(source(1000.0), edr_target), + "a stale cache key must never match the current frame" + ); + // So does the target black (`contrast_ratio`): 1000:1 vs 10000:1 on the + // same 203-nit target must not reuse the LUT. + let contrast_source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + let target_black = |contrast_ratio: f32| VideoRenderPipeline { + tone_map: ToneMapConfig { + contrast_ratio, + ..ToneMapConfig::default() + }, + ..VideoRenderPipeline::new(contrast_source, sdr_target) + }; + assert_ne!( + GamutLutKey::for_pipeline(&target_black(0.0)), + GamutLutKey::for_pipeline(&target_black(10_000.0)), + "a different target black changes the LUT's I axis" + ); + // Primaries on either side change the mapping. + assert_ne!( + base, + key( + source(1000.0), + TargetColorState::sdr_tone_map_target(ColorPrimaries::DisplayP3) + ) + ); + assert_ne!( + base, + key( + SourceColorState::new(ColorPrimaries::DisplayP3, TransferFunction::Pq), + sdr_target + ) + ); + // The luma quantization is monotonic and stable, and keeps sub-1-nit + // target blacks distinguishable. + assert_eq!(quantize_luma_pq(203.0), quantize_luma_pq(203.0)); + assert!(quantize_luma_pq(203.0) < quantize_luma_pq(400.0)); + assert!(quantize_luma_pq(0.1) < quantize_luma_pq(0.203)); + assert_eq!(quantize_luma_pq(10_000.0), 65535); + assert_eq!(quantize_luma_pq(-1.0), 0); + } + #[test] fn video_shader_reconstructs_packed_alpha_as_premultiplied_output() { assert!(VIDEO_SHADER_SOURCE.contains("uniforms.video_alpha_mode == 1")); @@ -3367,7 +4274,6 @@ mod tests { let decode = VIDEO_SHADER_SOURCE .find("rgb = transfer_to_source_reference_linear") .unwrap(); - let gamut = VIDEO_SHADER_SOURCE.find("rgb = apply_gamut_map").unwrap(); let source_nits = VIDEO_SHADER_SOURCE .find("rgb = source_reference_to_nits") .unwrap(); @@ -3378,20 +4284,27 @@ mod tests { let output = VIDEO_SHADER_SOURCE .find("rgb = target_reference_linear_to_output") .unwrap(); - assert!(decode < gamut); - assert!(gamut < source_nits); + assert!(decode < source_nits); assert!(source_nits < tone_map); assert!(tone_map < target_reference); assert!(target_reference < output); } #[test] - fn video_shader_applies_gamut_matrix_before_tone_mapping() { + fn video_shader_runs_the_ipt_tone_map_before_gamut_compression() { assert!(VIDEO_SHADER_SOURCE.contains("gamut_matrix_rows")); assert!(VIDEO_SHADER_SOURCE.contains("apply_gamut_map")); - let gamut = VIDEO_SHADER_SOURCE.find("rgb = apply_gamut_map").unwrap(); + assert!(VIDEO_SHADER_SOURCE.contains("ipt_matrix_rows")); + assert!(VIDEO_SHADER_SOURCE.contains("st2094_pick_knee")); + assert!(VIDEO_SHADER_SOURCE.contains("tone_map_curve_pq")); + // The primaries conversion happens inside the tone map (IPT + // roundtrip), so the fragment only calls tone_map_nits then the + // compression pass. let tone_map = VIDEO_SHADER_SOURCE.find("rgb = tone_map_nits").unwrap(); - assert!(gamut < tone_map); + let compress = VIDEO_SHADER_SOURCE.find("rgb = gamut_compress").unwrap(); + assert!(tone_map < compress); + // The separate matrix call site from the old flow is gone. + assert!(!VIDEO_SHADER_SOURCE.contains("rgb = apply_gamut_map")); } #[test] @@ -3507,6 +4420,90 @@ mod tests { .expect("dual-atlas danmaku pipeline"); } + #[test] + fn reported_display_headroom_drives_output_mode_negotiation() { + // The renderer negotiates on the render thread, where AppKit must not + // be touched, so it consumes whatever the host published from the main + // thread. A missing report negotiates as SDR, exactly like the AppKit + // fallback it replaced. + let config = crate::renderer::metal::MetalRendererConfig { + output_mode: crate::renderer::metal::MetalOutputMode::auto(1.0), + ..crate::renderer::metal::MetalRendererConfig::default() + }; + // Needs a real Metal device; skip rather than fail where there is none. + let Ok(mut renderer) = super::MetalRendererImpl::new(config) else { + eprintln!("skipping: no Metal device available"); + return; + }; + let hdr = crate::renderer::pipeline::SourceColorState::new( + ColorPrimaries::Bt2020, + TransferFunction::Pq, + ); + + assert_eq!(renderer.display_edr_headroom(), 1.0); + renderer.select_output_mode_for_source(hdr); + assert!( + !renderer.active_output_mode().is_edr(), + "an unreported display must not promote to EDR" + ); + + renderer.set_output_headroom(4.0, true); + assert_eq!(renderer.display_edr_headroom(), 4.0); + renderer.select_output_mode_for_source(hdr); + assert_eq!( + renderer.active_output_mode(), + crate::renderer::metal::MetalOutputMode::apple_edr(4.0) + ); + + // An unreported value clears the cache and returns to SDR. + renderer.set_output_headroom(4.0, false); + assert_eq!(renderer.display_edr_headroom(), 1.0); + renderer.select_output_mode_for_source(hdr); + assert!(!renderer.active_output_mode().is_edr()); + } + + #[test] + fn video_shader_compiles() { + // The video shader is the one carrying the tone-map curves, and it is + // only compiled at runtime, so a syntax error there would reach users. + // Needs a real Metal device; skip rather than fail where there is none. + let Ok(mut renderer) = + super::MetalRendererImpl::new(crate::renderer::metal::MetalRendererConfig::default()) + else { + eprintln!("skipping: no Metal device available"); + return; + }; + renderer.video_pipeline_state().expect("video pipeline"); + } + + #[test] + fn video_shader_evaluates_reinhard_and_mobius_in_linear_nits() { + // libplacebo runs these two curves in the linear PL_HDR_NORM domain + // and only the PQ-domain operators work on encoded values; evaluating + // them on PQ codes moved the Mobius knee far below its intended + // position and produced the mid-tone darkening the PR review measured. + for branch in ["uniforms.tone_map == 1", "uniforms.tone_map == 2"] { + let start = VIDEO_SHADER_SOURCE + .find(branch) + .unwrap_or_else(|| panic!("{branch} branch")); + let rest = &VIDEO_SHADER_SOURCE[start..]; + let end = rest.find("\n }").expect("branch end"); + let body = &rest[..end]; + assert!( + body.contains("float out_range_nits = max(dst_peak - dst_black, 0.000001);"), + "{branch} must scale in linear nits" + ); + assert!( + body.contains("nits_from_pq(x)"), + "{branch} must decode the PQ code before the curve" + ); + assert!( + body.contains("return pq_code("), + "{branch} must re-encode the result" + ); + } + } + #[test] fn overlay_uniforms_decode_libass_color() { let bitmap = crate::subtitle::SubtitleAlphaBitmap::new( @@ -3529,6 +4526,7 @@ mod tests { 100, layout, crate::renderer::pipeline::TargetColorState::default(), + false, ); assert_eq!(uniforms.rect, [12.0, 34.0, 56.0, 78.0]); @@ -3626,7 +4624,7 @@ mod tests { #[test] fn video_uniforms_keep_float4_fields_aligned() { - assert_eq!(std::mem::size_of::(), 144); + assert_eq!(std::mem::size_of::(), 3296); assert_eq!(std::mem::offset_of!(super::VideoUniforms, edr_output), 20); assert_eq!(std::mem::offset_of!(super::VideoUniforms, rect), 32); assert_eq!(std::mem::offset_of!(super::VideoUniforms, viewport), 48); @@ -3639,6 +4637,23 @@ mod tests { std::mem::offset_of!(super::VideoUniforms, gamut_matrix_rows), 96 ); + assert_eq!( + std::mem::offset_of!(super::VideoUniforms, ipt_matrix_rows), + 144 + ); + assert_eq!( + std::mem::offset_of!(super::VideoUniforms, tone_map_extra), + 288 + ); + assert_eq!( + std::mem::offset_of!(super::VideoUniforms, tone_map_coeffs), + 304 + ); + assert_eq!( + std::mem::offset_of!(super::VideoUniforms, gamut_lut_enabled), + 320 + ); + assert_eq!(std::mem::offset_of!(super::VideoUniforms, dovi), 336); } #[test] @@ -3669,6 +4684,7 @@ mod tests { 108, layout, crate::renderer::pipeline::TargetColorState::default(), + false, ); assert_eq!(uniforms.viewport, [1000.0, 1000.0]); diff --git a/crates/erika/src/renderer/metal/mod.rs b/crates/erika/src/renderer/metal/mod.rs index c652a555..63ee07ac 100644 --- a/crates/erika/src/renderer/metal/mod.rs +++ b/crates/erika/src/renderer/metal/mod.rs @@ -11,7 +11,8 @@ use crate::ffmpeg::{Frame, PlanarFrame}; use crate::overlay::OverlayFrame; pub use crate::renderer::pipeline::LumaUpscalerMode; use crate::renderer::pipeline::{ - ColorRange, HdrMetadata, MatrixCoefficients, SourceColorState, VideoRenderPipeline, + ColorRange, DoviSourceMetadata, HdrMetadata, MatrixCoefficients, SourceColorState, + VideoRenderPipeline, }; use crate::trace; @@ -142,7 +143,13 @@ pub(crate) fn metal_target_color( ) -> crate::renderer::pipeline::TargetColorState { match mode { MetalOutputMode::Sdr | MetalOutputMode::Auto { .. } => { - crate::renderer::pipeline::TargetColorState::sdr(ColorPrimaries::Bt709) + if source.is_hdr() { + crate::renderer::pipeline::TargetColorState::sdr_tone_map_target( + ColorPrimaries::Bt709, + ) + } else { + crate::renderer::pipeline::TargetColorState::sdr(ColorPrimaries::Bt709) + } } MetalOutputMode::AppleEdr { headroom } | MetalOutputMode::ExtendedLinear { headroom } => { #[cfg(any(target_os = "ios", target_os = "tvos"))] @@ -214,6 +221,8 @@ pub struct MetalRendererStats { pub edr_rendered_frames: u64, pub sdr_tonemap_frames: u64, pub output_mode_switches: u64, + /// Host headroom reports accepted by `set_output_headroom`. + pub headroom_updates: u64, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -306,14 +315,28 @@ impl ImportedVideoFrame { range: ColorRange, matrix: MatrixCoefficients, hdr_metadata: Option, + dovi_metadata: Option, ) { self.set_source_color( SourceColorState::new(primaries, transfer) .range(range) .matrix(matrix) - .hdr_metadata(hdr_metadata), + .hdr_metadata(hdr_metadata) + .dovi(dovi_metadata), ); } + + /// Attach a presenter-measured scene-average luminance so the tone map's + /// pivot follows the content like Dolby Vision L1 would. Only meaningful + /// when the source carries no dynamic L1 metadata of its own. + pub fn set_measured_scene_avg(&mut self, scene_avg_nits: Option) { + if scene_avg_nits.is_none() { + return; + } + let mut source = self.source_color(); + source = source.measured_scene_avg_nits(scene_avg_nits); + self.set_source_color(source); + } } pub struct VideoRenderFrame<'a> { @@ -612,6 +635,7 @@ impl MetalRenderer { frame.color_range(), frame.matrix_coefficients(), frame.hdr_metadata(), + frame.dovi_metadata(), ); Ok(imported) } @@ -853,7 +877,8 @@ impl RendererBackend for MetalRenderer { "Metal renderer received a non-VideoToolbox hardware payload".to_string(), ) })?; - let imported = self.import_player_frame(decoded)?; + let mut imported = self.import_player_frame(decoded)?; + imported.set_measured_scene_avg(frame.scene_avg_nits); if !decoded.is_videotoolbox() { self.software_upload_counter = self.software_upload_counter.wrapping_add(1); } @@ -1079,9 +1104,15 @@ impl RendererBackend for MetalRenderer { #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "tvos")))] let active_output_mode = self.output_mode.resolve_for_source(false); let extended = attached && active_output_mode.is_edr(); + #[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos"))] + let is_hdr10_pq = attached && self.inner.is_hdr10_pq(); + #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "tvos")))] + let is_hdr10_pq = false; OutputRuntimeStatus { requested_mode: self.output_mode, - active_encoding: if extended { + active_encoding: if is_hdr10_pq { + ActiveOutputEncoding::Hdr10Pq + } else if extended { ActiveOutputEncoding::AppleEdr } else { ActiveOutputEncoding::SdrSrgb @@ -1093,17 +1124,19 @@ impl RendererBackend for MetalRenderer { }, native_data_space: -1, requested_headroom: self.output_mode.headroom(), - active_headroom: if extended { + active_headroom: if is_hdr10_pq { + 10_000.0 / 203.0 + } else if extended { active_output_mode.headroom() } else { 1.0 }, active_headroom_known: attached, - extended_linear_active: extended, + extended_linear_active: extended && !is_hdr10_pq, fallback_reason: OutputFallbackReason::None, fallback_count: 0, data_space_failures: 0, - headroom_updates: 0, + headroom_updates: stats.headroom_updates, extended_linear_frames: stats.edr_rendered_frames, } } @@ -1118,6 +1151,22 @@ impl RendererBackend for MetalRenderer { let _ = mode; } } + + /// Caches the display headroom the host resolved on the main thread. + /// + /// `MetalRendererImpl::select_output_mode_for_source` runs on the render + /// thread and must not query AppKit, so the host owns the `NSView` / + /// `NSScreen` lookup and pushes the result here. + fn set_output_headroom(&mut self, headroom: f32, known: bool) { + #[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos"))] + { + self.inner.set_output_headroom(headroom, known); + } + #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "tvos")))] + { + let _ = (headroom, known); + } + } } #[cfg(test)] @@ -1346,6 +1395,7 @@ mod tests { ColorRange::Limited, MatrixCoefficients::Bt709, None, + None, ); assert_eq!(frame.source_color().range, ColorRange::Full); @@ -1372,6 +1422,7 @@ mod tests { ColorRange::Limited, MatrixCoefficients::Bt2020NonConstantLuminance, Some(metadata), + None, ); assert_eq!(frame.source_color().hdr_metadata, Some(metadata)); diff --git a/crates/erika/src/renderer/output.rs b/crates/erika/src/renderer/output.rs index 07ffae9b..6d185a9b 100644 --- a/crates/erika/src/renderer/output.rs +++ b/crates/erika/src/renderer/output.rs @@ -1,5 +1,5 @@ use crate::core::{ColorPrimaries, TransferFunction}; -use crate::renderer::pipeline::TargetColorState; +use crate::renderer::pipeline::{SourceColorState, TargetColorState}; /// Platform-neutral output request selected by the embedder. /// @@ -77,6 +77,57 @@ impl Default for OutputMode { } } +/// Clamp a resolved output mode to what the attached display can actually +/// present. +/// +/// A compositor cannot give an EDR/PQ layer meaningful headroom on a display +/// without EDR support: the HDR signal gets forced into the SDR range and the +/// picture looks blown out. HDR sources therefore tone map to SDR there, and +/// EDR requests are capped at the display's real headroom. +pub fn clamp_output_mode_to_display(mode: OutputMode, display_headroom: f32) -> OutputMode { + match mode { + OutputMode::AppleEdr { headroom } => { + if display_headroom <= 1.05 { + OutputMode::Sdr + } else { + OutputMode::apple_edr(headroom.min(display_headroom)) + } + } + OutputMode::ExtendedLinear { headroom } => { + OutputMode::extended_linear(headroom.min(display_headroom.max(1.0))) + } + other => other, + } +} + +/// Negotiate the output mode for a source against the display the playback +/// window is presented on. +/// +/// `Auto` is the per-screen negotiation path: an HDR source promotes to EDR +/// using the presenting display's *real* headroom when the screen can present +/// EDR at all, and stays on the SDR tone mapping path when it cannot — so the +/// same embedder configuration yields HDR on an HDR display and correctly +/// tone-mapped SDR on a non-HDR one. The embedder's configured headroom (when +/// set above SDR white) caps the negotiated value. Explicit EDR/ExtendedLinear +/// requests are clamped to the display's capability instead. +pub fn negotiate_output_mode( + requested: OutputMode, + source_is_hdr: bool, + display_headroom: f32, +) -> OutputMode { + match requested { + OutputMode::Auto { headroom } => { + if !source_is_hdr || display_headroom <= 1.05 { + OutputMode::Sdr + } else { + let cap = if headroom > 1.0 { headroom } else { f32::MAX }; + OutputMode::apple_edr(display_headroom.min(cap)) + } + } + other => clamp_output_mode_to_display(other, display_headroom), + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OutputColorSpace { Srgb, @@ -295,6 +346,22 @@ impl OutputDescription { pub fn target_transfer(self) -> TransferFunction { self.target.transfer } + + /// The color target an HDR source should render against for this output. + /// + /// An HDR source on a plain SDR output is tone-mapped against the + /// 203-nit HDR reference-white convention (`sdr_tone_map_target`) instead + /// of the 100-nit SDR mastering target. Extended-linear (EDR/scRGB) + /// outputs encode linear light above 1.0 and are *not* SDR outputs even + /// though their transfer reads `Srgb`, so they keep their negotiated + /// target; HDR10 PQ passthrough does its own conversion and is untouched. + pub fn tone_map_target_for(self, source: &SourceColorState) -> TargetColorState { + if source.is_hdr() && self.color_space == OutputColorSpace::Srgb { + TargetColorState::sdr_tone_map_target(self.target.primaries) + } else { + self.target + } + } } fn normalized_headroom(headroom: f32) -> f32 { @@ -327,6 +394,32 @@ mod tests { assert!(description.extended_linear); } + #[test] + fn tone_map_target_swaps_only_on_sdr_outputs() { + let hdr = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + let sdr = SourceColorState::new(ColorPrimaries::Bt709, TransferFunction::Srgb); + + // HDR source on an SDR output: the 203-nit tone-map target. + let swapped = OutputDescription::sdr().tone_map_target_for(&hdr); + assert_eq!(swapped.peak_nits, 203.0); + assert_eq!(swapped.reference_white_nits, 203.0); + + // HDR source on extended-linear/EDR outputs: the negotiated target is + // kept (its transfer is also Srgb, so the check must not key on it). + for mode in [OutputMode::extended_linear(4.0), OutputMode::apple_edr(2.0)] { + let description = OutputDescription::requested(mode); + let kept = description.tone_map_target_for(&hdr); + assert_eq!(kept, description.target, "mode {mode:?}"); + assert!(kept.peak_nits > 203.0); + } + + // HDR10 PQ passthrough and SDR sources are untouched. + let hdr10 = OutputDescription::hdr10().tone_map_target_for(&hdr); + assert_eq!(hdr10, OutputDescription::hdr10().target); + let sdr_target = OutputDescription::sdr().tone_map_target_for(&sdr); + assert_eq!(sdr_target, OutputDescription::sdr().target); + } + #[test] fn output_mode_rejects_invalid_headroom() { assert_eq!(OutputMode::extended_linear(f32::NAN).headroom(), 1.0); @@ -343,10 +436,17 @@ mod tests { automatic.resolve_for_source(true), OutputMode::apple_edr(4.0) ); + // An unconfigured (1.0) headroom keeps HDR sources on the SDR tone + // mapping path; EDR promotion requires the embedder to actually + // request headroom above SDR white. assert_eq!( OutputMode::auto(1.0).resolve_for_source(true), OutputMode::Sdr ); + assert_eq!( + OutputMode::auto(1.0).resolve_for_source(false), + OutputMode::Sdr + ); assert_eq!( OutputDescription::requested(automatic), OutputDescription::sdr() @@ -364,6 +464,71 @@ mod tests { assert_eq!(android.target.edr_headroom, 4.0); } + #[test] + fn auto_negotiates_hdr_per_presenting_display() { + // EDR display: an HDR source promotes to EDR using the display's real + // headroom; an unconfigured host headroom (1.0) defers to the display. + assert_eq!( + negotiate_output_mode(OutputMode::auto(1.0), true, 2.0), + OutputMode::apple_edr(2.0) + ); + // A configured headroom above SDR white caps the negotiated value. + assert_eq!( + negotiate_output_mode(OutputMode::auto(1.5), true, 2.0), + OutputMode::apple_edr(1.5) + ); + // SDR display: HDR sources stay on the tone mapping path, however + // large the embedder's headroom request was. + assert_eq!( + negotiate_output_mode(OutputMode::auto(4.0), true, 1.0), + OutputMode::Sdr + ); + // Non-HDR sources never promote, even on EDR displays. + assert_eq!( + negotiate_output_mode(OutputMode::auto(4.0), false, 2.0), + OutputMode::Sdr + ); + } + + #[test] + fn edr_requests_fall_back_to_sdr_on_displays_without_edr() { + let resolved = OutputMode::auto(4.0).resolve_for_source(true); + + assert_eq!(resolved, OutputMode::apple_edr(4.0)); + assert_eq!(clamp_output_mode_to_display(resolved, 1.0), OutputMode::Sdr); + assert_eq!( + clamp_output_mode_to_display(resolved, 1.04), + OutputMode::Sdr + ); + } + + #[test] + fn edr_requests_are_capped_at_real_display_headroom() { + let resolved = OutputMode::auto(4.0).resolve_for_source(true); + + assert_eq!( + clamp_output_mode_to_display(resolved, 2.0), + OutputMode::apple_edr(2.0) + ); + // A request below the display's capability stays untouched. + assert_eq!( + clamp_output_mode_to_display(OutputMode::apple_edr(1.5), 3.0), + OutputMode::apple_edr(1.5) + ); + } + + #[test] + fn extended_linear_requests_are_capped_but_survive_without_edr() { + assert_eq!( + clamp_output_mode_to_display(OutputMode::extended_linear(2.0), 1.0), + OutputMode::extended_linear(1.0) + ); + assert_eq!( + clamp_output_mode_to_display(OutputMode::Sdr, 1.0), + OutputMode::Sdr + ); + } + #[test] fn fallback_reason_codes_and_labels_are_stable() { let expected = [ diff --git a/crates/erika/src/renderer/pipeline.rs b/crates/erika/src/renderer/pipeline.rs index b7b08463..c057c1e0 100644 --- a/crates/erika/src/renderer/pipeline.rs +++ b/crates/erika/src/renderer/pipeline.rs @@ -21,12 +21,27 @@ impl HdrMetadata { } pub fn nominal_peak_nits(self) -> Option { - self.mastering_display - .and_then(|metadata| metadata.max_luminance_nits()) - .or_else(|| { - self.content_light - .and_then(|metadata| metadata.max_content_light_level_nits()) - }) + let mastering = self.mastering_display.and_then(|m| m.max_luminance_nits()); + let cll = self + .content_light + .and_then(|c| c.max_content_light_level_nits()); + match (mastering, cll) { + (Some(m), Some(c)) => { + if c >= 10.0 { + Some(m.min(c)) + } else { + Some(m) + } + } + (Some(m), None) => Some(m), + (None, Some(c)) => Some(c), + (None, None) => None, + } + } + + pub fn max_frame_average_light_level_nits(self) -> Option { + self.content_light + .and_then(|metadata| metadata.max_frame_average_light_level_nits()) } } @@ -58,6 +73,283 @@ impl ContentLightMetadata { Some(self.max_content_light_level_nits as f32) } } + + pub fn max_frame_average_light_level_nits(self) -> Option { + if self.max_frame_average_light_level_nits == 0 { + None + } else { + Some(self.max_frame_average_light_level_nits as f32) + } + } +} + +/// Maximum number of reshaping pieces per component in a Dolby Vision RPU +/// (`AV_DOVI_MAX_PIECES` in FFmpeg, `num_pivots - 1` segments). +pub const DOVI_MAX_PIECES: usize = 8; +/// Maximum number of MMR orders per piece (FFmpeg allows 1..=3). +pub const DOVI_MAX_MMR_ORDER: usize = 3; +/// Number of coefficients per MMR order: 3 linear terms plus the 4 cross +/// products (x·y, x·z, y·z, x·y·z). +pub const DOVI_MMR_COEFFS: usize = 7; + +/// One component's reshaping curve, converted from the RPU's fixed-point +/// representation into shader-ready floats. Pivots are normalized to the +/// base-layer signal range [0, 1] and coefficients by `2^-coef_log2_denom`, +/// exactly like libplacebo's `pl_map_dovi_metadata`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DoviComponentCurve { + /// 0 when this component carries no reshaping, otherwise 2..=9. + pub num_pivots: u8, + /// Sorted ascending, normalized to [0, 1]. Only the first `num_pivots` + /// entries are meaningful. + pub pivots: [f32; DOVI_MAX_PIECES + 1], + /// Polynomial coefficients per segment (x^0, x^1, x^2). Segments above + /// `poly_order` are zero-filled. + pub poly_coeffs: [[f32; 3]; DOVI_MAX_PIECES], + /// Per segment: 0 selects the polynomial, 1..=3 selects MMR of that order. + pub mmr_orders: [u8; DOVI_MAX_PIECES], + pub mmr_constants: [f32; DOVI_MAX_PIECES], + pub mmr_coeffs: [[[f32; DOVI_MMR_COEFFS]; DOVI_MAX_MMR_ORDER]; DOVI_MAX_PIECES], +} + +impl Default for DoviComponentCurve { + fn default() -> Self { + Self { + num_pivots: 0, + pivots: [0.0; DOVI_MAX_PIECES + 1], + poly_coeffs: [[0.0; 3]; DOVI_MAX_PIECES], + mmr_orders: [0; DOVI_MAX_PIECES], + mmr_constants: [0.0; DOVI_MAX_PIECES], + mmr_coeffs: [[[0.0; DOVI_MMR_COEFFS]; DOVI_MAX_MMR_ORDER]; DOVI_MAX_PIECES], + } + } +} + +/// Per-frame Dolby Vision dynamic metadata level 1: 12-bit PQ codes of the +/// frame's black, peak, and average luminance. The frame peak replaces the +/// static mastering peak (`source_max_pq`) in the tone map, matching +/// libplacebo's handling of the RPU's CIE-Y metadata (`pl_map_dovi_metadata` / +/// `pl_hdr_metadata_from_dovi_rpu`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DoviFramePq { + pub min_pq: u16, + pub max_pq: u16, + pub avg_pq: u16, +} + +/// Per-frame Dolby Vision RPU payload copied out of the decoder's +/// `AV_FRAME_DATA_DOVI_METADATA` side data before the frame is retired. +/// +/// The `nonlinear_matrix` is the RPU's "ycc_to_rgb" transform applied to the +/// reshaped (still PQ-encoded) signal; `rgb_to_lms` is the RPU's mastering +/// transform whose inverse converts PQ-linearized LMS back to BT.2020 RGB. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DoviSourceMetadata { + /// Per-component reshaping curves (luma, Cb, Cr). + pub reshaping: [DoviComponentCurve; 3], + /// RPU's ycc_to_rgb matrix applied to reshaped nonlinear signal. + pub nonlinear_matrix: RgbMatrix, + /// RPU signal offsets applied before the nonlinear matrix. + pub nonlinear_offset: [f32; 3], + /// RPU's rgb_to_lms mastering transform (inverted after PQ linearization). + pub rgb_to_lms: RgbMatrix, + /// 12-bit PQ code of the mastering display's black level (typically 0-100). + pub source_min_pq: u16, + /// 12-bit PQ code of the mastering display's peak level (typically 2000-4000 nits). + pub source_max_pq: u16, + /// Per-frame level 1 brightness metadata (dynamic DM block), or `None` + /// when the RPU carries no usable L1 block. + pub l1: Option, +} + +/// BT.2100 PQ constants shared by the code <-> nits conversions below. +const PQ_M1: f32 = 0.1593017578125; +const PQ_M2: f32 = 78.84375; +const PQ_C1: f32 = 0.8359375; +const PQ_C2: f32 = 18.8515625; +const PQ_C3: f32 = 18.6875; + +/// Decodes a 12-bit PQ code value into absolute nits, matching the PQ EOTF +/// used by the video shaders. +pub fn pq_code_to_nits(code: u16) -> f32 { + if code == 0 { + return 0.0; + } + let encoded = f32::from(code.min(4095)) / 4095.0; + nits_from_pq_code(encoded) +} + +/// PQ OETF: absolute luminance in nits to a PQ code over the 10 000-nit scale +/// (0.0 = black, 1.0 = 10 000 nits). Inverse of [`nits_from_pq_code`]. +pub(crate) fn pq_code_from_nits(nits: f32) -> f32 { + if !nits.is_finite() || nits <= 0.0 { + return 0.0; + } + let p = (nits / 10_000.0).clamp(0.0, 1.0).powf(PQ_M1); + ((PQ_C1 + PQ_C2 * p) / (1.0 + PQ_C3 * p)).powf(PQ_M2) +} + +/// PQ EOTF: a PQ code in [0, 1] over the 10 000-nit scale to absolute nits. +pub(crate) fn nits_from_pq_code(code: f32) -> f32 { + if !code.is_finite() || code <= 0.0 { + return 0.0; + } + let p = code.clamp(0.0, 1.0).powf(1.0 / PQ_M2); + let num = (p - PQ_C1).max(0.0); + let den = (PQ_C2 - PQ_C3 * p).max(0.000001); + 10_000.0 * (num / den).powf(1.0 / PQ_M1) +} + +/// Inverse of the no-crosstalk BT.2020-referred HPE RGB->LMS transform that +/// the RPU's `rgb_to_lms` output is fed into (hard-coded by libplacebo as +/// `dovi_lms2rgb`). +const DOVI_HPE_LMS_TO_RGB: RgbMatrix = RgbMatrix::new([ + [3.06441879, -2.16597676, 0.10155818], + [-0.65612108, 1.78554118, -0.12943749], + [0.01736321, -0.04725154, 1.03004253], +]); + +/// The composite LMS->RGB matrix applied after PQ linearization of a reshaped +/// Dolby Vision signal: the fixed HPE inverse multiplied by the RPU's +/// `rgb_to_lms` matrix, matching libplacebo's `dovi_lms2rgb` composition. +pub fn dovi_lms_to_rgb_matrix(rgb_to_lms: RgbMatrix) -> RgbMatrix { + DOVI_HPE_LMS_TO_RGB.mul(rgb_to_lms) +} + +/// Shader uniform block for Dolby Vision reshaping. All values are +/// vec4-aligned so the block can be appended to the shared video uniform +/// buffer across the WGSL, Metal and HLSL backends without packing tricks. +/// +/// **Size**: ~3KB total (144 vec4s for MMR + overhead). Modern GPUs support +/// this easily, but older mobile devices may have uniform buffer limits around +/// 16KB - this uses ~20% of that budget. +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq)] +#[cfg_attr(feature = "wgpu", derive(bytemuck::Pod, bytemuck::Zeroable))] +pub struct DoviUniforms { + /// x = 1.0 when the source is RPU-mapped; y/z/w = per-component segment + /// counts (`num_pivots - 1`, 0 when the component carries no curve). + pub flags: [f32; 4], + /// Interior pivots per component (two rows each, segments - 1 values, + /// padded with a quasi-infinite sentinel like libplacebo). + pub pivots: [[f32; 4]; 6], + /// Per-component `[first_pivot, last_pivot]` output clamp. + pub bounds: [[f32; 4]; 3], + /// Per-segment payload `[c0, c1, c2, kind]`: kind == 0 is the polynomial + /// `(c2·s + c1)·s + c0`, kind 1..=3 is MMR of that order with constant + /// `c0` and packed rows starting at offset `c1`. + pub coefficients: [[f32; 4]; 3 * DOVI_MAX_PIECES], + /// Packed MMR rows per component (two vec4 rows per order), addressed by + /// the segment's `c1` offset. + pub mmr: [[f32; 4]; 3 * 2 * DOVI_MAX_MMR_ORDER * DOVI_MAX_PIECES], + /// RPU "ycc_to_rgb" rows applied to the reshaped nonlinear signal. + pub nonlinear_matrix: [[f32; 4]; 3], + /// RPU signal offsets, pre-scaled so the shader's normalized 8-bit or + /// P010 samples subtract exactly (libplacebo folds 2^bits/(2^bits-1)). + pub nonlinear_offset: [f32; 4], + /// Composite LMS->RGB rows applied after PQ linearization. + pub lms_matrix: [[f32; 4]; 3], +} + +const DOVI_PIVOT_SENTINEL: f32 = 1e9; +/// RPU offsets are rational /1024-style values while shader samples are +/// normalized by n/(2^bits - 1). `DoviUniforms::of_for_representation` +/// applies the matching +/// 2^bits/(2^bits-1) correction for the actual uploaded representation. +impl DoviUniforms { + pub const fn disabled() -> Self { + Self { + flags: [0.0; 4], + pivots: [[0.0; 4]; 6], + bounds: [[0.0; 4]; 3], + coefficients: [[0.0; 4]; 3 * DOVI_MAX_PIECES], + mmr: [[0.0; 4]; 3 * 2 * DOVI_MAX_MMR_ORDER * DOVI_MAX_PIECES], + nonlinear_matrix: [[0.0; 4]; 3], + nonlinear_offset: [0.0; 4], + lms_matrix: [[0.0; 4]; 3], + } + } + + /// Builds uniforms using the historical 10-bit DV signal representation. + /// New callers with an explicit uploaded format should use + /// [`Self::of_for_representation`]. + pub fn of(source: &SourceColorState) -> Self { + Self::of_for_representation(source, true) + } + + /// Builds uniforms for the concrete plane representation used by the + /// renderer (`P010` when `is_p010` is true, otherwise 8-bit `NV12`). + pub fn of_for_representation(source: &SourceColorState, is_p010: bool) -> Self { + let Some(dovi) = &source.dovi else { + return Self::disabled(); + }; + let mut uniforms = Self::disabled(); + uniforms.flags[0] = 1.0; + for (component, curve) in dovi.reshaping.iter().enumerate() { + if curve.num_pivots < 2 { + continue; + } + let segments = (curve.num_pivots - 1) as usize; + uniforms.flags[1 + component] = segments as f32; + let mut interior = [DOVI_PIVOT_SENTINEL; DOVI_MAX_PIECES]; + interior[..segments - 1].copy_from_slice(&curve.pivots[1..segments]); + uniforms.pivots[2 * component] = [interior[0], interior[1], interior[2], interior[3]]; + uniforms.pivots[2 * component + 1] = + [interior[4], interior[5], interior[6], interior[7]]; + uniforms.bounds[component] = [ + curve.pivots[0].min(curve.pivots[segments]), + curve.pivots[0].max(curve.pivots[segments]), + 0.0, + 0.0, + ]; + + let mut mmr_row = 0usize; + for (segment, &kind) in curve.mmr_orders[..segments].iter().enumerate() { + let slot = DOVI_MAX_PIECES * component + segment; + if kind == 0 { + uniforms.coefficients[slot] = [ + curve.poly_coeffs[segment][0], + curve.poly_coeffs[segment][1], + curve.poly_coeffs[segment][2], + 0.0, + ]; + continue; + } + let order = (kind as usize).min(DOVI_MAX_MMR_ORDER); + let orders = &curve.mmr_coeffs[segment][..order]; + for (index, coefficients) in orders.iter().enumerate() { + let row = + DOVI_MAX_PIECES * 2 * DOVI_MAX_MMR_ORDER * component + mmr_row + 2 * index; + uniforms.mmr[row] = [coefficients[0], coefficients[1], coefficients[2], 0.0]; + uniforms.mmr[row + 1] = [ + coefficients[3], + coefficients[4], + coefficients[5], + coefficients[6], + ]; + } + uniforms.coefficients[slot] = [ + curve.mmr_constants[segment], + mmr_row as f32, + 0.0, + kind as f32, + ]; + mmr_row += 2 * order; + } + } + uniforms.nonlinear_matrix = dovi.nonlinear_matrix.row4s(); + let signal_bits = if is_p010 { 10 } else { 8 }; + let signal_max = ((1_u32 << signal_bits) - 1) as f32; + let signal_offset_scale = (1_u32 << signal_bits) as f32 / signal_max; + uniforms.nonlinear_offset = [ + dovi.nonlinear_offset[0] * signal_offset_scale, + dovi.nonlinear_offset[1] * signal_offset_scale, + dovi.nonlinear_offset[2] * signal_offset_scale, + 0.0, + ]; + uniforms.lms_matrix = dovi_lms_to_rgb_matrix(dovi.rgb_to_lms).row4s(); + uniforms + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -195,7 +487,7 @@ impl RgbMatrix { ] } - fn mul(self, rhs: Self) -> Self { + pub(crate) fn mul(self, rhs: Self) -> Self { let mut rows = [[0.0; 3]; 3]; for (row_index, row) in rows.iter_mut().enumerate() { for (col_index, value) in row.iter_mut().enumerate() { @@ -207,7 +499,7 @@ impl RgbMatrix { Self::new(rows) } - fn mul_vec(self, value: [f32; 3]) -> [f32; 3] { + pub(crate) fn mul_vec(self, value: [f32; 3]) -> [f32; 3] { [ self.rows[0][0] * value[0] + self.rows[0][1] * value[1] + self.rows[0][2] * value[2], self.rows[1][0] * value[0] + self.rows[1][1] * value[1] + self.rows[1][2] * value[2], @@ -215,7 +507,7 @@ impl RgbMatrix { ] } - fn inverse(self) -> Self { + pub(crate) fn inverse(self) -> Self { let m = self.rows; let det = m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) @@ -296,6 +588,36 @@ pub fn source_to_target_rgb_matrix(source: ColorPrimaries, target: ColorPrimarie xyz_to_rgb_matrix(target).mul(rgb_to_xyz_matrix(source)) } +/// RGB (source primaries, absolute D65-referred linear) → HPE-LMS, ported +/// from libplacebo's `pl_ipt_rgb2lms`: a 4% crosstalk mix of HPE XYZ→LMS +/// (D65) applied to the primaries RGB→XYZ matrix. The codebase's primaries +/// are all D65 so the chromatic-adaptation step to D65 is the identity and +/// is omitted. Tone mapping runs in this LMS-PQ-IPT space (see the shader +/// `tone_map_nits`), which converts primaries while mapping the intensity +/// axis instead of running a separate gamut matrix. +pub fn ipt_rgb2lms_matrix(primaries: ColorPrimaries) -> RgbMatrix { + const HPE: [[f32; 3]; 3] = [ + [0.40024, 0.70760, -0.08081], + [-0.22630, 1.16532, 0.04570], + [0.00000, 0.00000, 0.91822], + ]; + let c = 0.04_f32; + let crosstalk = RgbMatrix::new([ + [1.0 - 2.0 * c, c, c], + [c, 1.0 - 2.0 * c, c], + [c, c, 1.0 - 2.0 * c], + ]); + crosstalk + .mul(RgbMatrix::new(HPE)) + .mul(rgb_to_xyz_matrix(primaries)) +} + +/// Inverse of [`ipt_rgb2lms_matrix`] for the *target* primaries; this is +/// what converts the tone-mapped LMS signal back to display RGB. +pub fn ipt_lms2rgb_matrix(primaries: ColorPrimaries) -> RgbMatrix { + ipt_rgb2lms_matrix(primaries).inverse() +} + const D65_WHITE: Chromaticity = Chromaticity::new(0.3127, 0.3290); fn resolve_primaries(primaries: ColorPrimaries) -> ColorPrimaries { @@ -305,6 +627,30 @@ fn resolve_primaries(primaries: ColorPrimaries) -> ColorPrimaries { } } +fn primaries_code(primaries: ColorPrimaries) -> u32 { + match resolve_primaries(primaries) { + ColorPrimaries::Bt709 => 0, + ColorPrimaries::Bt2020 => 1, + ColorPrimaries::DisplayP3 => 2, + ColorPrimaries::Unknown => 0, + } +} + +#[allow(dead_code)] +pub(crate) fn code_to_primaries(code: u32) -> ColorPrimaries { + match code { + 1 => ColorPrimaries::Bt2020, + 2 => ColorPrimaries::DisplayP3, + _ => ColorPrimaries::Bt709, + } +} + +/// Source and target primaries (resolved) for the perceptual gamut LUT key, +/// packed into the uniforms' reserved word. +pub(crate) fn gamut_primaries_code(source: ColorPrimaries, target: ColorPrimaries) -> u32 { + (primaries_code(source) << 8) | primaries_code(target) +} + fn xy_to_xyz(value: Chromaticity) -> [f32; 3] { [value.x / value.y, 1.0, (1.0 - value.x - value.y) / value.y] } @@ -312,13 +658,32 @@ fn xy_to_xyz(value: Chromaticity) -> [f32; 3] { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ToneMapOperator { Clip, + /// Reinhard curve (libplacebo `pl_tone_map_reinhard`, output-relative). Reinhard, + /// Möbius transform with a linear region below the knee (libplacebo + /// `pl_tone_map_mobius`; `curve_param` is the knee, default 0.3). Mobius, + /// ITU-R BT.2390 EETF with black-point compensation (libplacebo + /// `pl_tone_map_bt2390`; `curve_param` is the knee offset, default 1.0). + Bt2390, + /// Perceptually linear single-pivot polynomial, the default in libplacebo + /// and mpv's gpu-next renderer (`curve_param` is the slope contrast, + /// default 0.30; the pivot follows the scene average luminance). + Spline, + /// ITU-R BT.2446 method A (log-domain Weber compression), described by + /// mpv as the recommended curve for well-mastered content. + Bt2446a, + /// SMPTE ST 2094-10 Annex B.2, the DolbyVision dynamic-metadata curve; + /// coefficients are solved per frame from the scene pivot. + St209410, } impl Default for ToneMapOperator { fn default() -> Self { - Self::Mobius + // Aligns with broadcast standard ITU-R BT.2390 and mpv's default + // tone-mapping curve: BT.2390 preserves 1:1 luminance for diffuse content + // below the knee without underexposing midtones, and rolls off highlights smoothly. + Self::Bt2390 } } @@ -364,6 +729,11 @@ pub struct SourceColorState { pub matrix: MatrixCoefficients, pub range: ColorRange, pub hdr_metadata: Option, + pub dovi: Option, + /// Measured per-frame scene-average luminance (nits) from CPU frame + /// statistics (HDR10 without Dolby Vision L1). Precedes the L1 average + /// when driving the tone-map pivot. + pub measured_scene_avg_nits: Option, pub nominal_peak_nits: f32, pub reference_white_nits: f32, } @@ -376,6 +746,8 @@ impl SourceColorState { matrix: MatrixCoefficients::default(), range: ColorRange::default(), hdr_metadata: None, + dovi: None, + measured_scene_avg_nits: None, nominal_peak_nits: nominal_peak_for_transfer(transfer), reference_white_nits: reference_white_for_transfer(transfer), } @@ -404,6 +776,85 @@ impl SourceColorState { self } + /// Attaches per-frame Dolby Vision RPU metadata. The RPU describes an + /// IPT/LMS signal referred to BT.2020 with a PQ transfer, so the primaries + /// and transfer are forced to BT.2020/PQ (matching libplacebo's + /// `pl_map_avdovi_metadata`). The RPU's `source_min_pq`/`source_max_pq` + /// replace static mastering luminance when present, while ordinary display + /// primaries and content-light metadata are retained. When the frame's + /// dynamic L1 block is present its frame peak replaces `source_max_pq` + /// for tone mapping (libplacebo uses the RPU's CIE-Y metadata the same + /// way); the static mastering display peak stays on the L0 value so + /// output-mode negotiation never reacts to per-frame brightness. Forcing + /// the transfer also repairs streams whose VUI tags are missing entirely. + pub fn dovi(mut self, metadata: Option) -> Self { + if let Some(dovi) = metadata { + self.primaries = ColorPrimaries::Bt2020; + self.transfer = TransferFunction::Pq; + self.reference_white_nits = reference_white_for_transfer(self.transfer).max(1.0); + let min_luminance = (dovi.source_min_pq != 0) + .then(|| pq_code_to_nits(dovi.source_min_pq)) + .filter(|value| value.is_finite() && *value >= 0.0); + let frame_peak = dovi + .l1 + .filter(|l1| l1.max_pq != 0) + .map(|l1| pq_code_to_nits(l1.max_pq)); + let mastering_peak = pq_code_to_nits(dovi.source_max_pq); + let peak = frame_peak.unwrap_or(mastering_peak); + if peak > 0.0 { + self.nominal_peak_nits = peak.max(1.0); + } else if self.nominal_peak_nits <= self.reference_white_nits { + self.nominal_peak_nits = nominal_peak_for_transfer(self.transfer); + } + // Keep ordinary mastering primaries/content-light metadata, but + // prefer the RPU's source luminance bounds when present. This lets + // native HDR10 outputs carry Dolby Vision black-level metadata too. + // The mastering peak stays on the static L0 value (never the + // per-frame L1 peak), so output-mode negotiation does not react + // to frame-by-frame brightness. + if min_luminance.is_some() + || (peak.is_finite() && peak > 0.0) + || self.hdr_metadata.is_some() + { + let mut hdr = self + .hdr_metadata + .unwrap_or_else(|| HdrMetadata::new(None, None)); + let mut mastering = hdr.mastering_display.unwrap_or(MasteringDisplayMetadata { + display_primaries: None, + white_point: None, + min_luminance_nits: None, + max_luminance_nits: None, + }); + if let Some(min_luminance) = min_luminance { + mastering.min_luminance_nits = Some(min_luminance); + } + if mastering_peak.is_finite() && mastering_peak > 0.0 { + mastering.max_luminance_nits = Some(mastering_peak); + } + hdr.mastering_display = Some(mastering); + self.hdr_metadata = Some(hdr); + } + self.dovi = Some(dovi); + } else { + self.dovi = None; + } + self + } + + /// Attach a measured scene-average luminance from frame statistics. Only + /// meaningful for HDR10 (PQ) sources without Dolby Vision L1 metadata; + /// the value drives the tone-map pivot exactly like L1's avg would. + pub fn measured_scene_avg_nits(mut self, scene_avg_nits: Option) -> Self { + if let Some(value) = scene_avg_nits { + if value.is_finite() && value > 0.0 { + self.measured_scene_avg_nits = Some(value); + return self; + } + } + self.measured_scene_avg_nits = None; + self + } + pub fn reference_white_nits(mut self, white: f32) -> Self { self.reference_white_nits = white.max(1.0); self @@ -441,6 +892,22 @@ impl TargetColorState { } } + /// SDR target for sources that require tone mapping (HDR → SDR). The tone + /// map peak and the encode reference white follow the HDR reference white + /// convention (BT.2408: 203 nits), matching libplacebo/mpv — NOT the + /// 100-nit SDR mastering value, which pushes the whole picture into the + /// top of the display range (measured: same frame, our render p50=89 vs + /// mpv 67 with identical source content). + pub fn sdr_tone_map_target(primaries: ColorPrimaries) -> Self { + Self { + primaries, + transfer: TransferFunction::Srgb, + peak_nits: 203.0, + reference_white_nits: 203.0, + edr_headroom: 1.0, + } + } + pub fn apple_edr(primaries: ColorPrimaries, headroom: f32) -> Self { Self::extended_linear(primaries, 203.0, headroom) } @@ -481,20 +948,59 @@ impl Default for TargetColorState { #[derive(Debug, Clone, Copy, PartialEq)] pub struct ToneMapConfig { pub operator: ToneMapOperator, - pub knee_start: f32, - pub desaturate: f32, + /// Per-operator curve parameter that mirrors libplacebo's + /// `pl_tone_map_constants` field the operator uses: spline contrast, + /// ST 2094-10 knee adaptation, BT.2390 knee offset, Mobius knee, Reinhard + /// contrast. A value of 0 selects the operator's default (see + /// `default_curve_param`). The shaders read it from `tone_map_extra.x`; + /// ST 2094-10 instead folds it into the coefficients solved on the CPU. + pub curve_param: f32, + /// Target display contrast used for black-point compensation + /// (`--target-contrast` in mpv). 0 selects the automatic value: 1000:1 for + /// SDR targets, infinite (0 black) for HDR/EDR targets. + pub contrast_ratio: f32, } impl Default for ToneMapConfig { fn default() -> Self { Self { - operator: ToneMapOperator::Mobius, - knee_start: 0.75, - desaturate: 0.0, + // Follow the operator enum's default so a default-operator change + // actually reaches the pipeline (a hardcoded value here silently + // overrides it). + operator: ToneMapOperator::default(), + curve_param: 0.0, + contrast_ratio: 0.0, } } } +impl ToneMapConfig { + /// The effective per-operator curve parameter, substituting the operator + /// default for 0. Mirrors libplacebo's per-function `param_def` where + /// mpv overrides it (spline contrast 0.30, mobius knee 0.30). + pub fn effective_curve_param(self) -> f32 { + if self.curve_param > 0.0 { + self.curve_param + } else { + default_curve_param(self.operator) + } + } +} + +fn default_curve_param(operator: ToneMapOperator) -> f32 { + match operator { + ToneMapOperator::Clip => 1.0, + ToneMapOperator::Reinhard => 0.5, + ToneMapOperator::Mobius => 0.3, + ToneMapOperator::Bt2390 => 1.0, + ToneMapOperator::Spline => 0.30, + ToneMapOperator::Bt2446a => 0.0, + // libplacebo's `pl_tone_map_st2094_10.param_def` (knee adaptation). + // mpv's manual documents 1.0 for the same knob; follow libplacebo. + ToneMapOperator::St209410 => 0.7, + } +} + #[derive(Debug, Clone, Copy, PartialEq)] pub struct ScalerConfig { pub kernel: ScalerKernel, @@ -516,6 +1022,7 @@ pub enum RenderPassKind { NeuralUpscale, PlaneSampling, ChromaReconstruction, + DoviReshape, TransferDecode, GamutMap, ToneMap, @@ -617,6 +1124,78 @@ impl VideoRenderPipeline { pub fn gamut_matrix(&self) -> RgbMatrix { source_to_target_rgb_matrix(self.source.primaries, self.target.primaries) } + + /// Rows 0-2: source-primaries RGB→LMS; rows 3-5: source-primaries + /// LMS→RGB; rows 6-8: target-primaries LMS→RGB. Tone mapping and + /// gamut mapping run in a unified single IPT pass (rows 0-2 in, optional + /// 3D LUT in IPT, rows 6-8 out to target primaries) — mirroring libplacebo. + pub fn ipt_matrix_rows(&self) -> [[f32; 4]; 9] { + let mut rows = [[0.0; 4]; 9]; + let source = ipt_rgb2lms_matrix(self.source.primaries); + for (index, row) in source + .rows() + .iter() + .chain(source.inverse().rows().iter()) + .chain(ipt_lms2rgb_matrix(self.target.primaries).rows().iter()) + .enumerate() + { + rows[index] = [row[0], row[1], row[2], 0.0]; + } + rows + } + + /// [curve parameter, scene average nits, target black nits, 0] for the + /// shaders' `tone_map_extra` uniform. + pub fn tone_map_extra(&self) -> [f32; 4] { + [ + self.tone_map.effective_curve_param(), + source_scene_avg_nits(&self.source), + // Black-point compensation only applies when the tone map is + // actually active; applying it to SDR->SDR would crush near-black + // content (mpv skips BPC when `pl_tone_map_params_noop`). + if requires_tone_mapping(self.source, self.target) { + target_black_nits(self.target, self.tone_map.contrast_ratio) + } else { + 0.0 + }, + 0.0, + ] + } + + /// SMPTE ST 2094-10 coefficients for the shaders' `tone_map_coeffs` + /// uniform; zeros when the operator is inactive. + /// Packed (source << 8 | target) resolved-primaries codes used to key + /// the perceptual gamut LUT cache and stored in `_gamut_reserved`. + pub fn gamut_primaries_code(&self) -> u32 { + gamut_primaries_code(self.source.primaries, self.target.primaries) + } + + pub fn tone_map_coeffs(&self) -> [f32; 4] { + st2094_10_coefficients_for(self) + } + + /// Whether the perceptual gamut-mapping LUT is needed: HDR sources + /// tone-mapped to a smaller gamut get the LUT; SDR passthrough and + /// same-gamut rendering keep the fast path (gamut_compress only). + pub fn gamut_lut_active(&self) -> bool { + requires_tone_mapping(self.source, self.target) + && resolve_primaries(self.source.primaries) != resolve_primaries(self.target.primaries) + } + + /// Operator code for the shaders' `tone_map` uniform. + /// + /// Zero (Clip, an identity curve) whenever this pipeline does not tone + /// map, so a Reinhard/Mobius selection cannot alter SDR->SDR content. + /// The configured operator would otherwise run unconditionally: the + /// shaders only branch on the code, and `tone_map_extra`'s black point + /// is already gated the same way. + pub fn tone_map_uniform_code(&self) -> u32 { + if requires_tone_mapping(self.source, self.target) { + tone_map_code(self.tone_map.operator) + } else { + tone_map_code(ToneMapOperator::Clip) + } + } } impl Default for VideoRenderPipeline { @@ -653,6 +1232,30 @@ pub struct VideoUniforms { pub nits: [f32; 4], pub luma_coefficients: [f32; 4], pub gamut_matrix_rows: [[f32; 4]; 3], + /// Rows 0-2: source-primaries RGB → HPE-LMS; rows 3-5: target-primaries + /// LMS → RGB (see [`ipt_rgb2lms_matrix`]/[`ipt_lms2rgb_matrix`]). The + /// shader runs tone mapping in LMS-PQ-IPT space so primaries convert as + /// part of the map instead of a separate gamut matrix. + pub ipt_matrix_rows: [[f32; 4]; 9], + /// x: per-operator curve parameter (see `ToneMapConfig::curve_param`; + /// unused by ST 2094-10, which solves its curve on the CPU); + /// y: per-frame scene average luminance in nits (Dolby Vision L1 avg, + /// 0 = unknown); z: target black point in nits + /// (`ToneMapConfig::contrast_ratio`); w: reserved. + pub tone_map_extra: [f32; 4], + /// SMPTE ST 2094-10 tone-map coefficients (c1, c2, c3) solved per frame + /// on the CPU; zero unless the ST2094-10 operator is active. + pub tone_map_coeffs: [f32; 4], + /// 1 when the perceptual gamut-mapping 3D LUT is bound and the shader + /// must sample it after tone mapping; 0 keeps the fast gamut_compress. + pub gamut_lut_enabled: u32, + /// Packed (source << 8 | target) resolved primaries for the LUT cache. + pub _gamut_primaries: u32, + /// Reserved for future per-LUT scaling; keeps the structure padded. + pub _gamut_reserved0: u32, + pub _gamut_reserved1: u32, + /// Dolby Vision reshaping payload; inert unless `flags[0]` is set. + pub dovi: DoviUniforms, } impl VideoUniforms { @@ -663,7 +1266,7 @@ impl VideoUniforms { full_range: u32::from(matches!(pipeline.source.range, ColorRange::Full)), source_transfer: transfer_code(pipeline.source.transfer), target_transfer: transfer_code(pipeline.target.transfer), - tone_map: tone_map_code(pipeline.tone_map.operator), + tone_map: pipeline.tone_map_uniform_code(), edr_output: u32::from(edr_output), input_mode: 0, scene_linear: 0, @@ -675,14 +1278,245 @@ impl VideoUniforms { ], luma_coefficients: [luma.kr, luma.kg, luma.kb, 0.0], gamut_matrix_rows: pipeline.gamut_matrix().row4s(), + ipt_matrix_rows: pipeline.ipt_matrix_rows(), + tone_map_extra: pipeline.tone_map_extra(), + tone_map_coeffs: pipeline.tone_map_coeffs(), + gamut_lut_enabled: u32::from(pipeline.gamut_lut_active()), + _gamut_primaries: gamut_primaries_code( + pipeline.source.primaries, + pipeline.target.primaries, + ), + _gamut_reserved0: 0, + _gamut_reserved1: 0, + dovi: DoviUniforms::of_for_representation(&pipeline.source, is_p010), + } + } +} + +/// Per-frame average luminance from the Dolby Vision L1 block in nits; 0 when +/// absent (libplacebo then falls back to the default knee fraction). +fn dovi_frame_average_nits(dovi: &DoviSourceMetadata) -> Option { + let l1 = dovi.l1?; + if l1.avg_pq == 0 { + return None; + } + let nits = pq_code_to_nits(l1.avg_pq); + (nits.is_finite() && nits > 0.0).then_some(nits) +} + +/// The scene-average luminance (nits) driving the tone-map pivot: the +/// presenter's measured average first, then the Dolby Vision L1 average, +/// then static MaxFALL, 0 when nothing is known. +fn source_scene_avg_nits(source: &SourceColorState) -> f32 { + source + .measured_scene_avg_nits + .or_else(|| source.dovi.as_ref().and_then(dovi_frame_average_nits)) + .or_else(|| { + source + .hdr_metadata + .and_then(|hdr| hdr.max_frame_average_light_level_nits()) + }) + .unwrap_or(0.0) +} + +/// Target black point for black-point compensation: mpv's `--target-contrast` +/// auto value is 1000:1 for SDR targets and infinite (0 black) for HDR/EDR +/// targets, which the encode stage maps back onto code 0. +fn target_black_nits(target: TargetColorState, contrast_ratio: f32) -> f32 { + if target.edr_headroom > 1.0 || target.transfer == TransferFunction::Pq { + return 0.0; + } + let ratio = if contrast_ratio > 0.0 { + contrast_ratio + } else { + 1000.0 + }; + target.peak_nits / ratio +} + +/// Per-frame ST 2094-10 coefficients for the current source/target +/// luminance envelope, following libplacebo's `pl_tone_map_st2094_10`: the +/// rational Möbius curve passes through (input min, output min), the scene +/// knee and (input max, output max), all in absolute nits. The knee itself is +/// picked in the PQ domain (libplacebo rescales internally even though +/// ST 2094-10 is a NITS-scaled function), and `ToneMapConfig::curve_param` +/// tunes libplacebo's `knee_adaptation` for this operator. +fn st2094_10_coefficients_for(pipeline: &VideoRenderPipeline) -> [f32; 4] { + if pipeline.tone_map.operator != ToneMapOperator::St209410 { + return [0.0; 4]; + } + let input_avg = source_scene_avg_nits(&pipeline.source); + let output_min = target_black_nits(pipeline.target, pipeline.tone_map.contrast_ratio); + let (src_knee, dst_knee) = st2094_pick_knee_nits( + 0.0, + pipeline.source.nominal_peak_nits, + input_avg, + output_min, + pipeline.target.peak_nits, + pipeline.tone_map.effective_curve_param(), + ); + solve_st2094_10( + 0.0, + src_knee, + pipeline.source.nominal_peak_nits, + output_min, + dst_knee, + pipeline.target.peak_nits, + ) +} + +/// The exact libplacebo `st2094_pick_knee` in the **PQ-code domain** (0.0 = +/// black, 1.0 = 10 000 nits). libplacebo always evaluates this in PQ, whatever +/// the tone-map function's own scaling is; callers holding absolute nits must +/// use [`st2094_pick_knee_nits`]. Constants: knee_adaptation 0.4 (the +/// `pl_tone_map_constants` default), knee_min 0.1, knee_max 0.8, knee_default +/// 0.4. Returns the source pivot and the adapted destination pivot. +pub fn st2094_pick_knee( + input_min: f32, + input_max: f32, + input_avg: f32, + output_min: f32, + output_max: f32, +) -> (f32, f32) { + st2094_pick_knee_impl( + input_min, + input_max, + input_avg, + output_min, + output_max, + KNEE_ADAPTATION_DEFAULT, + ) +} + +/// [`st2094_pick_knee`] for callers holding absolute nits: the knee is picked +/// in the PQ domain and rescaled back, mirroring libplacebo's +/// `pl_hdr_rescale(input_scaling, PL_HDR_PQ, ..)` round trip inside +/// `st2094_pick_knee`. `knee_adaptation` is libplacebo's +/// `pl_tone_map_constants.knee_adaptation`, which for ST 2094-10 is what +/// `--tone-mapping-param` tunes. +pub fn st2094_pick_knee_nits( + input_min: f32, + input_max: f32, + input_avg: f32, + output_min: f32, + output_max: f32, + knee_adaptation: f32, +) -> (f32, f32) { + let (src_knee, dst_knee) = st2094_pick_knee_impl( + pq_code_from_nits(input_min), + pq_code_from_nits(input_max), + pq_code_from_nits(input_avg), + pq_code_from_nits(output_min), + pq_code_from_nits(output_max), + knee_adaptation, + ); + (nits_from_pq_code(src_knee), nits_from_pq_code(dst_knee)) +} + +const KNEE_ADAPTATION_DEFAULT: f32 = 0.4; +const KNEE_MIN: f32 = 0.1; +const KNEE_MAX: f32 = 0.8; +const KNEE_DEFAULT: f32 = 0.4; + +fn st2094_pick_knee_impl( + input_min: f32, + input_max: f32, + input_avg: f32, + output_min: f32, + output_max: f32, + knee_adaptation: f32, +) -> (f32, f32) { + let mix = |a: f32, b: f32, x: f32| x * b + (1.0 - x) * a; + let src_knee_min = mix(input_min, input_max, KNEE_MIN); + let src_knee_max = mix(input_min, input_max, KNEE_MAX); + let dst_knee_min = mix(output_min, output_max, KNEE_MIN); + let dst_knee_max = mix(output_min, output_max, KNEE_MAX); + let fallback = mix(input_min, input_max, KNEE_DEFAULT); + let src_knee = + if input_avg > 0.0 { input_avg } else { fallback }.clamp(src_knee_min, src_knee_max); + let target = (src_knee - input_min) / (input_max - input_min).max(1e-6); + let adapted = mix(output_min, output_max, target); + let smooth = |edge0: f32, edge1: f32, x: f32| { + let t = ((x - edge0) / (edge1 - edge0)).clamp(0.0, 1.0); + t * t * (3.0 - 2.0 * t) + }; + let tuning = + 1.0 - smooth(KNEE_MAX, KNEE_DEFAULT, target) * smooth(KNEE_MIN, KNEE_DEFAULT, target); + let adaptation = mix(knee_adaptation.clamp(0.0, 1.0), 1.0, tuning); + let dst_knee = mix(src_knee, adapted, adaptation).clamp(dst_knee_min, dst_knee_max); + (src_knee, dst_knee) +} + +/// Solve the ST 2094-10 rational curve y = (c1 + c2 x) / (1 + c3 x) through +/// (x1,y1), (x2,y2) and (x3,y3) with Cramer's rule on the linear system +/// `c1 + xi*c2 - yi*xi*c3 = yi`. +/// +/// A degenerate anchor set (duplicated/collinear points) has no unique +/// solution; libplacebo divides by the zero determinant and produces +/// non-finite coefficients, which the shader would render as a black or NaN +/// frame. Fall back to the identity curve (c1 = 0, c2 = 1, c3 = 0 — the `Clip` +/// operator) so a malformed envelope degrades to no tone mapping instead. +fn solve_st2094_10(x1: f32, x2: f32, x3: f32, y1: f32, y2: f32, y3: f32) -> [f32; 4] { + const IDENTITY: [f32; 4] = [0.0, 1.0, 0.0, 0.0]; + let base = [ + [1.0, x1, -y1 * x1], + [1.0, x2, -y2 * x2], + [1.0, x3, -y3 * x3], + ]; + let det = |a: [[f32; 3]; 3]| { + a[0][0] * (a[1][1] * a[2][2] - a[1][2] * a[2][1]) + - a[0][1] * (a[1][0] * a[2][2] - a[1][2] * a[2][0]) + + a[0][2] * (a[1][0] * a[2][1] - a[1][1] * a[2][0]) + }; + let with_column = |column: usize, replacement: [f32; 3]| { + let mut m = base; + for row in 0..3 { + m[row][column] = replacement[row]; } + m + }; + let den = det(base); + if !den.is_finite() || den.abs() < 1e-12 { + return IDENTITY; + } + let rhs = [y1, y2, y3]; + let coefficients = [ + det(with_column(0, rhs)) / den, + det(with_column(1, rhs)) / den, + det(with_column(2, rhs)) / den, + 0.0, + ]; + if coefficients[..3].iter().all(|value| value.is_finite()) { + coefficients + } else { + IDENTITY } +} +impl VideoUniforms { pub fn rgb_texture_input(mut self) -> Self { self.input_mode = (self.input_mode & !VIDEO_INPUT_MODE_MASK) | 1; self } + /// Updates the decoded sample representation while keeping Dolby Vision + /// signal offsets in the same normalized domain as the texture samples. + /// This matters when a 10-bit P010 frame is down-converted to 8-bit NV12. + pub fn with_p010_representation(mut self, is_p010: bool) -> Self { + let old_bits = if self.is_p010 != 0 { 10 } else { 8 }; + let new_bits = if is_p010 { 10 } else { 8 }; + if old_bits != new_bits { + let old_scale = (1_u32 << old_bits) as f32 / ((1_u32 << old_bits) - 1) as f32; + let new_scale = (1_u32 << new_bits) as f32 / ((1_u32 << new_bits) - 1) as f32; + let ratio = new_scale / old_scale; + for offset in &mut self.dovi.nonlinear_offset[..3] { + *offset *= ratio; + } + } + self.is_p010 = u32::from(is_p010); + self + } + pub fn packed_d2s_luma_input(mut self) -> Self { self.input_mode = (self.input_mode & !VIDEO_INPUT_MODE_MASK) | 2; self @@ -727,6 +1561,10 @@ fn tone_map_code(operator: ToneMapOperator) -> u32 { ToneMapOperator::Clip => 0, ToneMapOperator::Reinhard => 1, ToneMapOperator::Mobius => 2, + ToneMapOperator::Bt2390 => 3, + ToneMapOperator::Spline => 4, + ToneMapOperator::Bt2446a => 5, + ToneMapOperator::St209410 => 6, } } @@ -752,6 +1590,12 @@ fn build_graph( RenderPassKind::ChromaReconstruction, "reconstruct chroma", )); + if source.dovi.is_some() { + graph.push(RenderPass::new( + RenderPassKind::DoviReshape, + "reshape dolby vision signal", + )); + } graph.push(RenderPass::new( RenderPassKind::TransferDecode, "decode transfer function", @@ -777,11 +1621,26 @@ fn build_graph( graph } +/// Peak luminance that is stable frame to frame: the mastering display (L0) +/// peak when the container/RPU provides one, otherwise the source's nominal +/// peak. Pass selection and output-mode decisions must use this instead of +/// `nominal_peak_nits`, which Dolby Vision replaces with the per-frame L1 peak +/// — a dark scene must not silently disable the tone map, the black-point +/// compensation, or the perceptual gamut LUT for one frame. +fn static_source_peak_nits(source: &SourceColorState) -> f32 { + source + .hdr_metadata + .and_then(|hdr| hdr.mastering_display) + .and_then(|mastering| mastering.max_luminance_nits) + .filter(|peak| peak.is_finite() && *peak > 0.0) + .unwrap_or(source.nominal_peak_nits) +} + fn requires_tone_mapping(source: SourceColorState, target: TargetColorState) -> bool { if !source.is_hdr() { return false; } - source.nominal_peak_nits > target.peak_nits * 1.05 + static_source_peak_nits(&source) > target.peak_nits * 1.05 } fn requires_gamut_mapping(source: SourceColorState, target: TargetColorState) -> bool { @@ -928,6 +1787,912 @@ mod tests { } } + /// PQ-code helpers matching the shaders; see `wgpu_video.wgsl`. + fn pq_code(nits: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = (nits / 10000.0).clamp(0.0, 1.0).powf(m1); + ((c1 + c2 * p) / (1.0 + c3 * p).max(0.000_001)).powf(m2) + } + + fn nits_from_pq(code: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = code.clamp(0.0, 1.0).powf(1.0 / m2); + let num = (p - c1).max(0.0); + let den = (c2 - c3 * p).max(0.000_001); + 10000.0 * (num / den).powf(1.0 / m1) + } + + /// Reference implementation of the shaders' `tone_map_curve_pq` spline + /// branch (tone_map code 4): the single-pivot polynomial from libplacebo's + /// `pl_tone_map_spline`, where the pivot follows the scene average. An + /// unknown average (0) takes the shaders' fallback of + /// `clamp(0.4 * src_peak, 100, 400)` nits before the knee pick. + fn spline_pq( + x: f32, + src_peak_nits: f32, + src_avg_nits: f32, + dst_peak_nits: f32, + dst_black_nits: f32, + contrast: f32, + ) -> f32 { + let in_min = 0.0; + let in_max = pq_code(src_peak_nits).max(0.000_001); + let out_min = pq_code(dst_black_nits); + let out_max = pq_code(dst_peak_nits).max(0.000_001); + let fallback_avg = (0.4 * src_peak_nits).clamp(100.0, 400.0); + let effective_src_avg = if src_avg_nits > 0.0 { + src_avg_nits + } else { + fallback_avg + }; + let (src_pivot, dst_pivot) = + st2094_pick_knee(in_min, in_max, pq_code(effective_src_avg), out_min, out_max); + let slope0 = (dst_pivot - out_min) / (src_pivot - in_min).max(0.000_001); + let ratio = (1.5 * (in_max / out_max - 1.0)).clamp(0.2, 1.2); + let slope = slope0.powf((1.0 - contrast) * ratio); + let (in_min0, in_max0) = (in_min - src_pivot, in_max - src_pivot); + let (out_min0, out_max0) = (out_min - dst_pivot, out_max - dst_pivot); + let pa = (out_min0 - slope * in_min0) / (in_min0 * in_min0); + let qa = (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0 * in_max0); + let qb = -3.0 * (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0); + let xr = x.clamp(in_min, in_max) - src_pivot; + let mapped = if xr > 0.0 { + ((qa * xr + qb) * xr + slope) * xr + } else { + (pa * xr + slope) * xr + }; + mapped + dst_pivot + } + + #[test] + fn spline_curve_anchors_and_monotonicity() { + let (src_peak, dst_peak, black) = (1000.0_f32, 100.0_f32, 0.0_f32); + let contrast = 0.3_f32; + // Endpoints are exact by construction: the quadratic anchors the lower + // endpoint, the cubic anchors the source peak on the target peak. + let lo = spline_pq(0.0, src_peak, 0.0, dst_peak, black, contrast); + assert!(nits_from_pq(lo).abs() < 1e-3, "black maps to {lo} PQ"); + let hi = spline_pq(pq_code(src_peak), src_peak, 0.0, dst_peak, black, contrast); + assert!(nits_from_pq(hi).abs() - dst_peak < 0.05, "peak = {hi} PQ"); + // Monotonic and bounded. + let mut previous = -1.0_f32; + for step in 0..=100 { + let x = pq_code(src_peak) * step as f32 / 100.0; + let mapped = spline_pq(x, src_peak, 0.0, dst_peak, black, contrast); + assert!(mapped >= previous, "not monotonic at {step}"); + assert!(mapped <= pq_code(dst_peak) + 1e-3); + previous = mapped; + } + // 10:1 compression puts 100-nit diffuse white below half of the + // mastered value. Without metadata the shaders fall back to a + // 400-nit average (0.4 * 1000 clamped), so the unknown-metadata + // curve is the darker one. + let diffuse = spline_pq(pq_code(100.0), src_peak, 0.0, dst_peak, black, contrast); + let diffuse_nits = nits_from_pq(diffuse); + let diffuse_known = spline_pq(pq_code(100.0), src_peak, 100.0, dst_peak, black, contrast); + let diffuse_known_nits = nits_from_pq(diffuse_known); + assert!( + diffuse_nits > 10.0 && diffuse_nits < 60.0, + "diffuse = {diffuse_nits}" + ); + assert!( + diffuse_nits < diffuse_known_nits, + "fallback avg {diffuse_nits} must sit below the 100-nit-avg curve {diffuse_known_nits}" + ); + } + + #[test] + fn spline_knee_follows_scene_average_brightness() { + // With a known scene average the pivot tracks the content, so a dark + // scene gets a lower source knee than a bright one (both stay within + // the [10%, 80%] of range clamp). + let (src_peak, dst_peak, black) = (1000.0_f32, 100.0_f32, 0.0_f32); + let dark_avg = 100.0_f32; + let bright_avg = 400.0_f32; + let dark_pivot = st2094_pick_knee( + 0.0, + pq_code(src_peak), + pq_code(dark_avg), + pq_code(black), + pq_code(dst_peak), + ) + .0; + let bright_pivot = st2094_pick_knee( + 0.0, + pq_code(src_peak), + pq_code(bright_avg), + pq_code(black), + pq_code(dst_peak), + ) + .0; + assert!( + dark_pivot < bright_pivot, + "dark {dark_pivot} vs bright {bright_pivot}" + ); + // Without metadata the pivot is the 40% default mix. + let default_pivot = st2094_pick_knee( + 0.0, + pq_code(src_peak), + 0.0, + pq_code(black), + pq_code(dst_peak), + ) + .0; + assert!((default_pivot - pq_code(src_peak) * 0.4).abs() < 1e-4); + } + + #[test] + fn pick_knee_clamps_stay_inside_the_fraction_range() { + // The pivot selection happens in the space the curve operates in + // (PQ for the shaders' spline, nits for ST 2094-10), so a mid-average + // input never escapes the [10%, 80%] of range clamp. + let (src_pq, dst_pq) = st2094_pick_knee( + 0.0, + pq_code(1000.0), + pq_code(500.0), + pq_code(0.0), + pq_code(100.0), + ); + assert!(src_pq >= pq_code(1000.0) * 0.1 - 1e-4); + assert!(src_pq <= pq_code(1000.0) * 0.8 + 1e-4); + assert!(dst_pq >= pq_code(100.0) * 0.1 - 1e-4); + assert!(dst_pq <= pq_code(100.0) * 0.8 + 1e-4); + // An out-of-range average clamps to the same fraction window. + let clamped = st2094_pick_knee( + 0.0, + pq_code(1000.0), + pq_code(9999.0), + pq_code(0.0), + pq_code(100.0), + ); + assert!((clamped.0 - pq_code(1000.0) * 0.8).abs() < 1e-4); + } + + /// Reference implementation of the shaders' BT.2390 branch with the + /// libplacebo black-point compensation (tone_map code 3). + fn bt2390_pq( + x: f32, + src_peak_nits: f32, + dst_peak_nits: f32, + dst_black_nits: f32, + knee_offset: f32, + ) -> f32 { + let in_max = pq_code(src_peak_nits).max(0.000_001); + let out_min = pq_code(dst_black_nits); + let out_max = pq_code(dst_peak_nits).max(0.000_001); + let max_lum = (out_max / in_max).clamp(0.0, 1.0); + let min_lum = out_min / in_max; + let ks = (1.0 + knee_offset) * max_lum - knee_offset; + let bp = (1.0 / min_lum.max(0.000_001)).min(4.0); + let mut u = x.clamp(0.0, in_max) / in_max; + if ks < 1.0 && u > ks { + let tb = (u - ks) / (1.0 - ks); + let tb2 = tb * tb; + let tb3 = tb2 * tb; + u = (2.0 * tb3 - 3.0 * tb2 + 1.0) * ks + + (tb3 - 2.0 * tb2 + tb) * (1.0 - ks) + + (-2.0 * tb3 + 3.0 * tb2) * max_lum; + } + if u < 1.0 { + u = u + min_lum * (1.0 - u).powf(bp); + let gain = if max_lum < 1.0 { + 1.0 / (1.0 + min_lum / max_lum * (1.0 - max_lum).powf(bp)) + } else { + 1.0 + }; + u = gain * (u - min_lum) + min_lum; + } + u * in_max + } + + /// Reference implementation of the shaders' `tone_map_curve_pq` Reinhard + /// branch (tone_map code 1). + /// + /// libplacebo evaluates this curve in the linear PL_HDR_NORM domain while + /// the PQ-domain operators (BT.2390, spline, ST 2094-10) work on encoded + /// values, so the reference rescales out of PQ codes before the curve and + /// re-encodes the result. Normalization cancels: `pl_tone_map_reinhard` + /// uses ratios and differences of same-unit linear values, so absolute + /// nits are equivalent to PL_HDR_NORM here. + fn reinhard_nits( + x: f32, + src_peak_nits: f32, + dst_peak_nits: f32, + dst_black_nits: f32, + contrast: f32, + ) -> f32 { + let in_max = src_peak_nits.max(0.000_001); + let out_min = dst_black_nits; + let out_range = (dst_peak_nits - dst_black_nits).max(0.000_001); + let peak = in_max / out_range; + let contrast = if contrast > 0.0 { contrast } else { 0.5 }; + let offset = (1.0 - contrast) / contrast.max(0.000_001); + let scale = (peak + offset) / peak; + let t = nits_from_pq(x).clamp(0.0, in_max) / out_range; + let mapped = t / (t + offset) * scale; + pq_code(mapped * out_range + out_min) + } + + /// Reference implementation of the shaders' `tone_map_curve_pq` Mobius + /// branch (tone_map code 2); linear domain as in [`reinhard_nits`]. The + /// knee `knee` is relative to the output range, so the 1:1 region ends at + /// `dst_black + knee * (dst_peak - dst_black)` nits. + fn mobius_nits( + x: f32, + src_peak_nits: f32, + dst_peak_nits: f32, + dst_black_nits: f32, + knee: f32, + ) -> f32 { + let in_max = src_peak_nits.max(0.000_001); + let out_min = dst_black_nits; + let out_range = (dst_peak_nits - dst_black_nits).max(0.000_001); + let peak = in_max / out_range; + let j = if knee > 0.0 { knee } else { 0.3 }; + let a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + let b = (j * j - 2.0 * j * peak + peak) / (peak - 1.0).max(0.000_001); + let scale = (b * b + 2.0 * b * j + j * j) / (b - a); + let t = nits_from_pq(x).clamp(0.0, in_max) / out_range; + let mapped = if t > j { scale * (t + a) / (t + b) } else { t }; + pq_code(mapped * out_range + out_min) + } + + #[test] + fn reinhard_and_mobius_run_in_linear_luminance() { + // The PR review measured a 1000->203 nit target with a near-zero + // black and the Mobius knee at 0.3: a 50-nit patch must pass through + // untouched, because the 1:1 region ends at 0.3 * 203 = 60.9 nits. + // Evaluating the curve on PQ codes instead pushed the knee far lower + // and darkened that patch to ~31.3 nits. + let (source_peak, target_peak, black) = (1000.0_f32, 203.0_f32, 0.0_f32); + let knee_nits = 0.3 * (target_peak - black); + assert!((knee_nits - 60.9).abs() < 0.05, "knee = {knee_nits}"); + + let below_knee = nits_from_pq(mobius_nits( + pq_code(50.0), + source_peak, + target_peak, + black, + 0.3, + )); + assert!( + (below_knee - 50.0).abs() < 0.5, + "50-nit patch below the knee = {below_knee} nits" + ); + + // Both curves anchor the source peak on the target peak. + for mapped in [ + nits_from_pq(reinhard_nits( + pq_code(source_peak), + source_peak, + target_peak, + black, + 0.0, + )), + nits_from_pq(mobius_nits( + pq_code(source_peak), + source_peak, + target_peak, + black, + 0.3, + )), + ] { + assert!( + (mapped - target_peak).abs() < 0.5, + "peak maps to {mapped} nits" + ); + } + + // Reinhard compresses the whole range, so it must darken the 50-nit + // patch slightly and stay monotonic. + let reinhard_50 = nits_from_pq(reinhard_nits( + pq_code(50.0), + source_peak, + target_peak, + black, + 0.0, + )); + assert!( + reinhard_50 > 40.0 && reinhard_50 < 50.0, + "reinhard 50-nit patch = {reinhard_50} nits" + ); + + for curve in [0, 1] { + let mut previous = -1.0_f32; + for step in 0..=100 { + let x = pq_code(source_peak) * step as f32 / 100.0; + let mapped = if curve == 0 { + reinhard_nits(x, source_peak, target_peak, black, 0.0) + } else { + mobius_nits(x, source_peak, target_peak, black, 0.3) + }; + assert!( + mapped >= previous, + "curve {curve} step {step} is not monotonic" + ); + previous = mapped; + } + } + } + + #[test] + fn tone_map_uniform_code_skips_the_curve_for_unmapped_content() { + // SDR -> SDR needs no tone mapping, so the shader must receive Clip + // (an identity curve) rather than the selected operator: running + // Reinhard here brightened an SDR 100-nit grey from ~130 to ~175 in + // the review's readback. + let source = SourceColorState::new(ColorPrimaries::Bt709, TransferFunction::Srgb); + let target = TargetColorState::sdr(ColorPrimaries::Bt709); + let mut pipeline = VideoRenderPipeline::new(source, target); + assert!(!pipeline.requires_tone_mapping()); + pipeline.tone_map.operator = ToneMapOperator::Reinhard; + assert_eq!( + pipeline.tone_map_uniform_code(), + tone_map_code(ToneMapOperator::Clip) + ); + assert_eq!( + VideoUniforms::from_pipeline(&pipeline, false, false).tone_map, + 0, + "the shader must see the identity curve" + ); + + // An HDR source that overshoots the target keeps the operator so the + // curve actually runs. + let hdr_source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + let mut hdr_pipeline = VideoRenderPipeline::new( + hdr_source, + TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709), + ); + assert!(hdr_pipeline.requires_tone_mapping()); + hdr_pipeline.tone_map.operator = ToneMapOperator::Reinhard; + assert_eq!( + hdr_pipeline.tone_map_uniform_code(), + tone_map_code(ToneMapOperator::Reinhard) + ); + } + + #[test] + fn bt2390_curve_anchors_and_monotonicity() { + let (source_peak, target_peak, black) = (1000.0_f32, 100.0_f32, 0.203_f32); + // Black maps to the compensated target black (0.203 nit), which the + // encode stage maps back onto code 0. + let black_mapped = nits_from_pq(bt2390_pq(0.0, source_peak, target_peak, black, 1.0)); + assert!( + (black_mapped - black).abs() < 0.05, + "black = {black_mapped}" + ); + let peak = nits_from_pq(bt2390_pq( + pq_code(source_peak), + source_peak, + target_peak, + black, + 1.0, + )); + // Black-point compensation perturbs the exact peak slightly; the + // endpoint stays within half a nit of the target. + assert!((peak - target_peak).abs() < 0.5, "peak = {peak}"); + let mut previous = -1.0_f32; + for step in 0..=100 { + let x = pq_code(source_peak) * step as f32 / 100.0; + let mapped = nits_from_pq(bt2390_pq(x, source_peak, target_peak, black, 1.0)); + assert!(mapped >= previous); + previous = mapped; + } + // BPC compensates: for a target with 0 black the curve would be + // unchanged, with 0.203 it lifts dark steps slightly. + let with_bpc = bt2390_pq(pq_code(1.0), source_peak, target_peak, 0.203, 1.0); + let without_bpc = bt2390_pq(pq_code(1.0), source_peak, target_peak, 0.0, 1.0); + assert!(with_bpc > without_bpc); + } + + #[test] + fn bt2390_hdr10_to_sdr_matches_broadcast_diffuse_white() { + let (source_peak, target_peak, black) = (1000.0_f32, 203.0_f32, 0.0_f32); + let diffuse_white_mapped = nits_from_pq(bt2390_pq( + pq_code(203.0), + source_peak, + target_peak, + black, + 1.0, + )); + // On a 203-nit SDR tone-map target, 203 nits diffuse white must map + // to >= 130 nits (sRGB value >= 210, matching Infuse/MPV and well above 188.1). + assert!( + diffuse_white_mapped >= 130.0, + "mapped diffuse white: {diffuse_white_mapped} nits" + ); + } + + /// Reference implementation of the shaders' BT.2446 method A branch + /// (tone_map code 5), evaluated in nits. + fn bt2446a_nits( + x_nits: f32, + src_peak_nits: f32, + dst_peak_nits: f32, + dst_black_nits: f32, + ) -> f32 { + let phdr = 1.0 + 32.0 * (src_peak_nits / 10000.0).powf(1.0 / 2.4); + let psdr = 1.0 + 32.0 * (dst_peak_nits / 10000.0).powf(1.0 / 2.4); + let mut t = (x_nits.clamp(0.0, src_peak_nits) / src_peak_nits).powf(1.0 / 2.4); + t = (1.0 + (phdr - 1.0) * t).ln() / phdr.ln(); + t = if t <= 0.7399 { + 1.0770 * t + } else if t < 0.9909 { + (-1.1510 * t + 2.7811) * t - 0.6302 + } else { + 0.5 * t + 0.5 + }; + t = (psdr.powf(t) - 1.0) / (psdr - 1.0); + let lb = dst_black_nits.max(0.0).powf(1.0 / 2.4); + let lw = dst_peak_nits.max(0.0).powf(1.0 / 2.4); + ((lw - lb) * t + lb).powf(2.4) + } + + #[test] + fn bt2446a_curve_anchors_and_monotonicity() { + let (src_peak, dst_peak, black) = (1000.0_f32, 100.0_f32, 0.203_f32); + let lo = bt2446a_nits(0.0, src_peak, dst_peak, black); + assert!((lo - black).abs() < 0.05, "black = {lo}"); + let hi = bt2446a_nits(src_peak, src_peak, dst_peak, black); + assert!((hi - dst_peak).abs() < 0.05, "peak = {hi}"); + let mut previous = -1.0_f32; + for step in 0..=100 { + let mapped = bt2446a_nits(src_peak * step as f32 / 100.0, src_peak, dst_peak, black); + assert!(mapped >= previous); + previous = mapped; + } + } + + #[test] + fn st2094_10_coefficients_interpolate_the_three_anchors() { + let (src_peak, dst_peak, black) = (1000.0_f32, 100.0_f32, 0.203_f32); + let (src_knee, dst_knee) = + st2094_pick_knee_nits(0.0, src_peak, 250.0, black, dst_peak, 0.4); + let [c1, c2, c3, _] = solve_st2094_10(0.0, src_knee, src_peak, black, dst_knee, dst_peak); + let eval = |x_nits: f32| (c1 + c2 * x_nits) / (1.0 + c3 * x_nits); + assert!((eval(0.0) - black).abs() < 1e-3); + assert!((eval(src_knee) - dst_knee).abs() < 1e-2); + assert!((eval(src_peak) - dst_peak).abs() < 1e-2); + let mut previous = -1.0_f32; + for step in 0..=100 { + let mapped = eval(src_peak * step as f32 / 100.0); + assert!(mapped >= previous); + previous = mapped; + } + } + + #[test] + fn st2094_10_degenerate_anchors_fall_back_to_the_identity_curve() { + // Duplicated or non-finite anchors have no unique solution. The curve + // must degrade to Clip (y = x); returning zero coefficients would make + // the shader map every pixel to PQ 0, i.e. a black frame. + let identity = [0.0, 1.0, 0.0, 0.0]; + assert_eq!(solve_st2094_10(0.0, 0.0, 0.0, 0.0, 1.0, 1.0), identity); + assert_eq!(solve_st2094_10(0.0, 1.0, 1.0, 0.0, 1.0, 1.0), identity); + assert_eq!(solve_st2094_10(f32::NAN, 1.0, 2.0, 0.0, 1.0, 2.0), identity); + assert_eq!( + solve_st2094_10(0.0, 1.0, 2.0, f32::INFINITY, 1.0, 2.0), + identity + ); + let [c1, c2, c3, _] = identity; + let eval = |x: f32| (c1 + c2 * x) / (1.0 + c3 * x); + assert!((eval(0.25) - 0.25).abs() < 1e-6); + assert!((eval(1.0) - 1.0).abs() < 1e-6); + } + + #[test] + fn st2094_10_knee_follows_measured_scene_average_for_hdr10() { + // ST 2094-10 must use the same scene-average fallback chain as the + // spline pivot (measured average, then DV L1, then MaxFALL), so an + // HDR10 stream with measured luma gets a content-following knee + // instead of the 40% default. + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let mut config = ToneMapConfig::default(); + config.operator = ToneMapOperator::St209410; + + let dark = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .measured_scene_avg_nits(Some(60.0)); + let bright = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .measured_scene_avg_nits(Some(400.0)); + let dark = VideoRenderPipeline { + tone_map: config, + ..VideoRenderPipeline::new(dark, target) + }; + let bright = VideoRenderPipeline { + tone_map: config, + ..VideoRenderPipeline::new(bright, target) + }; + let dark_knee = st2094_pick_knee_nits( + 0.0, + dark.source.nominal_peak_nits, + source_scene_avg_nits(&dark.source), + 0.203, + dark.target.peak_nits, + 0.4, + ) + .0; + let bright_knee = st2094_pick_knee_nits( + 0.0, + bright.source.nominal_peak_nits, + source_scene_avg_nits(&bright.source), + 0.203, + bright.target.peak_nits, + 0.4, + ) + .0; + assert!( + dark_knee < bright_knee, + "dark {dark_knee} vs bright {bright_knee}" + ); + + // MaxFALL is the third rung of the fallback chain. + let metadata = HdrMetadata::new( + None, + Some(ContentLightMetadata { + max_content_light_level_nits: 1000, + max_frame_average_light_level_nits: 239, + }), + ); + let fall = VideoRenderPipeline { + tone_map: config, + ..VideoRenderPipeline::new( + SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .hdr_metadata(Some(metadata)), + target, + ) + }; + assert!((source_scene_avg_nits(&fall.source) - 239.0).abs() < 1e-3); + } + + #[test] + fn st2094_pick_knee_nits_picks_the_knee_in_pq_space() { + // libplacebo always picks the ST 2094 knee in the PQ domain, even for + // the NITS-scaled functions. A 50-nit average on a 4000-nit source must + // land near 50 nits, not on the linear-space 10% floor (400 nits) that + // a nits-domain evaluation would clamp to. + let (src_knee, _) = st2094_pick_knee_nits(0.0, 4000.0, 50.0, 0.0, 203.0, 0.4); + assert!( + (src_knee - 50.0).abs() < 2.0, + "knee should follow the scene average in nits: {src_knee}" + ); + // The nits wrapper is exactly the PQ-domain call with a rescale. + let (pq_src, pq_dst) = st2094_pick_knee( + 0.0, + pq_code_from_nits(4000.0), + pq_code_from_nits(50.0), + 0.0, + pq_code_from_nits(203.0), + ); + let (nits_src, nits_dst) = st2094_pick_knee_nits(0.0, 4000.0, 50.0, 0.0, 203.0, 0.4); + assert!((nits_from_pq_code(pq_src) - nits_src).abs() < 1e-3); + assert!((nits_from_pq_code(pq_dst) - nits_dst).abs() < 1e-3); + } + + #[test] + fn st2094_10_curve_param_tunes_the_knee_adaptation() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .measured_scene_avg_nits(Some(120.0)); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let coefficients = |curve_param: f32| { + let config = ToneMapConfig { + operator: ToneMapOperator::St209410, + curve_param, + ..ToneMapConfig::default() + }; + st2094_10_coefficients_for(&VideoRenderPipeline { + tone_map: config, + ..VideoRenderPipeline::new(source, target) + }) + }; + let low = coefficients(0.1); + let high = coefficients(1.0); + assert_ne!( + low, high, + "curve_param must reach the ST 2094-10 knee adaptation" + ); + // 0 resolves to libplacebo's `pl_tone_map_st2094_10.param_def`. + assert_eq!(coefficients(0.0), coefficients(0.7)); + } + + #[test] + fn dovi_l1_peak_does_not_toggle_the_tone_map_or_gamut_lut() { + // `nominal_peak_nits` carries the per-frame L1 peak for Dolby Vision + // content. Pass selection must stay on the static L0 peak so a dark + // scene cannot drop the tone map, the black-point compensation, or the + // perceptual gamut LUT for a single frame. + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let dark = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq).dovi(Some( + sample_dovi_metadata_with_l1(0, pq_code_12(150.0), pq_code_12(80.0)), + )); + let bright = + SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq).dovi(Some( + sample_dovi_metadata_with_l1(0, pq_code_12(2200.0), pq_code_12(1800.0)), + )); + assert!( + dark.nominal_peak_nits < 200.0, + "dark frame L1 peak should sit below the SDR target peak" + ); + let dark = VideoRenderPipeline::new(dark, target); + let bright = VideoRenderPipeline::new(bright, target); + assert!(dark.requires_tone_mapping()); + assert!(bright.requires_tone_mapping()); + assert!(dark.gamut_lut_active()); + assert!(bright.gamut_lut_active()); + assert_eq!(dark.tone_map_extra()[2], bright.tone_map_extra()[2]); + } + + #[test] + fn tone_map_operator_defaults_and_codes() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .reference_white_nits(203.0); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + assert_eq!(pipeline.tone_map.operator, ToneMapOperator::Bt2390); + assert_eq!(tone_map_code(pipeline.tone_map.operator), 3); + // The curve parameter 0 resolves to the operator default. + assert!((pipeline.tone_map.effective_curve_param() - 1.0).abs() < 1e-6); + // Default contrast is 1000:1 for SDR targets -> 203 / 1000 black. + let extra = pipeline.tone_map_extra(); + assert!((extra[2] - 0.203).abs() < 1e-4, "black = {}", extra[2]); + assert_eq!(extra[0], 1.0); + } + + #[test] + fn hdr_output_target_black_is_zero() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + let target = TargetColorState::hdr10(ColorPrimaries::Bt2020); + let extra = VideoRenderPipeline::new(source, target).tone_map_extra(); + assert_eq!(extra[2], 0.0); + let edr = VideoRenderPipeline::new( + source, + TargetColorState::apple_edr(ColorPrimaries::Bt2020, 2.0), + ) + .tone_map_extra(); + assert_eq!(edr[2], 0.0); + // An explicit contrast ratio overrides the automatic default. + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let mut config = ToneMapConfig::default(); + config.contrast_ratio = 500.0; + let pipeline = VideoRenderPipeline::new(source, target); + let custom = VideoRenderPipeline { + tone_map: config, + ..pipeline + }; + assert!((custom.tone_map_extra()[2] - 203.0 / 500.0).abs() < 1e-4); + } + + #[test] + fn sdr_to_sdr_black_point_is_zero() { + // Black-point compensation must never run on SDR->SDR (near-black + // content would crush); the tone-map-only guard zeroes the black. + let source = SourceColorState::new(ColorPrimaries::Bt709, TransferFunction::Srgb); + let target = TargetColorState::sdr(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + assert!(!pipeline.requires_tone_mapping()); + assert_eq!(pipeline.tone_map_extra()[2], 0.0); + // An explicit contrast ratio is still ignored on the inactive path. + let mut config = ToneMapConfig::default(); + config.contrast_ratio = 500.0; + let custom = VideoRenderPipeline { + tone_map: config, + ..pipeline + }; + assert_eq!(custom.tone_map_extra()[2], 0.0); + } + + #[test] + fn ipt_matrices_match_libplacebo_values_and_white_is_invariant() { + // Values computed independently from libplacebo's pl_ipt_rgb2lms + // (4% crosstalk mix of HPE XYZ->LMS times the primaries RGB->XYZ). + let bt709 = ipt_rgb2lms_matrix(ColorPrimaries::Bt709); + let expected709 = [ + [0.2957641, 0.6230725, 0.0811667], + [0.1561920, 0.7272516, 0.1165579], + [0.0351023, 0.1565899, 0.8083030], + ]; + for (row, expected) in bt709.rows().iter().zip(expected709) { + for (value, expected) in row.iter().zip(expected) { + assert!((value - expected).abs() < 1e-5, "{value} != {expected}"); + } + } + // D65 white maps to equal LMS across primaries and inverts back. + for primaries in [ + ColorPrimaries::Bt709, + ColorPrimaries::Bt2020, + ColorPrimaries::DisplayP3, + ] { + let matrix = ipt_rgb2lms_matrix(primaries); + let lms = matrix.mul_vec([1.0, 1.0, 1.0]); + for value in lms { + assert!((value - 1.0).abs() < 1e-3, "white -> {lms:?}"); + } + let inverse = ipt_lms2rgb_matrix(primaries); + let back = inverse.mul_vec(lms); + for value in back { + assert!((value - 1.0).abs() < 1e-3, "roundtrip -> {back:?}"); + } + } + } + + #[test] + fn tone_map_pipeline_is_present_across_video_shaders() { + let shaders = [ + include_str!("wgpu_video.wgsl"), + include_str!("metal/apple.rs"), + include_str!("d3d11.rs"), + ]; + for shader in shaders { + assert!(shader.contains("st2094_pick_knee")); + assert!(shader.contains("tone_map_curve_pq")); + assert!(shader.contains("ipt_matrix_rows")); + assert!(shader.contains("tone_map_extra")); + assert!(shader.contains("tone_map_coeffs")); + assert!(shader.contains("0.0975689")); + assert!(shader.contains("tone_map == 4")); + assert!(shader.contains("tone_map == 5")); + assert!(shader.contains("SMPTE ST 2094-10")); + assert!(shader.contains("0.7399")); + // The IPT path applies the primaries conversion inside the tone + // map, so the old separate matrix call is gone. + assert!(!shader.contains("rgb = apply_gamut_map(rgb)")); + } + } + + #[test] + fn measured_scene_avg_precedes_dovi_l1_and_drives_the_pivot() { + let mut source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + // HDR10 without L1: attach a measured average, tone_map_extra.y follows. + source = source.measured_scene_avg_nits(Some(120.0)); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + let extra = pipeline.tone_map_extra(); + assert!((extra[1] - 120.0).abs() < 1e-3, "scene avg {}", extra[1]); + // A non-finite / zero value clears the channel. + let cleared_source = source.measured_scene_avg_nits(Some(0.0)); + assert_eq!(cleared_source.measured_scene_avg_nits, None); + let cleared = VideoRenderPipeline::new(cleared_source, target); + assert_eq!(cleared.tone_map_extra()[1], 0.0); + } + + #[test] + fn gamut_lut_active_only_for_tone_mapped_wide_gamut_sources() { + // BT.2020 PQ -> BT.709 SDR needs the perceptual LUT. + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + assert!(pipeline.gamut_lut_active()); + assert_eq!( + VideoUniforms::from_pipeline(&pipeline, false, false).gamut_lut_enabled, + 1 + ); + // Same-gamut HDR tone map keeps the fast path. + let same = VideoRenderPipeline::new( + source, + TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt2020), + ); + assert!(!same.gamut_lut_active()); + // SDR -> SDR passthrough never maps. + let sdr = VideoRenderPipeline::new( + SourceColorState::new(ColorPrimaries::Bt709, TransferFunction::Srgb), + TargetColorState::sdr(ColorPrimaries::Bt709), + ); + assert!(!sdr.gamut_lut_active()); + // HDR10 native output (PQ target, display does the mapping) needs no + // gamut LUT either. + let hdr10 = + VideoRenderPipeline::new(source, TargetColorState::hdr10(ColorPrimaries::Bt2020)); + assert!(!hdr10.gamut_lut_active()); + } + + #[test] + fn gamut_lut_primaries_code_round_trips() { + let code = gamut_primaries_code(ColorPrimaries::Bt2020, ColorPrimaries::Bt709); + assert_eq!(code >> 8, 1); + assert_eq!(code & 0xff, 0); + let p3 = gamut_primaries_code(ColorPrimaries::Bt709, ColorPrimaries::DisplayP3); + assert_eq!(p3 >> 8, 0); + assert_eq!(p3 & 0xff, 2); + } + + #[test] + fn gamut_lut_sampling_is_present_across_video_shaders() { + let shaders = [ + include_str!("wgpu_video.wgsl"), + include_str!("metal/apple.rs"), + include_str!("d3d11.rs"), + ]; + for shader in shaders { + assert!(shader.contains("gamut_lut_enabled")); + assert!(shader.contains("0.5 + 0.5 * atan2")); + assert!(shader.contains("ipt_matrix_rows[6].xyz")); + assert!(shader.contains("sampled.y - 0.5")); + // The LUT path replaces the fast gamut_compress. + assert!(shader.contains("gamut_compress")); + } + } + + #[cfg(feature = "wgpu")] + #[test] + fn wgsl_video_shader_parses_with_naga() { + // The wgpu backend compiles the WGSL only at render time; parse it + // here so syntax regressions fail in tests instead of on-device. + let source = include_str!("wgpu_video.wgsl"); + wgpu::naga::front::wgsl::parse_str(source) + .unwrap_or_else(|error| panic!("invalid WGSL: {error}")); + } + + /// Reference implementation of the shaders' `gamut_compress`: colors the + /// linear gamut matrix pushes out of the target gamut are blended + /// towards their naively-clipped version by an out-of-gamut smoothstep. + /// Slightly-out colors stay nearly intact; strongly-out BT.2020 + /// primaries land on the pure target primary with their hue intact — + /// luma blending would instead pull primary red towards grey and turn + /// it pink. Matches mpv's perceptual gamut mapping behavior. + fn gamut_compress(rgb: [f32; 3]) -> [f32; 3] { + let lo = rgb[0].min(rgb[1]).min(rgb[2]); + let outness = (-lo).max(0.0); + let x = (outness / 1.0).clamp(0.0, 1.0); + let k = x * x * (3.0 - 2.0 * x); + let mix = |a: f32, b: f32| a + (b - a) * k; + [ + mix(rgb[0], rgb[0].clamp(0.0, 1.0)), + mix(rgb[1], rgb[1].clamp(0.0, 1.0)), + mix(rgb[2], rgb[2].clamp(0.0, 1.0)), + ] + } + + #[test] + fn gamut_compress_preserves_hue_of_out_of_gamut_primaries() { + // In-gamut colors pass through untouched. + assert_eq!(gamut_compress([0.2, 0.7, 0.3]), [0.2, 0.7, 0.3]); + // A saturated BT.2020 teal-green that the gamut matrix pushes out of + // gamut: the compression never pushes a channel further out, and the + // channel ordering (hue) is preserved — hard clipping would have + // zeroed red/blue and turned it neon. (Residual negative components + // are clamped at encode time, as libplacebo clamps to its gamut + // floor.) + let out_of_gamut = [-0.08_f32, 0.9, -0.04]; + let mapped = gamut_compress(out_of_gamut); + assert!(mapped[0] > out_of_gamut[0] && mapped[2] > out_of_gamut[2]); + assert!(mapped[1] > mapped[2] && mapped[2] > mapped[0]); + // A strongly-out primary red (negative green and blue) maps to a + // hue-pure red: green and blue are compressed to ~0 together, and + // crucially blue is not lifted towards the luma grey — that is what + // turned wide-gamut red pink under luma blending. + let primary_red = [1.1_f32, -0.22, -0.07]; + let red = gamut_compress(primary_red); + assert!(red[1] < 0.01 && red[2] < 0.01, "red = {red:?}"); + assert!(red[0] > 0.9, "red = {red:?}"); + // Compression is monotonic: an out-of-gamut excursion of 1.0 maps fully + // onto the clip (k = 1), while mild excursions keep most of their + // range. + assert_eq!(gamut_compress([-1.0_f32, 0.9, -0.5]), [0.0, 0.9, 0.0]); + let mild = gamut_compress([-0.1_f32, 0.9, -0.05]); + assert!(mild[0] > -0.1 && mild[2] > -0.05); + } + + #[test] + fn gamut_compress_formula_is_present_across_video_shaders() { + let shaders = [ + include_str!("wgpu_video.wgsl"), + include_str!("metal/apple.rs"), + include_str!("d3d11.rs"), + ]; + for shader in shaders { + assert!(shader.contains("gamut_compress")); + assert!(shader.contains("smoothstep(0.0, 1.0, outness)")); + assert!(shader.contains("rgb = gamut_compress(rgb)")); + } + } + #[test] fn overlay_shaders_handle_sdr_ui_for_hdr_targets() { let metal = include_str!("metal/apple.rs"); @@ -1106,6 +2871,51 @@ mod tests { assert_eq!(source.nominal_peak_nits, 1000.0); } + #[test] + fn hdr_metadata_nominal_peak_bounds_by_max_cll_when_lower_than_mastering() { + let metadata = HdrMetadata::new( + Some(MasteringDisplayMetadata { + display_primaries: None, + white_point: None, + min_luminance_nits: Some(0.005), + max_luminance_nits: Some(1000.0), + }), + Some(ContentLightMetadata { + max_content_light_level_nits: 528, + max_frame_average_light_level_nits: 239, + }), + ); + + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .hdr_metadata(Some(metadata)); + + assert_eq!(metadata.nominal_peak_nits(), Some(528.0)); + assert_eq!(source.nominal_peak_nits, 528.0); + } + + #[test] + fn tone_map_extra_falls_back_to_max_fall_when_unmeasured() { + let metadata = HdrMetadata::new( + Some(MasteringDisplayMetadata { + display_primaries: None, + white_point: None, + min_luminance_nits: Some(0.005), + max_luminance_nits: Some(1000.0), + }), + Some(ContentLightMetadata { + max_content_light_level_nits: 528, + max_frame_average_light_level_nits: 239, + }), + ); + + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .hdr_metadata(Some(metadata)); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + let extra = pipeline.tone_map_extra(); + assert_eq!(extra[1], 239.0); + } + #[test] fn bt709_to_bt709_gamut_matrix_is_identity() { let matrix = source_to_target_rgb_matrix(ColorPrimaries::Bt709, ColorPrimaries::Bt709); @@ -1175,6 +2985,307 @@ mod tests { assert!(!uniforms.packed_alpha_right(false).has_packed_alpha_right()); } + /// A curve shaped like the RPU's default luma mapping: two polynomial + /// segments split at pivot 0.25, then one MMR segment of order 2. + fn sample_dovi_metadata() -> DoviSourceMetadata { + let mut reshaping = [DoviComponentCurve::default(); 3]; + reshaping[0].num_pivots = 4; + reshaping[0].pivots = [0.0, 0.25, 0.5, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + reshaping[0].poly_coeffs[0] = [0.0, 0.5, 0.0]; + reshaping[0].poly_coeffs[1] = [1.0, 1.0, 0.5]; + reshaping[0].mmr_orders[2] = 2; + reshaping[0].mmr_constants[2] = 0.25; + reshaping[0].mmr_coeffs[2][0] = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]; + reshaping[0].mmr_coeffs[2][1] = [0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4]; + DoviSourceMetadata { + reshaping, + nonlinear_matrix: RgbMatrix::new([ + [1.0, 0.5, 0.25], + [0.75, 1.0, 0.125], + [0.0625, 0.03125, 1.0], + ]), + nonlinear_offset: [0.25, 0.5, 0.5], + rgb_to_lms: RgbMatrix::new([ + [0.356742, 0.592257, 0.051081], + [0.156705, 0.747860, 0.095435], + [0.0, 0.041455, 0.958545], + ]), + source_min_pq: 62, + source_max_pq: 3079, + l1: None, + } + } + + #[test] + fn dovi_source_uses_per_frame_l1_peak_when_present() { + let mut source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + source = source.dovi(Some(sample_dovi_metadata_with_l1(1500, 2200, 1800))); + + assert_eq!( + source.nominal_peak_nits, + pq_code_to_nits(2200).max(1.0), + "per-frame L1 peak must replace the static RPU peak for tone mapping" + ); + // The static mastering display metadata keeps the L0 peak so + // output-mode negotiation stays stable frame to frame. + let mastering = source.hdr_metadata.unwrap().mastering_display.unwrap(); + assert_eq!(mastering.max_luminance_nits, Some(pq_code_to_nits(3079))); + } + + #[test] + fn dovi_source_falls_back_to_static_peak_when_l1_is_absent() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(sample_dovi_metadata_with_l1(0, 0, 0))); + assert_eq!( + source.nominal_peak_nits, + pq_code_to_nits(3079).max(1.0), + "an all-zero L1 block must not replace the static RPU peak" + ); + + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(sample_dovi_metadata())); + assert_eq!(source.nominal_peak_nits, pq_code_to_nits(3079).max(1.0)); + } + + fn sample_dovi_metadata_with_l1(min_pq: u16, max_pq: u16, avg_pq: u16) -> DoviSourceMetadata { + let mut metadata = sample_dovi_metadata(); + metadata.l1 = Some(DoviFramePq { + min_pq, + max_pq, + avg_pq, + }); + metadata + } + + /// 12-bit PQ code for an absolute luminance, for building RPU L1 blocks. + fn pq_code_12(nits: f32) -> u16 { + (pq_code_from_nits(nits) * 4095.0) + .round() + .clamp(0.0, 4095.0) as u16 + } + + #[test] + fn dovi_uniforms_are_disabled_without_metadata() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + assert_eq!( + DoviUniforms::of_for_representation(&source, false), + DoviUniforms::disabled() + ); + assert_eq!(DoviUniforms::disabled().flags[0], 0.0); + + let target = TargetColorState::sdr(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + let uniforms = VideoUniforms::from_pipeline(&pipeline, false, false).dovi; + + assert_eq!(uniforms, DoviUniforms::disabled()); + assert_eq!(uniforms.flags[0], 0.0); + } + + #[test] + fn dovi_uniforms_pack_pivots_poly_and_mmr() { + let metadata = sample_dovi_metadata(); + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(metadata)); + let uniforms = DoviUniforms::of_for_representation(&source, true); + + assert_eq!(uniforms.flags, [1.0, 3.0, 0.0, 0.0]); + // Interior pivots skip the endpoints; padding gets the sentinel. + assert_eq!( + uniforms.pivots[0], + [0.25, 0.5, DOVI_PIVOT_SENTINEL, DOVI_PIVOT_SENTINEL] + ); + assert_eq!(uniforms.pivots[1], [DOVI_PIVOT_SENTINEL; 4]); + assert_eq!(uniforms.bounds[0], [0.0, 1.0, 0.0, 0.0]); + assert_eq!(uniforms.coefficients[0], [0.0, 0.5, 0.0, 0.0]); + assert_eq!(uniforms.coefficients[1], [1.0, 1.0, 0.5, 0.0]); + // MMR rows start after two polynomial segments; the order rides in w. + assert_eq!(uniforms.coefficients[2], [0.25, 0.0, 0.0, 2.0]); + assert_eq!(uniforms.mmr[0], [0.1, 0.2, 0.3, 0.0]); + assert_eq!(uniforms.mmr[1], [0.4, 0.5, 0.6, 0.7]); + assert_eq!(uniforms.mmr[2], [0.8, 0.9, 1.0, 0.0]); + assert_eq!(uniforms.mmr[3], [1.1, 1.2, 1.3, 1.4]); + assert_eq!(uniforms.mmr[4], [0.0, 0.0, 0.0, 0.0]); + } + + #[test] + fn dovi_uniforms_apply_signal_offset_correction() { + let metadata = sample_dovi_metadata(); + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(metadata)); + let uniforms = DoviUniforms::of_for_representation(&source, true); + let correction = 1024.0_f32 / 1023.0; + + assert!((uniforms.nonlinear_offset[0] - 0.25 * correction).abs() < 1e-6); + assert!((uniforms.nonlinear_offset[1] - 0.5 * correction).abs() < 1e-6); + assert_eq!(uniforms.nonlinear_matrix[0], [1.0, 0.5, 0.25, 0.0]); + } + + #[test] + fn dovi_uniforms_use_the_uploaded_sample_depth_for_offsets() { + let metadata = sample_dovi_metadata(); + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(metadata)); + let p010 = DoviUniforms::of_for_representation(&source, true); + let nv12 = DoviUniforms::of_for_representation(&source, false); + assert!((p010.nonlinear_offset[0] - 0.25 * 1024.0 / 1023.0).abs() < 1e-6); + assert!((nv12.nonlinear_offset[0] - 0.25 * 256.0 / 255.0).abs() < 1e-6); + + let uniforms = VideoUniforms::from_pipeline( + &VideoRenderPipeline::new(source, TargetColorState::sdr(ColorPrimaries::Bt709)), + true, + false, + ); + let converted = uniforms.with_p010_representation(false); + assert_eq!(converted.is_p010, 0); + assert!((converted.dovi.nonlinear_offset[0] - nv12.nonlinear_offset[0]).abs() < 1e-6); + } + + #[test] + fn dovi_lms_matrix_composite_matches_libplacebo_default() { + // libplacebo composites its hard-coded HPE LMS->RGB matrix with the + // RPU's rgb_to_lms rows; for the RPU default matrix the product is + // this near-diagonal, white-preserving transform. + let matrix = dovi_lms_to_rgb_matrix(RgbMatrix::new([ + [5845.0 / 16384.0, 9702.0 / 16384.0, 837.0 / 16384.0], + [2568.0 / 16384.0, 12256.0 / 16384.0, 1561.0 / 16384.0], + [0.0, 679.0 / 16384.0, 15705.0 / 16384.0], + ])); + + let expected = [ + [0.753741425, 0.198592403, 0.047534181], + [0.045791140, 0.941773555, 0.012526896], + [-0.001211792, 0.017623405, 0.983739703], + ]; + for (row, expected_row) in matrix.rows().iter().zip(expected) { + for (value, expected_value) in row.iter().zip(expected_row) { + assert!((value - expected_value).abs() < 1e-5); + } + } + } + + #[test] + fn dovi_source_uses_rpu_peak_and_bt2020_primaries() { + let metadata = sample_dovi_metadata(); + let source = SourceColorState::new(ColorPrimaries::DisplayP3, TransferFunction::Pq) + .hdr_metadata(Some(HdrMetadata::new( + Some(MasteringDisplayMetadata { + display_primaries: None, + white_point: None, + min_luminance_nits: Some(0.005), + max_luminance_nits: Some(4000.0), + }), + None, + ))) + .dovi(Some(metadata)); + + // PQ code 3079 is the 12-bit encoding of ~1000 nits. + assert!((source.nominal_peak_nits - 1000.0).abs() < 5.0); + assert!( + (source + .hdr_metadata + .unwrap() + .mastering_display + .unwrap() + .min_luminance_nits + .unwrap() + - 0.005) + .abs() + < 0.0001 + ); + assert_eq!(source.primaries, ColorPrimaries::Bt2020); + assert!(source.is_hdr()); + assert_eq!(pq_code_to_nits(0), 0.0); + } + + #[test] + fn dovi_source_peak_falls_back_to_pq_default_when_rpu_max_pq_is_zero() { + let mut metadata = sample_dovi_metadata(); + metadata.source_max_pq = 0; + let source = SourceColorState::new(ColorPrimaries::Unknown, TransferFunction::Unknown) + .dovi(Some(metadata)); + + assert_eq!(source.transfer, TransferFunction::Pq); + assert_eq!(source.reference_white_nits, 203.0); + assert_eq!(source.nominal_peak_nits, 1000.0); + assert!(source.nominal_peak_nits > source.reference_white_nits); + } + + #[test] + fn dovi_source_forces_pq_when_stream_tags_are_missing() { + // libplacebo forces BT.2020/PQ from the RPU because P5/P8 VUI tags are + // unreliable; without this an unspecified trc would decode the + // reshaped PQ signal with an sRGB gamma. + let source = SourceColorState::new(ColorPrimaries::Unknown, TransferFunction::Unknown) + .dovi(Some(sample_dovi_metadata())); + + assert_eq!(source.transfer, TransferFunction::Pq); + assert_eq!(source.primaries, ColorPrimaries::Bt2020); + assert_eq!(source.reference_white_nits, 203.0); + assert_eq!(transfer_code(source.transfer), 3); + assert!(source.is_hdr()); + } + + #[test] + fn dovi_source_adds_reshape_pass_and_tone_maps_to_sdr() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(sample_dovi_metadata())); + let pipeline = + VideoRenderPipeline::new(source, TargetColorState::sdr(ColorPrimaries::Bt709)); + + assert!(pipeline.graph.contains(RenderPassKind::DoviReshape)); + assert!(pipeline.requires_tone_mapping()); + assert!(pipeline.requires_gamut_mapping()); + + let pipeline = + VideoRenderPipeline::new(source, TargetColorState::hdr10(ColorPrimaries::Bt2020)); + assert!(!pipeline.requires_tone_mapping()); + } + + #[test] + fn dovi_formulas_are_present_across_video_shaders() { + let shaders = [ + include_str!("wgpu_video.wgsl"), + include_str!("metal/apple.rs"), + include_str!("d3d11.rs"), + ]; + for shader in shaders { + assert!(shader.contains("dovi_flags")); + assert!(shader.contains("dovi_pivots")); + assert!(shader.contains("dovi_bounds")); + assert!(shader.contains("dovi_coefficients")); + assert!(shader.contains("dovi_mmr")); + assert!(shader.contains("dovi_nonlinear_matrix")); + assert!(shader.contains("dovi_nonlinear_offset")); + assert!(shader.contains("dovi_lms_matrix")); + assert!(shader.contains("dovi_reshaped_signal")); + assert!(shader.contains("dovi_signal_to_pq_rgb")); + assert!(shader.contains("dovi_lms_to_rgb")); + } + } + + #[test] + fn dovi_sdr_target_configures_bt709_output_matrix() { + let metadata = sample_dovi_metadata(); + let source = SourceColorState::default().dovi(Some(metadata)); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + let uniforms = VideoUniforms::from_pipeline(&pipeline, false, false); + + // Verify that ipt_matrix_rows[6..8] are properly configured to convert + // LMS directly to the display target primaries (BT.709), matching libplacebo. + let lms_to_target_r = [ + uniforms.ipt_matrix_rows[6][0], + uniforms.ipt_matrix_rows[6][1], + uniforms.ipt_matrix_rows[6][2], + ]; + // Target is BT.709: row 6 dot [1, 1, 1] should equal 1.0 (white point preservation) + let white_sum = lms_to_target_r[0] + lms_to_target_r[1] + lms_to_target_r[2]; + assert!( + (white_sum - 1.0).abs() < 1e-3, + "White point sum was {white_sum}" + ); + } + fn assert_matrix_close(actual: [[f32; 3]; 3], expected: [[f32; 3]; 3], epsilon: f32) { for row in 0..3 { for col in 0..3 { diff --git a/crates/erika/src/renderer/wgpu.rs b/crates/erika/src/renderer/wgpu.rs index 4a0a331b..c51f23fd 100644 --- a/crates/erika/src/renderer/wgpu.rs +++ b/crates/erika/src/renderer/wgpu.rs @@ -19,7 +19,6 @@ use wgpu::util::DeviceExt; #[cfg(target_os = "android")] use crate::android::{AndroidDataSpaceErrorKind, AndroidNativeWindow}; -#[cfg(target_os = "android")] use crate::core::ColorPrimaries; #[cfg(any( target_os = "android", @@ -31,7 +30,8 @@ use crate::core::ColorPrimaries; use crate::core::WgpuSurfaceKind; use crate::core::{ LumaUpscalerBackendStatus, PlatformSurface, PlayerError, PlayerVideoFrame, RenderFrameContext, - RendererBackend, RendererRuntimeStats, Result, SurfaceOutputCapabilities, WgpuSurfaceHandle, + RendererBackend, RendererRuntimeStats, Result, SurfaceOutputCapabilities, TransferFunction, + WgpuSurfaceHandle, }; use crate::danmaku::{ DanmakuAtlasUpdate, DanmakuGlyphAtlas, DanmakuGlyphInstance, DanmakuRenderPlan, @@ -43,6 +43,9 @@ use crate::renderer::android_vulkan::{ AndroidAhbConversionError, AndroidAhbCrop, AndroidAhbFrameDescription, AndroidAhbIntermediateFormat, AndroidVulkanInterop, retire_ahb_conversion_after_submission, }; +use crate::renderer::gamut::{ + GamutLut, GamutLutJob, GamutLutParams, LUT_SIZE_C, LUT_SIZE_H, LUT_SIZE_I, pack_rgba16f, +}; use crate::renderer::metal::{MetalRendererConfig, VideoAlphaMode}; #[cfg(target_env = "ohos")] use crate::renderer::ohos_vulkan::{ @@ -279,6 +282,38 @@ impl OverlayUniforms { } } +/// Decode the packed (source << 8 | target) primaries codes from +/// `VideoUniforms::_gamut_reserved`, used to key the perceptual LUT cache. +fn gamut_lut_key_of(uniforms: VideoUniforms) -> Option<(ColorPrimaries, ColorPrimaries, u32, u32)> { + if uniforms.gamut_lut_enabled == 0 { + return None; + } + let packed = uniforms._gamut_primaries; + let source = match packed >> 8 { + 1 => ColorPrimaries::Bt2020, + 2 => ColorPrimaries::DisplayP3, + _ => ColorPrimaries::Bt709, + }; + let target = match packed & 0xff { + 1 => ColorPrimaries::Bt2020, + 2 => ColorPrimaries::DisplayP3, + _ => ColorPrimaries::Bt709, + }; + // The LUT's I axis spans [target black, target peak] in PQ codes; any + // change to either endpoint changes the sampled range, so include both + // (`tone_map_extra.z` is the target black in nits). + let target_black_pq = quantize_luma_pq(uniforms.tone_map_extra[2]); + let target_peak_pq = quantize_luma_pq(uniforms.nits[1]); + Some((source, target, target_black_pq, target_peak_pq)) +} + +/// Quantize a luminance (nits) to its PQ code for the LUT cache key. The key +/// only has to change when the LUT's I axis changes, so 16-bit PQ resolution +/// is ample (and keeps the sub-1-nit target blacks of SDR targets distinct). +fn quantize_luma_pq(nits: f32) -> u32 { + (pq_code_for_lut(nits) * 65535.0) as u32 +} + /// Lazily-built GPU objects for the NV12/P010 video pipeline, tied to the color /// target format the pipeline was compiled for. struct VideoPipeline { @@ -365,7 +400,7 @@ fn prepare_planar_upload( } else { (frame, PlanarUploadPath::Native) }; - uniforms.is_p010 = u32::from(frame.format == PlanarPixelFormat::P010); + uniforms = uniforms.with_p010_representation(frame.format == PlanarPixelFormat::P010); Ok(PreparedPlanarUpload { frame, uniforms, @@ -373,6 +408,16 @@ fn prepare_planar_upload( }) } +fn pq_code_for_lut(nits: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = (nits / 10000.0).clamp(0.0, 1.0).powf(m1); + ((c1 + c2 * p) / (1.0 + c3 * p).max(0.000_001)).powf(m2) +} + fn source_color_for_player_frame(frame: &PlayerVideoFrame) -> SourceColorState { SourceColorState::new( frame.frame.color_primaries(), @@ -381,6 +426,8 @@ fn source_color_for_player_frame(frame: &PlayerVideoFrame) -> SourceColorState { .range(frame.frame.color_range()) .matrix(frame.frame.matrix_coefficients()) .hdr_metadata(frame.frame.hdr_metadata()) + .dovi(frame.frame.dovi_metadata()) + .measured_scene_avg_nits(frame.scene_avg_nits) } #[cfg(target_os = "android")] @@ -423,7 +470,7 @@ impl UploadedVideoFrame { let Some(source) = self.source_color else { return self.uniforms; }; - let pipeline = VideoRenderPipeline::new(source, output.target); + let pipeline = VideoRenderPipeline::new(source, output.tone_map_target_for(&source)); let uniforms = VideoUniforms::from_pipeline( &pipeline, self.uniforms.is_p010 != 0, @@ -499,6 +546,19 @@ pub struct WgpuRenderer { surface: Option, video_pipeline: Option, overlay_pipeline: Option, + /// Perceptual gamut-mapping 3D LUT cache, keyed by the source/target + /// primaries plus the target black/peak (PQ codes) that parametrize the + /// LUT's I axis. + gamut_lut: Option<( + (ColorPrimaries, ColorPrimaries, u32, u32), + wgpu::Texture, + wgpu::TextureView, + )>, + /// Background generation for a cache miss; the fast `gamut_compress` + /// path renders until the LUT lands. + gamut_lut_job: Option, + /// 1x1x1 fallback view so binding 4 always has a valid resource. + dummy_lut: Option, current_video: Option, current_video_visible: bool, upload_serial: u64, @@ -1425,6 +1485,9 @@ impl WgpuRenderer { output_status: OutputRuntimeStatus::requested(output_mode), output_headroom: OutputHeadroomState::default(), upscaler_mode: LumaUpscalerMode::Off, + gamut_lut: None, + gamut_lut_job: None, + dummy_lut: None, upscaler, upscaler_failed_frame_token: None, upscaler_active_frame_reported: false, @@ -1891,7 +1954,7 @@ impl WgpuRenderer { .surface .as_ref() .map_or_else(OutputDescription::sdr, |surface| surface.output); - let pipeline = VideoRenderPipeline::new(source, output.target); + let pipeline = VideoRenderPipeline::new(source, output.tone_map_target_for(&source)); if source.is_hdr() { self.stats.hdr_source_frames += 1; if !output.extended_linear && pipeline.requires_tone_mapping() { @@ -2345,6 +2408,127 @@ impl WgpuRenderer { })) } + /// Lazily create (and cache) the perceptual gamut LUT texture described + /// by the current uniforms. Returns `None` while the fast path is active + /// or the background generation has not finished yet; callers must then + /// mask `gamut_lut_enabled` off so the shader keeps the fast path. + fn gamut_lut_view(&mut self, uniforms: VideoUniforms) -> Option { + let (source, target, target_black_pq, target_peak_pq) = gamut_lut_key_of(uniforms)?; + let cached = self.gamut_lut.as_ref().is_some_and(|(key, _, _)| { + key.0 == source + && key.1 == target + && key.2 == target_black_pq + && key.3 == target_peak_pq + }); + if !cached { + let peak_nits = uniforms.nits[1].max(1.0); + let params = GamutLutParams { + source, + target, + // Same target black the shader derives from tone_map_extra.z. + min_luma: pq_code_for_lut(uniforms.tone_map_extra[2]), + max_luma: pq_code_for_lut(peak_nits), + }; + let job_params = self + .gamut_lut_job + .as_ref() + .map(GamutLutJob::params) + .filter(|job_params| *job_params == params); + match job_params { + // A matching job is running: take its result once it lands. + Some(_) => { + let lut = self.gamut_lut_job.as_ref().and_then(GamutLutJob::poll)?; + self.gamut_lut_job = None; + self.upload_gamut_lut(lut, source, target, target_black_pq, target_peak_pq); + } + None => { + // First request (or the key changed): spawn generation + // and keep the fast path for this frame. + self.gamut_lut_job = Some(GamutLutJob::spawn(params)); + return None; + } + } + } + Some( + self.gamut_lut + .as_ref() + .expect("gamut lut after upload") + .2 + .clone(), + ) + } + + fn upload_gamut_lut( + &mut self, + lut: GamutLut, + source: ColorPrimaries, + target: ColorPrimaries, + target_black_pq: u32, + target_peak_pq: u32, + ) { + let size = wgpu::Extent3d { + width: LUT_SIZE_I as u32, + height: LUT_SIZE_C as u32, + depth_or_array_layers: LUT_SIZE_H as u32, + }; + let texture = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("erika-wgpu-gamut-lut"), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D3, + format: wgpu::TextureFormat::Rgba16Float, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + // The texels are packed RGB (I, P+0.5, T+0.5); pad to RGBA16F. + let rgba16 = pack_rgba16f(&lut.texels, 1.0); + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &rgba16, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some((LUT_SIZE_I * 8) as u32), + // One "row" is a texel; one image (slice) is the C axis. + rows_per_image: Some(LUT_SIZE_C as u32), + }, + size, + ); + let view = texture.create_view(&wgpu::TextureViewDescriptor::default()); + self.gamut_lut = Some(( + (source, target, target_black_pq, target_peak_pq), + texture, + view, + )); + } + + /// A tiny 1x1x1 view for binding 4 when no LUT is in use. + fn dummy_lut_view(&mut self) -> Option { + if self.dummy_lut.is_none() { + let texture = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("erika-wgpu-gamut-lut-dummy"), + size: wgpu::Extent3d { + width: 1, + height: 1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D3, + format: wgpu::TextureFormat::Rgba16Float, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }); + self.dummy_lut = Some(texture.create_view(&wgpu::TextureViewDescriptor::default())); + } + self.dummy_lut.clone() + } + /// Encode and submit a render pass drawing the current video frame into /// `target_view`. The caller must have uploaded a frame and the video pipeline /// must be initialized. @@ -2525,6 +2709,19 @@ impl WgpuRenderer { .as_ref() .map_or(&native_luma_view, |output| &output.view); let chroma_view = &native_chroma_view; + // Create/cache the gamut LUT first: it needs `&mut self`, while the + // pipeline borrow below is immutable and would otherwise conflict. + // While the background generation is pending the uniform keeps the + // fast gamut_compress path and the dummy LUT holds binding 4. + let gamut_lut_view = self.gamut_lut_view(video_uniforms); + if gamut_lut_view.is_none() { + video_uniforms.gamut_lut_enabled = 0; + } + let dummy_lut_view = self.dummy_lut_view(); + let gamut_binding = match &gamut_lut_view { + Some(view) => wgpu::BindingResource::TextureView(view), + None => wgpu::BindingResource::TextureView(&dummy_lut_view.expect("dummy LUT view")), + }; let pipeline = self .video_pipeline .as_ref() @@ -2556,6 +2753,10 @@ impl WgpuRenderer { binding: 3, resource: wgpu::BindingResource::Sampler(&pipeline.sampler), }, + wgpu::BindGroupEntry { + binding: 4, + resource: gamut_binding, + }, ], }); @@ -3170,6 +3371,16 @@ impl WgpuRenderer { ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, + wgpu::BindGroupLayoutEntry { + binding: 4, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D3, + multisampled: false, + }, + count: None, + }, ], }); let layout = self @@ -4837,6 +5048,7 @@ fn retain_player_video_frame(frame: &PlayerVideoFrame) -> Result [f32; 3] { - let source_peak = u.nits[0].max(1.0); + // Modeled for the configurations this suite renders: Clip (code 0) and + // BT.2390 (code 3), which is the identity once the source peak already + // fits the target. Reinhard (1) and Mobius (2) are deliberately not + // approximated here — the shader evaluates them in IPT space with the + // curve in linear light, so a nits-domain stand-in would silently + // disagree with it. `wgpu_reinhard_and_mobius_run_in_linear_luminance` + // covers those against an independent reference instead. + assert!( + matches!(u.tone_map, 0 | 3), + "ref_tone_map models only clip and BT.2390, not code {}", + u.tone_map + ); let target_peak = u.nits[1].max(1.0); - let white = (source_peak / target_peak).max(1.0); let x = nits.map(|n| n.max(0.0) / target_peak); - match u.tone_map { - 1 => { - let white2 = white * white; - x.map(|xi| target_peak * (xi * (1.0 + xi / white2) / (1.0 + xi)).clamp(0.0, 1.0)) - } - 2 => { - let knee = 0.75; - let denom = (white - knee).max(0.0001); - x.map(|xi| { - let t = ((xi - knee) / denom).clamp(0.0, 1.0); - let shoulder = knee + (1.0 - knee) * (1.0 - (1.0 - t).powf(2.0)); - let s = if xi >= knee { shoulder } else { xi }; - target_peak * s - }) - } - _ => x.map(|xi| target_peak * xi.clamp(0.0, 1.0)), - } + x.map(|xi| target_peak * xi.clamp(0.0, 1.0)) } fn ref_output(rgb: [f32; 3], u: &VideoUniforms) -> [f32; 3] { @@ -5770,6 +5990,113 @@ mod tests { (luma, chroma) } + /// Renders a solid grey NV12 sample and returns the readback's red + /// channel. The uniforms below keep the shader output in + /// target-reference-linear space, so the channel value is + /// `255 * nits / target_reference_white`. + fn render_grey_sample( + renderer: &mut WgpuRenderer, + uniforms: &VideoUniforms, + tone_map: u32, + y: u8, + ) -> u8 { + let mut uniforms = *uniforms; + uniforms.tone_map = tone_map; + let (luma, chroma) = build_solid_nv12(4, 4, y, 128, 128); + let out = renderer + .render_nv12_offscreen(4, 4, &luma, &chroma, uniforms) + .unwrap(); + out.pixel(1, 1)[0] + } + + #[test] + fn wgpu_reinhard_and_mobius_run_in_linear_luminance() { + // The PR review's readback case: a 1000-nit source peak mapped onto a + // 203-nit target, so the Mobius knee sits at 0.3 * 203 = 60.9 nits. + // libplacebo evaluates Reinhard and Mobius in the linear PL_HDR_NORM + // domain; running them on PQ codes instead moved the knee far lower + // and darkened a 50-nit patch from ~50 to ~31 nits. + let mut renderer = WgpuRenderer::new().unwrap(); + let sdr = VideoUniforms::from_pipeline(&VideoRenderPipeline::sdr_default(), false, false); + + let mut uniforms = sdr; + uniforms.full_range = 1; + uniforms.source_transfer = 0; + uniforms.target_transfer = 0; + uniforms.scene_linear = 1; + uniforms.nits = [1000.0, 203.0, 203.0, 203.0]; + uniforms.tone_map_extra = [0.0, 0.0, 0.0, 0.0]; + uniforms.gamut_lut_enabled = 0; + uniforms.luma_coefficients = [0.2126, 0.7152, 0.0722, 0.0]; + uniforms.gamut_matrix_rows = [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + ]; + + // Y = 63 is 50.15 nits in this configuration, below the knee. + let below_knee_y = 63u8; + let clip = render_grey_sample(&mut renderer, &uniforms, 0, below_knee_y); + let mobius = render_grey_sample(&mut renderer, &uniforms, 2, below_knee_y); + let reinhard = render_grey_sample(&mut renderer, &uniforms, 1, below_knee_y); + + assert!( + (clip as i16 - 63).abs() <= 2, + "clip should reproduce the 50.15-nit input, got {clip}" + ); + // Mobius' 1:1 region covers everything below the knee, so this sample + // must come back untouched. A PQ-domain curve would return ~39 here. + assert!( + (mobius as i16 - clip as i16).abs() <= 1, + "mobius below the knee = {mobius}, clip = {clip}" + ); + // Reinhard compresses the whole range, so this sample darkens a + // little, but only a little. + assert!( + (clip as i16 - reinhard as i16) >= 1 && (clip as i16 - reinhard as i16) <= 3, + "reinhard below the knee = {reinhard}, clip = {clip}" + ); + + // Independent reference for the readback. libplacebo evaluates both + // curves in linear light, so the sample's nits go through the curve + // and come back as `255 * mapped_nits / target_reference_white`; the + // CPU tests in `renderer::pipeline` pin the same formulas. The + // PQ-domain port this replaces would miss these by tens of LSBs. + let expected = |nits: f32, tone_map: u32| -> i16 { + let (src_peak, dst_peak, dst_black) = (1000.0_f32, 203.0_f32, 0.0_f32); + let out_range = dst_peak - dst_black; + let peak = src_peak / out_range; + let t = nits / out_range; + let mapped = match tone_map { + 1 => { + // Reinhard, contrast 0.5 -> offset 1. + let offset = 1.0; + let scale = (peak + offset) / peak; + t / (t + offset) * scale + } + 2 => { + let j = 0.3; + let a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + let b = (j * j - 2.0 * j * peak + peak) / (peak - 1.0); + let scale = (b * b + 2.0 * b * j + j * j) / (b - a); + if t > j { scale * (t + a) / (t + b) } else { t } + } + _ => t, + }; + (255.0 * mapped).round() as i16 + }; + + for (y, tone_map) in [(below_knee_y, 1u32), (below_knee_y, 2), (200, 1), (200, 2)] { + let nits = f32::from(y) / 255.0 * 203.0; + let want = expected(nits, tone_map); + let got = i16::from(render_grey_sample(&mut renderer, &uniforms, tone_map, y)); + assert!( + (got - want).abs() <= 2, + "y={y} tone_map={tone_map} ({nits} nits) = {got}, expected {want}" + ); + } + } + #[test] fn wgpu_video_nv12_matches_cpu_reference() { let mut renderer = WgpuRenderer::new().unwrap(); diff --git a/crates/erika/src/renderer/wgpu_video.wgsl b/crates/erika/src/renderer/wgpu_video.wgsl index 04e1c04c..94ad7237 100644 --- a/crates/erika/src/renderer/wgpu_video.wgsl +++ b/crates/erika/src/renderer/wgpu_video.wgsl @@ -14,12 +14,28 @@ struct VideoUniforms { nits: vec4, luma_coefficients: vec4, gamut_matrix_rows: array, 3>, + ipt_matrix_rows: array, 9>, + tone_map_extra: vec4, + tone_map_coeffs: vec4, + gamut_lut_enabled: u32, + _gamut_primaries: u32, + _gamut_reserved0: u32, + _gamut_reserved1: u32, + dovi_flags: vec4, + dovi_pivots: array, 6>, + dovi_bounds: array, 3>, + dovi_coefficients: array, 24>, + dovi_mmr: array, 144>, + dovi_nonlinear_matrix: array, 3>, + dovi_nonlinear_offset: vec4, + dovi_lms_matrix: array, 3>, }; @group(0) @binding(0) var uniforms: VideoUniforms; @group(0) @binding(1) var luma_texture: texture_2d; @group(0) @binding(2) var chroma_texture: texture_2d; @group(0) @binding(3) var video_sampler: sampler; +@group(0) @binding(4) var gamut_lut: texture_3d; struct VertexOut { @builtin(position) position: vec4, @@ -113,23 +129,258 @@ fn source_reference_to_nits(rgb: vec3) -> vec3 { return max(rgb, vec3(0.0)) * source_reference_white_nits(); } -fn tone_map_nits(nits: vec3) -> vec3 { - let source_peak = source_peak_nits(); - let target_peak = target_peak_nits(); - let x = max(nits, vec3(0.0)) / target_peak; - let white = max(source_peak / target_peak, 1.0); +fn pq_code(nits: f32) -> f32 { + return pq_inverse_eotf(clamp(nits, 0.0, 10000.0) / 10000.0); +} + +fn nits_from_pq(code: f32) -> f32 { + return 10000.0 * pq_eotf(clamp(code, 0.0, 1.0)); +} + +// libplacebo pl_smoothstep with arbitrary edge order (WGSL `smoothstep` has +// undefined results when edge0 >= edge1, and libplacebo's knee tuning term +// deliberately uses reversed edges). +fn sstep(edge0: f32, edge1: f32, x: f32) -> f32 { + let t = clamp((x - edge0) / (edge1 - edge0), 0.0, 1.0); + return t * t * (3.0 - 2.0 * t); +} + +// libplacebo st2094_pick_knee evaluated on absolute PQ codes. The source +// pivot follows the scene average luminance when known and stays within +// [10%, 80%] of the range; the destination pivot rescales it into the output +// range and then adapts towards the 1:1 line (knee_adaptation 0.4). +fn st2094_pick_knee(src_min: f32, src_max: f32, src_avg: f32, dst_min: f32, dst_max: f32) -> vec2 { + let knee_adaptation = 0.4; + let min_knee = 0.1; + let max_knee = 0.8; + let def_knee = 0.4; + let src_knee_min = mix(src_min, src_max, min_knee); + let src_knee_max = mix(src_min, src_max, max_knee); + let dst_knee_min = mix(dst_min, dst_max, min_knee); + let dst_knee_max = mix(dst_min, dst_max, max_knee); + let fallback = mix(src_min, src_max, def_knee); + let src_knee = clamp(select(fallback, src_avg, src_avg > 0.0), src_knee_min, src_knee_max); + let knee_t = (src_knee - src_min) / max(src_max - src_min, 0.000001); + let adapted = mix(dst_min, dst_max, knee_t); + let tuning = 1.0 - sstep(max_knee, def_knee, knee_t) * sstep(min_knee, def_knee, knee_t); + let adaptation = mix(knee_adaptation, 1.0, tuning); + let dst_knee = clamp(mix(src_knee, adapted, adaptation), dst_knee_min, dst_knee_max); + return vec2(src_knee, clamp(dst_knee, dst_knee_min, dst_knee_max)); +} + +// The tone-map curve evaluated on the IPT intensity axis (PQ codes), +// mirroring libplacebo's tone-map functions. `param` is the per-operator +// curve parameter from ToneMapConfig::curve_param (0 = operator default). +fn tone_map_curve_pq(x_in: f32, param: f32) -> f32 { + let src_peak = source_peak_nits(); + let dst_peak = target_peak_nits(); + let src_avg = uniforms.tone_map_extra.y; + let dst_black = uniforms.tone_map_extra.z; + let in_min = 0.0; + let in_max = max(pq_code(src_peak), 0.000001); + let out_min = pq_code(dst_black); + let out_max = max(pq_code(dst_peak), 0.000001); + let out_range = max(out_max - out_min, 0.000001); + let x = clamp(x_in, in_min, in_max); + if (uniforms.tone_map == 0u) { + // Clip: values within the source range pass through untouched. + return x; + } if (uniforms.tone_map == 1u) { - let white2 = white * white; - return target_peak * clamp((x * (vec3(1.0) + x / white2)) / (vec3(1.0) + x), vec3(0.0), vec3(1.0)); + // Reinhard (output-relative, libplacebo pl_tone_map_reinhard). + // + // libplacebo evaluates this curve in the linear PL_HDR_NORM domain + // and only the PQ-domain operators (BT.2390, spline, ST 2094-10) + // work on encoded values. Running it on PQ codes bends the curve's + // meaning: against a 1000->203 nit target mid-tones darken badly, and + // the Mobius knee lands far below its intended position. Rescale to + // linear nits here and encode back at the end. + let in_max_nits = max(src_peak, 0.000001); + let out_min_nits = dst_black; + let out_range_nits = max(dst_peak - dst_black, 0.000001); + let peak = in_max_nits / out_range_nits; + let contrast = select(0.5, param, param > 0.0); + let offset = (1.0 - contrast) / max(contrast, 0.000001); + let scale = (peak + offset) / peak; + let t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; + let mapped = t / (t + offset) * scale; + return pq_code(mapped * out_range_nits + out_min_nits); } if (uniforms.tone_map == 2u) { - let knee = 0.75; - let denom = max(white - knee, 0.0001); - let t = clamp((x - vec3(knee)) / denom, vec3(0.0), vec3(1.0)); - let shoulder = knee + (1.0 - knee) * (vec3(1.0) - pow(vec3(1.0) - t, vec3(2.0))); - return target_peak * mix(x, shoulder, step(vec3(knee), x)); + // Mobius: Möbius transform with a 1:1 linear region below the knee, + // also evaluated in linear nits (see the Reinhard note above). The + // knee j is relative to the output range, so the linear region ends + // at dst_black + j * (dst_peak - dst_black). + let in_max_nits = max(src_peak, 0.000001); + let out_min_nits = dst_black; + let out_range_nits = max(dst_peak - dst_black, 0.000001); + let peak = in_max_nits / out_range_nits; + let j = select(0.3, param, param > 0.0); + let a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + let b = (j * j - 2.0 * j * peak + peak) / max(peak - 1.0, 0.000001); + let scale = (b * b + 2.0 * b * j + j * j) / (b - a); + let t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; + let mapped = select(t, scale * (t + a) / (t + b), t > j); + return pq_code(mapped * out_range_nits + out_min_nits); + } + if (uniforms.tone_map == 3u) { + // ITU-R BT.2390 EETF with black-point compensation (the libplacebo + // version also compensates target black; the earlier port skipped it). + let knee_offset = select(1.0, param, param > 0.0); + let max_lum = clamp(out_max / in_max, 0.0, 1.0); + let min_lum = out_min / in_max; + let ks = (1.0 + knee_offset) * max_lum - knee_offset; + let bp = min(max(1.0 / max(min_lum, 0.000001), 0.0), 4.0); + var u = x / in_max; + if (ks < 1.0 && u > ks) { + let tb = (u - ks) / (1.0 - ks); + let tb2 = tb * tb; + let tb3 = tb2 * tb; + u = (2.0 * tb3 - 3.0 * tb2 + 1.0) * ks + + (tb3 - 2.0 * tb2 + tb) * (1.0 - ks) + + (-2.0 * tb3 + 3.0 * tb2) * max_lum; + } + if (u < 1.0) { + u = u + min_lum * pow(1.0 - u, bp); + let gain = select(1.0, 1.0 / (1.0 + min_lum / max_lum * pow(1.0 - max_lum, bp)), max_lum < 1.0); + u = gain * (u - min_lum) + min_lum; + } + return u * in_max; + } + if (uniforms.tone_map == 4u) { + // Spline: perceptually linear single-pivot polynomial, the default + // tone map of libplacebo and mpv's gpu-next renderer. + let contrast = select(0.3, param, param > 0.0); + let fallback_avg = clamp(0.4 * src_peak, 100.0, 400.0); + let effective_src_avg = select(fallback_avg, src_avg, src_avg > 0.0); + let knee = st2094_pick_knee( + in_min, + in_max, + pq_code(effective_src_avg), + out_min, + out_max + ); + let src_pivot = knee.x; + let dst_pivot = knee.y; + let slope0 = (dst_pivot - out_min) / max(src_pivot - in_min, 0.000001); + let ratio = clamp(1.5 * (in_max / out_max - 1.0), 0.2, 1.2); + let slope = pow(slope0, (1.0 - contrast) * ratio); + let in_min0 = in_min - src_pivot; + let in_max0 = in_max - src_pivot; + let out_min0 = out_min - dst_pivot; + let out_max0 = out_max - dst_pivot; + let pa = (out_min0 - slope * in_min0) / (in_min0 * in_min0); + let qa = (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0 * in_max0); + let qb = -3.0 * (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0); + let xr = x - src_pivot; + let mapped = select((pa * xr + slope) * xr, ((qa * xr + qb) * xr + slope) * xr, xr > 0.0); + return mapped + dst_pivot; + } + if (uniforms.tone_map == 5u) { + // ITU-R BT.2446 method A: Weber-law log compression from the source + // peak envelope and a standardized S-curve (mpv's recommended curve + // for well-mastered content). + let phdr = 1.0 + 32.0 * pow(src_peak / 10000.0, 1.0 / 2.4); + let psdr = 1.0 + 32.0 * pow(dst_peak / 10000.0, 1.0 / 2.4); + var t = pow(nits_from_pq(x) / max(src_peak, 0.000001), 1.0 / 2.4); + t = log(1.0 + (phdr - 1.0) * t) / log(phdr); + if (t <= 0.7399) { + t = 1.0770 * t; + } else if (t < 0.9909) { + t = (-1.1510 * t + 2.7811) * t - 0.6302; + } else { + t = 0.5 * t + 0.5; + } + t = (pow(psdr, t) - 1.0) / (psdr - 1.0); + // BT.1886 EOTF from the target black point and peak. + let lb = pow(max(dst_black, 0.0), 1.0 / 2.4); + let lw = pow(max(dst_peak, 0.0), 1.0 / 2.4); + return pq_code(pow((lw - lb) * t + lb, 2.4)); + } + // SMPTE ST 2094-10 (DolbyVision's dynamic-metadata curve): rational + // Möbius interpolation in absolute nits; coefficients are solved per + // frame on the CPU from the same scene pivot. + let c1 = uniforms.tone_map_coeffs.x; + let c2 = uniforms.tone_map_coeffs.y; + let c3 = uniforms.tone_map_coeffs.z; + let x_nits = nits_from_pq(x); + let y_nits = (c1 + c2 * x_nits) / max(1.0 + c3 * x_nits, 0.000001); + return pq_code(clamp(y_nits, 0.0, 10000.0)); +} + +fn tone_map_nits(nits: vec3) -> vec3 { + if (uniforms.target_transfer == 3u) { + // HDR10 output: convert primaries by the gamut matrix and clamp to + // the PQ range (no tone mapping; the display does the HDR mapping). + return clamp( + apply_gamut_map(max(nits, vec3(0.0)) / source_reference_white_nits()) + * source_reference_white_nits(), + vec3(0.0), + vec3(10000.0) + ); + } + // libplacebo tone map: RGB in source primaries (absolute nits) to + // HPE-LMS, PQ-encode, IPT, map the intensity axis and apply the + // hue-preserving chroma rule, then decode back to RGB in the *source* + // primaries. The gamut LUT (below) performs the single source->target + // primaries conversion. + let rgb = max(nits, vec3(0.0)); + let lms = vec3( + dot(uniforms.ipt_matrix_rows[0].xyz, rgb), + dot(uniforms.ipt_matrix_rows[1].xyz, rgb), + dot(uniforms.ipt_matrix_rows[2].xyz, rgb) + ); + let lmspq = vec3(pq_code(lms.r), pq_code(lms.g), pq_code(lms.b)); + var ipt = vec3( + dot(vec3(0.4, 0.4, 0.2), lmspq), + dot(vec3(4.455, -4.851, 0.396), lmspq), + dot(vec3(0.8056, 0.3572, -1.1628), lmspq) + ); + let i_orig = ipt.x; + ipt.x = tone_map_curve_pq(ipt.x, uniforms.tone_map_extra.x); + // Libplacebo's chroma rule: clamp the saturation boost when brightening + // and desaturate (by the cubic hull term) when the mapping darkens. + let hull = vec2(i_orig, ipt.x); + let hull_c = ((hull - vec2(6.0)) * hull + vec2(9.0)) * hull; + let ratio = min(i_orig / max(ipt.x, 0.000001), hull_c.y / max(hull_c.x, 0.000001)); + ipt = vec3(ipt.x, ipt.y * ratio, ipt.z * ratio); + + if (uniforms.gamut_lut_enabled != 0u) { + // I axis spans the target's [black, peak] in PQ codes, matching + // libplacebo's gamut.min_luma/max_luma (tone_map_extra.z is the + // target black in nits, the same value the LUT was generated for). + let lut_min = pq_code(uniforms.tone_map_extra.z); + let lut_max = max(pq_code(target_peak_nits()), 0.000001); + let lut_range = max(lut_max - lut_min, 0.000001); + let pos = vec3( + clamp((ipt.x - lut_min) / lut_range, 0.0, 1.0), + clamp(2.0 * length(ipt.yz), 0.0, 1.0), + 0.5 + 0.5 * atan2(ipt.z, ipt.y) / 3.14159265 + ); + // libplacebo's texel_scale: lattice position -> texel-center coordinate. + let idx = vec3( + pos.x * (47.0 / 48.0) + 0.5 / 48.0, + pos.y * (31.0 / 32.0) + 0.5 / 32.0, + pos.z * (255.0 / 256.0) + 0.5 / 256.0 + ); + let sampled = textureSample(gamut_lut, video_sampler, idx).xyz; + ipt = vec3(sampled.x, sampled.y - 0.5, sampled.z - 0.5); } - return target_peak * clamp(x, vec3(0.0), vec3(1.0)); + let lmspq_out = vec3( + dot(vec3(1.0, 0.0975689, 0.205226), ipt), + dot(vec3(1.0, -0.113876, 0.133217), ipt), + dot(vec3(1.0, 0.0326151, -0.676887), ipt) + ); + let lms_out = vec3( + nits_from_pq(lmspq_out.r), + nits_from_pq(lmspq_out.g), + nits_from_pq(lmspq_out.b) + ); + return vec3( + dot(uniforms.ipt_matrix_rows[6].xyz, lms_out), + dot(uniforms.ipt_matrix_rows[7].xyz, lms_out), + dot(uniforms.ipt_matrix_rows[8].xyz, lms_out) + ); } fn apply_gamut_map(rgb: vec3) -> vec3 { @@ -140,8 +391,30 @@ fn apply_gamut_map(rgb: vec3) -> vec3 { ); } +// Hue-preserving gamut mapping: the linear gamut matrix can push highly +// saturated wide-gamut colors outside the target gamut (negative +// components). Blending those towards luma shifts hue — BT.2020 primary +// red picks up blue and turns pink. Instead blend towards the naive clip +// by an out-of-gamut smoothstep factor: slightly-out colors stay nearly +// intact, strongly-out primaries land on the pure target primary with +// their hue intact, matching mpv's perceptual gamut handling. Brightness +// overshoot (> 1) is left for the tone map. +fn gamut_compress(rgb: vec3) -> vec3 { + let lo = min(rgb.r, min(rgb.g, rgb.b)); + let outness = max(-lo, 0.0); + let k = smoothstep(0.0, 1.0, outness); + return mix(rgb, clamp(rgb, vec3(0.0), vec3(1.0)), k); +} + fn target_nits_to_reference_linear(nits: vec3) -> vec3 { - return max(nits, vec3(0.0)) / target_reference_white_nits(); + // libplacebo's encode maps [target black, target peak] onto [0, 1] where + // 1.0 is the target reference white, so the tone-map black-point + // compensation lands back on true black instead of lifting it. + let black = uniforms.tone_map_extra.z; + let peak = target_peak_nits(); + let range = max(peak - black, 0.0001); + return max(nits - vec3(black), vec3(0.0)) / range + * (range / target_reference_white_nits()); } fn target_reference_linear_to_output(rgb: vec3) -> vec3 { @@ -194,22 +467,110 @@ struct RangeExpandedYCbCr { }; fn expand_ycbcr_range(y_in: f32, cbcr_in: vec2) -> RangeExpandedYCbCr { + var y = y_in; + var cbcr = cbcr_in; + if (uniforms.is_p010 != 0u) { + // P010 stores 10-bit codes as code << 6 in a 16-bit UNORM texture. + let p010_scale = 65535.0 / 65472.0; + y *= p010_scale; + cbcr *= p010_scale; + } var out: RangeExpandedYCbCr; if (uniforms.full_range != 0u) { - out.y = y_in; - out.cbcr = cbcr_in - vec2(0.5); + out.y = y; + out.cbcr = cbcr - vec2(0.5); return out; } if (uniforms.is_p010 != 0u) { - out.y = (y_in - (64.0 / 1023.0)) * (1023.0 / 876.0); - out.cbcr = (cbcr_in - vec2(512.0 / 1023.0)) * (1023.0 / 896.0); + out.y = (y - (64.0 / 1023.0)) * (1023.0 / 876.0); + out.cbcr = (cbcr - vec2(512.0 / 1023.0)) * (1023.0 / 896.0); return out; } - out.y = (y_in - (16.0 / 255.0)) * (255.0 / 219.0); - out.cbcr = (cbcr_in - vec2(128.0 / 255.0)) * (255.0 / 224.0); + out.y = (y - (16.0 / 255.0)) * (255.0 / 219.0); + out.cbcr = (cbcr - vec2(128.0 / 255.0)) * (255.0 / 224.0); return out; } +// Dolby Vision RPU reshaping, ported from libplacebo's `pl_shader_dovi_reshape` +// (the renderer behind mpv's Dolby Vision mapping). The base-layer signal is +// reshaped per component through piecewise polynomial/MMR curves selected by +// pivot comparison, where MMR coefficients mix all three raw components. +fn dovi_reshaped_signal(sig_in: vec3) -> vec3 { + let sig = clamp(sig_in, vec3(0.0), vec3(1.0)); + var result: array; + result[0] = sig.r; + result[1] = sig.g; + result[2] = sig.b; + let flags = uniforms.dovi_flags; + for (var c = 0u; c < 3u; c = c + 1u) { + let segments = u32(flags[1u + c]); + if (segments == 0u) { + continue; + } + var s = result[c]; + var index = 0u; + for (var i = 0u; i < 7u; i = i + 1u) { + let pivot_row = uniforms.dovi_pivots[2u * c + i / 4u]; + let pivot = pivot_row[i % 4u]; + if (s >= pivot) { + index = index + 1u; + } + } + let coeff = uniforms.dovi_coefficients[8u * c + index]; + if (coeff.w < 0.5) { + s = (coeff.z * s + coeff.y) * s + coeff.x; + } else { + let base = 48u * c + u32(coeff.y); + let order = u32(coeff.w); + let sig_x = vec4( + sig.x * sig.y, + sig.x * sig.z, + sig.y * sig.z, + sig.x * sig.y * sig.z + ); + s = coeff.x; + s = s + dot(uniforms.dovi_mmr[base].xyz, sig); + s = s + dot(uniforms.dovi_mmr[base + 1u], sig_x); + if (order >= 2u) { + let sig2 = sig * sig; + let sig_x2 = sig_x * sig_x; + s = s + dot(uniforms.dovi_mmr[base + 2u].xyz, sig2); + s = s + dot(uniforms.dovi_mmr[base + 3u], sig_x2); + if (order >= 3u) { + s = s + dot(uniforms.dovi_mmr[base + 4u].xyz, sig2 * sig); + s = s + dot(uniforms.dovi_mmr[base + 5u], sig_x2 * sig_x); + } + } + } + let bounds = uniforms.dovi_bounds[c]; + result[c] = clamp(s, bounds.x, bounds.y); + } + return vec3(result[0], result[1], result[2]); +} + +// Reshaped nonlinear signal to PQ-encoded IPT via the RPU's ycc_to_rgb matrix +// and signal offsets. Applying the RPU offsets keeps integer offset codes +// exactly on sample codes (2^bits/(2^bits-1) folded in on the CPU). +fn dovi_signal_to_pq_rgb(sig: vec3) -> vec3 { + let reshaped = dovi_reshaped_signal(sig) - uniforms.dovi_nonlinear_offset.xyz; + return vec3( + dot(uniforms.dovi_nonlinear_matrix[0].xyz, reshaped), + dot(uniforms.dovi_nonlinear_matrix[1].xyz, reshaped), + dot(uniforms.dovi_nonlinear_matrix[2].xyz, reshaped) + ); +} + +// Linearized BT.2020-referred HPE LMS back to linear RGB, using the composite +// of the fixed HPE inverse with the RPU's rgb_to_lms matrix (premultiplied on +// the CPU, matching libplacebo's dovi_lms2rgb). +fn dovi_lms_to_rgb(linear: vec3) -> vec3 { + return vec3( + dot(uniforms.dovi_lms_matrix[0].xyz, linear), + dot(uniforms.dovi_lms_matrix[1].xyz, linear), + dot(uniforms.dovi_lms_matrix[2].xyz, linear) + ); +} + fn packed_luma_texel(virtual_coord_in: vec2, virtual_size: vec2) -> f32 { let virtual_coord = clamp(virtual_coord_in, vec2(0), virtual_size - vec2(1)); let packed_coord = virtual_coord / vec2(2); @@ -275,8 +636,24 @@ fn erika_video_fragment(in: VertexOut) -> @location(0) vec4 { color_coord.x *= 0.5; } let alpha_coord = vec2(0.5 + in.tex_coord.x * 0.5, in.tex_coord.y); + var y_sample = textureSample(luma_texture, video_sampler, color_coord).r; + if (input_mode == 2u) { + y_sample = sample_packed_luma(color_coord); + } + let cbcr_sample = textureSample(chroma_texture, video_sampler, color_coord).rg; var rgb: vec3; - if (input_mode == 1u) { + let dovi_enabled = uniforms.dovi_flags.x != 0.0; + let dovi_ycbcr_input = dovi_enabled && (input_mode == 0u || input_mode == 2u); + if (dovi_ycbcr_input) { + // The base layer carries the raw 12-bit DV signal (10-bit container, + // full range); range expansion and the YCbCr matrix are replaced by + // the RPU reshaping + ycc_to_rgb path. + var sig = vec3(y_sample, cbcr_sample.x, cbcr_sample.y); + if (uniforms.is_p010 != 0u) { + sig *= 65535.0 / 65472.0; + } + rgb = dovi_signal_to_pq_rgb(sig); + } else if (input_mode == 1u) { rgb = textureSample(luma_texture, video_sampler, color_coord).rgb; } else if (input_mode == 3u) { let original_rgb = textureSample(chroma_texture, video_sampler, color_coord).rgb; @@ -284,13 +661,6 @@ fn erika_video_fragment(in: VertexOut) -> @location(0) vec4 { let enhanced_luma = sample_packed_luma(color_coord); rgb = original_rgb + vec3(enhanced_luma - original_luma); } else { - var y_sample: f32; - if (input_mode == 2u) { - y_sample = sample_packed_luma(color_coord); - } else { - y_sample = textureSample(luma_texture, video_sampler, color_coord).r; - } - let cbcr_sample = textureSample(chroma_texture, video_sampler, color_coord).rg; let expanded = expand_ycbcr_range(y_sample, cbcr_sample); let y = expanded.y; let cbcr = expanded.cbcr; @@ -303,10 +673,13 @@ fn erika_video_fragment(in: VertexOut) -> @location(0) vec4 { rgb.g = (y - kr * rgb.r - kb * rgb.b) / kg; } rgb = transfer_to_source_reference_linear(rgb); - rgb = apply_gamut_map(rgb); + if (dovi_ycbcr_input) { + rgb = dovi_lms_to_rgb(rgb); + } rgb = source_reference_to_nits(rgb); rgb = tone_map_nits(rgb); rgb = target_nits_to_reference_linear(rgb); + rgb = gamut_compress(rgb); rgb = target_reference_linear_to_output(rgb); var alpha = 1.0; if (packed_alpha) { diff --git a/crates/erika_ffmpeg_sys/wrapper.h b/crates/erika_ffmpeg_sys/wrapper.h index 620d9533..71c8096f 100644 --- a/crates/erika_ffmpeg_sys/wrapper.h +++ b/crates/erika_ffmpeg_sys/wrapper.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include diff --git a/docs/architecture.md b/docs/architecture.md index 5da0bc62..af01c5f3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -167,7 +167,9 @@ The primary renderer for Apple platforms: - Zero-copy CVPixelBuffer → MTLTexture import via `CVMetalTextureCache`. - YCbCr sampling, transfer decode, gamut mapping (BT.2020→BT.709, Display P3→BT.709). -- Tone mapping: Mobius, Reinhard, clip operators with absolute nits. +- Tone mapping: libplacebo-style IPT-domain color map with BT.2390 EETF + default (plus spline, BT.2446 method A, ST 2094-10, Mobius, Reinhard, clip) + over absolute nits. - SDR output (`BGRA8Unorm`) and Apple EDR output (`RGBA16Float` with EDR headroom). - Neural luma upscaler (`LumaUpscalerMode`): ArtCNN C4F16/C4F16 DS/C4F32 2x doublers diff --git a/docs/dolby-vision.md b/docs/dolby-vision.md new file mode 100644 index 00000000..255bbfed --- /dev/null +++ b/docs/dolby-vision.md @@ -0,0 +1,286 @@ +# Dolby Vision HDR Mapping + +This document describes Erika's implementation of Dolby Vision RPU (Reference Processing Unit) metadata processing and color mapping. + +## Overview + +Dolby Vision is an HDR format that enhances video quality through per-frame metadata called RPU. The implementation follows [libplacebo](https://github.com/haasn/libplacebo)'s approach (the renderer behind mpv's Dolby Vision support) to ensure compatibility and correctness. + +## Supported Profiles + +| Profile | Description | Decode Strategy | RPU Available | +|---------|-------------|-----------------|---------------| +| **5** | Single layer, non-backward compatible (IPTPQc2) | Hardware decode on VideoToolbox & D3D11VA; software fallback on mobile backends | ✅ Yes | +| **8** | Single layer with backward-compatible base (e.g. 8.1 HDR10, 8.4 HLG) | Hardware decode allowed | ✅ Yes | + +> **Note on Profile Architecture**: Profile 8 is a single-layer profile where the base layer carries standard signaling (such as HDR10 PQ for 8.1 or HLG for 8.4) with Dolby Vision RPU metadata interleaved as NAL units. Profile 7 is the dual-layer profile (Base Layer + Enhancement Layer / FEL / MEL) primarily used on Ultra HD Blu-ray discs. + +### Decode Strategy and Metadata Extraction + +On desktop platforms: +- **macOS (VideoToolbox)** and **Windows (D3D11VA)** decoders preserve frame side data (`AV_FRAME_DATA_DOVI_METADATA`) alongside hardware texture surfaces (CVPixelBuffer / D3D11 texture). This enables hardware-accelerated decoding while feeding RPU uniforms directly into GPU shaders for per-frame reshaping and color mapping. + +On mobile/embedded backends: +- Hardware decoders like **MediaCodec** (Android) and generic **AvCodec** backends may not expose RPU side data attached to output frames. +- For **Profile 5** on those mobile backends, because the stream uses IPTPQc2 rather than standard YCbCr and lacks backward compatibility, playback falls back to software decode via FFmpeg's `avcodec` to reliably access `AV_FRAME_DATA_DOVI_METADATA`. Desktop VideoToolbox and D3D11VA keep hardware decode for Profile 5 (see the table above). +- For **Profile 8**, hardware decode can safely be preserved: if RPU side data is unavailable, the video still displays with correct HDR10/HLG colors because the base layer is backward compatible. + +See `dolby_vision_decode_fallback()` in `crates/erika/src/playback.rs`. + +## Pipeline Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ 1. Container (MP4/MKV) │ +│ - dvcC/dvvC box → Dolby Vision profile (5 or 8) │ +└────────────────────┬────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ 2. FFmpeg Decoder (hardware on desktop, software on mobile) │ +│ - Decodes compressed HEVC stream │ +│ - Parses RPU from NAL units │ +│ - Emits AV_FRAME_DATA_DOVI_METADATA side data │ +└────────────────────┬────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ 3. Metadata Extraction (ffmpeg.rs, `frame_dovi_metadata_result`) │ +│ - Reads AVDOVIMetadata via pointer arithmetic │ +│ - Normalizes pivots by base layer bit depth │ +│ - Scales coefficients by 2^(-coef_log2_denom) │ +│ - Produces DoviSourceMetadata │ +└────────────────────┬────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ 4. Uniform Packing (pipeline.rs, `DoviUniforms`) │ +│ - Converts to vec4-aligned DoviUniforms (~3KB) │ +│ - Packs polynomial and MMR coefficients │ +│ - Applies signal offset correction for the uploaded bit depth │ +└────────────────────┬────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ 5. GPU Shader (wgsl/metal/hlsl) │ +│ a. Reshaping: piecewise polynomial/MMR per component │ +│ b. Nonlinear matrix: RPU's ycc_to_rgb │ +│ c. PQ linearization: EOTF │ +│ d. LMS→RGB: composite HPE inverse × rgb_to_lms │ +│ e. Tone mapping to display │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Reshaping Algorithm + +The core of Dolby Vision mapping is **per-component piecewise reshaping**. Each component (Y, Cb, Cr) is transformed through curves defined by pivots and coefficients. + +### Polynomial Segments + +For a segment between pivots `p[i]` and `p[i+1]`, if the input signal `s` falls in that range: + +``` +output = (c2 · s + c1) · s + c0 +``` + +### MMR (Multivariate Polynomial Regression) + +For higher-order mapping, MMR mixes all three input components: + +```rust +// Order 1 +output = constant + dot([a, b, c], [R, G, B]) + dot([d, e, f, g], [R·G, R·B, G·B, R·G·B]) + +// Order 2: adds R², G², B² and squared cross terms +// Order 3: adds R³, G³, B³ and cubed cross terms +``` + +See `dovi_reshaped_signal()` in shaders for implementation. + +## Color Transform Flow + +After reshaping, the signal goes through: + +1. **Offset subtraction**: `reshaped - nonlinear_offset` +2. **Nonlinear matrix**: RPU's `ycc_to_rgb` (still PQ-encoded) +3. **PQ linearization**: Convert PQ code to linear light +4. **LMS to RGB**: `(HPE⁻¹ × rgb_to_lms) × linearized` +5. **Gamut/tone mapping**: Standard HDR pipeline continues + +## Tone Mapping (libplacebo color map) + +The tone map mirrors libplacebo's color map (`pl_shader_color_map`, the +engine behind mpv's `--vo=gpu-next`): RGB in source primaries (absolute +nits) — HPE-LMS — PQ encode — IPT. The operator's curve runs on the IPT +intensity axis, libplacebo's chroma rule (`hull` cubic on the pre/post +intensity pair) protects saturation, and the decode back to RGB lands in +the target primaries, so the primaries conversion happens inside the +roundtrip instead of a separate gamut matrix. + +Operators (`ToneMapOperator`, default `Bt2390`): + +| code | operator | curve parameter (`ToneMapConfig::curve_param`) | +|---|---|---| +| 0 | Clip | — | +| 1 | Reinhard | contrast (default 0.5) | +| 2 | Mobius | linear knee (default 0.3) | +| 3 | **BT.2390** (default) | knee offset (default 1.0) | +| 4 | Spline | slope contrast (default 0.30) | +| 5 | BT.2446 method A | — | +| 6 | SMPTE ST 2094-10 | knee adaptation (default 0.70; coefficients solved per frame on the CPU) | + +Black-point compensation uses `target black = target peak / contrast` +(`contrast_ratio`; auto 1000:1 for SDR, 0 for HDR/EDR targets) and is gated +so SDR → SDR rendering is untouched. The encode maps `[black, peak]` onto +`[0, 1]`, so the compensated floor lands back on code 0. + +Whether the tone map (and therefore the black-point compensation and the +perceptual gamut LUT) is active at all is decided from the **static** +mastering-display peak (L0), never the per-frame L1 peak: a dark scene must +not silently drop the tone map or the LUT for one frame. The per-frame L1 +peak still sets the curve's source peak (`tone_map_extra`/`nits.x`), which is +what libplacebo does with dynamic metadata. ST 2094-10 picks its knee in the +PQ domain even though the curve itself is solved in absolute nits, matching +libplacebo's internal `pl_hdr_rescale` round trip. + +## Scene-Adaptive HDR10 (measured luma) + +HDR10 streams carry only static ST.2086 mastering metadata, so without +further input the tone-map pivot would sit at the fixed 40% knee for every +scene. `crates/erika/src/luma_stats.rs` gives HDR10 the same treatment +Dolby Vision L1 gives Profile 5/8: for software-decoded PQ frames the +presenter samples a sparse grid of the luma plane (NV12/P010), linearizes +each sample, re-encodes to PQ and averages in that perceptual domain, then +smooths the running estimate with libplacebo's IIR filter +(`coeff = 1 - exp(-1/20)`). The smoothed scene average (nits) is attached to +the frame (`PlayerVideoFrame::scene_avg_nits`) and folded into +`SourceColorState.measured_scene_avg_nits`, which `tone_map_extra.y` +prefers over the L1 average — so the spline pivot follows the content. + +The samples are normalized with the frame's own color range: limited (TV) +range planes are expanded over the legal 16..235 / 64..940 span exactly like +the shaders' `expand_ycbcr_range`. HDR10 is normally limited range, so +measuring it as full range would lift black and clip the highlights before +the PQ re-encode. + +Hardware-decoded frames (VideoToolbox/D3D11VA/MediaCodec) have no CPU luma +plane and keep the static-metadata path. The estimator resets on generation +changes so seeks cannot carry the previous scene's brightness forward. + +## Perceptual Gamut Mapping (IPT 3D LUT) + +After the tone map, when an HDR source is tone-mapped into a smaller gamut +(BT.2020 → BT.709), the renderer applies a perceptual gamut map via a +CPU-generated 48 × 32 × 256 IPT-space LUT (`renderer::gamut`) instead of the +fast `gamut_compress`. Texels hold the perceptually mapped +`(I, P + 0.5, T + 0.5)` color — the I axis spans the target's +`[black, peak]` in PQ codes (libplacebo's `gamut.min_luma`/`max_luma`, i.e. +the tone map's output range). Generation follows libplacebo's IPT LUT layout +and index mapping so the shaders sample the same lattice; the chroma rolloff +is currently a simplified dead-zone blend plus Möbius soft clip rather than +libplacebo's full per-hue boundary search, so highly saturated BT.2020 colors +can diverge slightly from mpv. The shaders rebuild RGB in the source +primaries, run the LMS-PQ-IPT roundtrip, sample the LUT in ICh space and +decode back to the target primaries; the LUT texture is cached per (source, +target, target black, target peak) and bound as binding/slot 4 (WGSL), +texture 2 (Metal) or t2 (D3D11), always with a dummy 1×1×1 fallback so the +fast path keeps a valid layout. + +## Per-Frame L1 Brightness Metadata + +The RPU's dynamic DM extension blocks carry **level 1** per-frame brightness +metadata: `min_pq` / `max_pq` / `avg_pq` in 12-bit PQ codes. FFmpeg's RPU +decoder copies these blocks into the frame side data (`AVDOVIMetadata` +`ext_block_offset` region); `frame_dovi_level1` validates that region and +extracts the level 1 block. + +The frame's `max_pq` is used as the tone-map source peak when L1 metadata is present, and the frame's `avg_pq` becomes the scene average that drives the tone-map pivot. Static mastering-display metadata remains responsible for output-mode negotiation and tone-map/gamut-LUT enablement decisions. This makes the default BT.2390 curve +scene-adaptive: a dark scene is not compressed against a 4000-nit mastering +peak, so its highlights stay distinct. The **static mastering display peak +is untouched** — output-mode negotiation (SDR/EDR per display) and the +decision whether to tone map / bind the perceptual gamut LUT both read the +static L0 peak, so per-frame brightness can never toggle them. + +An absent, all-zero, or inverted level 1 block falls back to the static +`source_max_pq` (and the fixed 40% knee when no average is known); the RPU +itself is never rejected over L1. + +## Forced BT.2020/PQ + +Dolby Vision Profile 5/8 VUI tags are **unreliable**. The implementation forces: + +```rust +self.primaries = ColorPrimaries::Bt2020; +self.transfer = TransferFunction::Pq; +``` + +Without this, a stream tagged as "unspecified transfer" would decode PQ-encoded samples with sRGB gamma, producing completely wrong brightness. + +See `SourceColorState::dovi()` in `pipeline.rs`. + +## Testing Strategy + +### Unit Tests + +- `frame_reads_dovi_side_data`: Verifies FFmpeg side data parsing +- `dovi_uniforms_pack_pivots_poly_and_mmr`: Validates uniform packing +- `dovi_source_forces_pq_when_stream_tags_are_missing`: Confirms PQ forcing +- `dovi_l1_brightness_metadata_is_parsed`: Verifies level 1 ext-block parsing and invalid-L1 fallback +- `dovi_source_uses_per_frame_l1_peak_when_present`: Confirms L1 max_pq drives the tone-map source peak while mastering metadata remains static for output-mode decisions +- `fel_residual_keeps_the_base_layer_mapping`: Confirms a Profile 7 FEL RPU keeps its base-layer mapping and reports `DoviElStatus` instead of being rejected +- `dolby_vision_profile_5_stays_on_hardware_for_videotoolbox_and_d3d11va`: Verifies desktop hardware decoders stay on hardware for Profile 5 +- `dolby_vision_profile_5_falls_back_to_software_on_mobile_backends`: Verifies mobile backends fall back to software decode for Profile 5 +- `dolby_vision_profile_8_stays_on_hardware_decode`: Profile 8 hardware decode preservation + +### Integration Tests (require samples) + +Set environment variables to enable: + +- `ERIKA_DV_SAMPLE`: Profile 5 sample (RPU mapping verification) +- `ERIKA_DV_PROFILE_8_SAMPLE`: Profile 8 sample (hardware decode test) + +## Enhancement-layer residual (Profile 7 FEL / MEL) + +Profile 7 keeps its enhancement layer in a second HEVC layer that this renderer +never decodes, so an RPU with `disable_residual_flag == 0` describes a residual +that has nothing to be added to. Dropping the whole RPU in that case would also +discard the base-layer reshaping curves, color matrices, and L1 trims, which +still apply to the base layer on their own — so the RPU is mapped as usual and +only the un-composable part is reported. + +`Frame::dovi_el_status()` returns a `DoviElStatus` (`residual_requested`, +`nlq_nontrivial`), and playback emits one throttled `dovi_el_not_composed` +diagnostic per stream. This mirrors libplacebo, which exposes `nlq_active` but +documents that "consumers that have not bound an enhancement layer must not +look at these fields": the NLQ fields are parsed for reporting only and never +feed the shaders. + +## Known Limitations + +1. **Dual-layer FEL / MEL residual composition not supported**: Profile 7 + enhancement layers are never decoded, so the NLQ residual an RPU asks for + cannot be composed. The base layer still gets the full Dolby Vision mapping + (curves, matrices, L1 trims) and the gap is reported through the + `dovi_el_not_composed` diagnostic — see "Enhancement-layer residual" above. + Real Profile 7 content therefore renders like mpv/libplacebo rather than + like a plain HDR10 base layer, but without the extra detail the + enhancement layer carries. +2. **Mobile hardware decode RPU extraction**: On mobile backends (MediaCodec), hardware decoders do not expose RPU side data, requiring software decode for Profile 5. +3. **Uniform buffer size**: ~3KB may exceed limits on very old mobile GPUs (pre-2015) +4. **CPU plane upload on software decode**: When software decode fallback is used, software planes incur CPU-to-GPU texture upload overhead. + +## References + +- [Dolby Vision Specification](https://professional.dolby.com/dolby-vision/) +- [libplacebo dovi_reshape implementation](https://github.com/haasn/libplacebo/blob/master/src/shaders/dovi.c) +- [FFmpeg AVDOVIMetadata](https://ffmpeg.org/doxygen/trunk/structAVDOVIMetadata.html) +- [BT.2100 PQ EOTF](https://www.itu.int/rec/R-REC-BT.2100) + +## Implementation Files + +| File | Purpose | +|------|---------| +| `crates/erika/src/ffmpeg.rs` | Container and frame metadata extraction (`frame_dovi_metadata_result`) | +| `crates/erika/src/playback.rs` | Profile-based decode fallback (`dolby_vision_decode_fallback`) | +| `crates/erika/src/renderer/pipeline.rs` | Data structures and uniform packing (`DoviUniforms`) | +| `crates/erika/src/renderer/gamut.rs` | Perceptual gamut LUT generation | +| `crates/erika/src/luma_stats.rs` | Scene-adaptive HDR10 luma measurement | +| `crates/erika/src/renderer/wgpu_video.wgsl` | WGSL shader implementation | +| `crates/erika/src/renderer/metal/apple.rs` | Metal shader implementation | +| `crates/erika/src/renderer/d3d11.rs` | HLSL shader implementation | diff --git a/examples/danmaku_perf_lab/native/DanmakuPerfLab.m b/examples/danmaku_perf_lab/native/DanmakuPerfLab.m index f2911770..a7fa5eda 100644 --- a/examples/danmaku_perf_lab/native/DanmakuPerfLab.m +++ b/examples/danmaku_perf_lab/native/DanmakuPerfLab.m @@ -63,6 +63,7 @@ - (instancetype)initWithFrame:(NSRect)frameRect { if ([self.metalLayer respondsToSelector:@selector(setDisplaySyncEnabled:)]) { self.metalLayer.displaySyncEnabled = !erika_perf_lab_uncapped(); } + self.metalLayer.delegate = (id)self; self.layer = self.metalLayer; self.startTime = CACurrentMediaTime(); } diff --git a/examples/macos_native_demo/native/ErikaMetalDemo.m b/examples/macos_native_demo/native/ErikaMetalDemo.m index 229a1aef..5a2b0b1c 100644 --- a/examples/macos_native_demo/native/ErikaMetalDemo.m +++ b/examples/macos_native_demo/native/ErikaMetalDemo.m @@ -4,6 +4,7 @@ extern void erika_demo_attach_layer(void *layer, unsigned int width, unsigned int height, double scale); extern void erika_demo_resize_layer(unsigned int width, unsigned int height, double scale); +extern void erika_demo_set_display_headroom(float headroom, bool known); extern void erika_demo_render_frame(double time_seconds); extern void erika_demo_toggle_play_pause(void); extern void erika_demo_seek_seconds(double seconds); @@ -12,6 +13,23 @@ extern bool erika_demo_is_playing(void); extern double erika_demo_smoke_seconds(void); +/// Potential EDR headroom of a screen: what it can do regardless of the +/// current brightness setting. 1.0 means no EDR. Main thread only. +static float ErikaPotentialEdrHeadroom(NSScreen *screen) { + if (screen == nil) { + return 1.0f; + } + SEL selector = NSSelectorFromString(@"maximumPotentialExtendedDynamicRangeColorComponentValue"); + if (![screen respondsToSelector:selector]) { + return 1.0f; + } + NSNumber *value = [screen valueForKey:@"maximumPotentialExtendedDynamicRangeColorComponentValue"]; + if (![value isKindOfClass:[NSNumber class]]) { + return 1.0f; + } + return MAX(1.0f, value.floatValue); +} + static NSString *ErikaFormatTime(double seconds) { if (!isfinite(seconds) || seconds < 0.0) { seconds = 0.0; @@ -42,6 +60,7 @@ - (instancetype)initWithFrame:(NSRect)frameRect { self.metalLayer.pixelFormat = MTLPixelFormatBGRA8Unorm; self.metalLayer.framebufferOnly = YES; self.metalLayer.opaque = YES; + self.metalLayer.delegate = (id)self; self.layer = self.metalLayer; self.startTime = CACurrentMediaTime(); } @@ -90,6 +109,11 @@ - (void)updateDrawableSizeAndAttach:(BOOL)attach { self.metalLayer.frame = self.bounds; unsigned int pixelWidth = (unsigned int)MAX(1.0, round(drawableSize.width)); unsigned int pixelHeight = (unsigned int)MAX(1.0, round(drawableSize.height)); + // The renderer negotiates its output mode off the main thread and no longer + // reads AppKit itself, so publish the presenting screen's EDR capability + // from here: on attach, and again whenever the window moves to a screen with + // different backing properties. + erika_demo_set_display_headroom(ErikaPotentialEdrHeadroom(self.window.screen), true); if (attach) { erika_demo_attach_layer((__bridge void *)self.metalLayer, pixelWidth, pixelHeight, scale); } else { @@ -282,6 +306,9 @@ - (void)applicationDidFinishLaunching:(NSNotification *)notification { [self.window center]; [self.window makeKeyAndOrderFront:nil]; double smokeSeconds = erika_demo_smoke_seconds(); + // Always activate: an inactive app gets App-Napped by macOS, which + // throttles the render timer and stalls the presentation-driven decode. + [NSApp activateIgnoringOtherApps:YES]; if (smokeSeconds > 0.0) { self.smokeTimer = [NSTimer scheduledTimerWithTimeInterval:smokeSeconds target:self @@ -289,8 +316,6 @@ - (void)applicationDidFinishLaunching:(NSNotification *)notification { userInfo:nil repeats:NO]; [[NSRunLoop mainRunLoop] addTimer:self.smokeTimer forMode:NSRunLoopCommonModes]; - } else { - [NSApp activateIgnoringOtherApps:YES]; } } diff --git a/examples/macos_native_demo/src/main.rs b/examples/macos_native_demo/src/main.rs index 0c271fea..4044b3f8 100644 --- a/examples/macos_native_demo/src/main.rs +++ b/examples/macos_native_demo/src/main.rs @@ -17,6 +17,7 @@ static SUBTITLE_PATH: OnceLock = OnceLock::new(); static DANMAKU_PATH: OnceLock = OnceLock::new(); static SMOKE_SECONDS: OnceLock = OnceLock::new(); static EDR_HEADROOM: OnceLock = OnceLock::new(); +static SEEK_SEQUENCE: OnceLock> = OnceLock::new(); unsafe extern "C" { fn erika_demo_run_app(); @@ -30,6 +31,7 @@ struct DemoState { presenter: PresenterRuntime, load_attempted: bool, overlay_logged: bool, + seek_cursor: usize, } impl DemoState { @@ -44,10 +46,32 @@ impl DemoState { })?, load_attempted: false, overlay_logged: false, + seek_cursor: 0, }) } fn render(&mut self, time_seconds: f64) { + if let Some(sequence) = SEEK_SEQUENCE.get() { + while self.seek_cursor < sequence.len() && time_seconds >= sequence[self.seek_cursor].0 + { + let (trigger, position) = sequence[self.seek_cursor]; + match self + .presenter + .seek(Duration::from_secs_f64(position.max(0.0))) + { + Ok(()) => eprintln!( + "Erika demo seek #{}(at {trigger:.1}s) -> {position:.1}s ok", + self.seek_cursor + 1 + ), + Err(error) => eprintln!( + "Erika demo seek #{} -> {:.1}s failed: {error}", + self.seek_cursor + 1, + position + ), + } + self.seek_cursor += 1; + } + } if !self.load_attempted { self.load_attempted = true; if let Some(uri) = MEDIA_URI.get() { @@ -82,6 +106,16 @@ impl DemoState { eprintln!("Erika demo overlay active through presenter runtime"); self.overlay_logged = true; } + if SEEK_SEQUENCE.get().is_some() { + eprintln!( + "Erika demo seek stats: decoded={} rendered={} import_failures={} render_failures={} backpressure_drops={}", + stats.decoded_video_frames, + stats.rendered_video_frames, + stats.import_failures, + stats.render_failures, + stats.video_frame_backpressure_drops + ); + } } Err(error) => eprintln!("Erika demo render failed: {error}"), } @@ -164,6 +198,20 @@ pub extern "C" fn erika_demo_attach_layer(layer: *mut c_void, width: u32, height }); } +/// Publishes the presenting display's EDR headroom. +/// +/// The renderer negotiates its output mode on the render thread and no longer +/// probes AppKit itself, so the AppKit side (the demo view) resolves the value +/// on the main thread and pushes it here. +#[unsafe(no_mangle)] +pub extern "C" fn erika_demo_set_display_headroom(headroom: f32, known: bool) { + DEMO.with(|demo| { + demo.borrow_mut() + .presenter + .set_output_headroom(headroom, known); + }); +} + #[unsafe(no_mangle)] pub extern "C" fn erika_demo_resize_layer(width: u32, height: u32, scale: f64) { DEMO.with(|demo| { @@ -223,7 +271,7 @@ fn main() { let options = parse_args(&args).unwrap_or_else(|error| { eprintln!("{error}"); eprintln!( - "usage: cargo run -p macos_native_demo -- [--edr [HEADROOM]] [--smoke-seconds N] [--subtitle PATH] [--ass-subtitle PATH] [--danmaku PATH] [media-path-or-uri]" + "usage: cargo run -p macos_native_demo -- [--edr [HEADROOM]] [--smoke-seconds N] [--subtitle PATH] [--ass-subtitle PATH] [--danmaku PATH] [--seek-sequence t:pos,t:pos] [media-path-or-uri]" ); process::exit(2); }); @@ -234,6 +282,21 @@ fn main() { DANMAKU_PATH.set(path).expect("danmaku path is set once"); } + if let Some(sequence) = options.seek_sequence { + let parsed: Vec<(f64, f64)> = sequence + .split(',') + .filter(|entry| !entry.trim().is_empty()) + .map(|entry| { + let (trigger, position) = entry.split_once(':').unwrap_or((entry, "0")); + ( + trigger.trim().parse::().unwrap_or(0.0), + position.trim().parse::().unwrap_or(0.0), + ) + }) + .collect(); + eprintln!("Erika demo seek sequence: {parsed:?}"); + SEEK_SEQUENCE.set(parsed).expect("seek sequence set once"); + } if let Some(headroom) = options.edr_headroom { EDR_HEADROOM .set(headroom) @@ -257,6 +320,7 @@ struct DemoOptions { media_uri: Option, smoke_seconds: Option, edr_headroom: Option, + seek_sequence: Option, subtitle_path: Option, danmaku_path: Option, } @@ -265,11 +329,20 @@ fn parse_args(args: &[String]) -> Result { let mut media_uri = None; let mut smoke_seconds = None; let mut edr_headroom = None; + let mut seek_sequence = None; let mut subtitle_path = None; let mut danmaku_path = None; let mut index = 0; while index < args.len() { match args[index].as_str() { + "--seek-sequence" => { + index += 1; + seek_sequence = Some( + args.get(index) + .cloned() + .ok_or("--seek-sequence requires a value like 3:14,5:16")?, + ); + } "--edr" => { let mut headroom = 4.0; if let Some(value) = args.get(index + 1) { @@ -341,6 +414,7 @@ fn parse_args(args: &[String]) -> Result { media_uri, smoke_seconds, edr_headroom, + seek_sequence, subtitle_path, danmaku_path, }) diff --git a/examples/wgpu_decode_png/src/main.rs b/examples/wgpu_decode_png/src/main.rs index b421cf85..f8b8d5fd 100644 --- a/examples/wgpu_decode_png/src/main.rs +++ b/examples/wgpu_decode_png/src/main.rs @@ -102,14 +102,34 @@ fn render_frame(renderer: &mut WgpuRenderer, frame: Frame, out: &str) { media_time: pts.unwrap_or_default(), late_by: None, generation: 1, + scene_avg_nits: std::env::var("ERIKA_TEST_SCENE_AVG_NITS") + .ok() + .and_then(|value| value.parse::().ok()), }; renderer .upload_player_frame(&player_frame) .expect("upload decoded frame"); - let readback = renderer + // The perceptual gamut LUT is generated on a background thread and only + // bound once ready, so the first render uses the fast path. Repeat the + // render (ERIKA_TEST_RENDER_PASSES, default 1) to exercise the LUT path + // the player reaches after the first frames. + let passes = std::env::var("ERIKA_TEST_RENDER_PASSES") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(1); + let mut readback = renderer .render_current_offscreen(None) .expect("render current frame") .expect("a frame was uploaded"); + for _ in 1..passes { + // TODO: replace fixed-delay waiting with explicit LUT-ready synchronization. + std::thread::sleep(Duration::from_millis(200)); + readback = renderer + .render_current_offscreen(None) + .expect("render current frame") + .expect("a frame was uploaded"); + } write_png(out, readback.width, readback.height, &readback.rgba); println!("wrote {out} ({}x{})", readback.width, readback.height); } diff --git a/examples/wgpu_overlay_png/src/main.rs b/examples/wgpu_overlay_png/src/main.rs index 14dbb1ae..ff04d864 100644 --- a/examples/wgpu_overlay_png/src/main.rs +++ b/examples/wgpu_overlay_png/src/main.rs @@ -3,6 +3,7 @@ //! the wgpu overlay/subtitle compositing path (alpha-blended over the video). use erika::overlay::{OverlayFrame, OverlayViewport}; +use erika::renderer::pipeline::VideoRenderPipeline; use erika::renderer::wgpu::{VideoUniforms, WgpuRenderer}; use erika::subtitle::{SubtitleAlphaBitmap, SubtitleBitmapPlacement, SubtitleBitmapPlane}; @@ -153,21 +154,6 @@ fn color_bars_nv12() -> (Vec, Vec) { } fn bars_uniforms() -> VideoUniforms { - VideoUniforms { - is_p010: 0, - full_range: 0, - source_transfer: 0, - target_transfer: 0, - tone_map: 0, - edr_output: 0, - input_mode: 0, - scene_linear: 0, - nits: [100.0, 100.0, 100.0, 100.0], - luma_coefficients: [0.2126, 0.7152, 0.0722, 0.0], - gamut_matrix_rows: [ - [1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - ], - } + let pipeline = VideoRenderPipeline::sdr_default(); + VideoUniforms::from_pipeline(&pipeline, false, false) } diff --git a/examples/wgpu_video_png/src/main.rs b/examples/wgpu_video_png/src/main.rs index e93a414f..dfb16b2c 100644 --- a/examples/wgpu_video_png/src/main.rs +++ b/examples/wgpu_video_png/src/main.rs @@ -2,6 +2,7 @@ //! pipeline and writes the result to a PNG. A visual smoke test for the wgpu //! YCbCr->RGB path: the output PNG should show clean SMPTE-style color bars. +use erika::renderer::pipeline::VideoRenderPipeline; use erika::renderer::wgpu::{VideoUniforms, WgpuRenderer}; const WIDTH: u32 = 256; @@ -63,23 +64,8 @@ fn build_color_bars_nv12() -> (Vec, Vec) { /// A faithful BT.709 limited-range round-trip: linear in/out, clip tone map, /// matched nits, identity gamut. The decoded RGB should match the source bars. fn bt709_limited_uniforms() -> VideoUniforms { - VideoUniforms { - is_p010: 0, - full_range: 0, - source_transfer: 0, - target_transfer: 0, - tone_map: 0, - edr_output: 0, - input_mode: 0, - scene_linear: 0, - nits: [100.0, 100.0, 100.0, 100.0], - luma_coefficients: [0.2126, 0.7152, 0.0722, 0.0], - gamut_matrix_rows: [ - [1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - ], - } + let pipeline = VideoRenderPipeline::sdr_default(); + VideoUniforms::from_pipeline(&pipeline, false, false) } fn main() { diff --git a/examples/wgpu_window_check/src/main.rs b/examples/wgpu_window_check/src/main.rs index 06e8dffd..45711374 100644 --- a/examples/wgpu_window_check/src/main.rs +++ b/examples/wgpu_window_check/src/main.rs @@ -8,6 +8,7 @@ use std::ffi::c_void; use std::process; use std::time::Duration; +use erika::renderer::pipeline::VideoRenderPipeline; use erika::renderer::wgpu::{VideoUniforms, WgpuRenderer}; use erika::{ PlatformSurface, RenderFrameContext, RendererBackend, WgpuSurfaceHandle, WgpuSurfaceKind, @@ -139,21 +140,6 @@ fn color_bars_nv12() -> (Vec, Vec) { } fn bars_uniforms() -> VideoUniforms { - VideoUniforms { - is_p010: 0, - full_range: 0, - source_transfer: 0, - target_transfer: 0, - tone_map: 0, - edr_output: 0, - input_mode: 0, - scene_linear: 0, - nits: [100.0, 100.0, 100.0, 100.0], - luma_coefficients: [0.2126, 0.7152, 0.0722, 0.0], - gamut_matrix_rows: [ - [1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - ], - } + let pipeline = VideoRenderPipeline::sdr_default(); + VideoUniforms::from_pipeline(&pipeline, false, false) } diff --git a/packages/erika_flutter/ios/Classes/ErikaFlutterPlugin.swift b/packages/erika_flutter/ios/Classes/ErikaFlutterPlugin.swift index 9385b537..e71a84fc 100644 --- a/packages/erika_flutter/ios/Classes/ErikaFlutterPlugin.swift +++ b/packages/erika_flutter/ios/Classes/ErikaFlutterPlugin.swift @@ -2872,6 +2872,14 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan } private func presenterConfigForNewPlayer(arguments: Any?, hdrDebug: Bool) -> ErikaPresenterConfigC { + // An explicit disable must become SDR, not `auto(headroom: 1.0)`: under + // the per-presenting-display Auto negotiation a headroom of 1.0 means "no + // embedder cap, defer to the display", so an EDR display would promote + // again and ignore ERIKA_DISABLE_EDR. + let edrDisabled = boolEnvironmentFlag( + "ERIKA_DISABLE_EDR", + environment: ProcessInfo.processInfo.environment + ) if let args = arguments as? [String: Any], let explicitMode = int32Value(args["outputMode"]) { let headroom = floatValue(args["edrHeadroom"]) ?? 4.0 let config: ErikaPresenterConfigC @@ -2881,7 +2889,7 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan case 2: config = ErikaPresenterConfigC(outputMode: 2, edrHeadroom: max(1.0, headroom)) case 3: - config = .auto(headroom: headroom) + config = edrDisabled ? .sdr : .auto(headroom: headroom) default: config = .sdr } @@ -2891,7 +2899,10 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan ) return config } - let headroom = resolvedEdrHeadroom(hdrDebug: hdrDebug) + guard let headroom = resolvedEdrHeadroom(hdrDebug: hdrDebug) else { + erikaHdrLog(hdrDebug, "ERIKA_DISABLE_EDR is set; using SDR output") + return .sdr + } let config = ErikaPresenterConfigC.auto(headroom: headroom) erikaHdrLog( hdrDebug, @@ -2900,11 +2911,13 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan return config } - private func resolvedEdrHeadroom(hdrDebug: Bool) -> Float { + /// EDR headroom to request for a new player, or `nil` when + /// `ERIKA_DISABLE_EDR` disables EDR and the player must use SDR. + private func resolvedEdrHeadroom(hdrDebug: Bool) -> Float? { let environment = ProcessInfo.processInfo.environment if boolEnvironmentFlag("ERIKA_DISABLE_EDR", environment: environment) { erikaHdrLog(hdrDebug, "EDR disabled by ERIKA_DISABLE_EDR") - return 1.0 + return nil } if let override = floatEnvironmentValue("ERIKA_EDR_HEADROOM", environment: environment), override > 1.0 { erikaHdrLog(hdrDebug, "EDR headroom override ERIKA_EDR_HEADROOM=\(String(format: "%.3f", override))") diff --git a/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift b/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift index 5ba49d6e..1b33d8a4 100644 --- a/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift +++ b/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift @@ -647,6 +647,7 @@ private final class ErikaNativeLibrary { ) -> Int32 typealias ResizeSurfaceFn = @convention(c) (UnsafeMutableRawPointer?, UInt32, UInt32, Double) -> Int32 typealias RenderTickFn = @convention(c) (UnsafeMutableRawPointer?, Double, UnsafeMutableRawPointer?) -> Int32 + typealias SetOutputHeadroomFn = @convention(c) (UnsafeMutableRawPointer?, Float, Bool) -> Int32 typealias CaptureFrameRgbaFn = @convention(c) (UnsafeMutableRawPointer?, UInt32, UInt32, UnsafeMutableRawPointer?, Int) -> Int32 typealias PollEventFn = @convention(c) (UnsafeMutableRawPointer?, UnsafeMutableRawPointer?) -> Int32 typealias LastErrorMessageFn = @convention(c) () -> UnsafeMutablePointer? @@ -712,6 +713,7 @@ private final class ErikaNativeLibrary { let renderTick: RenderTickFn let captureFrameRgba: CaptureFrameRgbaFn? let pollEvent: PollEventFn + let setOutputHeadroom: SetOutputHeadroomFn? let lastErrorMessage: LastErrorMessageFn let stringFree: StringFreeFn @@ -783,6 +785,7 @@ private final class ErikaNativeLibrary { renderTick = try Self.load("erika_presenter_render_tick", from: libraryHandle, as: RenderTickFn.self) captureFrameRgba = Self.loadOptional("erika_presenter_capture_frame_rgba", from: libraryHandle, as: CaptureFrameRgbaFn.self) pollEvent = try Self.load("erika_presenter_poll_event", from: libraryHandle, as: PollEventFn.self) + setOutputHeadroom = Self.loadOptional("erika_presenter_set_output_headroom", from: libraryHandle, as: SetOutputHeadroomFn.self) lastErrorMessage = try Self.load("erika_last_error_message", from: libraryHandle, as: LastErrorMessageFn.self) stringFree = try Self.load("erika_string_free", from: libraryHandle, as: StringFreeFn.self) } @@ -1822,6 +1825,7 @@ private final class ErikaPlayerHost { operation: "attach_metal_layer" ) } + reportDisplayHeadroom() } else { try withNativeCall { try check( @@ -1832,6 +1836,23 @@ private final class ErikaPlayerHost { } } + /// Publishes the presenting display's EDR headroom to the renderer. + /// + /// The renderer negotiates its output mode on the render thread (the + /// CVDisplayLink callback), where AppKit must not be touched, so the value + /// is resolved here and pushed through the C ABI instead. Main thread only: + /// it reads the attached `NSView`'s screen. + private func reportDisplayHeadroom() { + guard let setOutputHeadroom = library.setOutputHeadroom, + let view = attachedView else { + return + } + let headroom = view.displayEdrHeadroom() + withNativeCall { + _ = setOutputHeadroom(handle, headroom, true) + } + } + private func startDisplayDriverIfNeeded(resetClock: Bool) { if resetClock { startTimeSeconds = CACurrentMediaTime() @@ -1884,6 +1905,10 @@ private final class ErikaPlayerHost { object: nil, queue: .main ) { [weak self] _ in + // Screen parameters change without the display ID changing (the EDR + // capability of the same screen can move), so republish the headroom + // before the display-link retarget short-circuits. + self?.reportDisplayHeadroom() self?.retargetDisplayDriverIfScreenChanged() } displayConfigurationObservers.append(observer) @@ -2170,6 +2195,23 @@ private func withOptionalCString(_ value: String?, _ body: (UnsafePointer Float { + guard let screen else { + return 1.0 + } + let key = "maximumPotentialExtendedDynamicRangeColorComponentValue" + guard screen.responds(to: Selector((key))), + let number = screen.value(forKey: key) as? NSNumber else { + return 1.0 + } + return max(1.0, number.floatValue) +} + private protocol ErikaMetalSurfaceView: AnyObject { var platformViewId: Int64 { get } var metalLayer: CAMetalLayer { get } @@ -2179,6 +2221,11 @@ private protocol ErikaMetalSurfaceView: AnyObject { func updateDrawableSize() func pngSnapshotData() -> Data? + + /// EDR headroom of the display this view is presented on. Main thread only: + /// the renderer consumes the reported value on its own thread and must not + /// query AppKit itself. + func displayEdrHeadroom() -> Float } private final class WeakErikaVideoPlatformViewBox { @@ -2279,6 +2326,10 @@ final class ErikaVideoPlatformView: NSView, ErikaMetalSurfaceView { func pngSnapshotData() -> Data? { snapshotPngData(of: self) } + + func displayEdrHeadroom() -> Float { + screenPotentialEdrHeadroom(window?.screen) + } } final class ErikaWindowOverlayView: NSView, ErikaMetalSurfaceView { @@ -2408,6 +2459,10 @@ final class ErikaWindowOverlayView: NSView, ErikaMetalSurfaceView { func pngSnapshotData() -> Data? { snapshotPngData(of: self) } + + func displayEdrHeadroom() -> Float { + screenPotentialEdrHeadroom(window?.screen) + } } private func snapshotPngData(of view: NSView) -> Data? { @@ -3399,6 +3454,14 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan private func presenterConfigForNewPlayer(arguments: Any?) throws -> ErikaPresenterConfigC { let alphaMode = (arguments as? [String: Any]) .flatMap { int32Value($0["videoAlphaMode"]) } ?? 0 + // An explicit disable must become SDR, not `auto(headroom: 1.0)`: under + // the per-presenting-display Auto negotiation a headroom of 1.0 means "no + // embedder cap, defer to the display", so an EDR display would promote + // again and ignore ERIKA_DISABLE_EDR. + let edrDisabled = boolEnvironmentFlag( + "ERIKA_DISABLE_EDR", + environment: ProcessInfo.processInfo.environment + ) if let args = arguments as? [String: Any], let explicitMode = int32Value(args["outputMode"]) { let headroom = floatValue(args["edrHeadroom"]) ?? 4.0 @@ -3409,7 +3472,7 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan case 2: config = ErikaPresenterConfigC(outputMode: 2, edrHeadroom: max(1.0, headroom)) case 3: - config = .auto(headroom: headroom) + config = edrDisabled ? .sdr : .auto(headroom: headroom) default: config = .sdr } @@ -3417,7 +3480,12 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan return config } - let headroom = resolvedEdrHeadroom() + guard let headroom = resolvedEdrHeadroom() else { + NSLog("ErikaFlutterPlugin: ERIKA_DISABLE_EDR is set; using SDR output") + var config = ErikaPresenterConfigC.sdr + config.videoAlphaMode = alphaMode + return config + } NSLog("ErikaFlutterPlugin: using automatic Apple output, headroom \(headroom)x") let config = ErikaPresenterConfigC.auto(headroom: headroom) var alphaConfig = config @@ -3425,10 +3493,12 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan return alphaConfig } - private func resolvedEdrHeadroom() -> Float { + /// EDR headroom to request for a new player, or `nil` when + /// `ERIKA_DISABLE_EDR` disables EDR and the player must use SDR. + private func resolvedEdrHeadroom() -> Float? { let environment = ProcessInfo.processInfo.environment if boolEnvironmentFlag("ERIKA_DISABLE_EDR", environment: environment) { - return 1.0 + return nil } if let override = floatEnvironmentValue("ERIKA_EDR_HEADROOM", environment: environment), override > 1.0 { @@ -3451,16 +3521,7 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan NSApp.keyWindow?.screen ?? NSApp.mainWindow?.screen ?? NSScreen.main - guard let screen else { - return 1.0 - } - - let key = "maximumPotentialExtendedDynamicRangeColorComponentValue" - guard screen.responds(to: Selector((key))), - let number = screen.value(forKey: key) as? NSNumber else { - return 1.0 - } - return max(1.0, number.floatValue) + return screenPotentialEdrHeadroom(screen) } private func boolEnvironmentFlag( diff --git a/packages/erika_flutter/tvos/Classes/ErikaFlutterPlugin.swift b/packages/erika_flutter/tvos/Classes/ErikaFlutterPlugin.swift index fd38a3ec..912dad82 100644 --- a/packages/erika_flutter/tvos/Classes/ErikaFlutterPlugin.swift +++ b/packages/erika_flutter/tvos/Classes/ErikaFlutterPlugin.swift @@ -2656,6 +2656,14 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan } private func presenterConfigForNewPlayer(arguments: Any?, hdrDebug: Bool) -> ErikaPresenterConfigC { + // An explicit disable must become SDR, not `auto(headroom: 1.0)`: under + // the per-presenting-display Auto negotiation a headroom of 1.0 means "no + // embedder cap, defer to the display", so an EDR display would promote + // again and ignore ERIKA_DISABLE_EDR. + let edrDisabled = boolEnvironmentFlag( + "ERIKA_DISABLE_EDR", + environment: ProcessInfo.processInfo.environment + ) if let args = arguments as? [String: Any], let explicitMode = int32Value(args["outputMode"]) { let headroom = floatValue(args["edrHeadroom"]) ?? 4.0 let config: ErikaPresenterConfigC @@ -2665,7 +2673,7 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan case 2: config = ErikaPresenterConfigC(outputMode: 2, edrHeadroom: max(1.0, headroom)) case 3: - config = .auto(headroom: headroom) + config = edrDisabled ? .sdr : .auto(headroom: headroom) default: config = .sdr } @@ -2675,7 +2683,10 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan ) return config } - let headroom = resolvedEdrHeadroom(hdrDebug: hdrDebug) + guard let headroom = resolvedEdrHeadroom(hdrDebug: hdrDebug) else { + erikaHdrLog(hdrDebug, "ERIKA_DISABLE_EDR is set; using SDR output") + return .sdr + } let config = ErikaPresenterConfigC.auto(headroom: headroom) erikaHdrLog( hdrDebug, @@ -2684,11 +2695,13 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan return config } - private func resolvedEdrHeadroom(hdrDebug: Bool) -> Float { + /// EDR headroom to request for a new player, or `nil` when + /// `ERIKA_DISABLE_EDR` disables EDR and the player must use SDR. + private func resolvedEdrHeadroom(hdrDebug: Bool) -> Float? { let environment = ProcessInfo.processInfo.environment if boolEnvironmentFlag("ERIKA_DISABLE_EDR", environment: environment) { erikaHdrLog(hdrDebug, "EDR disabled by ERIKA_DISABLE_EDR") - return 1.0 + return nil } if let override = floatEnvironmentValue("ERIKA_EDR_HEADROOM", environment: environment), override > 1.0 { erikaHdrLog(hdrDebug, "EDR headroom override ERIKA_EDR_HEADROOM=\(String(format: "%.3f", override))")