From 1694e552ee868f357e3bc0c9e4420f17afe1f0f5 Mon Sep 17 00:00:00 2001 From: jumusu Date: Thu, 10 Sep 2026 12:27:06 +0800 Subject: [PATCH 1/6] feat(dovi): libplacebo-style Dolby Vision RPU mapping and HDR tone/gamut pipeline Implements end-to-end Dolby Vision and HDR color mapping aligned with libplacebo/mpv gpu-next. RPU path - Parse AV_FRAME_DATA_DOVI_METADATA with bounds/alignment checks; scale pivots by base-layer bit depth and coefficients by 2^-coef_log2_denom. - Piecewise polynomial + MMR reshaping, nonlinear ycc_to_rgb, PQ linearization, and HPE-LMS composite (forced BT.2020/PQ for P5/P8). - Per-frame L1 drives the tone-map peak/scene average; static L0 still decides output-mode negotiation and tone-map/LUT enablement. - Profile 5 falls back to software decode on MediaCodec/AvCodec; desktop VideoToolbox/D3D11VA keep hardware decode. - Profile 7 FEL residual is reported (DoviElStatus) rather than dropping the whole RPU; base-layer mapping still applies. - Structured reject reasons + throttled diagnostics. Tone and gamut mapping - IPT-domain single-pass tone map with BT.2390 default, plus spline, BT.2446 method A, ST 2094-10, Mobius, Reinhard, clip. Black-point compensation via contrast_ratio (auto 1000:1 for SDR). - ST 2094-10 knee picked in the PQ domain; degenerate anchors fall back to Clip instead of a black frame. - Perceptual gamut mapping via a CPU-generated 48x32x256 IPT 3D LUT (RGBA16F) shared by Metal/wgpu/D3D11, generated off-thread, cached by (source, target, target black, target peak). Chroma rolloff is a simplified dead-zone blend rather than libplacebo's full boundary search (documented). - HDR10 scene-adaptive pivot via CPU luma measurement (luma_stats), honoring limited range and yuv420p10le vs P010 packing. Platform - Metal: negotiate EDR/Auto HDR from the presenting screen; linearize subtitles/danmaku into EDR drawables; do not bind the placeholder LUT while it regenerates. - D3D11: upload the LUT as true RGBA16F Texture3D; mask gamut_lut_enabled while generation is pending (constants are passed into draw_video, not a discarded local). - CI: build/upload wgpu_decode_png and erika-capi bundle on macOS arm64. Docs - New docs/dolby-vision.md; CHANGELOG Unreleased covers the feature set; architecture.md default operator corrected to BT.2390. Tests: cargo test -p erika --lib (583) pass locally; fmt clean. --- .github/workflows/ci.yml | 27 + CHANGELOG.md | 71 + crates/erika/src/core.rs | 5 + crates/erika/src/ffmpeg.rs | 1195 +++++++++- crates/erika/src/lib.rs | 1 + crates/erika/src/luma_stats.rs | 403 ++++ crates/erika/src/playback.rs | 253 ++- crates/erika/src/presenter.rs | 48 +- crates/erika/src/renderer.rs | 2 + crates/erika/src/renderer/d3d11.rs | 731 +++++- crates/erika/src/renderer/frame.rs | 14 +- crates/erika/src/renderer/gamut.rs | 751 +++++++ crates/erika/src/renderer/metal/apple.rs | 1142 +++++++++- crates/erika/src/renderer/metal/mod.rs | 47 +- crates/erika/src/renderer/output.rs | 167 +- crates/erika/src/renderer/pipeline.rs | 1962 ++++++++++++++++- crates/erika/src/renderer/wgpu.rs | 236 +- crates/erika/src/renderer/wgpu_video.wgsl | 415 +++- crates/erika_ffmpeg_sys/wrapper.h | 1 + docs/architecture.md | 4 +- docs/dolby-vision.md | 286 +++ .../danmaku_perf_lab/native/DanmakuPerfLab.m | 1 + .../macos_native_demo/native/ErikaMetalDemo.m | 6 +- examples/macos_native_demo/src/main.rs | 62 +- examples/wgpu_decode_png/src/main.rs | 22 +- examples/wgpu_overlay_png/src/main.rs | 20 +- examples/wgpu_video_png/src/main.rs | 20 +- examples/wgpu_window_check/src/main.rs | 20 +- 28 files changed, 7686 insertions(+), 226 deletions(-) create mode 100644 crates/erika/src/luma_stats.rs create mode 100644 crates/erika/src/renderer/gamut.rs create mode 100644 docs/dolby-vision.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e061349d..4f9a048e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,6 +131,33 @@ jobs: ERIKA_NATIVE_TARGET="${{ matrix.target }}" cargo clippy --locked -p erika --all-targets --all-features --no-deps + - name: Build wgpu_decode_png example + if: matrix.arch == 'arm64' + run: | + # Link against libclang_rt.osx to resolve __isPlatformVersionAtLeast, + # a symbol the sanitizer runtime needs but that may be weakly referenced. + # See commits acae10f and 3498b8e for context. + RT_DIR="$(dirname "$(find "$(dirname "$(xcrun -f clang)")/../lib/clang" -name libclang_rt.osx.a | head -1)")" + echo "clang_rt dir: $RT_DIR" + ERIKA_NATIVE_PROFILE="$PROFILE" \ + ERIKA_NATIVE_TARGET="${{ matrix.target }}" \ + RUSTFLAGS="-C link-arg=-L$RT_DIR -C link-arg=-lclang_rt.osx" \ + cargo build --locked -p wgpu_decode_png --release --target "${{ matrix.target }}" + - name: Package erika-capi bundle + if: matrix.arch == 'arm64' + run: >- + bash packaging/bundle.sh erika-capi-macos-arm64 + dist/erika-capi-macos-arm64.zip + "target/${{ matrix.target }}/release/liberika_capi.a" + - name: Upload test artifacts + if: matrix.arch == 'arm64' + uses: actions/upload-artifact@v4 + with: + name: dv-test-macos-arm64 + path: | + target/aarch64-apple-darwin/release/wgpu_decode_png + dist/erika-capi-macos-arm64.zip + if-no-files-found: error ios: name: iOS device staticlib diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b828ce1..cad77013 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,77 @@ ## Unreleased +### Renderer + +#### Dolby Vision and HDR tone mapping + +- Dolby Vision RPU mapping via libplacebo-style piecewise polynomial/MMR + reshaping, followed by the RPU nonlinear matrix, PQ linearization, and + LMS→RGB composite. Profile 5/8 VUI tags are forced to BT.2020/PQ. +- Profile 5 falls back to software decode on mobile backends (MediaCodec / + generic AvCodec) that cannot attach RPU side data to hardware frames; + VideoToolbox and D3D11VA keep hardware decode. +- Profile 7 FEL/MEL RPUs are no longer rejected outright. An RPU whose + `disable_residual_flag` asks for an enhancement layer still applies its + base-layer reshaping curves, color matrices, and L1 trims — dropping the + whole RPU threw those away and left the frame as plain HDR10. The + un-composable residual is reported through the new `Frame::dovi_el_status()` + and one throttled `dovi_el_not_composed` diagnostic per stream, matching + libplacebo's split between `nlq_active` and "consumers that have not bound an + enhancement layer must not look at these fields". +- Per-frame Dolby Vision L1 brightness (`min_pq`/`max_pq`/`avg_pq`) drives the + tone-map source peak and scene-average pivot; the static mastering-display + (L0) peak still decides output-mode negotiation and whether tone mapping / + the perceptual gamut LUT are enabled. +- Tone mapping switched to a libplacebo-style IPT-domain color map with + **BT.2390 EETF as the default** operator, plus spline, BT.2446 method A, + SMPTE ST 2094-10, Mobius, Reinhard, and clip. Black-point compensation uses + `target black = target peak / contrast_ratio` (auto 1000:1 for SDR). +- Perceptual gamut mapping via a CPU-generated 48×32×256 IPT 3D LUT + (following libplacebo's lattice and index mapping) when a tone-mapped HDR + source is compressed into a smaller gamut; generated on a background thread + so the first wide-gamut frame does not stall the render thread. Chroma + rolloff is currently a simplified dead-zone blend rather than libplacebo's + full per-hue boundary search. +- Scene-adaptive HDR10: software-decoded PQ frames without DoVi L1 get a + CPU-measured scene-average luminance (`luma_stats`) that drives the + tone-map pivot, mirroring mpv's `--hdr-compute-peak`. +- Metal EDR negotiation reads the presenting screen rather than + `NSScreen.mainScreen`, and Auto HDR is decided from that display's + capability. Callers must set `CAMetalLayer.delegate` (examples updated). +- Metal: the perceptual gamut LUT is only marked active when the texture + resolved for the frame actually matches its (source, target, target-peak) + key. A headroom or primaries change while a new LUT generates used to bind + the 1x1x1 placeholder with `gamut_lut_enabled` still set, rendering black + frames until generation finished. D3D11 now masks the uniform the same way + while its LUT is still generating. +- ST 2094-10 tone mapping picks its knee in the PQ domain, matching + libplacebo's internal rescale, and `ToneMapConfig::curve_param` now tunes + its knee adaptation (default 0.70, libplacebo's `param_def`). Previously the + parameter had no effect and the knee was selected in linear nits. +- The tone map, its black-point compensation, and the perceptual gamut LUT are + enabled from the static mastering-display (L0) peak rather than the + per-frame Dolby Vision L1 peak, so dark scenes no longer drop them for a + frame and then restore them. +- Metal EDR / extended-linear output linearizes subtitle and danmaku colors + before compositing into the linear drawable; previously they were written + gamma-encoded, making mid-tones and colored text too bright. +- HDR10 scene-luma measurement honors the frame's color range: limited (TV) + range luma planes are expanded over the legal code span like the shaders' + `expand_ycbcr_range`, instead of being measured as full range. The + measurement also distinguishes `yuv420p10le` (10-bit code in bits `[9:0]`) + from P010 (left-aligned in `[15:6]`); treating both as P010 measured + software Main10 HDR10 as near-black. +- The perceptual gamut LUT's I axis now spans the target's `[black, peak]` in + PQ codes (libplacebo's `gamut.min_luma`/`max_luma`) instead of + `[0, peak]`; generation and all three backends' samplers agree, and the + cache key includes the target black, so a `contrast_ratio` change + regenerates the LUT. SDR targets shift their gamut mapping by up to ~3.5% + of the I axis; HDR/EDR targets are unchanged (black = 0). +- A degenerate ST 2094-10 anchor set (duplicated/non-finite points) falls back + to the identity curve (Clip) instead of zero coefficients, which the shader + would have rendered as a black frame. + ## 0.1.8 - 2026-09-07 ### Compatibility diff --git a/crates/erika/src/core.rs b/crates/erika/src/core.rs index c8a26fe3..42a289bf 100644 --- a/crates/erika/src/core.rs +++ b/crates/erika/src/core.rs @@ -463,6 +463,10 @@ pub struct PlayerVideoFrame { pub media_time: Duration, pub late_by: Option, pub generation: u64, + /// Measured scene-average luminance (nits) attached by the presenter for + /// HDR10 software frames without Dolby Vision L1 metadata. `None` for + /// every other source; renderers fold it into the tone-map pivot. + pub scene_avg_nits: Option, } impl PlayerVideoFrame { @@ -481,6 +485,7 @@ impl PlayerVideoFrame { media_time, late_by, generation, + scene_avg_nits: None, }) } } diff --git a/crates/erika/src/ffmpeg.rs b/crates/erika/src/ffmpeg.rs index beaea6fa..7772e481 100644 --- a/crates/erika/src/ffmpeg.rs +++ b/crates/erika/src/ffmpeg.rs @@ -12,9 +12,10 @@ use std::time::Duration; use crate::core::{ColorPrimaries, FrameRate, TrackInfo, TrackKind, TransferFunction, VideoParams}; use crate::renderer::pipeline::{ - Chromaticity, ColorRange, ContentLightMetadata, HdrMetadata, MasteringDisplayMetadata, - MatrixCoefficients, + Chromaticity, ColorRange, ContentLightMetadata, DoviComponentCurve, DoviFramePq, + DoviSourceMetadata, HdrMetadata, MasteringDisplayMetadata, MatrixCoefficients, RgbMatrix, }; +use crate::renderer::pipeline::{DOVI_MAX_MMR_ORDER, DOVI_MAX_PIECES}; use crate::source::{ByteRange, MediaSource}; use crate::subtitle::{ AssTrackResources, DecodedSubtitleFrame, SubtitleBitmapPlane, SubtitleFontAttachment, @@ -544,6 +545,13 @@ impl CodecParameters<'_> { pub fn kind(self) -> Option { unsafe { track_kind((*self.ptr).codec_type) } } + + /// Container-signalled Dolby Vision configuration profile (`dvcC`/`dvvC` + /// carried as `AV_PKT_DATA_DOVI_CONF` codec parameters side data), used to + /// steer decode backend selection. + pub fn dolby_vision_profile(self) -> Option { + unsafe { codec_parameters_dolby_vision_profile(self.ptr) } + } } pub struct OwnedCodecParameters { @@ -555,6 +563,13 @@ pub struct OwnedCodecParameters { unsafe impl Send for OwnedCodecParameters {} impl OwnedCodecParameters { + /// Container-signalled Dolby Vision configuration profile (`dvcC`/`dvvC` + /// carried as `AV_PKT_DATA_DOVI_CONF` codec parameters side data), used to + /// steer decode backend selection. + pub fn dolby_vision_profile(&self) -> Option { + unsafe { codec_parameters_dolby_vision_profile(self.ptr) } + } + fn copy_from(parameters: CodecParameters<'_>) -> Result { let ptr = unsafe { sys::avcodec_parameters_alloc() }; if ptr.is_null() { @@ -2149,6 +2164,53 @@ impl Frame { } } + /// Zero-copy view of a software frame's luma plane. + /// + /// `yuv420p10le` and `p010le` both store two bytes per sample but pack + /// the 10-bit code differently: software Main10 (`yuv420p10le`) keeps the + /// code in bits `[9:0]`, while P010 left-aligns it in bits `[15:6]`. + /// Callers must respect [`LumaPlaneView::sample_layout`] — treating both + /// as P010 silently mis-measures software HDR10. + pub fn luma_plane_view(&self) -> Option> { + let width = self.width() as usize; + let height = self.height() as usize; + if width == 0 || height == 0 { + return None; + } + let format = self.raw_pixel_format(); + let (bytes_per_sample, layout) = if format == sys::AVPixelFormat_AV_PIX_FMT_YUV420P + || format == sys::AVPixelFormat_AV_PIX_FMT_NV12 + { + (1, LumaSampleLayout::U8) + } else if format == sys::AVPixelFormat_AV_PIX_FMT_YUV420P10LE { + (2, LumaSampleLayout::Packed10) + } else if format == sys::AVPixelFormat_AV_PIX_FMT_P010LE { + (2, LumaSampleLayout::P010) + } else { + return None; + }; + unsafe { + let frame = &*self.ptr; + let data = frame.data[0] as *const u8; + if data.is_null() { + return None; + } + let stride = usize::try_from(frame.linesize[0]).ok()?; + if stride < width.checked_mul(bytes_per_sample)? { + return None; + } + Some(LumaPlaneView { + data, + stride, + width, + height, + bytes_per_sample, + layout, + _frame: std::marker::PhantomData, + }) + } + } + pub fn is_videotoolbox(&self) -> bool { self.raw_pixel_format() == sys::AVPixelFormat_AV_PIX_FMT_VIDEOTOOLBOX } @@ -2386,6 +2448,61 @@ impl Frame { unsafe { frame_hdr_metadata(self.ptr) } } + /// Per-frame Dolby Vision RPU metadata parsed by the HEVC decoder + /// (`AV_FRAME_DATA_DOVI_METADATA`). Present on RPU-carrying streams such + /// as profiles 5, 7, and 8, on both software and hardware decoded frames + /// (FFmpeg parses the RPU on the CPU and attaches it regardless). + pub fn dovi_metadata(&self) -> Option { + unsafe { + frame_dovi_metadata_result(self.ptr) + .ok() + .map(|(metadata, _)| metadata) + } + } + + /// Reports the enhancement-layer residual an RPU asks for but this renderer + /// cannot compose (Profile 7 FEL/MEL). The base-layer mapping still applies + /// to such frames, so this is diagnostic signal rather than a rejection. + /// Returns `None` when the frame carries no usable RPU at all. + pub fn dovi_el_status(&self) -> Option { + unsafe { frame_dovi_metadata_result(self.ptr).ok().map(|(_, el)| el) } + } + + /// Returns `None` when this frame's Dolby Vision metadata can feed the + /// mapping path. Otherwise reports why it cannot. Frames without any RPU + /// side data report [`DoviRejectReason::MissingSideData`], which callers + /// must interpret against the stream's Dolby Vision profile: absent RPUs + /// are normal on Profile 8, but Profile 5 has no HDR10-compatible base + /// layer, so a missing RPU leaves the frame unmappable. + pub fn dovi_unavailable_reason(&self) -> Option { + match self.dovi_mapping_probe() { + Ok(_) => None, + Err(reason) => Some(reason), + } + } + + /// One-pass probe of this frame's RPU: `Ok` carries the enhancement-layer + /// status when the mapping path can consume the RPU, `Err` reports why it + /// cannot. Prefer this over calling [`Frame::dovi_unavailable_reason`] + /// and [`Frame::dovi_el_status`] back-to-back, which re-parses the side + /// data. + pub fn dovi_mapping_probe(&self) -> std::result::Result { + if self.ptr.is_null() { + return Err(DoviRejectReason::MissingSideData); + } + let has_side_data = unsafe { + !sys::av_frame_get_side_data( + self.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + ) + .is_null() + }; + if !has_side_data { + return Err(DoviRejectReason::MissingSideData); + } + unsafe { frame_dovi_metadata_result(self.ptr).map(|(_, el)| el) } + } + pub fn transfer_to_system_memory(&self) -> Result { let frame = Frame::alloc(self.time_base)?; check( @@ -2636,6 +2753,169 @@ pub struct Nv12Frame { pub chroma: Vec, } +/// How a luma plane stores each sample. 10-bit frames share two bytes per +/// sample but not the same bit packing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LumaSampleLayout { + /// 8-bit `yuv420p` / `nv12`. + U8, + /// 10-bit packed LE (`yuv420p10le`): code lives in bits `[9:0]`. + Packed10, + /// 10-bit P010 LE: code left-aligned in bits `[15:6]`. + P010, +} + +impl LumaSampleLayout { + /// Decode one little-endian sample into a 10-bit code in `[0, 1023]`. + /// Returns `None` for the 8-bit layout. + pub fn decode_10bit(self, sample: u16) -> Option { + match self { + Self::U8 => None, + Self::Packed10 => Some(sample & 0x03FF), + Self::P010 => Some(sample >> 6), + } + } +} + +/// Strided, zero-copy view of a software frame's luma plane (see +/// [`Frame::luma_plane_view`]). +#[derive(Clone, Copy)] +pub struct LumaPlaneView<'a> { + data: *const u8, + stride: usize, + width: usize, + height: usize, + bytes_per_sample: usize, + layout: LumaSampleLayout, + _frame: std::marker::PhantomData<&'a Frame>, +} + +impl<'a> LumaPlaneView<'a> { + pub fn width(&self) -> u32 { + self.width as u32 + } + + pub fn height(&self) -> u32 { + self.height as u32 + } + + /// 10-bit samples (two bytes per sample)? + pub fn is_10bit(&self) -> bool { + self.bytes_per_sample == 2 + } + + pub fn sample_layout(&self) -> LumaSampleLayout { + self.layout + } + + /// Test-only view over a packed luma buffer: `stride == width * + /// bytes_per_sample`. Tests that care about stride padding lay the rows + /// out manually and point `data` at row 0. + #[cfg(test)] + pub(crate) fn from_packed( + data: &'a [u8], + width: u32, + height: u32, + is_10bit: bool, + ) -> Option { + let layout = if is_10bit { + LumaSampleLayout::P010 + } else { + LumaSampleLayout::U8 + }; + Self::from_packed_with_layout(data, width, height, layout) + } + + /// Test-only view with an explicit sample layout. + #[cfg(test)] + pub(crate) fn from_packed_with_layout( + data: &'a [u8], + width: u32, + height: u32, + layout: LumaSampleLayout, + ) -> Option { + let width = width as usize; + let height = height as usize; + let bytes_per_sample = match layout { + LumaSampleLayout::U8 => 1, + LumaSampleLayout::Packed10 | LumaSampleLayout::P010 => 2, + }; + if width == 0 || height == 0 || data.len() < width * height * bytes_per_sample { + return None; + } + let stride = width * bytes_per_sample; + Some(Self::from_strided_with_layout( + data, + stride, + width as u32, + height as u32, + layout, + )) + } + + /// Test-only view over rows of `width` samples separated by `stride` + /// bytes (the decoder-alignment layout `Frame::luma_plane_view` exposes). + #[cfg(test)] + pub(crate) fn from_strided( + data: &'a [u8], + stride: usize, + width: u32, + height: u32, + is_10bit: bool, + ) -> Self { + let layout = if is_10bit { + LumaSampleLayout::P010 + } else { + LumaSampleLayout::U8 + }; + Self::from_strided_with_layout(data, stride, width, height, layout) + } + + /// Test-only view with an explicit sample layout. + #[cfg(test)] + pub(crate) fn from_strided_with_layout( + data: &'a [u8], + stride: usize, + width: u32, + height: u32, + layout: LumaSampleLayout, + ) -> Self { + assert!(data.len() >= stride * height as usize); + Self { + data: data.as_ptr(), + stride, + width: width as usize, + height: height as usize, + bytes_per_sample: match layout { + LumaSampleLayout::U8 => 1, + LumaSampleLayout::Packed10 | LumaSampleLayout::P010 => 2, + }, + layout, + _frame: std::marker::PhantomData, + } + } + + /// Row `index` of the luma plane: `width` samples of 1 or 2 bytes each. + /// `None` when the index is out of bounds. + pub fn row(&self, index: usize) -> Option<&'a [u8]> { + if index >= self.height { + return None; + } + let row_bytes = self.width.checked_mul(self.bytes_per_sample)?; + let offset = index.checked_mul(self.stride)?.checked_add(row_bytes)?; + let total = self.stride.checked_mul(self.height)?; + if offset > total { + return None; + } + // SAFETY: `data` points at a valid luma plane whose rows hold at + // least `row_bytes` bytes every `stride` bytes (validated when the + // view was built), and the frame outlives `'a`. + Some(unsafe { std::slice::from_raw_parts(self.data.add(index * self.stride), row_bytes) }) + } +} + +unsafe impl Send for LumaPlaneView<'_> {} + /// GPU upload format for a repacked planar frame. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PlanarPixelFormat { @@ -4316,6 +4596,401 @@ unsafe fn frame_hdr_metadata(frame: *const sys::AVFrame) -> Option Some(HdrMetadata::new(mastering_display, content_light)) } +/// Reads the container's Dolby Vision configuration record from codec +/// parameters side data (`AV_PKT_DATA_DOVI_CONF` in ffmpeg 8), returning the +/// Dolby Vision profile number. +unsafe fn codec_parameters_dolby_vision_profile( + parameters: *const sys::AVCodecParameters, +) -> Option { + if parameters.is_null() { + return None; + } + let side_data = unsafe { + sys::av_packet_side_data_get( + (*parameters).coded_side_data, + (*parameters).nb_coded_side_data, + sys::AVPacketSideDataType_AV_PKT_DATA_DOVI_CONF, + ) + }; + if side_data.is_null() { + return None; + } + let data = unsafe { (*side_data).data }; + if data.is_null() { + return None; + } + let size = unsafe { (*side_data).size }; + if usize::try_from(size).ok()? < mem::size_of::() { + return None; + } + let record = data as *const sys::AVDOVIDecoderConfigurationRecord; + Some(unsafe { (*record).dv_profile }) +} + +/// Why a decoded frame cannot feed the Dolby Vision mapping path. +/// +/// Reportable through [`Frame::dovi_unavailable_reason`] so a stream whose +/// RPUs are missing or rejected stays diagnosable instead of silently falling +/// back to the base layer — Profile 5 in particular has no HDR10-compatible +/// base layer, and an unmapped Profile 8 frame with a rejected RPU displays +/// with wrong colors. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DoviRejectReason { + /// The frame carries no RPU side data at all. Normal for profiles whose + /// base layer is HDR10-compatible (e.g. Profile 8); always wrong for + /// Profile 5. + MissingSideData, + /// Side data exists but failed structural validation (size, alignment, + /// sub-structure offsets). + MalformedSideData, + /// Base-layer bit depth outside FFmpeg's 8..=16 range. + UnsupportedBitDepth, + /// `coef_log2_denom` beyond FFmpeg's fixed-point range (above 32). + UnsupportedCoefDenom, + /// Reshaping curves invalid or absent (pivots, orders, unknown methods). + InvalidCurves, + /// Color matrices/offsets invalid or the RGB→LMS matrix is singular. + InvalidColorMetadata, +} + +impl DoviRejectReason { + pub fn label(self) -> &'static str { + match self { + Self::MissingSideData => "rpu side data missing", + Self::MalformedSideData => "rpu side data malformed", + Self::UnsupportedBitDepth => "base layer bit depth outside 8..=16", + Self::UnsupportedCoefDenom => "coef_log2_denom above 32", + Self::InvalidCurves => "invalid reshaping curves", + Self::InvalidColorMetadata => "invalid color metadata", + } + } +} + +/// Enhancement-layer information an RPU carries that this renderer does not +/// compose. Profile 7 keeps its enhancement layer in a second HEVC layer that +/// this renderer never decodes, so the NLQ residual these fields describe has +/// nothing to be added to. libplacebo draws the same line: it exposes +/// `nlq_active` and documents that "consumers that have not bound an +/// enhancement layer must not look at these fields", while still applying the +/// base-layer reshaping, color matrices, and trims. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct DoviElStatus { + /// `disable_residual_flag == 0`: the RPU expects its residual to be + /// composed with an enhancement layer (Profile 7 FEL/MEL). + pub residual_requested: bool, + /// The RPU carries a non-trivial NLQ definition for that residual. + pub nlq_nontrivial: bool, +} + +impl DoviElStatus { + /// Whether the RPU asked for anything this renderer cannot compose. + pub fn is_active(self) -> bool { + self.residual_requested || self.nlq_nontrivial + } +} + +/// Reads the decoder's parsed Dolby Vision RPU side data and converts it into +/// shader-ready floats, mirroring libplacebo's `pl_map_dovi_metadata`: pivots +/// are normalized by the base-layer bit depth and curve coefficients by +/// `2^-coef_log2_denom`. The second value reports any enhancement-layer +/// residual the RPU asked for but this renderer cannot compose; it never +/// rejects an otherwise valid RPU. +unsafe fn frame_dovi_metadata_result( + frame: *const sys::AVFrame, +) -> std::result::Result<(DoviSourceMetadata, DoviElStatus), DoviRejectReason> { + if frame.is_null() { + return Err(DoviRejectReason::MissingSideData); + } + let side_data = unsafe { + sys::av_frame_get_side_data(frame, sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA) + }; + if side_data.is_null() { + return Err(DoviRejectReason::MissingSideData); + } + let data = unsafe { (*side_data).data }; + if data.is_null() { + return Err(DoviRejectReason::MalformedSideData); + } + let size = usize::try_from(unsafe { (*side_data).size }) + .ok() + .ok_or(DoviRejectReason::MalformedSideData)?; + if size < mem::size_of::() { + return Err(DoviRejectReason::MalformedSideData); + } + if (data as usize) % mem::align_of::() != 0 { + return Err(DoviRejectReason::MalformedSideData); + } + // The sub-structures live behind byte offsets inside this side data buffer + // (`av_dovi_get_header` and friends are C inline helpers bindgen does not + // emit), so resolve them by the same pointer arithmetic here with bounds checks. + let metadata = unsafe { *data.cast::() }; + let data_address = data as usize; + let aligned = |offset: usize, alignment: usize| { + data_address + .checked_add(offset) + .is_some_and(|address| address % alignment == 0) + }; + let in_bounds = |offset: usize, structure: usize| { + offset.checked_add(structure).is_some_and(|end| end <= size) + }; + if !in_bounds( + metadata.header_offset, + mem::size_of::(), + ) || !in_bounds( + metadata.mapping_offset, + mem::size_of::(), + ) || !in_bounds( + metadata.color_offset, + mem::size_of::(), + ) || !aligned( + metadata.header_offset, + mem::align_of::(), + ) || !aligned( + metadata.mapping_offset, + mem::align_of::(), + ) || !aligned( + metadata.color_offset, + mem::align_of::(), + ) { + return Err(DoviRejectReason::MalformedSideData); + } + let header = unsafe { + &*data + .add(metadata.header_offset) + .cast::() + }; + let mapping = unsafe { + &*data + .add(metadata.mapping_offset) + .cast::() + }; + let color = unsafe { + &*data + .add(metadata.color_offset) + .cast::() + }; + + let bl_bit_depth = usize::from(header.bl_bit_depth); + let coef_denom = u32::from(header.coef_log2_denom); + // Validate bit depth and coefficient denominator before using them in + // shifts below; malformed side data must be rejected, never panic. + // FFmpeg uses coef_log2_denom up to 32 for float RPU coefficients. + if !(8..=16).contains(&bl_bit_depth) { + return Err(DoviRejectReason::UnsupportedBitDepth); + } + if coef_denom > 32 { + return Err(DoviRejectReason::UnsupportedCoefDenom); + } + // This renderer has no enhancement-layer input, so the NLQ residual below + // is never composed. That does not make the RPU unusable: the base-layer + // reshaping curves, color matrices, and L1 trims still apply, and dropping + // the whole RPU would throw them away for every Profile 7 FEL frame. + // libplacebo keeps the same separation (it exposes `nlq_active` but only + // composes it when an enhancement layer is bound). + // + // FFmpeg marks an absent NLQ with the AV_DOVI_NLQ_NONE sentinel (-1); + // method 0 with all-neutral parameters is likewise an identity mapping. + let nlq_nontrivial = match mapping.nlq_method_idc as i32 { + -1 => false, + 0 => mapping.nlq.iter().any(|params| { + params.nlq_offset != 0 + || params.linear_deadzone_slope != 0 + || params.linear_deadzone_threshold != 0 + || (params.vdr_in_max != 0 + && params.vdr_in_max != (1_u64 << u32::from(header.coef_log2_denom))) + }), + _ => true, + }; + let el = DoviElStatus { + residual_requested: header.disable_residual_flag == 0, + nlq_nontrivial, + }; + + let pivot_scale = 1.0_f32 / (((1_usize << bl_bit_depth) - 1) as f32); + let coefficient_scale = 2.0_f32.powi(-(coef_denom as i32)); + + let mut reshaping = [DoviComponentCurve::default(); 3]; + let mut has_curve = false; + for (component, curve) in reshaping.iter_mut().enumerate() { + let source = &mapping.curves[component]; + let num_pivots = usize::from(source.num_pivots); + if num_pivots == 0 { + continue; + } + has_curve = true; + if !(2..=DOVI_MAX_PIECES + 1).contains(&num_pivots) { + return Err(DoviRejectReason::InvalidCurves); + } + let max_pivot = ((1_usize << bl_bit_depth) - 1) as u16; + let mut previous = None; + for &pivot in source.pivots[..num_pivots].iter() { + if pivot > max_pivot || previous.is_some_and(|previous| pivot <= previous) { + return Err(DoviRejectReason::InvalidCurves); + } + previous = Some(pivot); + } + curve.num_pivots = num_pivots as u8; + for (slot, &pivot) in curve.pivots[..num_pivots] + .iter_mut() + .zip(source.pivots[..num_pivots].iter()) + { + *slot = pivot_scale * f32::from(pivot); + } + for segment in 0..num_pivots - 1 { + match source.mapping_idc[segment] { + sys::AVDOVIMappingMethod_AV_DOVI_MAPPING_MMR => { + let order = usize::from(source.mmr_order[segment]); + if order == 0 || order > DOVI_MAX_MMR_ORDER { + return Err(DoviRejectReason::InvalidCurves); + } + curve.mmr_orders[segment] = order as u8; + curve.mmr_constants[segment] = + coefficient_scale * source.mmr_constant[segment] as f32; + let destination = &mut curve.mmr_coeffs[segment][..order]; + for (order_index, coefficients) in destination.iter_mut().enumerate() { + for (index, coefficient) in coefficients.iter_mut().enumerate() { + *coefficient = coefficient_scale + * source.mmr_coef[segment][order_index][index] as f32; + } + } + } + sys::AVDOVIMappingMethod_AV_DOVI_MAPPING_POLYNOMIAL => { + let poly_order = usize::from(source.poly_order[segment]); + if !(1..=2).contains(&poly_order) { + return Err(DoviRejectReason::InvalidCurves); + } + let coefficients = &source.poly_coef[segment]; + for (order_index, slot) in curve.poly_coeffs[segment].iter_mut().enumerate() { + *slot = if order_index <= poly_order { + coefficient_scale * coefficients[order_index] as f32 + } else { + 0.0 + }; + } + } + _ => return Err(DoviRejectReason::InvalidCurves), + } + } + } + if !has_curve { + return Err(DoviRejectReason::InvalidCurves); + } + + let rational_matrix = |values: &[sys::AVRational; 9]| -> Option<[[f32; 3]; 3]> { + let mut matrix = [[0.0_f32; 3]; 3]; + for row in 0..3 { + for col in 0..3 { + let value = &values[row * 3 + col]; + if value.den == 0 { + return None; + } + let value = value.num as f32 / value.den as f32; + if !value.is_finite() { + return None; + } + matrix[row][col] = value; + } + } + Some(matrix) + }; + + let mut nonlinear_offset = [0.0_f32; 3]; + for (index, slot) in nonlinear_offset.iter_mut().enumerate() { + let value = &color.ycc_to_rgb_offset[index]; + if value.den == 0 { + return Err(DoviRejectReason::InvalidColorMetadata); + } + *slot = value.num as f32 / value.den as f32; + if !slot.is_finite() { + return Err(DoviRejectReason::InvalidColorMetadata); + } + } + + if color.source_min_pq > 4095 + || color.source_max_pq > 4095 + || (color.source_max_pq != 0 && color.source_min_pq > color.source_max_pq) + { + return Err(DoviRejectReason::InvalidColorMetadata); + } + + let nonlinear_matrix = + rational_matrix(&color.ycc_to_rgb_matrix).ok_or(DoviRejectReason::InvalidColorMetadata)?; + let rgb_to_lms = + rational_matrix(&color.rgb_to_lms_matrix).ok_or(DoviRejectReason::InvalidColorMetadata)?; + let determinant = rgb_to_lms[0][0] + * (rgb_to_lms[1][1] * rgb_to_lms[2][2] - rgb_to_lms[1][2] * rgb_to_lms[2][1]) + - rgb_to_lms[0][1] + * (rgb_to_lms[1][0] * rgb_to_lms[2][2] - rgb_to_lms[1][2] * rgb_to_lms[2][0]) + + rgb_to_lms[0][2] + * (rgb_to_lms[1][0] * rgb_to_lms[2][1] - rgb_to_lms[1][1] * rgb_to_lms[2][0]); + if !determinant.is_finite() || determinant == 0.0 { + return Err(DoviRejectReason::InvalidColorMetadata); + } + + // Level 1 per-frame brightness metadata lives in the DM extension blocks + // that the RPU decoder appends right after the color structure. `av_dovi_find_level` + // is exported by FFmpeg but performs an unchecked pointer walk, so validate + // the block region here like the sub-structures above. L1 is optional + // signal quality metadata: an invalid or absent block never rejects the RPU. + let l1 = unsafe { frame_dovi_level1(data, &metadata, size) }; + + Ok(( + DoviSourceMetadata { + reshaping, + nonlinear_matrix: RgbMatrix::new(nonlinear_matrix), + nonlinear_offset, + rgb_to_lms: RgbMatrix::new(rgb_to_lms), + source_min_pq: color.source_min_pq, + source_max_pq: color.source_max_pq, + l1, + }, + el, + )) +} + +/// Reads the dynamic DM level 1 block (per-frame min/max/avg luminance in +/// 12-bit PQ codes) from the validated ext-block region, matching +/// `av_dovi_get_ext`'s pointer arithmetic. +unsafe fn frame_dovi_level1( + data: *const u8, + metadata: &sys::AVDOVIMetadata, + size: usize, +) -> Option { + let count = usize::try_from(metadata.num_ext_blocks).ok()?; + if count == 0 { + return None; + } + let block_size = metadata.ext_block_size; + if block_size < mem::size_of::() { + return None; + } + let total = block_size.checked_mul(count)?; + let end = metadata.ext_block_offset.checked_add(total)?; + if end > size { + return None; + } + if (metadata.ext_block_offset as usize) % mem::align_of::() != 0 { + return None; + } + let ext = unsafe { data.add(metadata.ext_block_offset) }; + for index in 0..count { + let block = unsafe { &*ext.add(block_size * index).cast::() }; + if block.level != 1 { + continue; + } + let l1 = unsafe { block.__bindgen_anon_1.l1 }; + if l1.max_pq == 0 || (l1.min_pq != 0 && l1.min_pq > l1.max_pq) { + return None; + } + return Some(DoviFramePq { + min_pq: l1.min_pq, + max_pq: l1.max_pq, + avg_pq: l1.avg_pq, + }); + } + None +} + unsafe fn mastering_display_metadata( frame: *const sys::AVFrame, ) -> Option { @@ -5071,6 +5746,59 @@ mod tests { ); } + #[test] + fn playback_fixture_has_no_dolby_vision_profile() { + let path = std::env::var_os("ERIKA_PLAYBACK_FIXTURE") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| { + Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/playback/playback-fixture.mkv") + }); + let demuxer = Demuxer::open_path(&path).unwrap(); + let video = demuxer + .probe() + .tracks + .iter() + .find(|track| track.kind == TrackKind::Video) + .unwrap(); + + let parameters = demuxer.owned_codec_parameters(video.id as i32).unwrap(); + assert_eq!(parameters.dolby_vision_profile(), None); + } + + #[test] + fn dv_sample_reports_dolby_vision_profile() { + let Some(path) = std::env::var_os("ERIKA_DV_SAMPLE") else { + return; + }; + let demuxer = Demuxer::open_path(&path).unwrap(); + let video = demuxer + .probe() + .tracks + .iter() + .find(|track| track.kind == TrackKind::Video) + .expect("DV sample must contain a video stream"); + + let parameters = demuxer.owned_codec_parameters(video.id as i32).unwrap(); + assert_eq!(parameters.dolby_vision_profile(), Some(5)); + } + + #[test] + fn dv_profile_8_sample_reports_profile() { + let Some(path) = std::env::var_os("ERIKA_DV_PROFILE_8_SAMPLE") else { + return; + }; + let demuxer = Demuxer::open_path(&path).unwrap(); + let video = demuxer + .probe() + .tracks + .iter() + .find(|track| track.kind == TrackKind::Video) + .expect("DV Profile 8 sample must contain a video stream"); + + let parameters = demuxer.owned_codec_parameters(video.id as i32).unwrap(); + assert_eq!(parameters.dolby_vision_profile(), Some(8)); + } + #[test] fn real_ass_container_preserves_header_fonts_and_matroska_chunk_when_env_is_set() { let Ok(path) = std::env::var("ERIKA_ASS_SAMPLE") else { @@ -5308,6 +6036,469 @@ mod tests { sys::AVRational { num, den } } + #[test] + fn frame_reads_dovi_side_data() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { + let mut size = 0_usize; + let metadata = sys::av_dovi_metadata_alloc(&mut size); + assert!(!metadata.is_null()); + assert!(size >= mem::size_of::()); + let header = (metadata as *mut u8) + .add((*metadata).header_offset) + .cast::(); + let mapping = (metadata as *mut u8) + .add((*metadata).mapping_offset) + .cast::(); + let color = (metadata as *mut u8) + .add((*metadata).color_offset) + .cast::(); + + (*header).bl_bit_depth = 10; + (*header).el_bit_depth = 10; + (*header).coef_log2_denom = 13; + (*header).disable_residual_flag = 1; + + let curve = &mut (*mapping).curves[0]; + curve.num_pivots = 3; + curve.pivots[0] = 0; + curve.pivots[1] = 256; + curve.pivots[2] = 1023; + curve.mapping_idc[0] = sys::AVDOVIMappingMethod_AV_DOVI_MAPPING_POLYNOMIAL; + curve.poly_order[0] = 1; + curve.poly_coef[0][0] = 0; + curve.poly_coef[0][1] = 1 << 13; + curve.poly_coef[0][2] = 0; + curve.mapping_idc[1] = sys::AVDOVIMappingMethod_AV_DOVI_MAPPING_MMR; + curve.mmr_order[1] = 1; + curve.mmr_constant[1] = 1024; + curve.mmr_coef[1][0][0] = 4096; + curve.mmr_coef[1][0][6] = -8192; + + (*color).ycc_to_rgb_matrix = [ + rational(9575, 8192), + rational(0, 8192), + rational(14742, 8192), + rational(9575, 8192), + rational(1754, 8192), + rational(4383, 8192), + rational(9575, 8192), + rational(17372, 8192), + rational(0, 8192), + ]; + (*color).ycc_to_rgb_offset = [rational(1, 4), rational(2, 1), rational(2, 1)]; + (*color).rgb_to_lms_matrix = [ + rational(5845, 16384), + rational(9702, 16384), + rational(837, 16384), + rational(2568, 16384), + rational(12256, 16384), + rational(1561, 16384), + rational(0, 16384), + rational(679, 16384), + rational(15705, 16384), + ]; + (*color).source_min_pq = 62; + (*color).source_max_pq = 3079; + + let side_data = sys::av_frame_new_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + size, + ); + assert!(!side_data.is_null()); + ptr::copy_nonoverlapping(metadata.cast::(), (*side_data).data, size); + sys::av_free(metadata.cast()); + } + + let dovi = frame.dovi_metadata().unwrap(); + let luma = &dovi.reshaping[0]; + assert_eq!(luma.num_pivots, 3); + assert_close(luma.pivots[1], 256.0 / 1023.0); + assert_close(luma.pivots[2], 1.0); + assert_close(luma.poly_coeffs[0][1], 1.0); + assert_eq!(luma.mmr_orders[1], 1); + assert_close(luma.mmr_constants[1], 0.125); + assert_close(luma.mmr_coeffs[1][0][0], 0.5); + assert_close(luma.mmr_coeffs[1][0][6], -1.0); + // Chroma and luma curves without pivots stay empty. + assert_eq!(dovi.reshaping[1].num_pivots, 0); + assert_close(dovi.nonlinear_matrix.rows()[0][0], 9575.0 / 8192.0); + assert_close(dovi.nonlinear_offset[0], 0.25); + assert_close(dovi.rgb_to_lms.rows()[2][2], 15705.0 / 16384.0); + assert_eq!(dovi.source_min_pq, 62); + assert_eq!(dovi.source_max_pq, 3079); + + unsafe { + let side_data = sys::av_frame_get_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + ); + assert!(!side_data.is_null()); + let metadata = *(*side_data).data.cast::(); + let header = &mut *((*side_data) + .data + .add(metadata.header_offset) + .cast::()); + let mapping = &mut *((*side_data) + .data + .add(metadata.mapping_offset) + .cast::()); + mapping.curves[0].mmr_order[1] = 0; + assert_eq!(frame.dovi_metadata(), None); + header.disable_residual_flag = 0; + assert_eq!(frame.dovi_metadata(), None); + } + } + + #[test] + fn frame_rejects_truncated_or_invalid_dovi_side_data() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { + let side_data = sys::av_frame_new_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + 4, + ); + assert!(!side_data.is_null()); + } + assert_eq!(frame.dovi_metadata(), None); + } + + /// Attaches a known-good RPU: 10-bit base layer, `coef_log2_denom = 32` + /// with the identity polynomial scaled to match, no residual, identity + /// matrices, valid PQ range. + unsafe fn attach_valid_dovi_rpu(frame: &Frame) { + unsafe { + let mut size = 0_usize; + let metadata = sys::av_dovi_metadata_alloc(&mut size); + assert!(!metadata.is_null()); + let header = (metadata as *mut u8) + .add((*metadata).header_offset) + .cast::(); + let mapping = (metadata as *mut u8) + .add((*metadata).mapping_offset) + .cast::(); + let color = (metadata as *mut u8) + .add((*metadata).color_offset) + .cast::(); + + (*header).bl_bit_depth = 10; + (*header).el_bit_depth = 10; + (*header).coef_log2_denom = 32; + (*header).disable_residual_flag = 1; + + let curve = &mut (*mapping).curves[0]; + curve.num_pivots = 2; + curve.pivots[0] = 0; + curve.pivots[1] = 1023; + curve.mapping_idc[0] = sys::AVDOVIMappingMethod_AV_DOVI_MAPPING_POLYNOMIAL; + curve.poly_order[0] = 1; + curve.poly_coef[0][0] = 0; + curve.poly_coef[0][1] = 1_i64 << 32; + curve.poly_coef[0][2] = 0; + + (*color).ycc_to_rgb_matrix = [ + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + ]; + (*color).ycc_to_rgb_offset = [rational(0, 1), rational(0, 1), rational(0, 1)]; + (*color).rgb_to_lms_matrix = [ + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + ]; + (*color).source_min_pq = 0; + (*color).source_max_pq = 3079; + + let side_data = sys::av_frame_new_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + size, + ); + assert!(!side_data.is_null()); + ptr::copy_nonoverlapping(metadata.cast::(), (*side_data).data, size); + sys::av_free(metadata.cast()); + } + } + + unsafe fn mutate_dovi_rpu( + frame: &Frame, + mutate: impl FnOnce( + &mut sys::AVDOVIRpuDataHeader, + &mut sys::AVDOVIDataMapping, + &mut sys::AVDOVIColorMetadata, + ), + ) { + unsafe { + let side_data = sys::av_frame_get_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + ); + assert!(!side_data.is_null()); + let metadata = *(*side_data).data.cast::(); + let header = &mut *((*side_data) + .data + .add(metadata.header_offset) + .cast::()); + let mapping = &mut *((*side_data) + .data + .add(metadata.mapping_offset) + .cast::()); + let color = &mut *((*side_data) + .data + .add(metadata.color_offset) + .cast::()); + mutate(header, mapping, color); + } + } + + /// Writes a single dynamic DM level 1 ext block into the side data. + /// `av_dovi_metadata_alloc` reserves the full ext block array inside the + /// same allocation, so the first block lands at `ext_block_offset`. + unsafe fn with_dovi_l1(frame: &Frame, min_pq: u16, max_pq: u16, avg_pq: u16) { + unsafe { + let side_data = sys::av_frame_get_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + ); + assert!(!side_data.is_null()); + let metadata = &mut *(*side_data).data.cast::(); + let block = &mut *((*side_data) + .data + .add(metadata.ext_block_offset) + .cast::()); + block.level = 1; + block.__bindgen_anon_1.l1 = sys::AVDOVIDmLevel1 { + min_pq, + max_pq, + avg_pq, + }; + metadata.num_ext_blocks = 1; + } + } + + #[test] + fn dovi_l1_brightness_metadata_is_parsed() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { attach_valid_dovi_rpu(&frame) }; + assert_eq!(frame.dovi_metadata().unwrap().l1, None); + + unsafe { with_dovi_l1(&frame, 62, 2200, 1500) }; + let l1 = frame + .dovi_metadata() + .unwrap() + .l1 + .expect("level 1 block should be parsed"); + assert_eq!( + l1, + DoviFramePq { + min_pq: 62, + max_pq: 2200, + avg_pq: 1500 + } + ); + + // Inverted min/max and an all-zero block are treated as absent. + unsafe { with_dovi_l1(&frame, 2200, 100, 1500) }; + assert_eq!(frame.dovi_metadata().unwrap().l1, None); + unsafe { with_dovi_l1(&frame, 0, 0, 0) }; + assert_eq!(frame.dovi_metadata().unwrap().l1, None); + } + + #[test] + fn dovi_malformed_ext_region_skips_l1_but_keeps_rpu() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { + attach_valid_dovi_rpu(&frame); + let side_data = sys::av_frame_get_side_data( + frame.ptr, + sys::AVFrameSideDataType_AV_FRAME_DATA_DOVI_METADATA, + ); + assert!(!side_data.is_null()); + let metadata = &mut *(*side_data).data.cast::(); + metadata.ext_block_offset = usize::MAX; + metadata.ext_block_size = mem::size_of::(); + metadata.num_ext_blocks = 5; + } + let dovi = frame + .dovi_metadata() + .expect("RPU must remain usable when the ext region is malformed"); + assert_eq!(dovi.l1, None); + } + + #[test] + fn frame_reads_dovi_side_data_with_32bit_coef_denom() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { attach_valid_dovi_rpu(&frame) }; + + let dovi = frame + .dovi_metadata() + .expect("32-bit coef denominator should be accepted"); + let luma = &dovi.reshaping[0]; + assert_eq!(luma.num_pivots, 2); + assert_close(luma.poly_coeffs[0][1], 1.0); + + // Test with 31-bit denominator + unsafe { + mutate_dovi_rpu(&frame, |header, mapping, _| { + header.coef_log2_denom = 31; + mapping.curves[0].poly_coef[0][1] = 1_i64 << 31; + }); + } + let dovi31 = frame + .dovi_metadata() + .expect("31-bit coef denominator should be accepted"); + assert_close(dovi31.reshaping[0].poly_coeffs[0][1], 1.0); + + // Test with invalid 33-bit denominator (should be rejected) + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.coef_log2_denom = 33); + } + assert_eq!(frame.dovi_metadata(), None); + } + + #[test] + fn dovi_unavailable_reason_classifies_rejections() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + assert_eq!( + frame.dovi_unavailable_reason(), + Some(DoviRejectReason::MissingSideData) + ); + + unsafe { attach_valid_dovi_rpu(&frame) }; + assert_eq!(frame.dovi_unavailable_reason(), None); + + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.coef_log2_denom = 33); + } + assert_eq!( + frame.dovi_unavailable_reason(), + Some(DoviRejectReason::UnsupportedCoefDenom) + ); + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.coef_log2_denom = 32); + } + + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.disable_residual_flag = 0); + } + // A residual request is reported through `dovi_el_status`, not rejected: + // the base-layer mapping still applies (see + // `fel_residual_keeps_the_base_layer_mapping`). + assert_eq!(frame.dovi_unavailable_reason(), None); + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.disable_residual_flag = 1); + } + + unsafe { + mutate_dovi_rpu(&frame, |_, mapping, _| mapping.curves[0].pivots[1] = 0); + } + assert_eq!( + frame.dovi_unavailable_reason(), + Some(DoviRejectReason::InvalidCurves) + ); + unsafe { + mutate_dovi_rpu(&frame, |_, mapping, _| mapping.curves[0].pivots[1] = 1023); + } + + unsafe { + mutate_dovi_rpu(&frame, |_, _, color| { + color.rgb_to_lms_matrix = [ + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + ]; + }); + } + assert_eq!( + frame.dovi_unavailable_reason(), + Some(DoviRejectReason::InvalidColorMetadata) + ); + + // Restoring the matrix makes the RPU usable again, and the reason + // tracks the metadata rather than sticky stream state. + unsafe { + mutate_dovi_rpu(&frame, |_, _, color| { + color.rgb_to_lms_matrix = [ + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + rational(0, 1), + rational(0, 1), + rational(0, 1), + rational(1, 1), + ]; + }); + } + assert_eq!(frame.dovi_unavailable_reason(), None); + assert!(frame.dovi_metadata().is_some()); + } + + #[test] + fn fel_residual_keeps_the_base_layer_mapping() { + let frame = Frame::alloc(TimeBase { num: 1, den: 1 }).unwrap(); + unsafe { attach_valid_dovi_rpu(&frame) }; + + // Profile 7 FEL: the RPU asks for residual composition, which this + // renderer cannot do (no enhancement-layer input). The RPU itself is + // still valid, so the base-layer reshaping must survive; only the + // un-composable part is reported. + unsafe { + mutate_dovi_rpu(&frame, |header, _, _| header.disable_residual_flag = 0); + } + assert_eq!(frame.dovi_unavailable_reason(), None); + let metadata = frame.dovi_metadata().expect("FEL RPU stays usable"); + assert_eq!(metadata.reshaping[0].num_pivots, 2); + assert_close(metadata.reshaping[0].pivots[1], 1.0); + let status = frame.dovi_el_status().expect("FEL status is reported"); + assert!(status.residual_requested); + assert!(!status.nlq_nontrivial); + assert!(status.is_active()); + + // A non-trivial NLQ definition is likewise reported, never rejected. + unsafe { + mutate_dovi_rpu(&frame, |_, mapping, _| mapping.nlq_method_idc = 1); + } + assert_eq!(frame.dovi_unavailable_reason(), None); + assert!(frame.dovi_metadata().is_some()); + let status = frame.dovi_el_status().expect("NLQ status is reported"); + assert!(status.nlq_nontrivial); + assert!(status.is_active()); + + // Profile 8 style RPUs (no residual, no NLQ) report an inactive status + // so callers do not diagnose them. + unsafe { + mutate_dovi_rpu(&frame, |header, mapping, _| { + header.disable_residual_flag = 1; + mapping.nlq_method_idc = -1; + }); + } + assert_eq!(frame.dovi_el_status(), Some(DoviElStatus::default())); + assert!(!frame.dovi_el_status().unwrap().is_active()); + } + #[test] fn frame_rate_normalizes_positive_rationals() { assert_eq!( diff --git a/crates/erika/src/lib.rs b/crates/erika/src/lib.rs index d37bdf56..4586f974 100644 --- a/crates/erika/src/lib.rs +++ b/crates/erika/src/lib.rs @@ -6,6 +6,7 @@ pub mod core; pub mod danmaku; pub mod debug_hud; pub mod ffmpeg; +pub(crate) mod luma_stats; #[cfg(target_env = "ohos")] pub mod ohos; pub mod overlay; diff --git a/crates/erika/src/luma_stats.rs b/crates/erika/src/luma_stats.rs new file mode 100644 index 00000000..b3f9e50e --- /dev/null +++ b/crates/erika/src/luma_stats.rs @@ -0,0 +1,403 @@ +//! Frame-luminance statistics feeding the tone map's scene-adaptive pivot. +//! +//! mpv/libplacebo measure per-frame average/peak brightness on the GPU +//! (`--hdr-compute-peak`) and feed it to the tone-map curve, which is what +//! makes HDR10 (static-metadata-only) content behave like Dolby Vision's +//! per-frame L1. This module computes the same signals on the CPU for +//! software-decoded frames, sampling a sparse grid of rows/columns so a 4K +//! frame costs well under a millisecond, then smooths them with the same +//! IIR filter libplacebo uses (τ = smoothing_period). +//! +//! Only the luma plane is needed: HDR10 software frames arrive as NV12 +//! (8-bit), `yuv420p10le` (10-bit packed LE, code in bits `[9:0]`), or P010 +//! (10-bit MSB-aligned in 16-bit LE). The view reports which packing it +//! carries; treating `yuv420p10le` as P010 would measure near-black. The +//! frame's color range decides the sample normalization — HDR10 streams are +//! normally limited (TV) range, so measuring them as full range would lift +//! black and compress the highlights before the PQ re-encode. + +/// How many rows/columns of the frame are sampled (stride = dim / SAMPLES). +const SAMPLES: usize = 48; +/// libplacebo `pl_peak_detect_default_params.smoothing_period` (frames). +const SMOOTHING_PERIOD: f32 = 20.0; + +const PQ_M1: f32 = 2610.0 / 4096.0 * 1.0 / 4.0; +const PQ_M2: f32 = 2523.0 / 4096.0 * 128.0; +const PQ_C1: f32 = 3424.0 / 4096.0; +const PQ_C2: f32 = 2413.0 / 4096.0 * 32.0; +const PQ_C3: f32 = 2392.0 / 4096.0 * 32.0; + +/// One frame's measured luminance in PQ code (12-bit-ish precision). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FrameLumaStats { + /// Mean luma in PQ code over the sampled grid. + pub avg_pq: f32, + /// Maximum sampled luma in PQ code. + pub max_pq: f32, +} + +/// Stateful IIR smoothing of the per-frame signals, mirroring libplacebo's +/// `update_peak_buf`: `state += coeff * (measured - state)` with +/// `coeff = 1 - exp(-1/smoothing_period)`. +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct LumaSmoother { + pub avg_pq: f32, + pub max_pq: f32, + /// 0 until the first frame is pushed. + initialized: bool, +} + +impl LumaSmoother { + pub fn new() -> Self { + Self::default() + } + + /// Fold a frame's measurements into the running estimate and return the + /// smoothed (avg, max) in PQ code. + pub fn push(&mut self, measured: FrameLumaStats) -> (f32, f32) { + let coeff = 1.0 - (-1.0 / SMOOTHING_PERIOD).exp(); + if !self.initialized { + self.avg_pq = measured.avg_pq; + self.max_pq = measured.max_pq; + self.initialized = true; + } else { + self.avg_pq += coeff * (measured.avg_pq - self.avg_pq); + self.max_pq += coeff * (measured.max_pq - self.max_pq); + } + (self.avg_pq, self.max_pq) + } + + pub fn reset(&mut self) { + *self = Self::default(); + } +} + +/// Convert a normalized luma sample in [0, 1] (after range expansion and +/// bit-depth normalization) to its PQ code. +fn pq_code_of_luma(luma: f32) -> f32 { + let p = luma.clamp(0.0, 1.0).powf(PQ_M1); + ((PQ_C1 + PQ_C2 * p) / (1.0 + PQ_C3 * p)).powf(PQ_M2) +} + +/// Convert a normalized PQ-encoded luma sample to a PQ code over 10 k nits. +/// The luma plane of an HDR10 frame holds PQ-encoded Y'; decode it to linear +/// nits first (PQ EOTF), then re-encode so the average is perceptual. +fn sample_code(normalized: f32) -> f32 { + let linear_nits = pq_eotf(normalized.clamp(0.0, 1.0)) * 10000.0; + pq_code_of_luma(linear_nits / 10000.0) +} + +/// Measure a strided luma plane in place (see +/// `crate::ffmpeg::Frame::luma_plane_view`): 8-bit samples, or 10-bit samples +/// in either the `yuv420p10le` packed layout or the P010 MSB-aligned layout, +/// sampled on the same sparse grid as the packed variants. `full_range` +/// selects the sample normalization: full-range planes normalize by the code +/// maximum, limited (TV) planes by the legal 16..235 / 64..940 span, exactly +/// like the shaders' `expand_ycbcr_range`. This is the presenter's per-frame +/// path — repacking the whole frame with `to_planar_frame` would copy +/// megabytes to sample a 48x48 grid. +pub fn measure_luma_plane_view( + view: &crate::ffmpeg::LumaPlaneView<'_>, + full_range: bool, +) -> Option { + let w = view.width() as usize; + let h = view.height() as usize; + if w == 0 || h == 0 { + return None; + } + let is_10bit = view.is_10bit(); + let sample_bytes = if is_10bit { 2 } else { 1 }; + let x_step = (w / SAMPLES).max(1); + let y_step = (h / SAMPLES).max(1); + let mut sum = 0.0_f64; + let mut max_code = 0.0_f32; + let mut count = 0_u64; + let mut row_index = 0usize; + while row_index < h { + let Some(row) = view.row(row_index) else { + break; + }; + let sample_count = row.len() / sample_bytes; + let mut x = 0usize; + while x < sample_count { + let sample = if is_10bit { + let raw = u16::from_le_bytes([row[x * 2], row[x * 2 + 1]]); + // yuv420p10le keeps the code in bits [9:0]; p010le left-aligns + // it in [15:6]. Treating both as P010 crushes software Main10. + view.sample_layout() + .decode_10bit(raw) + .expect("10-bit layout decodes a 10-bit code") as f32 + } else { + row[x] as f32 + }; + let normalized = expand_luma_sample(sample, is_10bit, full_range); + let code = sample_code(normalized); + sum += code as f64; + if code > max_code { + max_code = code; + } + count += 1; + x += x_step; + } + row_index += y_step; + } + if count == 0 { + return None; + } + Some(FrameLumaStats { + avg_pq: (sum / count as f64) as f32, + max_pq: max_code, + }) +} + +/// Normalize one luma code to [0, 1] in the encoded domain, applying the +/// limited-range expansion when the plane is TV range. +fn expand_luma_sample(sample: f32, is_10bit: bool, full_range: bool) -> f32 { + let (black, span, peak) = if is_10bit { + (64.0, 876.0, 1023.0) + } else { + (16.0, 219.0, 255.0) + }; + if full_range { + return (sample / peak).clamp(0.0, 1.0); + } + ((sample - black) / span).clamp(0.0, 1.0) +} + +fn pq_eotf(code: f32) -> f32 { + let p = code.clamp(0.0, 1.0).powf(1.0 / PQ_M2); + let num = (p - PQ_C1).max(0.0); + let den = (PQ_C2 - PQ_C3 * p).max(1e-9); + (num / den).powf(1.0 / PQ_M1) +} + +/// Convert a PQ code back to nits (for diagnostics and for feeding the +/// `tone_map_extra.y` scene-average slot, which expects nits). +pub fn pq_code_to_nits(code: f32) -> f32 { + 10000.0 * pq_eotf(code) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ffmpeg::LumaPlaneView; + + fn view(luma: &[u8], w: u32, h: u32, is_10bit: bool) -> LumaPlaneView<'_> { + LumaPlaneView::from_packed(luma, w, h, is_10bit).unwrap() + } + + #[test] + fn nv12_black_frame_measures_near_zero() { + let (w, h) = (1920_u32, 1080_u32); + let luma = vec![0_u8; (w * h) as usize]; + let stats = measure_luma_plane_view(&view(&luma, w, h, false), true).unwrap(); + assert!(pq_code_to_nits(stats.avg_pq) < 0.5, "avg {}", stats.avg_pq); + assert!(pq_code_to_nits(stats.max_pq) < 0.5, "max {}", stats.max_pq); + } + + #[test] + fn nv12_full_white_measures_ten_k_nits() { + let (w, h) = (1920_u32, 1080_u32); + let luma = vec![255_u8; (w * h) as usize]; + let stats = measure_luma_plane_view(&view(&luma, w, h, false), true).unwrap(); + assert!((pq_code_to_nits(stats.avg_pq) - 10000.0).abs() < 20.0); + assert!((pq_code_to_nits(stats.max_pq) - 10000.0).abs() < 20.0); + } + + #[test] + fn p010_white_measures_ten_k_nits() { + let (w, h) = (1920_u32, 1080_u32); + let mut luma = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + luma.extend_from_slice(&(1023_u16 << 6).to_le_bytes()); + } + let stats = measure_luma_plane_view(&view(&luma, w, h, true), true).unwrap(); + assert!((pq_code_to_nits(stats.avg_pq) - 10000.0).abs() < 20.0); + } + + #[test] + fn yuv420p10le_white_measures_ten_k_nits() { + // Software Main10 packs the 10-bit code in bits [9:0], not P010's + // [15:6]. Measuring a full-range white plane as P010 would yield + // sample 15 and a near-black scene average. + let (w, h) = (1920_u32, 1080_u32); + let mut luma = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + luma.extend_from_slice(&1023_u16.to_le_bytes()); + } + let packed = LumaPlaneView::from_packed_with_layout( + &luma, + w, + h, + crate::ffmpeg::LumaSampleLayout::Packed10, + ) + .unwrap(); + let stats = measure_luma_plane_view(&packed, true).unwrap(); + assert!( + (pq_code_to_nits(stats.avg_pq) - 10000.0).abs() < 20.0, + "avg {}", + stats.avg_pq + ); + } + + #[test] + fn yuv420p10le_limited_white_hits_ten_k_nits() { + // Limited-range white (940) must expand to the code peak, not be + // crushed by a mistaken >> 6. + let (w, h) = (64_u32, 64_u32); + let mut luma = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + luma.extend_from_slice(&940_u16.to_le_bytes()); + } + let packed = LumaPlaneView::from_packed_with_layout( + &luma, + w, + h, + crate::ffmpeg::LumaSampleLayout::Packed10, + ) + .unwrap(); + let stats = measure_luma_plane_view(&packed, false).unwrap(); + assert!( + (pq_code_to_nits(stats.avg_pq) - 10000.0).abs() < 20.0, + "avg {}", + stats.avg_pq + ); + } + + #[test] + fn strided_rows_measure_like_packed_rows() { + // A stride larger than the visible row (decoder alignment padding) + // must not change the measurement. + let (w, h) = (64_u32, 48_u32); + let row_bytes = w as usize; + let stride = row_bytes + 64; + let mut padded = vec![255_u8; stride * h as usize]; // padding bright + for y in 0..h as usize { + for x in 0..row_bytes { + padded[y * stride + x] = 16; + } + } + let strided_view = LumaPlaneView::from_strided(&padded, stride, w, h, false); + // Same visible samples, tightly packed. + let packed: Vec = (0..h as usize) + .flat_map(|y| padded[y * stride..y * stride + row_bytes].to_vec()) + .collect(); + let strided = measure_luma_plane_view(&strided_view, true).unwrap(); + let packed_stats = measure_luma_plane_view(&view(&packed, w, h, false), true).unwrap(); + assert!((strided.avg_pq - packed_stats.avg_pq).abs() < 1e-6); + assert!((strided.max_pq - packed_stats.max_pq).abs() < 1e-6); + assert!( + pq_code_to_nits(strided.max_pq) < 100.0, + "padding leaked: {}", + pq_code_to_nits(strided.max_pq) + ); + } + + #[test] + fn bright_highlights_raise_the_peak_but_not_the_mean() { + let (w, h) = (1920_u32, 1080_u32); + // Mostly dim (16/255) with a small bright region (255). + let mut luma = vec![16_u8; (w * h) as usize]; + for y in (h / 4)..(h / 2) { + for x in (w / 4)..(w / 2) { + luma[(y * w + x) as usize] = 255; + } + } + let stats = measure_luma_plane_view(&view(&luma, w, h, false), true).unwrap(); + let avg = pq_code_to_nits(stats.avg_pq); + let max = pq_code_to_nits(stats.max_pq); + assert!(avg < 100.0, "avg {avg}"); + assert!(max > 9000.0, "max {max}"); + } + + #[test] + fn limited_range_black_and_white_hit_the_code_ends() { + // HDR10 streams are normally limited (TV) range: 64 is the 10-bit PQ + // black code and 940 is the 10 000-nit code, so measuring them as full + // range would lift black and clip the highlights before the PQ + // re-encode. + let (w, h) = (1920_u32, 1080_u32); + let mut black = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + black.extend_from_slice(&(64_u16 << 6).to_le_bytes()); + } + let stats = measure_luma_plane_view(&view(&black, w, h, true), false).unwrap(); + assert!( + pq_code_to_nits(stats.avg_pq) < 0.5, + "avg {} -> {} nits", + stats.avg_pq, + pq_code_to_nits(stats.avg_pq) + ); + + let mut white = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + white.extend_from_slice(&(940_u16 << 6).to_le_bytes()); + } + let stats = measure_luma_plane_view(&view(&white, w, h, true), false).unwrap(); + assert!( + (pq_code_to_nits(stats.max_pq) - 10000.0).abs() < 20.0, + "limited-range white should reach 10 k nits: {}", + pq_code_to_nits(stats.max_pq) + ); + } + + #[test] + fn limited_range_mid_gray_matches_the_shader_expansion() { + // 10-bit limited-range 512 expands to (512 - 64) / 876, the same + // normalization `expand_ycbcr_range` applies in the video shaders. + let (w, h) = (16_u32, 16_u32); + let mut luma = Vec::with_capacity((w * h * 2) as usize); + for _ in 0..(w * h) as usize { + luma.extend_from_slice(&(512_u16 << 6).to_le_bytes()); + } + let stats = measure_luma_plane_view(&view(&luma, w, h, true), false).unwrap(); + let expected = (512.0 - 64.0) / 876.0; + let code = sample_code(expected); + assert!((stats.avg_pq - code).abs() < 1e-5, "avg {}", stats.avg_pq); + // The full-range reading is visibly different, so the range matters. + let full = measure_luma_plane_view(&view(&luma, w, h, true), true).unwrap(); + assert!((full.avg_pq - stats.avg_pq).abs() > 1e-3); + } + + #[test] + fn smoother_converges_and_tracks_step() { + let mut smoother = LumaSmoother::new(); + let dim = FrameLumaStats { + avg_pq: 0.2, + max_pq: 0.5, + }; + // First sample initializes directly. + let (avg, max) = smoother.push(dim); + assert_eq!(avg, 0.2); + assert_eq!(max, 0.5); + // A sustained new level converges asymptotically (τ=20). + let bright = FrameLumaStats { + avg_pq: 0.6, + max_pq: 0.9, + }; + let mut last = 0.0_f32; + for _ in 0..300 { + let (avg, _) = smoother.push(bright); + last = avg; + } + assert!((last - 0.6).abs() < 0.01, "converged avg {last}"); + // A single bright frame only nudges the estimate a little. + let mut s2 = LumaSmoother::new(); + s2.push(dim); + s2.push(bright); + let (avg, max) = (s2.avg_pq, s2.max_pq); + assert!(avg < 0.25 && avg > 0.2, "avg {avg}"); + assert!(max < 0.55 && max > 0.5, "max {max}"); + } + + #[test] + fn reset_clears_state() { + let mut smoother = LumaSmoother::new(); + smoother.push(FrameLumaStats { + avg_pq: 0.8, + max_pq: 0.9, + }); + smoother.reset(); + assert_eq!(smoother, LumaSmoother::default()); + } +} diff --git a/crates/erika/src/playback.rs b/crates/erika/src/playback.rs index 313c5e06..b6f595ef 100644 --- a/crates/erika/src/playback.rs +++ b/crates/erika/src/playback.rs @@ -15,7 +15,8 @@ use crate::core::{ }; use crate::ffmpeg::{ self, AudioResampler, Decoder, DecoderBackend, DecoderConfig, DecoderOutputFrame, Demuxer, - Frame, OwnedCodecParameters, PcmAudioFrame, PcmFormat, StreamSelection, SubtitleDecoder, + DoviRejectReason, Frame, OwnedCodecParameters, PcmAudioFrame, PcmFormat, StreamSelection, + SubtitleDecoder, }; use crate::source::{self, source_from_uri_with_hint, source_from_uri_with_options}; use crate::subtitle::{ @@ -731,6 +732,8 @@ pub struct PlaybackSession { audio_seek_dropped_packets: usize, mediacodec_surface_disabled: bool, video_decoder_fallbacks: u64, + dolby_vision_profile: Option, + dovi_rpu_diagnostics: DoviRpuDiagnostics, video_decoder_events: VecDeque, queue_limits: PlaybackQueueLimits, buffer_recovery_audio: Duration, @@ -877,25 +880,29 @@ impl PlaybackSession { let mut video_decoder = None; let mut video_decoder_fallbacks = 0u64; let mut video_decoder_events = VecDeque::new(); + let mut dolby_vision_profile = None; let mut selected_streams = Vec::new(); if let Some(stream_index) = selected_video_track { selected_streams.push(stream_index); let parameters = codec_parameters_for(&codec_parameters, stream_index)?; let codec = parameters.codec_name(); - let decoder_config = config.video_decode.decoder_config(); + let requested_config = config.video_decode.decoder_config(); + dolby_vision_profile = parameters.dolby_vision_profile(); + let (decoder_config, decode_fallback_reason) = + dolby_vision_decode_fallback(dolby_vision_profile, requested_config); video_decoder = Some( match open_video_decoder(parameters, decoder_config, &decoder_resources) { Ok(decoder) => { let event = VideoDecoderEvent { stage: video_decoder_open_stage(decoder_config).to_string(), - requested_backend: decoder_config.backend, + requested_backend: requested_config.backend, previous_backend: None, active_backend: decoder.backend(), fallback_count: 0, codec: codec.clone(), pixel_format: None, line_sizes: None, - reason: None, + reason: decode_fallback_reason, }; trace::diagnostic(event.structured_message()); video_decoder_events.push_back(event); @@ -1232,6 +1239,8 @@ impl PlaybackSession { audio_seek_dropped_packets: 0, mediacodec_surface_disabled, video_decoder_fallbacks, + dolby_vision_profile, + dovi_rpu_diagnostics: DoviRpuDiagnostics::default(), video_decoder_events, queue_limits, buffer_recovery_audio: buffer_recovery_audio_for_request(request), @@ -2756,6 +2765,8 @@ impl PlaybackSession { let status = drain_video_frames( self.video_decoder.as_mut().expect("video decoder exists"), &mut self.video_frames, + self.dolby_vision_profile, + &mut self.dovi_rpu_diagnostics, )?; let video_frame_limit = self.active_video_frame_queue_limit(); if demand != PlaybackPumpDemand::BufferingAudio { @@ -5767,12 +5778,20 @@ fn trace_clock_correction( fn drain_video_frames( decoder: &mut Decoder, frames: &mut VecDeque, + dolby_vision_profile: Option, + dovi_rpu_diagnostics: &mut DoviRpuDiagnostics, ) -> Result { let decode_backend = decoder.backend(); let mut status = DecoderDrainStatus::default(); loop { match decoder.receive_frame()? { DecoderOutputFrame::Frame(frame) => { + diagnose_dovi_frame( + dolby_vision_profile, + dovi_rpu_diagnostics, + &frame, + decode_backend, + ); frames.push_back(DecodedVideoFrame { frame, decode_backend, @@ -5788,6 +5807,99 @@ fn drain_video_frames( } } +/// Per-stream accounting for decoded frames whose Dolby Vision RPU cannot +/// feed the mapping path, and for frames whose RPU asks for an enhancement +/// layer this renderer never decodes. +#[derive(Debug, Default)] +struct DoviRpuDiagnostics { + unavailable_frames: u64, + el_frames: u64, +} + +/// Re-report interval after the first diagnostic, so a broken stream stays +/// visible over long playback without logging at frame rate. +const DOVI_RPU_DIAGNOSTIC_INTERVAL: u64 = 1024; + +impl DoviRpuDiagnostics { + fn record_unavailable_frame(&mut self) -> u64 { + self.unavailable_frames = self.unavailable_frames.saturating_add(1); + self.unavailable_frames + } + + fn record_el_frame(&mut self) -> u64 { + self.el_frames = self.el_frames.saturating_add(1); + self.el_frames + } + + fn should_report(count: u64) -> bool { + count == 1 || count % DOVI_RPU_DIAGNOSTIC_INTERVAL == 0 + } +} + +/// Decides whether a frame lacking usable RPU metadata warrants a diagnostic. +/// A missing RPU is only fatal on Profile 5, whose base layer is not +/// HDR10-compatible; other profiles either repeat RPUs across frames or fall +/// back to a base layer that displays correctly. A *present but rejected* +/// RPU always warrants one: the stream intended to carry Dolby Vision mapping +/// and the frame will silently display unmapped. +fn dovi_rpu_diagnostic_warranted(profile: Option, reason: DoviRejectReason) -> bool { + reason != DoviRejectReason::MissingSideData || profile == Some(5) +} + +fn diagnose_dovi_frame( + profile: Option, + diagnostics: &mut DoviRpuDiagnostics, + frame: &Frame, + decode_backend: DecoderBackend, +) { + // One parse: both the reject reason and the EL residual status come from + // the same `frame_dovi_metadata_result` result. + let el_status = match frame.dovi_mapping_probe() { + Ok(status) => status, + Err(reason) => { + if !dovi_rpu_diagnostic_warranted(profile, reason) { + return; + } + let count = diagnostics.record_unavailable_frame(); + if !DoviRpuDiagnostics::should_report(count) { + return; + } + trace::diagnostic( + serde_json::json!({ + "event": "dovi_rpu_unavailable", + "profile": profile, + "decodeBackend": decode_backend.as_str(), + "reason": reason.label(), + "framesAffected": count, + }) + .to_string(), + ); + return; + } + }; + // The RPU is usable, but it may ask for an enhancement layer this renderer + // never decodes (Profile 7 FEL/MEL). The base-layer mapping still applies, + // so report the gap instead of pretending the stream was composed. + if !el_status.is_active() { + return; + } + let count = diagnostics.record_el_frame(); + if !DoviRpuDiagnostics::should_report(count) { + return; + } + trace::diagnostic( + serde_json::json!({ + "event": "dovi_el_not_composed", + "profile": profile, + "decodeBackend": decode_backend.as_str(), + "residualRequested": el_status.residual_requested, + "nlqNontrivial": el_status.nlq_nontrivial, + "framesAffected": count, + }) + .to_string(), + ); +} + fn pop_matching_audio_frame( frames: &mut VecDeque, keep_frame: &mut impl FnMut(&mut PcmAudioFrame) -> bool, @@ -5936,6 +6048,30 @@ fn should_fallback_video_decoder_open_error(backend: DecoderBackend, codec: Opti && codec.is_some_and(|codec| codec.eq_ignore_ascii_case("av1"))) } +/// VideoToolbox and D3D11VA hardware decoders retain the Dolby Vision RPU +/// metadata side data on the hardware AVFrame, allowing GPU textures + RPU +/// reshaping via shaders without software decode. +/// +/// Backends without GPU shader RPU mapping (e.g. Android MediaCodec with +/// direct surface output, or OpenHarmony AVCodec) fall back to software +/// decoding for Profile 5 streams to preserve color mapping. +fn dolby_vision_decode_fallback( + profile: Option, + requested: DecoderConfig, +) -> (DecoderConfig, Option) { + let mobile_hardware_fallback = matches!( + requested.backend, + DecoderBackend::MediaCodec | DecoderBackend::AvCodec + ); + if !mobile_hardware_fallback || profile != Some(5) { + return (requested, None); + } + ( + DecoderConfig::software(), + Some("dolby vision profile 5 on mobile/embedded backend requires software decode for RPU mapping".to_string()), + ) +} + fn video_decoder_open_stage(config: DecoderConfig) -> &'static str { match (config.backend, config.mediacodec_surface) { (DecoderBackend::MediaCodec, true) => "open_surface", @@ -8509,4 +8645,113 @@ mod tests { assert_ne!(first.vsyncs, second.vsyncs); assert!(first.residual_error_nanos.signum() != second.residual_error_nanos.signum()); } + + #[test] + fn dolby_vision_profile_5_stays_on_hardware_for_videotoolbox_and_d3d11va() { + let vt_config = DecoderConfig { + backend: DecoderBackend::VideoToolbox, + mediacodec_surface: false, + }; + let (config, reason) = dolby_vision_decode_fallback(Some(5), vt_config); + assert_eq!(config.backend, DecoderBackend::VideoToolbox); + assert_eq!(reason, None); + + let d3d11_config = DecoderConfig { + backend: DecoderBackend::D3d11va, + mediacodec_surface: false, + }; + let (config, reason) = dolby_vision_decode_fallback(Some(5), d3d11_config); + assert_eq!(config.backend, DecoderBackend::D3d11va); + assert_eq!(reason, None); + } + + #[test] + fn dolby_vision_profile_5_falls_back_to_software_decode_on_mobile_backends() { + let avcodec_config = DecoderConfig { + backend: DecoderBackend::AvCodec, + mediacodec_surface: false, + }; + let (config, reason) = dolby_vision_decode_fallback(Some(5), avcodec_config); + assert_eq!(config.backend, DecoderBackend::Software); + assert!(reason.is_some()); + assert!(reason.unwrap().contains("profile 5")); + + let mediacodec_config = DecoderConfig { + backend: DecoderBackend::MediaCodec, + mediacodec_surface: true, + }; + let (config, reason) = dolby_vision_decode_fallback(Some(5), mediacodec_config); + assert_eq!(config.backend, DecoderBackend::Software); + assert!(reason.is_some()); + assert!(reason.unwrap().contains("profile 5")); + } + + #[test] + fn dolby_vision_profile_8_stays_on_hardware_decode() { + let hardware_config = DecoderConfig { + backend: DecoderBackend::VideoToolbox, + mediacodec_surface: false, + }; + let (config, reason) = dolby_vision_decode_fallback(Some(8), hardware_config); + + assert_eq!(config.backend, DecoderBackend::VideoToolbox); + assert_eq!(reason, None); + } + + #[test] + fn dolby_vision_software_decode_stays_software() { + let software_config = DecoderConfig::software(); + let (config, reason) = dolby_vision_decode_fallback(Some(5), software_config); + + assert_eq!(config.backend, DecoderBackend::Software); + assert_eq!(reason, None); + } + + #[test] + fn dovi_rpu_missing_is_diagnostic_only_on_profile_5() { + assert!(dovi_rpu_diagnostic_warranted( + Some(5), + DoviRejectReason::MissingSideData + )); + assert!(!dovi_rpu_diagnostic_warranted( + Some(8), + DoviRejectReason::MissingSideData + )); + assert!(!dovi_rpu_diagnostic_warranted( + None, + DoviRejectReason::MissingSideData + )); + assert!(dovi_rpu_diagnostic_warranted( + None, + DoviRejectReason::UnsupportedCoefDenom + )); + assert!(dovi_rpu_diagnostic_warranted( + Some(8), + DoviRejectReason::InvalidCurves + )); + } + + #[test] + fn dovi_rpu_diagnostics_report_first_then_spaced() { + assert!(DoviRpuDiagnostics::should_report(1)); + assert!(!DoviRpuDiagnostics::should_report(2)); + assert!(!DoviRpuDiagnostics::should_report( + DOVI_RPU_DIAGNOSTIC_INTERVAL - 1 + )); + assert!(DoviRpuDiagnostics::should_report( + DOVI_RPU_DIAGNOSTIC_INTERVAL + )); + assert!(!DoviRpuDiagnostics::should_report( + DOVI_RPU_DIAGNOSTIC_INTERVAL + 1 + )); + + let mut diagnostics = DoviRpuDiagnostics::default(); + assert_eq!(diagnostics.record_unavailable_frame(), 1); + assert_eq!(diagnostics.record_unavailable_frame(), 2); + // The enhancement-layer counter is independent, so a stream that both + // loses RPUs and carries FEL residual reports each at its own cadence. + assert_eq!(diagnostics.record_el_frame(), 1); + assert_eq!(diagnostics.record_unavailable_frame(), 3); + assert_eq!(diagnostics.record_el_frame(), 2); + } } diff --git a/crates/erika/src/presenter.rs b/crates/erika/src/presenter.rs index 1351ad49..3df6f489 100644 --- a/crates/erika/src/presenter.rs +++ b/crates/erika/src/presenter.rs @@ -42,6 +42,7 @@ use crate::danmaku::{ }; use crate::debug_hud::{DebugHud, DebugHudSnapshot}; use crate::ffmpeg::DecoderBackend; +use crate::luma_stats::LumaSmoother; #[cfg(target_env = "ohos")] use crate::ohos::ohaudio::{OHAudioOutput, OHAudioOutputConfig}; use crate::overlay::{OverlayFrame, OverlayTimeline, OverlayViewport}; @@ -306,6 +307,11 @@ pub struct PresenterRuntime { current_danmaku_prepared: Option, danmaku_plan_replacement_pending: bool, rejected_video_import_route: Option, + /// Rolling scene-average luminance (HDR10 without DoVi L1), injected + /// into each software frame's `scene_avg_nits` to drive the tone-map + /// pivot like Dolby Vision L1 would. + scene_luma_smoother: LumaSmoother, + last_scene_luma_generation: u64, current_media_time: Duration, current_generation: u64, current_surface_metrics: Option, @@ -358,6 +364,33 @@ fn should_reject_video_import( rejected == Some(candidate) } +/// Measure the scene-average luminance of an HDR10 (PQ, no Dolby Vision L1) +/// software-decoded frame and fold it into `smoother`, returning the smoothed +/// average in nits. Hardware frames (no CPU plane) and non-HDR10 sources keep +/// the previous estimate (`None` when nothing is known yet). The luma plane is +/// sampled in place with its native stride — no frame repack happens here. +fn measure_frame_scene_avg(frame: &PlayerVideoFrame, smoother: &mut LumaSmoother) -> Option { + use crate::core::TransferFunction; + use crate::luma_stats::{measure_luma_plane_view, pq_code_to_nits}; + use crate::renderer::pipeline::ColorRange; + // Hardware payloads have no CPU plane to sample. + let decoded = frame.frame.decoded_frame()?; + // Only PQ (HDR10) sources. When the RPU already carries L1 brightness, + // prefer that over measuring the base layer. + if decoded.transfer_function() != TransferFunction::Pq { + return None; + } + if decoded.dovi_metadata().and_then(|dovi| dovi.l1).is_some() { + return None; + } + let full_range = matches!(decoded.color_range(), ColorRange::Full); + let plane = decoded.luma_plane_view()?; + let measured = measure_luma_plane_view(&plane, full_range)?; + let (avg_pq, _max_pq) = smoother.push(measured); + let nits = pq_code_to_nits(avg_pq); + (nits.is_finite() && nits > 0.0).then_some(nits) +} + fn should_report_video_frame_backpressure(drop_count: u64) -> bool { drop_count == 1 || drop_count.is_power_of_two() } @@ -717,6 +750,8 @@ impl PresenterRuntime { current_danmaku_prepared: None, danmaku_plan_replacement_pending: false, rejected_video_import_route: None, + scene_luma_smoother: LumaSmoother::new(), + last_scene_luma_generation: 0, current_media_time: Duration::ZERO, current_generation: 1, current_surface_metrics: None, @@ -1943,10 +1978,21 @@ impl PresenterRuntime { break; } match self.video_frames.try_recv() { - Ok(frame) => { + Ok(mut frame) => { if frame.generation != self.player.playback_generation() { continue; } + if frame.generation != self.last_scene_luma_generation { + // New playback: start the smoothing state fresh so a + // seek cannot carry the old scene's brightness over. + self.scene_luma_smoother.reset(); + self.last_scene_luma_generation = frame.generation; + } + if let Some(scene_avg) = + measure_frame_scene_avg(&frame, &mut self.scene_luma_smoother) + { + frame.scene_avg_nits = Some(scene_avg); + } #[cfg(target_os = "android")] let mediacodec_surface = frame.frame.is_mediacodec(); #[cfg(not(target_os = "android"))] diff --git a/crates/erika/src/renderer.rs b/crates/erika/src/renderer.rs index 16977504..d0b2c4be 100644 --- a/crates/erika/src/renderer.rs +++ b/crates/erika/src/renderer.rs @@ -7,6 +7,8 @@ pub mod d3d11; #[cfg(target_os = "windows")] mod d3d11_artcnn; mod frame; +/// CPU-side perceptual gamut LUT generation (libplacebo-compatible). +pub mod gamut; pub mod metal; #[cfg(all(feature = "wgpu", target_env = "ohos"))] pub(crate) mod ohos_vulkan; diff --git a/crates/erika/src/renderer/d3d11.rs b/crates/erika/src/renderer/d3d11.rs index 166560bb..7b8851dd 100644 --- a/crates/erika/src/renderer/d3d11.rs +++ b/crates/erika/src/renderer/d3d11.rs @@ -19,9 +19,9 @@ use ::windows::Win32::Graphics::Direct3D11::{ D3D11_RENDER_TARGET_BLEND_DESC, D3D11_RESOURCE_MISC_SHARED, D3D11_SAMPLER_DESC, D3D11_SDK_VERSION, D3D11_SHADER_RESOURCE_VIEW_DESC, D3D11_SHADER_RESOURCE_VIEW_DESC_0, D3D11_SUBRESOURCE_DATA, D3D11_TEX2D_ARRAY_SRV, D3D11_TEX2D_SRV, D3D11_TEXTURE2D_DESC, - D3D11_USAGE_DEFAULT, D3D11_VIEWPORT, D3D11CreateDevice, ID3D11BlendState, ID3D11Buffer, - ID3D11Device, ID3D11DeviceContext, ID3D11InputLayout, ID3D11Multithread, ID3D11PixelShader, - ID3D11Query, ID3D11RenderTargetView, ID3D11Resource, ID3D11SamplerState, + D3D11_TEXTURE3D_DESC, D3D11_USAGE_DEFAULT, D3D11_VIEWPORT, D3D11CreateDevice, ID3D11BlendState, + ID3D11Buffer, ID3D11Device, ID3D11DeviceContext, ID3D11InputLayout, ID3D11Multithread, + ID3D11PixelShader, ID3D11Query, ID3D11RenderTargetView, ID3D11Resource, ID3D11SamplerState, ID3D11ShaderResourceView, ID3D11Texture2D, ID3D11VertexShader, }; use ::windows::Win32::Graphics::Dxgi::Common::{ @@ -49,9 +49,12 @@ use crate::core::{ use crate::danmaku::{ DanmakuAtlasUpdate, DanmakuGlyphAtlas, DanmakuGlyphInstance, DanmakuRenderPlan, }; -use crate::ffmpeg::Frame; +use crate::ffmpeg::{Frame, PlanarPixelFormat}; use crate::overlay::OverlayFrame; use crate::renderer::d3d11_artcnn::D3d11ArtCnn; +use crate::renderer::gamut::{ + GamutLut, GamutLutJob, GamutLutParams, LUT_SIZE_C, LUT_SIZE_H, LUT_SIZE_I, pack_rgba16f, +}; use crate::renderer::metal::{MetalRendererConfig, VideoAlphaMode}; use crate::renderer::output::{ ActiveOutputEncoding, OutputFallbackReason, OutputRuntimeStatus, OutputSurfaceFormat, @@ -76,6 +79,17 @@ struct VsOut { float2 texcoord : TEXCOORD0; }; +struct DoviUniforms { + float4 dovi_flags; + float4 dovi_pivots[6]; + float4 dovi_bounds[3]; + float4 dovi_coefficients[24]; + float4 dovi_mmr[144]; + float4 dovi_nonlinear_matrix[3]; + float4 dovi_nonlinear_offset; + float4 dovi_lms_matrix[3]; +}; + cbuffer VideoConstants : register(b0) { uint is_p010; uint full_range; @@ -88,6 +102,14 @@ cbuffer VideoConstants : register(b0) { float4 nits; float4 luma_coefficients; float4 gamut_matrix_rows[3]; + float4 ipt_matrix_rows[9]; + float4 tone_map_extra; + float4 tone_map_coeffs; + uint gamut_lut_enabled; + uint gamut_primaries; + uint gamut_reserved0; + uint gamut_reserved1; + DoviUniforms dovi; // xy scales native/packed luma coordinates; zw scales native chroma. // D3D11VA textures can be allocation-aligned beyond the visible frame. float4 texture_scales; @@ -95,6 +117,7 @@ cbuffer VideoConstants : register(b0) { Texture2D lumaTex : register(t0); Texture2D chromaTex : register(t1); +Texture3D gamutLut : register(t2); SamplerState videoSampler : register(s0); float source_peak_nits() { @@ -184,26 +207,24 @@ float3 source_reference_to_nits(float3 rgb) { return max(rgb, float3(0.0, 0.0, 0.0)) * source_reference_white_nits(); } -float3 tone_map_nits(float3 input_nits) { - float source_peak = source_peak_nits(); - float target_peak = target_peak_nits(); - float3 x = max(input_nits, float3(0.0, 0.0, 0.0)) / target_peak; - float white = max(source_peak / target_peak, 1.0); - if (tone_map == 1u) { - float white2 = white * white; - return target_peak * clamp((x * (float3(1.0, 1.0, 1.0) + x / white2)) / (float3(1.0, 1.0, 1.0) + x), float3(0.0, 0.0, 0.0), float3(1.0, 1.0, 1.0)); - } - if (tone_map == 2u) { - float knee = 0.75; - float denom = max(white - knee, 0.0001); - float3 knee3 = float3(knee, knee, knee); - float3 t = clamp((x - knee3) / denom, float3(0.0, 0.0, 0.0), float3(1.0, 1.0, 1.0)); - float3 shoulder = knee3 + (1.0 - knee) * (float3(1.0, 1.0, 1.0) - pow(float3(1.0, 1.0, 1.0) - t, float3(2.0, 2.0, 2.0))); - return target_peak * lerp(x, shoulder, step(knee3, x)); - } - return target_peak * clamp(x, float3(0.0, 0.0, 0.0), float3(1.0, 1.0, 1.0)); +float pq_code(float nits) { + return pq_inverse_eotf(clamp(nits, 0.0, 10000.0) / 10000.0); +} + +float nits_from_pq(float code) { + return 10000.0 * pq_eotf(clamp(code, 0.0, 1.0)); +} + +// libplacebo pl_smoothstep with arbitrary edge order (HLSL smoothstep has +// undefined results when edge0 >= edge1, and libplacebo's knee tuning term +// deliberately uses reversed edges). +float sstep(float edge0, float edge1, float x) { + float t = clamp((x - edge0) / (edge1 - edge0), 0.0, 1.0); + return t * t * (3.0 - 2.0 * t); } +// Simple primaries conversion (HDR10 output path); the tone-mapped path +// converts primaries inside the IPT roundtrip instead. float3 apply_gamut_map(float3 rgb) { return float3( dot(gamut_matrix_rows[0].xyz, rgb), @@ -212,8 +233,254 @@ float3 apply_gamut_map(float3 rgb) { ); } +// libplacebo st2094_pick_knee evaluated on absolute PQ codes. The source +// pivot follows the scene average luminance when known and stays within +// [10%, 80%] of the range; the destination pivot rescales it into the output +// range and then adapts towards the 1:1 line (knee_adaptation 0.4). +float2 st2094_pick_knee(float src_min, float src_max, float src_avg, float dst_min, float dst_max) { + const float knee_adaptation = 0.4; + const float min_knee = 0.1; + const float max_knee = 0.8; + const float def_knee = 0.4; + float src_knee_min = lerp(src_min, src_max, min_knee); + float src_knee_max = lerp(src_min, src_max, max_knee); + float dst_knee_min = lerp(dst_min, dst_max, min_knee); + float dst_knee_max = lerp(dst_min, dst_max, max_knee); + float fallback = lerp(src_min, src_max, def_knee); + float src_knee = clamp(src_avg > 0.0 ? src_avg : fallback, src_knee_min, src_knee_max); + float target = (src_knee - src_min) / max(src_max - src_min, 0.000001); + float adapted = lerp(dst_min, dst_max, target); + float tuning = 1.0 - sstep(max_knee, def_knee, target) * sstep(min_knee, def_knee, target); + float adaptation = lerp(knee_adaptation, 1.0, tuning); + float dst_knee = clamp(lerp(src_knee, adapted, adaptation), dst_knee_min, dst_knee_max); + return float2(src_knee, dst_knee); +} + +// The tone-map curve evaluated on the IPT intensity axis (PQ codes), +// mirroring libplacebo's tone-map functions. `param` is the per-operator +// curve parameter from ToneMapConfig::curve_param (0 = operator default). +float tone_map_curve_pq(float x_in, float param) { + float src_peak = source_peak_nits(); + float dst_peak = target_peak_nits(); + float src_avg = tone_map_extra.y; + float dst_black = tone_map_extra.z; + float in_min = 0.0; + float in_max = max(pq_code(src_peak), 0.000001); + float out_min = pq_code(dst_black); + float out_max = max(pq_code(dst_peak), 0.000001); + float out_range = max(out_max - out_min, 0.000001); + float x = clamp(x_in, in_min, in_max); + if (tone_map == 0u) { + // Clip: values within the source range pass through untouched. + return x; + } + if (tone_map == 1u) { + // Reinhard (output-relative, libplacebo pl_tone_map_reinhard). + float peak = in_max / out_range; + float contrast = param > 0.0 ? param : 0.5; + float offset = (1.0 - contrast) / max(contrast, 0.000001); + float scale = (peak + offset) / peak; + float t = x / out_range; + float mapped = t / (t + offset) * scale; + return mapped * out_range + out_min; + } + if (tone_map == 2u) { + // Mobius: Mobius transform with a 1:1 linear region below the knee. + float peak = in_max / out_range; + float j = param > 0.0 ? param : 0.3; + float a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + float b = (j * j - 2.0 * j * peak + peak) / max(peak - 1.0, 0.000001); + float scale = (b * b + 2.0 * b * j + j * j) / (b - a); + float t = x / out_range; + float mapped = t > j ? scale * (t + a) / (t + b) : t; + return mapped * out_range + out_min; + } + if (tone_map == 3u) { + // ITU-R BT.2390 EETF with black-point compensation (the libplacebo + // version also compensates target black; the earlier port skipped it). + float knee_offset = param > 0.0 ? param : 1.0; + float max_lum = clamp(out_max / in_max, 0.0, 1.0); + float min_lum = out_min / in_max; + float ks = (1.0 + knee_offset) * max_lum - knee_offset; + float bp = min(max(1.0 / max(min_lum, 0.000001), 0.0), 4.0); + float u = x / in_max; + if (ks < 1.0 && u > ks) { + float tb = (u - ks) / (1.0 - ks); + float tb2 = tb * tb; + float tb3 = tb2 * tb; + u = (2.0 * tb3 - 3.0 * tb2 + 1.0) * ks + + (tb3 - 2.0 * tb2 + tb) * (1.0 - ks) + + (-2.0 * tb3 + 3.0 * tb2) * max_lum; + } + if (u < 1.0) { + u = u + min_lum * pow(1.0 - u, bp); + float gain = max_lum < 1.0 + ? 1.0 / (1.0 + min_lum / max_lum * pow(1.0 - max_lum, bp)) + : 1.0; + u = gain * (u - min_lum) + min_lum; + } + return u * in_max; + } + if (tone_map == 4u) { + // Spline: perceptually linear single-pivot polynomial, the default + // tone map of libplacebo and mpv's gpu-next renderer. + float contrast = param > 0.0 ? param : 0.3; + float fallback_avg = clamp(0.4 * src_peak, 100.0, 400.0); + float effective_src_avg = src_avg > 0.0 ? src_avg : fallback_avg; + float2 knee = st2094_pick_knee( + in_min, + in_max, + pq_code(effective_src_avg), + out_min, + out_max + ); + float src_pivot = knee.x; + float dst_pivot = knee.y; + float slope0 = (dst_pivot - out_min) / max(src_pivot - in_min, 0.000001); + float ratio = clamp(1.5 * (in_max / out_max - 1.0), 0.2, 1.2); + float slope = pow(slope0, (1.0 - contrast) * ratio); + float in_min0 = in_min - src_pivot; + float in_max0 = in_max - src_pivot; + float out_min0 = out_min - dst_pivot; + float out_max0 = out_max - dst_pivot; + float pa = (out_min0 - slope * in_min0) / (in_min0 * in_min0); + float qa = (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0 * in_max0); + float qb = -3.0 * (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0); + float xr = x - src_pivot; + float mapped = xr > 0.0 + ? ((qa * xr + qb) * xr + slope) * xr + : (pa * xr + slope) * xr; + return mapped + dst_pivot; + } + if (tone_map == 5u) { + // ITU-R BT.2446 method A: Weber-law log compression from the source + // peak envelope and a standardized S-curve (mpv's recommended curve + // for well-mastered content). + float phdr = 1.0 + 32.0 * pow(src_peak / 10000.0, 1.0 / 2.4); + float psdr = 1.0 + 32.0 * pow(dst_peak / 10000.0, 1.0 / 2.4); + float t = pow(nits_from_pq(x) / max(src_peak, 0.000001), 1.0 / 2.4); + t = log(1.0 + (phdr - 1.0) * t) / log(phdr); + if (t <= 0.7399) { + t = 1.0770 * t; + } else if (t < 0.9909) { + t = (-1.1510 * t + 2.7811) * t - 0.6302; + } else { + t = 0.5 * t + 0.5; + } + t = (pow(psdr, t) - 1.0) / (psdr - 1.0); + // BT.1886 EOTF from the target black point and peak. + float lb = pow(max(dst_black, 0.0), 1.0 / 2.4); + float lw = pow(max(dst_peak, 0.0), 1.0 / 2.4); + return pq_code(pow((lw - lb) * t + lb, 2.4)); + } + // SMPTE ST 2094-10 (DolbyVision's dynamic-metadata curve): rational + // Mobius interpolation in absolute nits; coefficients are solved per + // frame on the CPU from the same scene pivot. + float c1 = tone_map_coeffs.x; + float c2 = tone_map_coeffs.y; + float c3 = tone_map_coeffs.z; + float x_nits = nits_from_pq(x); + float y_nits = (c1 + c2 * x_nits) / max(1.0 + c3 * x_nits, 0.000001); + return pq_code(clamp(y_nits, 0.0, 10000.0)); +} + +float3 tone_map_nits(float3 input_nits) { + if (target_transfer == 3u) { + // HDR10 output: convert primaries by the gamut matrix and clamp to + // the PQ range (no tone mapping; the display does the HDR mapping). + return clamp(apply_gamut_map(max(input_nits, float3(0.0, 0.0, 0.0)) / source_reference_white_nits()) + * source_reference_white_nits(), float3(0.0, 0.0, 0.0), float3(10000.0, 10000.0, 10000.0)); + } + // libplacebo color map: RGB in source primaries (absolute nits) to + // HPE-LMS, PQ-encode, IPT, map the intensity axis and apply the + // hue-preserving chroma rule, then decode back to RGB in the target + // primaries. The primaries conversion happens inside this roundtrip. + float3 rgb = max(input_nits, float3(0.0, 0.0, 0.0)); + float3 lms = float3( + dot(ipt_matrix_rows[0].xyz, rgb), + dot(ipt_matrix_rows[1].xyz, rgb), + dot(ipt_matrix_rows[2].xyz, rgb) + ); + float3 lmspq = float3(pq_code(lms.r), pq_code(lms.g), pq_code(lms.b)); + float3 ipt = float3( + dot(float3(0.4, 0.4, 0.2), lmspq), + dot(float3(4.455, -4.851, 0.396), lmspq), + dot(float3(0.8056, 0.3572, -1.1628), lmspq) + ); + float i_orig = ipt.x; + ipt.x = tone_map_curve_pq(ipt.x, tone_map_extra.x); + // Libplacebo's chroma rule: clamp the saturation boost when brightening + // and desaturate (by the cubic hull term) when the mapping darkens. + float2 hull = float2(i_orig, ipt.x); + float2 hull_c = ((hull - float2(6.0, 6.0)) * hull + float2(9.0, 9.0)) * hull; + float ratio = min(i_orig / max(ipt.x, 0.000001), hull_c.y / max(hull_c.x, 0.000001)); + ipt.yz = ipt.yz * ratio; + if (gamut_lut_enabled != 0u) { + // I axis spans the target's [black, peak] in PQ codes, matching + // libplacebo's gamut.min_luma/max_luma (tone_map_extra.z is the + // target black in nits, the same value the LUT was generated for). + float lut_min = pq_code(tone_map_extra.z); + float lut_max = max(pq_code(target_peak_nits()), 0.000001); + float lut_range = max(lut_max - lut_min, 0.000001); + float3 pos = float3( + clamp((ipt.x - lut_min) / lut_range, 0.0, 1.0), + clamp(2.0 * length(ipt.yz), 0.0, 1.0), + 0.5 + 0.5 * atan2(ipt.z, ipt.y) / 3.14159265 + ); + // libplacebo's texel_scale: the lattice position must be remapped to + // the texel-center coordinate, otherwise the low end of the chroma + // axis (whose first texel stores zero chroma) leaks in and crushes + // saturation. + float3 idx = float3( + pos.x * (47.0 / 48.0) + 0.5 / 48.0, + pos.y * (31.0 / 32.0) + 0.5 / 32.0, + pos.z * (255.0 / 256.0) + 0.5 / 256.0 + ); + float3 sampled = gamutLut.Sample(videoSampler, idx).xyz; + ipt = float3(sampled.x, sampled.y - 0.5, sampled.z - 0.5); + } + float3 lmspq_out = float3( + dot(float3(1.0, 0.0975689, 0.205226), ipt), + dot(float3(1.0, -0.113876, 0.133217), ipt), + dot(float3(1.0, 0.0326151, -0.676887), ipt) + ); + float3 lms_out = float3( + nits_from_pq(lmspq_out.r), + nits_from_pq(lmspq_out.g), + nits_from_pq(lmspq_out.b) + ); + return float3( + dot(ipt_matrix_rows[6].xyz, lms_out), + dot(ipt_matrix_rows[7].xyz, lms_out), + dot(ipt_matrix_rows[8].xyz, lms_out) + ); +} + +// Hue-preserving gamut mapping: the linear gamut matrix can push highly +// saturated wide-gamut colors outside the target gamut (negative +// components). Blending those towards luma shifts hue - BT.2020 primary +// red picks up blue and turns pink. Instead blend towards the naive clip +// by an out-of-gamut smoothstep factor: slightly-out colors stay nearly +// intact, strongly-out primaries land on the pure target primary with +// their hue intact, matching mpv's perceptual gamut handling. Mirrors the +// WGSL/MSL gamut_compress and the Rust reference in pipeline.rs tests. +// Brightness overshoot (> 1) is left for the tone map. +float3 gamut_compress(float3 rgb) { + float lo = min(rgb.r, min(rgb.g, rgb.b)); + float outness = max(-lo, 0.0); + float k = smoothstep(0.0, 1.0, outness); + return lerp(rgb, clamp(rgb, 0.0, 1.0), k); +} + float3 target_nits_to_reference_linear(float3 input_nits) { - return max(input_nits, float3(0.0, 0.0, 0.0)) / target_reference_white_nits(); + // libplacebo's encode maps [target black, target peak] onto [0, 1] where + // 1.0 is the target reference white, so the tone-map black-point + // compensation lands back on true black instead of lifting it. + float black = tone_map_extra.z; + float peak = target_peak_nits(); + float range = max(peak - black, 0.0001); + return max(input_nits - float3(black, black, black), float3(0.0, 0.0, 0.0)) / range + * (range / target_reference_white_nits()); } float3 target_reference_linear_to_output(float3 rgb) { @@ -261,6 +528,12 @@ float4 final_output(float3 rgb, float alpha) { } void expand_ycbcr_range(float y_in, float2 cbcr_in, out float y, out float2 cbcr) { + if (is_p010 != 0u) { + // P010 stores 10-bit codes as code << 6 in a 16-bit UNORM texture. + const float p010_scale = 65535.0 / 65472.0; + y_in *= p010_scale; + cbcr_in *= p010_scale; + } if (full_range != 0u) { y = y_in; cbcr = cbcr_in - float2(0.5, 0.5); @@ -275,6 +548,83 @@ void expand_ycbcr_range(float y_in, float2 cbcr_in, out float y, out float2 cbcr cbcr = (cbcr_in - float2(128.0 / 255.0, 128.0 / 255.0)) * (255.0 / 224.0); } +// Dolby Vision RPU reshaping, ported from libplacebo's `pl_shader_dovi_reshape` +// (the renderer behind mpv's Dolby Vision mapping). The base-layer signal is +// reshaped per component through piecewise polynomial/MMR curves selected by +// pivot comparison, where MMR coefficients mix all three raw components. +float3 dovi_reshaped_signal(float3 sig_in) { + float3 sig = clamp(sig_in, 0.0, 1.0); + float result[3] = { sig.r, sig.g, sig.b }; + float4 flags = dovi.dovi_flags; + for (uint c = 0; c < 3u; c++) { + uint segments = uint(flags[1u + c]); + if (segments == 0u) { + continue; + } + float s = result[c]; + uint index = 0u; + for (uint i = 0u; i < 7u; i++) { + float4 pivot_row = dovi.dovi_pivots[2u * c + i / 4u]; + float pivot = pivot_row[i % 4u]; + if (s >= pivot) { + index = index + 1u; + } + } + float4 coeff = dovi.dovi_coefficients[8u * c + index]; + if (coeff.w < 0.5) { + s = (coeff.z * s + coeff.y) * s + coeff.x; + } else { + uint base = 48u * c + uint(coeff.y); + uint order = uint(coeff.w); + float4 sig_x = float4( + sig.x * sig.y, + sig.x * sig.z, + sig.y * sig.z, + sig.x * sig.y * sig.z + ); + s = coeff.x; + s = s + dot(dovi.dovi_mmr[base].xyz, sig); + s = s + dot(dovi.dovi_mmr[base + 1u], sig_x); + if (order >= 2u) { + float3 sig2 = sig * sig; + float4 sig_x2 = sig_x * sig_x; + s = s + dot(dovi.dovi_mmr[base + 2u].xyz, sig2); + s = s + dot(dovi.dovi_mmr[base + 3u], sig_x2); + if (order >= 3u) { + s = s + dot(dovi.dovi_mmr[base + 4u].xyz, sig2 * sig); + s = s + dot(dovi.dovi_mmr[base + 5u], sig_x2 * sig_x); + } + } + } + float4 bounds = dovi.dovi_bounds[c]; + result[c] = clamp(s, bounds.x, bounds.y); + } + return float3(result[0], result[1], result[2]); +} + +// Reshaped nonlinear signal to PQ-encoded IPT via the RPU's ycc_to_rgb matrix +// and signal offsets. Applying the RPU offsets keeps integer offset codes +// exactly on sample codes (2^bits/(2^bits-1) folded in on the CPU). +float3 dovi_signal_to_pq_rgb(float3 sig) { + float3 reshaped = dovi_reshaped_signal(sig) - dovi.dovi_nonlinear_offset.xyz; + return float3( + dot(dovi.dovi_nonlinear_matrix[0].xyz, reshaped), + dot(dovi.dovi_nonlinear_matrix[1].xyz, reshaped), + dot(dovi.dovi_nonlinear_matrix[2].xyz, reshaped) + ); +} + +// Linearized BT.2020-referred HPE LMS back to linear RGB, using the composite +// of the fixed HPE inverse with the RPU's rgb_to_lms matrix (premultiplied on +// the CPU, matching libplacebo's dovi_lms2rgb). +float3 dovi_lms_to_rgb(float3 lin) { + return float3( + dot(dovi.dovi_lms_matrix[0].xyz, lin), + dot(dovi.dovi_lms_matrix[1].xyz, lin), + dot(dovi.dovi_lms_matrix[2].xyz, lin) + ); +} + VsOut vs_main(VsIn input) { VsOut output; output.position = float4(input.position, 0.0, 1.0); @@ -335,22 +685,38 @@ float4 ps_main(VsOut input) : SV_Target { ? sample_packed_luma(luma_coord) : lumaTex.Sample(videoSampler, luma_coord).r; float2 cbcr_sample = chromaTex.Sample(videoSampler, chroma_coord).rg; - float y; - float2 cbcr; - expand_ycbcr_range(y_sample, cbcr_sample, y, cbcr); - - float kr = luma_coefficients.x; - float kg = max(luma_coefficients.y, 0.000001); - float kb = luma_coefficients.z; + bool dovi_enabled = dovi.dovi_flags.x != 0.0; + bool dovi_ycbcr_input = dovi_enabled && (base_input_mode == 0u || base_input_mode == 2u); float3 rgb; - rgb.r = y + 2.0 * (1.0 - kr) * cbcr.y; - rgb.b = y + 2.0 * (1.0 - kb) * cbcr.x; - rgb.g = (y - kr * rgb.r - kb * rgb.b) / kg; + if (dovi_ycbcr_input) { + // The base layer carries the raw 12-bit DV signal (10-bit container, + // full range); range expansion and the YCbCr matrix are replaced by + // the RPU reshaping + ycc_to_rgb path. + float3 sig = float3(y_sample, cbcr_sample.x, cbcr_sample.y); + if (is_p010 != 0u) { + sig *= 65535.0 / 65472.0; + } + rgb = dovi_signal_to_pq_rgb(sig); + } else { + float y; + float2 cbcr; + expand_ycbcr_range(y_sample, cbcr_sample, y, cbcr); + + float kr = luma_coefficients.x; + float kg = max(luma_coefficients.y, 0.000001); + float kb = luma_coefficients.z; + rgb.r = y + 2.0 * (1.0 - kr) * cbcr.y; + rgb.b = y + 2.0 * (1.0 - kb) * cbcr.x; + rgb.g = (y - kr * rgb.r - kb * rgb.b) / kg; + } rgb = transfer_to_source_reference_linear(rgb); - rgb = apply_gamut_map(rgb); + if (dovi_ycbcr_input) { + rgb = dovi_lms_to_rgb(rgb); + } rgb = source_reference_to_nits(rgb); rgb = tone_map_nits(rgb); rgb = target_nits_to_reference_linear(rgb); + rgb = gamut_compress(rgb); rgb = target_reference_linear_to_output(rgb); float alpha = 1.0; if (packed_alpha) { @@ -560,6 +926,7 @@ pub struct D3d11RendererStats { pub surface_width: u32, pub surface_height: u32, pub rendered_frames: u64, + pub software_video_frames: u64, pub hardware_video_frames: u64, pub zero_copy_video_frames: u64, pub direct_zero_copy_video_frames: u64, @@ -630,6 +997,7 @@ impl AttachedSurface { struct ImportedVideoFrame { _frame: Frame, _texture: ID3D11Texture2D, + _chroma_texture: Option, luma: ID3D11ShaderResourceView, chroma: ID3D11ShaderResourceView, width: u32, @@ -794,8 +1162,12 @@ impl D3d11OutputMode { } fn target_color_for_source(self, source: SourceColorState) -> TargetColorState { - let _ = source; - self.target_color() + match self { + Self::Sdr if source.is_hdr() => { + TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709) + } + _ => self.target_color(), + } } } @@ -840,6 +1212,12 @@ pub struct D3d11Renderer { upscaler: D3d11ArtCnn, next_frame_token: u64, hdr10_output_unavailable: bool, + /// Cached perceptual gamut LUT (3D RGBA16F SRV), keyed the same way as + /// the wgpu/Metal caches: (source, target, target black PQ, target peak PQ). + gamut_lut: Option<(u32, u32, u32, u32, ID3D11ShaderResourceView)>, + /// Background generation for a cache miss; the fast `gamut_compress` + /// path renders until the LUT lands. + gamut_lut_job: Option, stats: D3d11RendererStats, } @@ -861,6 +1239,8 @@ impl D3d11Renderer { upscaler: D3d11ArtCnn::default(), next_frame_token: 0, hdr10_output_unavailable: false, + gamut_lut: None, + gamut_lut_job: None, stats: D3d11RendererStats::default(), }) } @@ -1198,6 +1578,7 @@ impl D3d11Renderer { let imported = ImportedVideoFrame { _frame: retained_frame, _texture: texture, + _chroma_texture: None, luma, chroma, width: visible_width, @@ -1573,6 +1954,95 @@ impl D3d11Renderer { Ok(()) } + /// Return a cached (or freshly generated) perceptual gamut LUT SRV for + /// the given uniforms, or `None` while the fast path is active or the + /// background generation is still pending. Callers must then mask + /// `gamut_lut_enabled` off so the shader keeps the fast path. + fn gamut_lut_view( + &mut self, + uniforms: &VideoUniforms, + ) -> Result> { + if uniforms.gamut_lut_enabled == 0 { + return Ok(None); + } + let packed = uniforms._gamut_primaries; + let source = packed >> 8; + let target = packed & 0xff; + // The LUT's I axis spans [target black, target peak] in PQ codes; + // either endpoint changing invalidates the cached LUT. + let black_pq = d3d_quantize_luma_pq(uniforms.tone_map_extra[2]); + let peak_pq = d3d_quantize_luma_pq(uniforms.nits[1]); + if let Some((s, t, b, p, srv)) = &self.gamut_lut { + if *s == source && *t == target && *b == black_pq && *p == peak_pq { + return Ok(Some(srv.clone())); + } + } + let params = GamutLutParams { + source: d3d_code_to_primaries(source), + target: d3d_code_to_primaries(target), + // Same target black the shader derives from tone_map_extra.z. + min_luma: d3d_pq_code_for_lut(uniforms.tone_map_extra[2]), + max_luma: d3d_pq_code_for_lut(uniforms.nits[1]), + }; + let job_params = self + .gamut_lut_job + .as_ref() + .map(GamutLutJob::params) + .filter(|job_params| *job_params == params); + if job_params.is_none() { + // First request (or the key changed): spawn generation and keep + // the fast path for this frame. + self.gamut_lut_job = Some(GamutLutJob::spawn(params)); + return Ok(None); + } + let Some(lut) = self.gamut_lut_job.as_ref().and_then(GamutLutJob::poll) else { + return Ok(None); + }; + self.gamut_lut_job = None; + let state = self.state.as_ref().expect("device ensured for gamut lut"); + // Pack (I, P+0.5, T+0.5) into RGBA16F bytes laid out I x C x H. + let texels = pack_rgba16f(&lut.texels, 1.0); + let initial = [D3D11_SUBRESOURCE_DATA { + pSysMem: texels.as_ptr() as *const c_void, + SysMemPitch: (LUT_SIZE_I * 4 * 2) as u32, + SysMemSlicePitch: (LUT_SIZE_I * LUT_SIZE_C * 4 * 2) as u32, + }]; + let desc = D3D11_TEXTURE3D_DESC { + Width: LUT_SIZE_I as u32, + Height: LUT_SIZE_C as u32, + Depth: LUT_SIZE_H as u32, + MipLevels: 1, + Format: DXGI_FORMAT_R16G16B16A16_FLOAT, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let mut texture = None; + unsafe { + state + .device + .CreateTexture3D(&desc, Some(initial.as_ptr()), Some(&mut texture)) + .map_err(|error| d3d_error("ID3D11Device::CreateTexture3D(gamut lut)", error))?; + } + let resource = texture + .expect("gamut lut texture created") + .cast::() + .map_err(|error| d3d_error("cast to ID3D11Resource", error))?; + let mut srv = None; + unsafe { + state + .device + .CreateShaderResourceView(&resource, None, Some(&mut srv)) + .map_err(|error| { + d3d_error("ID3D11Device::CreateShaderResourceView(gamut lut)", error) + })?; + } + let srv = srv.expect("gamut lut srv created"); + self.gamut_lut = Some((source, target, black_pq, peak_pq, srv.clone())); + Ok(Some(srv)) + } + fn render_video(&mut self, context: RenderFrameContext<'_>) -> Result { if self.current_video.is_none() { return Ok(false); @@ -1649,6 +2119,16 @@ impl D3d11Renderer { } else { None }; + let mut video_constants = { + let video = self.current_video.as_ref().expect("video checked"); + video.constants + }; + let gamut_lut = self.gamut_lut_view(&video_constants)?; + if gamut_lut.is_none() { + // Background generation pending: keep the fast gamut_compress + // path until the LUT lands. + video_constants.gamut_lut_enabled = 0; + } let video = self.current_video.as_ref().expect("video checked"); let state = self.state.as_ref().expect("device ensured"); let surface = self.surface.as_ref().expect("surface ensured"); @@ -1684,7 +2164,14 @@ impl D3d11Renderer { ], ); } - state.draw_video(video, upscaled_luma.as_ref(), scene_rtv, target_rect)?; + state.draw_video( + video, + video_constants, + upscaled_luma.as_ref(), + gamut_lut.as_ref(), + scene_rtv, + target_rect, + )?; if !overlay_draws.is_empty() { // Subtitle coordinates are produced in the video-frame viewport. // Composite them through the same aspect-fit viewport as the video @@ -1902,10 +2389,104 @@ impl RendererBackend for D3d11Renderer { "d3d11: hardware frame is not importable as D3D11VA".to_string(), )); } - self.stats.cpu_video_frame_fallbacks += 1; - Err(PlayerError::Renderer( - "d3d11: software frames require WgpuFallback or a CPU upload path".to_string(), - )) + let decoded = frame.frame.decoded_frame().ok_or_else(|| { + PlayerError::Renderer("d3d11: video payload has no CPU-readable frame".to_string()) + })?; + let planar = decoded.to_planar_frame().ok_or_else(|| { + PlayerError::Renderer(format!( + "d3d11: unsupported software video frame format {}", + decoded + .pixel_format() + .unwrap_or_else(|| "unknown".to_string()) + )) + })?; + self.ensure_default_device()?; + let source = source_color_for_frame(frame); + let output_mode = self.select_output_mode_for_source(source)?; + let target = output_mode.target_color_for_source(source); + let (texture_format, bytes_per_sample) = match planar.format { + PlanarPixelFormat::Nv12 => (D3d11VideoTextureFormat::Nv12, 1_u32), + PlanarPixelFormat::P010 => (D3d11VideoTextureFormat::P010, 2_u32), + }; + let width = planar.width.max(1); + let height = planar.height.max(1); + let chroma_width = width.div_ceil(2); + let chroma_height = height.div_ceil(2); + let luma_pitch = width + .checked_mul(bytes_per_sample) + .ok_or_else(|| PlayerError::Renderer("d3d11: luma row pitch overflowed".to_string()))?; + let chroma_pitch = chroma_width + .checked_mul(2) + .and_then(|pitch| pitch.checked_mul(bytes_per_sample)) + .ok_or_else(|| { + PlayerError::Renderer("d3d11: chroma row pitch overflowed".to_string()) + })?; + let sample_bytes = bytes_per_sample as usize; + let expected_luma = (width as usize) + .checked_mul(height as usize) + .and_then(|samples| samples.checked_mul(sample_bytes)) + .ok_or_else(|| { + PlayerError::Renderer("d3d11: luma plane size overflowed".to_string()) + })?; + let expected_chroma = (chroma_width as usize) + .checked_mul(chroma_height as usize) + .and_then(|samples| samples.checked_mul(2)) + .and_then(|bytes| bytes.checked_mul(sample_bytes)) + .ok_or_else(|| { + PlayerError::Renderer("d3d11: chroma plane size overflowed".to_string()) + })?; + if planar.luma.len() != expected_luma || planar.chroma.len() != expected_chroma { + return Err(PlayerError::Renderer(format!( + "d3d11: invalid {:?} plane sizes (luma {}, expected {}; chroma {}, expected {})", + planar.format, + planar.luma.len(), + expected_luma, + planar.chroma.len(), + expected_chroma, + ))); + } + let (luma, chroma) = { + let state = self.state.as_ref().expect("device ensured"); + ( + create_overlay_texture( + state, + width, + height, + texture_format.luma_srv(), + &planar.luma, + luma_pitch, + )?, + create_overlay_texture( + state, + chroma_width, + chroma_height, + texture_format.chroma_srv(), + &planar.chroma, + chroma_pitch, + )?, + ) + }; + let retained_frame = decoded.try_clone_ref().map_err(|error| { + PlayerError::Renderer(format!("d3d11: av_frame_ref failed: {error}")) + })?; + self.stats.software_video_frames += 1; + let frame_token = self.next_frame_token; + self.next_frame_token = self.next_frame_token.wrapping_add(1); + self.current_video = Some(ImportedVideoFrame { + _frame: retained_frame, + _texture: luma._texture, + _chroma_texture: Some(chroma._texture), + luma: luma.view, + chroma: chroma.view, + width, + height, + tex_rect: D3d11TexRect::FULL, + _array_index: 0, + frame_token, + constants: constants_for_frame(source, texture_format, target) + .packed_alpha_right(self.video_alpha_mode.has_alpha()), + }); + Ok(()) } fn clear_current_frame(&mut self) -> Result<()> { @@ -1937,7 +2518,7 @@ impl RendererBackend for D3d11Renderer { danmaku_draw_items: self.stats.danmaku_items, overlay_alpha_atlas_uploads: self.stats.overlay_alpha_atlas_uploads, overlay_alpha_atlas_reuses: self.stats.overlay_alpha_atlas_reuses, - software_video_frames: 0, + software_video_frames: self.stats.software_video_frames, hardware_video_frames: self.stats.hardware_video_frames, zero_copy_video_frames: self.stats.zero_copy_video_frames, direct_zero_copy_video_frames: self.stats.direct_zero_copy_video_frames, @@ -2113,10 +2694,17 @@ impl D3d11DeviceState { }) } + /// Draw the imported video frame. `constants` is the per-frame uniform + /// payload the caller already resolved (including any + /// `gamut_lut_enabled` mask for a still-generating LUT); the stored + /// `ImportedVideoFrame.constants` is left untouched so a later present + /// can re-enable the LUT once it lands. fn draw_video( &self, video: &ImportedVideoFrame, + constants: VideoUniforms, upscaled_luma: Option<&ID3D11ShaderResourceView>, + gamut_lut: Option<&ID3D11ShaderResourceView>, render_target: &ID3D11RenderTargetView, target: D3d11DrawRect, ) -> Result<()> { @@ -2131,7 +2719,7 @@ impl D3d11DeviceState { let stride = mem::size_of::() as u32; let offset = 0u32; let vertices = video_vertices(D3d11TexRect::FULL); - let mut common = video.constants; + let mut common = constants; if upscaled_luma.is_some() { common = common.packed_d2s_luma_input(); } @@ -2184,20 +2772,20 @@ impl D3d11DeviceState { self.context.PSSetShader(&self.pixel_shader, None); self.context .PSSetConstantBuffers(0, Some(&[Some(self.constants.clone())])); - self.context.PSSetShaderResources( - 0, - Some(&[ - Some(upscaled_luma.cloned().unwrap_or_else(|| video.luma.clone())), - Some(video.chroma.clone()), - ]), - ); + let srvs = [ + Some(upscaled_luma.cloned().unwrap_or_else(|| video.luma.clone())), + Some(video.chroma.clone()), + gamut_lut.cloned(), + ]; + self.context.PSSetShaderResources(0, Some(&srvs)); self.context .PSSetSamplers(0, Some(&[Some(self.sampler.clone())])); self.context .OMSetRenderTargets(Some(&[Some(render_target.clone())]), None); self.context.OMSetBlendState(None, None, u32::MAX); self.context.Draw(6, 0); - self.context.PSSetShaderResources(0, Some(&[None, None])); + self.context + .PSSetShaderResources(0, Some(&[None, None, None])); } Ok(()) } @@ -2395,6 +2983,31 @@ fn aspect_fit_rect( } } +fn d3d_pq_code_for_lut(nits: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = (nits / 10000.0).clamp(0.0, 1.0).powf(m1); + ((c1 + c2 * p) / (1.0 + c3 * p).max(0.000_001)).powf(m2) +} + +/// Quantize a luminance (nits) to its PQ code for the LUT cache key, matching +/// the wgpu/Metal quantizers. 16-bit PQ resolution keeps the sub-1-nit target +/// blacks of SDR targets distinct. +fn d3d_quantize_luma_pq(nits: f32) -> u32 { + (d3d_pq_code_for_lut(nits) * 65535.0) as u32 +} + +fn d3d_code_to_primaries(code: u32) -> ColorPrimaries { + match code { + 1 => ColorPrimaries::Bt2020, + 2 => ColorPrimaries::DisplayP3, + _ => ColorPrimaries::Bt709, + } +} + fn create_default_device() -> Result<(ID3D11Device, ID3D11DeviceContext)> { trace("create_default_device: D3D11CreateDevice"); let feature_levels = [ @@ -3094,6 +3707,8 @@ fn source_color_for_frame(frame: &PlayerVideoFrame) -> SourceColorState { .range(frame.frame.color_range()) .matrix(frame.frame.matrix_coefficients()) .hdr_metadata(frame.frame.hdr_metadata()) + .dovi(frame.frame.dovi_metadata()) + .measured_scene_avg_nits(frame.scene_avg_nits) } fn constants_for_frame( @@ -3732,6 +4347,20 @@ mod tests { assert_eq!(sdr.scene_linear, 0); } + #[test] + fn hlsl_gamut_lut_samples_the_target_black_to_peak_axis() { + // libplacebo's LUT I axis is [target black, target peak]; a backend + // that samples `ipt.x / peak` again would diverge from the generated + // LUT. The lattice position must also be remapped to the texel-center + // coordinate (libplacebo's `texel_scale`), or the zero-chroma texel at + // the low end of the C axis crushes saturation. + let source = std::str::from_utf8(SHADER_SOURCE).unwrap(); + assert!(source.contains("float lut_min = pq_code(tone_map_extra.z);")); + assert!(source.contains("clamp((ipt.x - lut_min) / lut_range, 0.0, 1.0)")); + assert!(source.contains("pos.y * (31.0 / 32.0) + 0.5 / 32.0")); + assert!(source.contains("pos.x * (47.0 / 48.0) + 0.5 / 48.0")); + } + #[test] fn overlay_uniforms_enable_scene_linear_only_for_hdr10() { let hdr10 = diff --git a/crates/erika/src/renderer/frame.rs b/crates/erika/src/renderer/frame.rs index 402e92bc..3b99fd65 100644 --- a/crates/erika/src/renderer/frame.rs +++ b/crates/erika/src/renderer/frame.rs @@ -1,6 +1,6 @@ use crate::core::{ColorPrimaries, TransferFunction}; use crate::ffmpeg::{D3d11vaTexture, Frame, PlanarFrame, Result as FfmpegResult}; -use crate::renderer::pipeline::{ColorRange, HdrMetadata, MatrixCoefficients}; +use crate::renderer::pipeline::{ColorRange, DoviSourceMetadata, HdrMetadata, MatrixCoefficients}; #[cfg(any(target_os = "android", target_env = "ohos"))] use std::sync::Arc; @@ -26,6 +26,7 @@ pub struct VideoFrameDescriptor { pub range: ColorRange, pub matrix: MatrixCoefficients, pub hdr_metadata: Option, + pub dovi_metadata: Option, } impl VideoFrameDescriptor { @@ -41,6 +42,7 @@ impl VideoFrameDescriptor { range: frame.color_range(), matrix: frame.matrix_coefficients(), hdr_metadata: frame.hdr_metadata(), + dovi_metadata: frame.dovi_metadata(), } } } @@ -214,6 +216,16 @@ impl VideoFramePayload { } } + pub fn dovi_metadata(&self) -> Option { + match self { + Self::Decoded(frame) => frame.dovi_metadata(), + #[cfg(target_os = "android")] + Self::AndroidHardwareBuffer(frame) => frame.descriptor.dovi_metadata, + #[cfg(target_env = "ohos")] + Self::OhosNativeBuffer(frame) => frame.descriptor.dovi_metadata, + } + } + pub fn has_hw_frames_context(&self) -> bool { match self { Self::Decoded(frame) => frame.has_hw_frames_context(), diff --git a/crates/erika/src/renderer/gamut.rs b/crates/erika/src/renderer/gamut.rs new file mode 100644 index 00000000..413eead1 --- /dev/null +++ b/crates/erika/src/renderer/gamut.rs @@ -0,0 +1,751 @@ +//! Perceptual gamut mapping via a precomputed IPT-space 3D LUT, following +//! libplacebo's `pl_gamut_map_perceptual` LUT layout (the default gamut map +//! of libplacebo/mpv). The mapping converts a color from the source primaries +//! into the destination primaries *inside* IPT and then rolls off +//! out-of-gamut chroma towards the destination gamut, protecting in-gamut +//! colors (dead zone) with a Möbius soft clip. +//! +//! The chroma rolloff is currently a simplified dead-zone blend plus +//! softclip rather than libplacebo's full per-hue boundary search, so highly +//! saturated BT.2020 colors can diverge slightly from mpv. +//! +//! Because the boundary search is expensive, libplacebo precomputes it once +//! into a 3D LUT over the IPT intensity I, the IPT chroma magnitude C and the +//! IPT hue angle h; the renderers sample it per pixel. This module generates +//! the same LUT lattice on the CPU (48 x 32 x 256 = 393216 texels), and the +//! shaders do the lookup with the same index mapping. +//! +//! Note: libplacebo currently *samples the perceptual LUT in ICh space* with +//! the I channel *in absolute PQ units* over the target display range +//! `[min_luma, max_luma]`. We reproduce that layout exactly so the WGSL/MSL +//! and HLSL samplers match the reference pixels. + +use crate::core::ColorPrimaries; +use crate::renderer::pipeline::{RgbMatrix, ipt_rgb2lms_matrix}; + +/// LUT dimensions, matching libplacebo's `pl_color_map_default_params` +/// `lut3d_size = {48, 32, 256}` (I, C, h). +pub const LUT_SIZE_I: usize = 48; +pub const LUT_SIZE_C: usize = 32; +pub const LUT_SIZE_H: usize = 256; +/// Per-texel component count (RGB with a padded w). +pub const LUT_COMPONENTS: usize = 4; + +const PQ_M1: f32 = 2610.0 / 4096.0 * 1.0 / 4.0; +const PQ_M2: f32 = 2523.0 / 4096.0 * 128.0; +const PQ_C1: f32 = 3424.0 / 4096.0; +const PQ_C2: f32 = 2413.0 / 4096.0 * 32.0; +const PQ_C3: f32 = 2392.0 / 4096.0 * 32.0; + +#[allow(dead_code)] +/// 4% crosstalk HPE matrix shared with the tone map. +fn hpe_crosstalk() -> RgbMatrix { + let c = 0.04_f32; + RgbMatrix::new([ + [1.0 - 2.0 * c, c, c], + [c, 1.0 - 2.0 * c, c], + [c, c, 1.0 - 2.0 * c], + ]) +} + +fn rgb2lms(primaries: ColorPrimaries) -> RgbMatrix { + ipt_rgb2lms_matrix(primaries) +} + +#[allow(dead_code)] +fn lms2rgb(primaries: ColorPrimaries) -> RgbMatrix { + rgb2lms(primaries).inverse() +} + +/// Perceptual gamut mapping parameters that change how the LUT is computed. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct GamutLutParams { + /// Source primaries of the tone-mapped (still source-referred) signal. + pub source: ColorPrimaries, + /// Destination (display) primaries. + pub target: ColorPrimaries, + /// Minimum display luminance in PQ code (absolute), i.e. libplacebo's + /// `gamut.min_luma`: the target's black point, which is also the tone + /// map's output minimum (0 for HDR/EDR targets, `peak / contrast` for SDR + /// ones). + pub min_luma: f32, + /// Maximum display luminance in PQ code (libplacebo's `gamut.max_luma`, + /// the tone map's output peak). + pub max_luma: f32, +} + +/// A generated 3D perceptual gamut LUT. Texels are (I, P, T) triples with a +/// padded component, normalized to [0, 1] like libplacebo's uint16 upload +/// (`I`, `P + 0.5`, `T + 0.5` scaled). +#[derive(Debug, Clone, PartialEq)] +pub struct GamutLut { + /// size = LUT_SIZE_I * LUT_SIZE_C * LUT_SIZE_H, stored as float triples + /// (R=IPT.I, G=IPT.P + 0.5, B=IPT.T + 0.5) in [0, 1]. + pub texels: Vec<[f32; 3]>, + /// `min_luma`/`max_luma` in PQ that the LUT's I axis spans. + pub params: GamutLutParams, +} + +impl GamutLut { + /// Generate the perceptual LUT, mirroring libplacebo's + /// `pl_gamut_map_perceptual` evaluated over the lattice texels. + /// + /// libplacebo caches the per-hue boundary peak (`saturate`); we instead + /// precompute the source/destination peaks for each of the `LUT_SIZE_H` + /// hue slices once, which the texel loop then reuses — same math, no + /// repeated golden-section searches. + pub fn generate(params: GamutLutParams) -> Self { + let src = GamutState::new(params.source, params); + let dst = GamutState::new(params.target, params); + let mut texels = Vec::with_capacity(LUT_SIZE_I * LUT_SIZE_C * LUT_SIZE_H); + // Lattice order must match the 3D texture addressing used by the + // shaders: width = I (48, stride 1), height = C (32, stride width), + // depth = h (256, stride width * height). The loop runs h outer, + // C middle, I inner so that width (I) is contiguous in memory, and + // the expensive per-hue `saturate` search runs 256 times instead of + // 393,216 times. + for hx in 0..LUT_SIZE_H { + let h = -std::f32::consts::PI + + 2.0 * std::f32::consts::PI * hx as f32 / (LUT_SIZE_H - 1) as f32; + let src_peak = saturate(h, &src); + let dst_peak = saturate(h, &dst); + let max_c = src_peak[1].max(dst_peak[1]).max(1e-9); + let soft = Softclip { + knee: 0.70, + _desat: 0.35, + }; + for cx in 0..LUT_SIZE_C { + let c = 0.5 * cx as f32 / (LUT_SIZE_C - 1) as f32; + for ix in 0..LUT_SIZE_I { + let i = params.min_luma + + (params.max_luma - params.min_luma) * ix as f32 / (LUT_SIZE_I - 1) as f32; + let mapped = perceptual_map_at( + [i, c, h], + &src, + &dst, + src_peak, + dst_peak, + max_c, + &soft, + params, + ); + texels.push([mapped[0], mapped[1] + 0.5, mapped[2] + 0.5]); + } + } + } + Self { texels, params } + } +} + +/// A `GamutLut::generate` running on a background thread, shared by the +/// Metal/wgpu/D3D11 renderers. Generation costs hundreds of milliseconds +/// (393k texels of PQ roundtrips plus 256 hue-boundary searches); running it +/// on the render thread stalls the first HDR wide-gamut frame. The renderer +/// polls `poll()` each frame and keeps the fast `gamut_compress` path until +/// the LUT lands. +pub struct GamutLutJob { + params: GamutLutParams, + result: std::sync::mpsc::Receiver, +} + +impl GamutLutJob { + /// Spawn generation for `params`. The thread is detached: it writes into + /// the channel and exits, so dropping the job simply discards an unused + /// result. + pub fn spawn(params: GamutLutParams) -> Self { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + std::thread::Builder::new() + .name("erika-gamut-lut".to_string()) + .spawn(move || { + let lut = GamutLut::generate(params); + let _ = sender.send(lut); + }) + .expect("spawn gamut LUT generation thread"); + Self { + params, + result: receiver, + } + } + + /// The parameters this job is generating for. + pub fn params(&self) -> GamutLutParams { + self.params + } + + /// Take the finished LUT when generation has completed. + pub fn poll(&self) -> Option { + self.result.try_recv().ok() + } +} + +#[cfg(test)] +mod job_tests { + use super::*; + + #[test] + fn job_produces_the_same_lut_as_inline_generation() { + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 0.75, + }; + let job = GamutLutJob::spawn(params); + let lut = loop { + if let Some(lut) = job.poll() { + break lut; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + }; + assert_eq!(lut.params, params); + assert_eq!(lut.texels.len(), LUT_SIZE_I * LUT_SIZE_C * LUT_SIZE_H); + // Polling after completion yields nothing further. + assert!(job.poll().is_none()); + } +} + +fn pq_oetf(x: f32) -> f32 { + let x = x.max(0.0).powf(PQ_M1); + ((PQ_C1 + PQ_C2 * x) / (1.0 + PQ_C3 * x)).powf(PQ_M2) +} + +fn pq_eotf(x: f32) -> f32 { + let x = x.max(0.0).powf(1.0 / PQ_M2); + let num = (x - PQ_C1).max(0.0); + let den = (PQ_C2 - PQ_C3 * x).max(1e-9); + (num / den).powf(1.0 / PQ_M1) +} + +fn rgb2ipt(rgb: [f32; 3], gamut: &GamutState) -> [f32; 3] { + let lms = gamut.rgb2lms.mul_vec(rgb); + let lp = pq_oetf(lms[0]); + let mp = pq_oetf(lms[1]); + let sp = pq_oetf(lms[2]); + [ + 0.4000 * lp + 0.4000 * mp + 0.2000 * sp, + 4.4550 * lp - 4.8510 * mp + 0.3960 * sp, + 0.8056 * lp + 0.3572 * mp - 1.1628 * sp, + ] +} + +fn ipt2rgb(ipt: [f32; 3], gamut: &GamutState) -> [f32; 3] { + let lp = ipt[0] + 0.0975689 * ipt[1] + 0.205226 * ipt[2]; + let mp = ipt[0] - 0.1138760 * ipt[1] + 0.133217 * ipt[2]; + let sp = ipt[0] + 0.0326151 * ipt[1] - 0.676887 * ipt[2]; + let l = pq_eotf(lp); + let m = pq_eotf(mp); + let s = pq_eotf(sp); + gamut.lms2rgb.mul_vec([l, m, s]) +} + +/// Perceptual map of one IPT color given as (I in PQ code, chroma, hue), +/// mirroring libplacebo's `perceptual()` body with the per-hue peaks already +/// computed. `src`/`dst` are the gamut states, `src_peak`/`dst_peak` the +/// maximally-saturated boundary colors at this hue, `max_c` their chroma +/// max (the dead-zone denominator). +fn perceptual_map_at( + ich: [f32; 3], + src: &GamutState, + dst: &GamutState, + _src_peak: [f32; 3], + _dst_peak: [f32; 3], + max_c: f32, + soft: &Softclip, + _params: GamutLutParams, +) -> [f32; 3] { + let ipt_in = ich2ipt(ich); + let mapped = rgb2ipt(ipt2rgb(ipt_in, src), dst); + + // Protect in-gamut region: blend only colors whose chroma exceeds the + // perceptual dead zone (30% of the peak), scaling to full strength. + let deadzone = 0.30_f32; + let strength = 0.80_f32; + let k = pl_smoothstep(deadzone, 1.0, ich[1] / max_c) * strength; + let ipt = [ + ipt_in[0] + (mapped[0] - ipt_in[0]) * k, + ipt_in[1] + (mapped[1] - ipt_in[1]) * k, + ipt_in[2] + (mapped[2] - ipt_in[2]) * k, + ]; + + let rgb = ipt2rgb(ipt, dst); + let max_rgb = rgb[0].max(rgb[1]).max(rgb[2]); + let out = [ + softclip(rgb[0], max_rgb, dst.max_rgb, soft).max(dst.min_rgb), + softclip(rgb[1], max_rgb, dst.max_rgb, soft).max(dst.min_rgb), + softclip(rgb[2], max_rgb, dst.max_rgb, soft).max(dst.min_rgb), + ]; + rgb2ipt(out, dst) +} + +fn softclip(value: f32, source: f32, target: f32, c: &Softclip) -> f32 { + if target == 0.0 { + return 0.0; + } + let peak = source / target; + let x = (value / target).min(peak); + if x <= c.knee || peak <= 1.0 { + return value; + } + let j = c.knee; + let a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + let b = (j * j - 2.0 * j * peak + peak) / (peak - 1.0).max(1e-6); + let scale = (b * b + 2.0 * b * j + j * j) / (b - a); + scale * (x + a) / (x + b) * target +} + +struct Softclip { + knee: f32, + _desat: f32, +} + +fn pl_smoothstep(edge0: f32, edge1: f32, x: f32) -> f32 { + let t = ((x - edge0) / (edge1 - edge0)).clamp(0.0, 1.0); + t * t * (3.0 - 2.0 * t) +} + +struct GamutState { + rgb2lms: RgbMatrix, + lms2rgb: RgbMatrix, + min_rgb: f32, + max_rgb: f32, + min_luma: f32, + max_luma: f32, +} + +impl GamutState { + fn new(primaries: ColorPrimaries, params: GamutLutParams) -> Self { + let m = rgb2lms(primaries); + Self { + lms2rgb: m.inverse(), + rgb2lms: m, + min_rgb: pq_eotf(params.min_luma) - 1e-6, + max_rgb: pq_eotf(params.max_luma) + 1e-6, + min_luma: params.min_luma, + max_luma: params.max_luma, + } + } +} + +#[allow(dead_code)] +fn ipt2ich(ipt: [f32; 3]) -> [f32; 3] { + [ + ipt[0], + (ipt[1] * ipt[1] + ipt[2] * ipt[2]).sqrt(), + ipt[2].atan2(ipt[1]), + ] +} + +fn ich2ipt(ich: [f32; 3]) -> [f32; 3] { + [ich[0], ich[1] * ich[2].cos(), ich[1] * ich[2].sin()] +} + +/// Returns the maximally saturated in-gamut color of `gamut` at `hue`, +/// using a golden-section search over I with a bounded binary search for the +/// C boundary (mirrors libplacebo's `saturate`). +fn saturate(hue: f32, gamut: &GamutState) -> [f32; 3] { + let inv_phi = 0.618_033_988_749_894_8_f32; + let inv_phi2 = 0.381_966_011_250_105_15_f32; + + // Golden-section bracket over I, keeping the full (I, C, h) points so + // each iteration re-bounds the boundary search like the C version. + let (mut lo_i, mut lo_c) = (gamut.min_luma, 0.0_f32); + let (hi_i, mut hi_c) = (gamut.max_luma, 0.0_f32); + let mut de = hi_i - lo_i; + let mut a = [lo_i + inv_phi2 * de, 0.0, hue]; + let mut b = [lo_i + inv_phi * de, 0.0, hue]; + a[1] = desat_bounded(a[0], hue, 0.0, 0.5, gamut)[1]; + b[1] = desat_bounded(b[0], hue, 0.0, 0.5, gamut)[1]; + + while de > 5e-5 { + de *= inv_phi; + if a[1] > b[1] { + hi_c = b[1]; + b = a; + a[0] = lo_i + inv_phi2 * de; + a[1] = desat_bounded(a[0], hue, lo_c - 5e-5, 0.5, gamut)[1]; + } else { + lo_i = a[0]; + lo_c = a[1]; + a = b; + b[0] = lo_i + inv_phi * de; + b[1] = desat_bounded(b[0], hue, hi_c - 5e-5, 0.5, gamut)[1]; + } + } + + if a[1] > b[1] { + [a[0], a[1], hue] + } else { + [b[0], b[1], hue] + } +} + +/// Find the gamut boundary at luminance `i` and hue `h` within `[cmin, cmax]`. +fn desat_bounded(i: f32, h: f32, cmin: f32, cmax: f32, gamut: &GamutState) -> [f32; 3] { + if i <= gamut.min_luma { + return [gamut.min_luma, 0.0, h]; + } + if i >= gamut.max_luma { + return [gamut.max_luma, 0.0, h]; + } + let max_di = i * 5e-5; + let mut lo = cmin; + let mut hi = cmax; + loop { + let c = (lo + hi) / 2.0; + if ingamut(ich2ipt([i, c, h]), gamut) { + lo = c; + } else { + hi = c; + } + if hi - lo <= max_di { + return [i, (lo + hi) / 2.0, h]; + } + } +} + +fn ingamut(ipt: [f32; 3], gamut: &GamutState) -> bool { + let rgb = ipt2rgb(ipt, gamut); + rgb[0] >= gamut.min_rgb + && rgb[0] <= gamut.max_rgb + && rgb[1] >= gamut.min_rgb + && rgb[1] <= gamut.max_rgb + && rgb[2] >= gamut.min_rgb + && rgb[2] <= gamut.max_rgb +} + +/// Convert an f32 to IEEE-754 binary16 (round-to-nearest-even) for packing +/// RGBA16F textures. The gamut LUT texels carry values in [-0.5, 1.0], so a +/// half-float texture is exact enough and is filterable on every backend +/// (Rgba32Float is not filterable on wgpu). +pub fn f32_to_f16(value: f32) -> u16 { + let bits = value.to_bits(); + let sign = (bits >> 16) & 0x8000; + let exp = ((bits >> 23) & 0xff) as i32; + let mant = bits & 0x7fffff; + if exp == 0xff { + return (sign | 0x7c00 | u32::from(mant != 0)) as u16; + } + let e = exp - 127 + 15; + if e >= 0x1f { + return (sign | 0x7c00) as u16; // overflow -> inf + } + if e <= 0 { + return if e <= -10 { + sign as u16 // underflow -> signed zero + } else { + let m = mant | 0x800000; + let shift = (14 - e) as u32; + let rounded = (m >> shift) + u32::from((m >> (shift - 1)) & 1 == 1); + (sign | rounded) as u16 + }; + } + let rounded = mant + 0x1000 + ((mant >> 13) & 1); + let mut e = e as u32; + if rounded & 0x800000 != 0 { + e += 1; + } + (sign | (e << 10) | ((rounded >> 13) & 0x3ff)) as u16 +} + +/// Pack the f32 texel triples into interleaved RGBA16F little-endian bytes, +/// matching the `Rgba16Float` textures of the Metal/wgpu/D3D11 backends. +pub fn pack_rgba16f(texels: &[[f32; 3]], alpha: f32) -> Vec { + let mut bytes = Vec::with_capacity(texels.len() * 8); + for texel in texels { + for channel in texel.iter().copied().chain(std::iter::once(alpha)) { + bytes.extend_from_slice(&f32_to_f16(channel).to_le_bytes()); + } + } + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn f16_packing_roundtrips_and_matches_half_layout() { + // Known IEEE-754 binary16 bit patterns. + assert_eq!(f32_to_f16(0.0), 0x0000); + assert_eq!(f32_to_f16(1.0), 0x3c00); + assert_eq!(f32_to_f16(0.5), 0x3800); + assert_eq!(f32_to_f16(-0.5), 0xb800); + assert_eq!(f32_to_f16(10000.0), 0x70e2); + assert_eq!(f32_to_f16(-1.0), 0xbc00); + // Padded alpha keeps texel pitch at 8 bytes. + let packed = pack_rgba16f(&[[0.0, 0.5, 1.0]], 1.0); + assert_eq!(packed.len(), 8); + assert_eq!(&packed[0..2], &0x0000_u16.to_le_bytes()); + assert_eq!(&packed[2..4], &0x3800_u16.to_le_bytes()); + assert_eq!(&packed[4..6], &0x3c00_u16.to_le_bytes()); + assert_eq!(&packed[6..8], &0x3c00_u16.to_le_bytes()); + } + + #[test] + fn lut_generation_is_deterministic_and_bounded() { + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 1.0, + }; + let a = GamutLut::generate(params); + let b = GamutLut::generate(params); + assert_eq!(a.texels.len(), LUT_SIZE_I * LUT_SIZE_C * LUT_SIZE_H); + assert_eq!(a, b); + for texel in &a.texels { + assert!( + texel[0] >= 0.0 && texel[0] <= 1.0, + "I out of range: {texel:?}" + ); + assert!( + texel[1] >= 0.0 && texel[1] <= 1.0, + "P out of range: {texel:?}" + ); + assert!( + texel[2] >= 0.0 && texel[2] <= 1.0, + "T out of range: {texel:?}" + ); + } + } + + #[test] + fn in_gamut_colors_stay_near_identity_in_dead_zone() { + // A low-chroma, mid-intensity color (inside both gamuts) maps close + // to itself: the dead-zone blend keeps k small. + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 1.0, + }; + // I=0.5 (mid), C=0.02, h=0 + let src = GamutState::new(params.source, params); + let dst = GamutState::new(params.target, params); + let h = 0.0_f32; + let src_peak = saturate(h, &src); + let dst_peak = saturate(h, &dst); + let soft = Softclip { + knee: 0.70, + _desat: 0.35, + }; + let max_c = src_peak[1].max(dst_peak[1]).max(1e-9); + let out = perceptual_map_at( + [0.5, 0.02, h], + &src, + &dst, + src_peak, + dst_peak, + max_c, + &soft, + params, + ); + assert!( + (out[0] - 0.5).abs() < 0.05 && (out[1] - 0.02).abs() < 0.1, + "out={out:?}" + ); + } + + #[test] + fn saturated_wide_gamut_color_is_brought_in_gamut() { + // BT.2020 primary green is far outside BT.709; the map must reduce + // its chroma without flipping the hue wildly. + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.5, + max_luma: 0.9, + }; + // I at mid-display, high chroma, hue of ~primary green in IPT. + let src = GamutState::new(params.source, params); + let dst = GamutState::new(params.target, params); + let h = 0.9_f32; + let src_peak = saturate(h, &src); + let dst_peak = saturate(h, &dst); + let soft = Softclip { + knee: 0.70, + _desat: 0.35, + }; + let max_c = src_peak[1].max(dst_peak[1]).max(1e-9); + let out = perceptual_map_at( + [0.7, 0.4, h], + &src, + &dst, + src_peak, + dst_peak, + max_c, + &soft, + params, + ); + let rgb = ipt2rgb([out[0], out[1], out[2]], &dst); + assert!( + rgb.iter().all(|v| *v >= -1e-4 && *v <= 1.0 + 1e-4), + "still out of gamut: {rgb:?}" + ); + } +} + +#[cfg(test)] +mod anchor_audit { + use super::*; + use crate::core::ColorPrimaries; + + #[test] + fn grayscale_anchors_survive_perceptual_map() { + // Black/mid-gray/white (chroma ≈ 0) must keep chroma near zero after + // the perceptual map; a LUT numeric bug shows up here first. + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 1.0, + }; + let src = GamutState::new(params.source, params); + let dst = GamutState::new(params.target, params); + // IPT mid-gray: any I, P = T = 0 + for i in [0.0, 0.1, 0.5, 0.9] { + let mapped = perceptual_map_at( + [i, 0.0, 0.0], + &src, + &dst, + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + 1.0, + &Softclip { + knee: 0.7, + _desat: 0.35, + }, + params, + ); + assert!( + mapped[1].abs() < 1e-3 && mapped[2].abs() < 1e-3, + "chroma leak on gray: {mapped:?}" + ); + } + } + + #[test] + fn lut_gray_column_keeps_chroma_near_zero_and_rises_with_i() { + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 1.0, + }; + let lut = GamutLut::generate(params); + // C = 0 (chroma axis 0), mid hue: chroma should stay near zero and + // I should increase along the lattice axis. + let mut previous_i = None; + for ix in [0, 8, 24, 47] { + let idx = 128 * LUT_SIZE_C * LUT_SIZE_I + ix; + let t = lut.texels[idx]; + // Texels pack (I, P + 0.5, T + 0.5); zero chroma is 0.5/0.5. + assert!( + (t[1] - 0.5).abs() < 1e-3 && (t[2] - 0.5).abs() < 1e-3, + "chroma leak at I index {ix}: {t:?}" + ); + if let Some(previous) = previous_i { + let current = t[0]; + assert!( + current >= previous, + "I not monotone at index {ix}: {current} < {previous}" + ); + } + previous_i = Some(t[0]); + } + } +} + +#[cfg(test)] +mod layout_audit { + use super::*; + use crate::core::ColorPrimaries; + + #[test] + fn texel_layout_is_i_major_within_h_slice() { + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: 1.0, + }; + let lut = GamutLut::generate(params); + // Lattice order is h-outer / C-mid / I-inner: walking one I step must + // stay inside the same (h, C) cell, so the mid-h gray column rises in + // I and stays chroma-free, while the C=1 row starts after LUT_SIZE_I. + let mid_h_start = 128 * LUT_SIZE_C * LUT_SIZE_I; + let first_i = lut.texels[mid_h_start]; + let last_i = lut.texels[mid_h_start + LUT_SIZE_I - 1]; + assert!( + last_i[0] >= first_i[0], + "I axis must increase along the inner lattice: {first_i:?} -> {last_i:?}" + ); + // Next C row begins exactly after one full I axis. + let next_c = lut.texels[mid_h_start + LUT_SIZE_I]; + assert!( + (next_c[1] - 0.5).abs() > (first_i[1] - 0.5).abs(), + "C=0 and C=1 rows should differ in chroma packing: {first_i:?} vs {next_c:?}" + ); + } +} + +#[cfg(test)] +mod dim_audit { + use super::*; + use crate::core::ColorPrimaries; + + #[test] + fn map_keeps_mid_brightness() { + // Tone-mapped mid-grays must not collapse to black: I stays near the + // input band even with a little chroma. + let params = GamutLutParams { + source: ColorPrimaries::Bt2020, + target: ColorPrimaries::Bt709, + min_luma: 0.0, + max_luma: pq_code_for_t(203.0), + }; + let src = GamutState::new(params.source, params); + let dst = GamutState::new(params.target, params); + for (i, c) in [ + (0.3, 0.0), + (0.3, 0.02), + (0.3, 0.05), + (0.3, 0.1), + (0.5, 0.05), + ] { + let h = 1.0_f32; + let sp = saturate(h, &src); + let dp = saturate(h, &dst); + let maxc = sp[1].max(dp[1]).max(1e-9); + let out = perceptual_map_at( + [i, c, h], + &src, + &dst, + sp, + dp, + maxc, + &Softclip { + knee: 0.7, + _desat: 0.35, + }, + params, + ); + assert!( + out[0] > i * 0.5, + "mid-gray I collapsed: in I={i} C={c} -> out {:?}", + out + ); + assert!(out[0].is_finite() && out[1].is_finite() && out[2].is_finite()); + } + } + + fn pq_code_for_t(n: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = (n / 10000.0).clamp(0.0, 1.0).powf(m1); + ((c1 + c2 * p) / (1.0 + c3 * p).max(1e-6)).powf(m2) + } +} diff --git a/crates/erika/src/renderer/metal/apple.rs b/crates/erika/src/renderer/metal/apple.rs index b881a15f..fb04e42e 100644 --- a/crates/erika/src/renderer/metal/apple.rs +++ b/crates/erika/src/renderer/metal/apple.rs @@ -32,7 +32,7 @@ use objc2_foundation::NSString; use objc2_metal::{ MTLBlendFactor, MTLBlendOperation, MTLClearColor, MTLCreateSystemDefaultDevice, MTLLoadAction, MTLOrigin, MTLPixelFormat, MTLRegion, MTLResourceOptions, MTLSize, MTLStorageMode, - MTLStoreAction, MTLTextureDescriptor, MTLTextureUsage, + MTLStoreAction, MTLTextureDescriptor, MTLTextureType, MTLTextureUsage, }; use objc2_metal::{ MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLCommandBufferStatus, MTLCommandEncoder, @@ -49,6 +49,9 @@ use objc2_quartz_core::{kCAContentsFormatRGBA8Uint, kCAContentsFormatRGBA16Float use crate::core::{ColorPrimaries, RendererResourceStats, SurfaceMetrics, TransferFunction}; use crate::danmaku::{DanmakuAtlasUpdate, DanmakuGlyphAtlas, DanmakuRenderPlan}; +use crate::renderer::gamut::{ + GamutLut, GamutLutJob, GamutLutParams, LUT_SIZE_C, LUT_SIZE_H, LUT_SIZE_I, pack_rgba16f, +}; use crate::renderer::metal::upscaler::LumaUpscaler; use crate::renderer::metal::{ ClearColor, DanmakuRenderFrame, ImportedVideoFormat, ImportedVideoFrameInfo, @@ -57,8 +60,9 @@ use crate::renderer::metal::{ VideoFrameTextureSource, VideoRenderFrame, fourcc_string, metal_drawable_pixel_format, metal_target_color, }; -use crate::renderer::pipeline::{ColorRange, LumaUpscalerMode, ToneMapOperator}; -use crate::renderer::pipeline::{SourceColorState, TargetColorState}; +use crate::renderer::output::negotiate_output_mode; +use crate::renderer::pipeline::{ColorRange, DoviUniforms, LumaUpscalerMode, ToneMapOperator}; +use crate::renderer::pipeline::{SourceColorState, TargetColorState, VideoRenderPipeline}; use crate::renderer::presentation::PresentationLayout as VideoPresentationLayout; use crate::subtitle::{AssColor, SubtitleAlphaBitmap}; use crate::trace; @@ -114,6 +118,49 @@ pub struct ImportedVideoFrameResult { pub textures: ImportedVideoFrameTextures, } +/// Identity of a perceptual gamut LUT: it is only valid for one +/// (source, target, target-black, target-peak) combination. The black/peak +/// pair sets the LUT's I range. A cached LUT whose key does not match the +/// current frame must never be bound — the shader would sample a LUT built for +/// a different display/gamut. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct GamutLutKey { + source: u32, + target: u32, + target_black_pq: u32, + target_peak_pq: u32, +} + +impl GamutLutKey { + /// The key for a frame's color pipeline. It deliberately reads only the + /// static target/primaries state: per-frame content brightness (Dolby + /// Vision L1, measured scene average) must not force LUT regeneration. + fn for_pipeline(pipeline: &VideoRenderPipeline) -> Self { + let packed = pipeline.gamut_primaries_code(); + let target_black_nits = pipeline.tone_map_extra()[2]; + Self { + source: packed >> 8, + target: packed & 0xff, + target_black_pq: quantize_luma_pq(target_black_nits), + target_peak_pq: quantize_luma_pq(pipeline.target.peak_nits), + } + } +} + +/// Quantize a luminance (nits) to its PQ code for the LUT cache key. The key +/// only has to change when the LUT's I axis changes, so 16-bit PQ resolution +/// is ample (and resolves the sub-1-nit target blacks of SDR targets, which a +/// linear nits quantization would collapse together). +fn quantize_luma_pq(nits: f32) -> u32 { + (pq_code_for_lut(nits) * 65535.0) as u32 +} + +/// Cache of the generated perceptual gamut LUT for [`GamutLutKey`]. +struct GamutLutCache { + key: GamutLutKey, + texture: Retained>, +} + pub struct MetalRendererImpl { device: Retained>, queue: Retained>, @@ -139,6 +186,13 @@ pub struct MetalRendererImpl { pending_gpu_timing: Option>>, stats: MetalRendererStats, layer_color_space_label: &'static str, + /// Perceptual gamut LUT (3D RGBA16Float) cached per (source, target, + /// peak) key; `None` when the fast path is in use. + gamut_lut: Option, + /// Background generation for a cache miss; the fast `gamut_compress` + /// path renders until the LUT lands. + gamut_lut_job: Option, + dummy_gamut_lut: Option>>, logged_first_video_frame: bool, } @@ -154,6 +208,24 @@ fn hdr_debug_enabled() -> bool { .unwrap_or(false) } +fn pq_code_for_lut(nits: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = (nits / 10000.0).clamp(0.0, 1.0).powf(m1); + ((c1 + c2 * p) / (1.0 + c3 * p).max(0.000_001)).powf(m2) +} + +fn code_to_primaries(code: u32) -> ColorPrimaries { + match code { + 1 => ColorPrimaries::Bt2020, + 2 => ColorPrimaries::DisplayP3, + _ => ColorPrimaries::Bt709, + } +} + impl MetalRendererImpl { pub fn new(config: MetalRendererConfig) -> Result { let device = MTLCreateSystemDefaultDevice().ok_or_else(|| { @@ -192,6 +264,9 @@ impl MetalRendererImpl { pending_gpu_timing: None, stats: MetalRendererStats::default(), layer_color_space_label: "unconfigured", + gamut_lut: None, + gamut_lut_job: None, + dummy_gamut_lut: None, logged_first_video_frame: false, }) } @@ -367,6 +442,78 @@ impl MetalRendererImpl { self.output_mode } + pub fn is_hdr10_pq(&self) -> bool { + self.output_mode.is_edr() + && matches!( + self.layer_color_space_label, + "itur-2100-pq" | "display-p3-pq" + ) + } + + /// EDR headroom of the display the player window is presented on. + /// + /// The *potential* value is used deliberately: it reports what the display + /// can do regardless of the current brightness setting, so playback does + /// not flip between SDR and EDR while the brightness slider moves. Falls + /// back to 1.0 (no EDR) when AppKit cannot answer. Resolved through the + /// layer's hosting window: `NSScreen.mainScreen` tracks the systemwide + /// key window, which belongs to a *different* app whenever this one is + /// inactive — negotiating from it then enables PQ passthrough while the + /// layer sits on an SDR display, rendering washed-out colors. AppKit + /// makes the hosting NSView the delegate of a view-assigned backing + /// layer, so prefer delegate→window→screen and fall back to mainScreen + /// when that chain is unavailable (e.g. detached layers). + #[cfg(target_os = "macos")] + fn display_edr_headroom(&self) -> f32 { + use objc2::msg_send; + use objc2::runtime::{AnyClass, AnyObject}; + use objc2::sel; + + unsafe { + let screen: Option> = self + .layer + .as_ref() + .and_then(|layer| { + let layer_obj: &AnyObject = layer; + if let Some(screen) = screen_from_layer_delegate(layer_obj) { + return Some(screen); + } + let mut curr: Option> = msg_send![layer_obj, superlayer]; + while let Some(parent) = curr { + if let Some(screen) = screen_from_layer_delegate(&parent) { + return Some(screen); + } + curr = msg_send![&parent, superlayer]; + } + if let Some(screen) = screen_from_app_windows(layer_obj) { + return Some(screen); + } + None + }) + .or_else(|| { + let class = AnyClass::get(c"NSScreen")?; + msg_send![class, mainScreen] + }); + let Some(screen) = screen else { + return 1.0; + }; + let selector = sel!(maximumPotentialExtendedDynamicRangeColorComponentValue); + let responds: bool = msg_send![&screen, respondsToSelector: selector]; + if !responds { + return 1.0; + } + let potential: f64 = msg_send![ + &screen, + maximumPotentialExtendedDynamicRangeColorComponentValue + ]; + if potential.is_finite() && potential > 0.0 { + potential as f32 + } else { + 1.0 + } + } + } + fn select_output_mode_for_source(&mut self, source: SourceColorState) { let source_is_hdr = source.is_hdr(); if source_is_hdr { @@ -378,7 +525,18 @@ impl MetalRendererImpl { let selected = if self.flutter_texture_attached { MetalOutputMode::Sdr } else { - self.requested_output_mode.resolve_for_source(source_is_hdr) + #[cfg(target_os = "macos")] + { + negotiate_output_mode( + self.requested_output_mode, + source_is_hdr, + self.display_edr_headroom(), + ) + } + #[cfg(not(target_os = "macos"))] + { + self.requested_output_mode.resolve_for_source(source_is_hdr) + } }; if selected != self.output_mode { self.set_output_mode(selected); @@ -586,6 +744,159 @@ impl MetalRendererImpl { Ok(()) } + /// Return a cached (or freshly generated) perceptual gamut LUT texture + /// for the frame's color pipeline, or `None` when the fast path is used + /// or the background generation is still pending. `Some` is returned only + /// when the texture matches the frame's key; callers must mask + /// `gamut_lut_enabled` off on `None` so the shader keeps the fast path + /// instead of sampling the 1x1x1 placeholder. + fn gamut_lut_texture( + &mut self, + frame: &VideoRenderFrame<'_>, + ) -> Result>>> { + if !frame.pipeline.gamut_lut_active() { + return Ok(None); + } + let key = GamutLutKey::for_pipeline(&frame.pipeline); + if let Some(cached) = &self.gamut_lut { + if cached.key == key { + return Ok(Some(cached.texture.clone())); + } + } + let params = GamutLutParams { + source: code_to_primaries(key.source), + target: code_to_primaries(key.target), + // Same target black the shader derives from tone_map_extra.z. + min_luma: pq_code_for_lut(frame.pipeline.tone_map_extra()[2]), + max_luma: pq_code_for_lut(frame.pipeline.target.peak_nits), + }; + let job_params = self + .gamut_lut_job + .as_ref() + .map(GamutLutJob::params) + .filter(|job_params| *job_params == params); + if job_params.is_none() { + // First request (or the key changed): spawn generation and keep + // the fast path for this frame. + self.gamut_lut_job = Some(GamutLutJob::spawn(params)); + return Ok(None); + } + let Some(lut) = self.gamut_lut_job.as_ref().and_then(GamutLutJob::poll) else { + return Ok(None); + }; + self.gamut_lut_job = None; + let texture = self.upload_gamut_lut(&lut)?; + self.gamut_lut = Some(GamutLutCache { + key, + texture: texture.clone(), + }); + Ok(Some(texture)) + } + + /// Pack (I, P+0.5, T+0.5) into a fresh 3D RGBA16Float texture. + fn upload_gamut_lut( + &mut self, + lut: &GamutLut, + ) -> Result>> { + // Metal lacks a 3D convenience constructor in this binding; build the + // descriptor from the 2D factory and switch the type/depth. + let descriptor = unsafe { + MTLTextureDescriptor::texture2DDescriptorWithPixelFormat_width_height_mipmapped( + MTLPixelFormat::RGBA16Float, + LUT_SIZE_I, + LUT_SIZE_C, + false, + ) + }; + descriptor.setTextureType(MTLTextureType::Type3D); + unsafe { + descriptor.setDepth(LUT_SIZE_H); + } + descriptor.setUsage(MTLTextureUsage::ShaderRead); + descriptor.setResourceOptions(MTLResourceOptions::StorageModeShared); + let texture = self + .device + .newTextureWithDescriptor(&descriptor) + .ok_or_else(|| { + PlayerError::Renderer( + "newTextureWithDescriptor (gamut LUT) returned nil".to_string(), + ) + })?; + let rgba16 = pack_rgba16f(&lut.texels, 1.0); + let region = MTLRegion { + origin: objc2_metal::MTLOrigin { x: 0, y: 0, z: 0 }, + size: objc2_metal::MTLSize { + width: LUT_SIZE_I, + height: LUT_SIZE_C, + depth: LUT_SIZE_H, + }, + }; + let bytes_per_row = LUT_SIZE_I * 4 * 2; // RGBA16F = 8 bytes/texel + unsafe { + texture.replaceRegion_mipmapLevel_slice_withBytes_bytesPerRow_bytesPerImage( + region, + 0, + 0, + NonNull::new(rgba16.as_ptr().cast::().cast_mut()) + .expect("gamut lut pointer is non-null"), + bytes_per_row, + LUT_SIZE_C * bytes_per_row, // one z-slice = width * bytes_per_row + ); + } + Ok(texture) + } + + fn dummy_gamut_lut_texture(&mut self) -> Result>> { + if let Some(dummy) = &self.dummy_gamut_lut { + return Ok(dummy.clone()); + } + let descriptor = unsafe { + MTLTextureDescriptor::texture2DDescriptorWithPixelFormat_width_height_mipmapped( + MTLPixelFormat::RGBA16Float, + 1, + 1, + false, + ) + }; + descriptor.setTextureType(MTLTextureType::Type3D); + unsafe { + descriptor.setDepth(1); + } + descriptor.setUsage(MTLTextureUsage::ShaderRead); + descriptor.setResourceOptions(MTLResourceOptions::StorageModeShared); + let texture = self + .device + .newTextureWithDescriptor(&descriptor) + .ok_or_else(|| { + PlayerError::Renderer( + "newTextureWithDescriptor (dummy gamut LUT) returned nil".to_string(), + ) + })?; + // 1 texel: (I=0, P=0, T=0) stored with P+0.5, T+0.5 -> [0.0, 0.5, 0.5, 1.0] + let dummy_data: [u16; 4] = [0x0000, 0x3800, 0x3800, 0x3c00]; + let region = MTLRegion { + origin: objc2_metal::MTLOrigin { x: 0, y: 0, z: 0 }, + size: objc2_metal::MTLSize { + width: 1, + height: 1, + depth: 1, + }, + }; + unsafe { + texture.replaceRegion_mipmapLevel_slice_withBytes_bytesPerRow_bytesPerImage( + region, + 0, + 0, + NonNull::new(dummy_data.as_ptr().cast::().cast_mut()) + .expect("dummy lut pointer is non-null"), + 8, + 8, + ); + } + self.dummy_gamut_lut = Some(texture.clone()); + Ok(texture) + } + pub fn render_video_frame(&mut self, frame: VideoRenderFrame<'_>) -> Result<()> { self.render_video_frame_inner(frame, None, None) } @@ -855,6 +1166,17 @@ impl MetalRendererImpl { "renderCommandEncoderWithDescriptor returned nil".to_string(), )); }; + // Resolve the LUT before building the uniforms: while a new key's + // background generation is pending, the shader keeps the fast + // gamut_compress path (the dummy texture holds texture 2). + // Readiness comes from the returned texture, not from the cache: + // a stale cache entry for another key must not enable the LUT. + let cached_gamut_lut = self.gamut_lut_texture(&frame)?; + let gamut_lut_ready = cached_gamut_lut.is_some(); + let gamut_lut = match cached_gamut_lut { + Some(lut) => lut, + None => self.dummy_gamut_lut_texture()?, + }; let uniforms = VideoUniforms { is_p010: matches!(frame.frame.info.format, ImportedVideoFormat::P010) as u32, full_range: matches!(frame.pipeline.source.range, ColorRange::Full) as u32, @@ -874,10 +1196,26 @@ impl MetalRendererImpl { ], luma_coefficients: luma_coefficients(frame.pipeline.luma_coefficients()), gamut_matrix_rows: frame.pipeline.gamut_matrix().row4s(), + ipt_matrix_rows: frame.pipeline.ipt_matrix_rows(), + tone_map_extra: frame.pipeline.tone_map_extra(), + tone_map_coeffs: frame.pipeline.tone_map_coeffs(), + gamut_lut_enabled: if gamut_lut_ready { + frame.pipeline.gamut_lut_active() as u32 + } else { + 0 + }, + gamut_primaries: frame.pipeline.gamut_primaries_code(), + gamut_reserved0: 0, + gamut_reserved1: 0, + dovi: DoviUniforms::of_for_representation( + &frame.pipeline.source, + matches!(frame.frame.info.format, ImportedVideoFormat::P010), + ), }; encoder.setRenderPipelineState(&pipeline); encoder.setFragmentTexture_atIndex(Some(luma), 0); encoder.setFragmentTexture_atIndex(Some(chroma), 1); + encoder.setFragmentTexture_atIndex(Some(&gamut_lut), 2); encoder.setFragmentSamplerState_atIndex(Some(&sampler), 0); encoder.setVertexBytes_length_atIndex( NonNull::new( @@ -1052,6 +1390,14 @@ impl MetalRendererImpl { )); }; + // Resolve the LUT before building the uniforms (same gating as + // the live path above). + let cached_gamut_lut = self.gamut_lut_texture(&frame)?; + let gamut_lut_ready = cached_gamut_lut.is_some(); + let gamut_lut = match cached_gamut_lut { + Some(lut) => lut, + None => self.dummy_gamut_lut_texture()?, + }; let uniforms = VideoUniforms { is_p010: matches!(frame.frame.info.format, ImportedVideoFormat::P010) as u32, full_range: matches!(frame.pipeline.source.range, ColorRange::Full) as u32, @@ -1071,10 +1417,26 @@ impl MetalRendererImpl { ], luma_coefficients: luma_coefficients(frame.pipeline.luma_coefficients()), gamut_matrix_rows: frame.pipeline.gamut_matrix().row4s(), + ipt_matrix_rows: frame.pipeline.ipt_matrix_rows(), + tone_map_extra: frame.pipeline.tone_map_extra(), + tone_map_coeffs: frame.pipeline.tone_map_coeffs(), + gamut_lut_enabled: if gamut_lut_ready { + frame.pipeline.gamut_lut_active() as u32 + } else { + 0 + }, + gamut_primaries: frame.pipeline.gamut_primaries_code(), + gamut_reserved0: 0, + gamut_reserved1: 0, + dovi: DoviUniforms::of_for_representation( + &frame.pipeline.source, + matches!(frame.frame.info.format, ImportedVideoFormat::P010), + ), }; encoder.setRenderPipelineState(&pipeline); encoder.setFragmentTexture_atIndex(Some(luma), 0); encoder.setFragmentTexture_atIndex(Some(chroma), 1); + encoder.setFragmentTexture_atIndex(Some(&gamut_lut), 2); encoder.setFragmentSamplerState_atIndex(Some(&sampler), 0); encoder.setVertexBytes_length_atIndex( NonNull::new( @@ -1208,7 +1570,15 @@ impl MetalRendererImpl { &plane.rgba, )?; let (x, y, width, height) = plane.scaled_rect(viewport_width, viewport_height); - let uniforms = OverlayUniforms::from_plane(x, y, width, height, layout, target); + let uniforms = OverlayUniforms::from_plane( + x, + y, + width, + height, + layout, + target, + self.output_mode.is_edr(), + ); unsafe { encoder.setRenderPipelineState(&pipeline); encoder.setFragmentTexture_atIndex(Some(&*texture), 0); @@ -1248,6 +1618,7 @@ impl MetalRendererImpl { atlas_height, layout, target, + self.output_mode.is_edr(), ); unsafe { encoder.setRenderPipelineState(&pipeline); @@ -1417,7 +1788,7 @@ impl MetalRendererImpl { viewport: layout.overlay_viewport(), target_transfer: transfer_code(target.transfer), _reserved0: 0, - ui_nits: [ui_reference_white_nits(target), 0.0, 0.0, 0.0], + ui_nits: ui_output_nits(target, self.output_mode.is_edr()), }; unsafe { encoder.setRenderPipelineState(pipeline); @@ -2144,6 +2515,86 @@ fn configure_layer_dynamic_range(layer: &CAMetalLayer, enabled: bool) { } } +#[cfg(target_os = "macos")] +unsafe fn screen_from_layer_delegate( + layer: &objc2::runtime::AnyObject, +) -> Option> { + use objc2::msg_send; + use objc2::rc::Retained; + use objc2::runtime::{AnyClass, AnyObject}; + let delegate: Option> = msg_send![layer, delegate]; + let delegate = delegate?; + let view_class = AnyClass::get(c"NSView")?; + let is_view: bool = msg_send![&delegate, isKindOfClass: view_class]; + if !is_view { + return None; + } + let window: Option> = msg_send![&delegate, window]; + let window = window?; + let screen: Option> = msg_send![&window, screen]; + screen +} + +#[cfg(target_os = "macos")] +unsafe fn screen_from_app_windows( + target_layer: &objc2::runtime::AnyObject, +) -> Option> { + use objc2::msg_send; + use objc2::rc::Retained; + use objc2::runtime::{AnyClass, AnyObject}; + let app_class = AnyClass::get(c"NSApplication")?; + let app: Option> = msg_send![app_class, sharedApplication]; + let app = app?; + let windows: Option> = msg_send![&app, windows]; + let windows = windows?; + let count: usize = msg_send![&windows, count]; + for i in 0..count { + let window: Retained = msg_send![&windows, objectAtIndex: i]; + let content_view: Option> = msg_send![&window, contentView]; + if let Some(content_view) = content_view { + if unsafe { view_contains_layer(&content_view, target_layer) } { + let screen: Option> = msg_send![&window, screen]; + return screen; + } + } + } + None +} + +#[cfg(target_os = "macos")] +unsafe fn view_contains_layer( + view: &objc2::runtime::AnyObject, + target_layer: &objc2::runtime::AnyObject, +) -> bool { + use objc2::msg_send; + use objc2::rc::Retained; + use objc2::runtime::AnyObject; + let view_layer: Option> = msg_send![view, layer]; + if let Some(vl) = view_layer { + if Retained::as_ptr(&vl) == target_layer as *const AnyObject { + return true; + } + let mut curr: Option> = msg_send![target_layer, superlayer]; + while let Some(parent) = curr { + if Retained::as_ptr(&parent) == Retained::as_ptr(&vl) { + return true; + } + curr = msg_send![&parent, superlayer]; + } + } + let subviews: Option> = msg_send![view, subviews]; + if let Some(subviews) = subviews { + let count: usize = msg_send![&subviews, count]; + for i in 0..count { + let subview: Retained = msg_send![&subviews, objectAtIndex: i]; + if unsafe { view_contains_layer(&subview, target_layer) } { + return true; + } + } + } + false +} + #[repr(C)] #[derive(Debug, Clone, Copy)] struct VideoUniforms { @@ -2160,6 +2611,14 @@ struct VideoUniforms { nits: [f32; 4], luma_coefficients: [f32; 4], gamut_matrix_rows: [[f32; 4]; 3], + ipt_matrix_rows: [[f32; 4]; 9], + tone_map_extra: [f32; 4], + tone_map_coeffs: [f32; 4], + gamut_lut_enabled: u32, + gamut_primaries: u32, + gamut_reserved0: u32, + gamut_reserved1: u32, + dovi: DoviUniforms, } fn metal_pixel_format(format: MetalDrawablePixelFormat) -> MTLPixelFormat { @@ -2264,11 +2723,37 @@ fn ui_reference_white_nits(target: TargetColorState) -> f32 { } } +/// `ui_nits.y`: non-zero when the drawable holds linear light (Apple EDR / +/// extended-linear output), in which case the UI color must be linearized +/// before compositing — the video pass writes linear values into the same +/// buffer. PQ targets encode the UI inside the shader and SDR targets +/// composite in the output transfer, so both keep 0 here. +fn ui_linear_reference_white_nits(target: TargetColorState, edr_output: bool) -> f32 { + if !edr_output || matches!(target.transfer, TransferFunction::Pq) { + 0.0 + } else { + target.reference_white_nits.max(1.0) + } +} + +fn ui_output_nits(target: TargetColorState, edr_output: bool) -> [f32; 4] { + [ + ui_reference_white_nits(target), + ui_linear_reference_white_nits(target, edr_output), + 0.0, + 0.0, + ] +} + fn tone_map_code(operator: ToneMapOperator) -> u32 { match operator { ToneMapOperator::Clip => 0, ToneMapOperator::Reinhard => 1, ToneMapOperator::Mobius => 2, + ToneMapOperator::Bt2390 => 3, + ToneMapOperator::Spline => 4, + ToneMapOperator::Bt2446a => 5, + ToneMapOperator::St209410 => 6, } } @@ -2330,6 +2815,7 @@ impl OverlayUniforms { height: u32, layout: VideoPresentationLayout, target: TargetColorState, + edr_output: bool, ) -> Self { Self { rect: layout.map_source_rect(x as f32, y as f32, width as f32, height as f32), @@ -2338,7 +2824,7 @@ impl OverlayUniforms { overlay_mode: 0, target_transfer: transfer_code(target.transfer), color: [1.0, 1.0, 1.0, 1.0], - ui_nits: [ui_reference_white_nits(target), 0.0, 0.0, 0.0], + ui_nits: ui_output_nits(target, edr_output), } } @@ -2349,6 +2835,7 @@ impl OverlayUniforms { atlas_height: usize, layout: VideoPresentationLayout, target: TargetColorState, + edr_output: bool, ) -> Self { let color = AssColor::from_libass_rgba(bitmap.color_rgba); let atlas_width = atlas_width.max(1) as f32; @@ -2375,7 +2862,7 @@ impl OverlayUniforms { color.blue as f32 / 255.0, color.alpha as f32 / 255.0, ], - ui_nits: [ui_reference_white_nits(target), 0.0, 0.0, 0.0], + ui_nits: ui_output_nits(target, edr_output), } } } @@ -2792,6 +3279,21 @@ struct VideoUniforms { float4 nits; float4 luma_coefficients; float4 gamut_matrix_rows[3]; + float4 ipt_matrix_rows[9]; + float4 tone_map_extra; + float4 tone_map_coeffs; + uint gamut_lut_enabled; + uint gamut_primaries; + uint gamut_reserved0; + uint gamut_reserved1; + float4 dovi_flags; + float4 dovi_pivots[6]; + float4 dovi_bounds[3]; + float4 dovi_coefficients[24]; + float4 dovi_mmr[144]; + float4 dovi_nonlinear_matrix[3]; + float4 dovi_nonlinear_offset; + float4 dovi_lms_matrix[3]; }; float source_peak_nits(constant VideoUniforms& uniforms) { @@ -2881,25 +3383,16 @@ float3 source_reference_to_nits(float3 rgb, constant VideoUniforms& uniforms) { return max(rgb, float3(0.0)) * source_reference_white_nits(uniforms); } -float3 tone_map_nits(float3 nits, constant VideoUniforms& uniforms) { - float source_peak = source_peak_nits(uniforms); - float target_peak = target_peak_nits(uniforms); - float3 x = max(nits, float3(0.0)) / target_peak; - float white = max(source_peak / target_peak, 1.0); - if (uniforms.tone_map == 1) { - float white2 = white * white; - return target_peak * clamp((x * (float3(1.0) + x / white2)) / (float3(1.0) + x), 0.0, 1.0); - } - if (uniforms.tone_map == 2) { - constexpr float knee = 0.75; - float denom = max(white - knee, 0.0001); - float3 t = clamp((x - float3(knee)) / denom, 0.0, 1.0); - float3 shoulder = knee + (1.0 - knee) * (float3(1.0) - pow(float3(1.0) - t, float3(2.0))); - return target_peak * mix(x, shoulder, step(float3(knee), x)); - } - return target_peak * clamp(x, 0.0, 1.0); +float pq_code(float nits) { + return pq_inverse_eotf(clamp(nits, 0.0, 10000.0) / 10000.0); } +float nits_from_pq(float code) { + return 10000.0 * pq_eotf(clamp(code, 0.0, 1.0)); +} + +// Simple primaries conversion (HDR10 output path); the tone-mapped path +// converts primaries inside the IPT roundtrip instead. float3 apply_gamut_map(float3 rgb, constant VideoUniforms& uniforms) { return float3( dot(uniforms.gamut_matrix_rows[0].xyz, rgb), @@ -2908,8 +3401,270 @@ float3 apply_gamut_map(float3 rgb, constant VideoUniforms& uniforms) { ); } +// libplacebo pl_smoothstep with arbitrary edge order (Metal smoothstep has +// undefined results when edge0 >= edge1, and libplacebo's knee tuning term +// deliberately uses reversed edges). +float sstep(float edge0, float edge1, float x) { + float t = clamp((x - edge0) / (edge1 - edge0), 0.0, 1.0); + return t * t * (3.0 - 2.0 * t); +} + +// libplacebo st2094_pick_knee evaluated on absolute PQ codes. The source +// pivot follows the scene average luminance when known and stays within +// [10%, 80%] of the range; the destination pivot rescales it into the output +// range and then adapts towards the 1:1 line (knee_adaptation 0.4). +float2 st2094_pick_knee(float src_min, float src_max, float src_avg, float dst_min, float dst_max) { + constexpr float knee_adaptation = 0.4; + constexpr float min_knee = 0.1; + constexpr float max_knee = 0.8; + constexpr float def_knee = 0.4; + float src_knee_min = mix(src_min, src_max, min_knee); + float src_knee_max = mix(src_min, src_max, max_knee); + float dst_knee_min = mix(dst_min, dst_max, min_knee); + float dst_knee_max = mix(dst_min, dst_max, max_knee); + float fallback = mix(src_min, src_max, def_knee); + float src_knee = clamp(src_avg > 0.0 ? src_avg : fallback, src_knee_min, src_knee_max); + float target = (src_knee - src_min) / max(src_max - src_min, 0.000001); + float adapted = mix(dst_min, dst_max, target); + float tuning = 1.0 - sstep(max_knee, def_knee, target) * sstep(min_knee, def_knee, target); + float adaptation = mix(knee_adaptation, 1.0, tuning); + float dst_knee = clamp(mix(src_knee, adapted, adaptation), dst_knee_min, dst_knee_max); + return float2(src_knee, dst_knee); +} + +// The tone-map curve evaluated on the IPT intensity axis (PQ codes), +// mirroring libplacebo's tone-map functions. `param` is the per-operator +// curve parameter from ToneMapConfig::curve_param (0 = operator default). +float tone_map_curve_pq(float x_in, float param, constant VideoUniforms& uniforms) { + float src_peak = source_peak_nits(uniforms); + float dst_peak = target_peak_nits(uniforms); + float src_avg = uniforms.tone_map_extra.y; + float dst_black = uniforms.tone_map_extra.z; + float in_min = 0.0; + float in_max = max(pq_code(src_peak), 0.000001); + float out_min = pq_code(dst_black); + float out_max = max(pq_code(dst_peak), 0.000001); + float out_range = max(out_max - out_min, 0.000001); + float x = clamp(x_in, in_min, in_max); + if (uniforms.tone_map == 0) { + // Clip: values within the source range pass through untouched. + return x; + } + if (uniforms.tone_map == 1) { + // Reinhard (output-relative, libplacebo pl_tone_map_reinhard). + float peak = in_max / out_range; + float contrast = param > 0.0 ? param : 0.5; + float offset = (1.0 - contrast) / max(contrast, 0.000001); + float scale = (peak + offset) / peak; + float t = x / out_range; + float mapped = t / (t + offset) * scale; + return mapped * out_range + out_min; + } + if (uniforms.tone_map == 2) { + // Mobius: Mobius transform with a 1:1 linear region below the knee. + float peak = in_max / out_range; + float j = param > 0.0 ? param : 0.3; + float a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + float b = (j * j - 2.0 * j * peak + peak) / max(peak - 1.0, 0.000001); + float scale = (b * b + 2.0 * b * j + j * j) / (b - a); + float t = x / out_range; + float mapped = t > j ? scale * (t + a) / (t + b) : t; + return mapped * out_range + out_min; + } + if (uniforms.tone_map == 3) { + // ITU-R BT.2390 EETF with black-point compensation (the libplacebo + // version also compensates target black; the earlier port skipped it). + float knee_offset = param > 0.0 ? param : 1.0; + float max_lum = clamp(out_max / in_max, 0.0, 1.0); + float min_lum = out_min / in_max; + float ks = (1.0 + knee_offset) * max_lum - knee_offset; + float bp = min(max(1.0 / max(min_lum, 0.000001), 0.0), 4.0); + float u = x / in_max; + if (ks < 1.0 && u > ks) { + float tb = (u - ks) / (1.0 - ks); + float tb2 = tb * tb; + float tb3 = tb2 * tb; + u = (2.0 * tb3 - 3.0 * tb2 + 1.0) * ks + + (tb3 - 2.0 * tb2 + tb) * (1.0 - ks) + + (-2.0 * tb3 + 3.0 * tb2) * max_lum; + } + if (u < 1.0) { + u = u + min_lum * pow(1.0 - u, bp); + float gain = max_lum < 1.0 + ? 1.0 / (1.0 + min_lum / max_lum * pow(1.0 - max_lum, bp)) + : 1.0; + u = gain * (u - min_lum) + min_lum; + } + return u * in_max; + } + if (uniforms.tone_map == 4) { + // Spline: perceptually linear single-pivot polynomial, the default + // tone map of libplacebo and mpv's gpu-next renderer. + float contrast = param > 0.0 ? param : 0.3; + float fallback_avg = clamp(0.4 * src_peak, 100.0, 400.0); + float effective_src_avg = src_avg > 0.0 ? src_avg : fallback_avg; + float2 knee = st2094_pick_knee( + in_min, + in_max, + pq_code(effective_src_avg), + out_min, + out_max + ); + float src_pivot = knee.x; + float dst_pivot = knee.y; + float slope0 = (dst_pivot - out_min) / max(src_pivot - in_min, 0.000001); + float ratio = clamp(1.5 * (in_max / out_max - 1.0), 0.2, 1.2); + float slope = pow(slope0, (1.0 - contrast) * ratio); + float in_min0 = in_min - src_pivot; + float in_max0 = in_max - src_pivot; + float out_min0 = out_min - dst_pivot; + float out_max0 = out_max - dst_pivot; + float pa = (out_min0 - slope * in_min0) / (in_min0 * in_min0); + float qa = (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0 * in_max0); + float qb = -3.0 * (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0); + float xr = x - src_pivot; + float mapped = xr > 0.0 + ? ((qa * xr + qb) * xr + slope) * xr + : (pa * xr + slope) * xr; + return mapped + dst_pivot; + } + if (uniforms.tone_map == 5) { + // ITU-R BT.2446 method A: Weber-law log compression from the source + // peak envelope and a standardized S-curve (mpv's recommended curve + // for well-mastered content). + float phdr = 1.0 + 32.0 * pow(src_peak / 10000.0, 1.0 / 2.4); + float psdr = 1.0 + 32.0 * pow(dst_peak / 10000.0, 1.0 / 2.4); + float t = pow(nits_from_pq(x) / max(src_peak, 0.000001), 1.0 / 2.4); + t = log(1.0 + (phdr - 1.0) * t) / log(phdr); + if (t <= 0.7399) { + t = 1.0770 * t; + } else if (t < 0.9909) { + t = (-1.1510 * t + 2.7811) * t - 0.6302; + } else { + t = 0.5 * t + 0.5; + } + t = (pow(psdr, t) - 1.0) / (psdr - 1.0); + // BT.1886 EOTF from the target black point and peak. + float lb = pow(max(dst_black, 0.0), 1.0 / 2.4); + float lw = pow(max(dst_peak, 0.0), 1.0 / 2.4); + return pq_code(pow((lw - lb) * t + lb, 2.4)); + } + // SMPTE ST 2094-10 (DolbyVision's dynamic-metadata curve): rational + // Mobius interpolation in absolute nits; coefficients are solved per + // frame on the CPU from the same scene pivot. + float c1 = uniforms.tone_map_coeffs.x; + float c2 = uniforms.tone_map_coeffs.y; + float c3 = uniforms.tone_map_coeffs.z; + float x_nits = nits_from_pq(x); + float y_nits = (c1 + c2 * x_nits) / max(1.0 + c3 * x_nits, 0.000001); + return pq_code(clamp(y_nits, 0.0, 10000.0)); +} + +float3 tone_map_nits( + float3 input_nits, + texture3d gamut_lut, + sampler video_sampler, + constant VideoUniforms& uniforms +) { + if (uniforms.target_transfer == 3) { + // HDR10 output: convert primaries by the gamut matrix and clamp to + // the PQ range (no tone mapping; the display does the HDR mapping). + return clamp(apply_gamut_map(max(input_nits, float3(0.0)) / source_reference_white_nits(uniforms), uniforms) + * source_reference_white_nits(uniforms), float3(0.0), float3(10000.0)); + } + // libplacebo color map: RGB in source primaries (absolute nits) to + // HPE-LMS, PQ-encode, IPT, map the intensity axis and apply the + // hue-preserving chroma rule, optionally sample the 3D gamut LUT in + // IPT space, then decode back to RGB in the target primaries (rows 6-8). + // The primaries conversion and gamut mapping happen in this single IPT pass. + float3 rgb = max(input_nits, float3(0.0)); + float3 lms = float3( + dot(uniforms.ipt_matrix_rows[0].xyz, rgb), + dot(uniforms.ipt_matrix_rows[1].xyz, rgb), + dot(uniforms.ipt_matrix_rows[2].xyz, rgb) + ); + float3 lmspq = float3(pq_code(lms.r), pq_code(lms.g), pq_code(lms.b)); + float3 ipt = float3( + dot(float3(0.4, 0.4, 0.2), lmspq), + dot(float3(4.455, -4.851, 0.396), lmspq), + dot(float3(0.8056, 0.3572, -1.1628), lmspq) + ); + float i_orig = ipt.x; + ipt.x = tone_map_curve_pq(ipt.x, uniforms.tone_map_extra.x, uniforms); + // Libplacebo's chroma rule: clamp the saturation boost when brightening + // and desaturate (by the cubic hull term) when the mapping darkens. + float2 hull = float2(i_orig, ipt.x); + float2 hull_c = ((hull - float2(6.0)) * hull + float2(9.0)) * hull; + float ratio = min(i_orig / max(ipt.x, 0.000001), hull_c.y / max(hull_c.x, 0.000001)); + ipt.yz = ipt.yz * ratio; + + if (uniforms.gamut_lut_enabled != 0) { + // I axis spans the target's [black, peak] in PQ codes, matching + // libplacebo's gamut.min_luma/max_luma (tone_map_extra.z is the + // target black in nits, the same value the LUT was generated for). + float lut_min = pq_code(uniforms.tone_map_extra.z); + float lut_max = max(pq_code(target_peak_nits(uniforms)), 0.000001); + float lut_range = max(lut_max - lut_min, 0.000001); + float3 pos = float3( + clamp((ipt.x - lut_min) / lut_range, 0.0, 1.0), + clamp(2.0 * length(ipt.yz), 0.0, 1.0), + 0.5 + 0.5 * atan2(ipt.z, ipt.y) / 3.14159265 + ); + // libplacebo's texel_scale: the lattice position must be remapped to + // the texel-center coordinate, otherwise the low end of the chroma + // axis (whose first texel stores zero chroma) leaks in and crushes + // saturation. + float3 idx = float3( + pos.x * (47.0 / 48.0) + 0.5 / 48.0, + pos.y * (31.0 / 32.0) + 0.5 / 32.0, + pos.z * (255.0 / 256.0) + 0.5 / 256.0 + ); + float3 sampled = gamut_lut.sample(video_sampler, idx).xyz; + ipt = float3(sampled.x, sampled.y - 0.5, sampled.z - 0.5); + } + + float3 lmspq_out = float3( + dot(float3(1.0, 0.0975689, 0.205226), ipt), + dot(float3(1.0, -0.113876, 0.133217), ipt), + dot(float3(1.0, 0.0326151, -0.676887), ipt) + ); + float3 lms_out = float3( + nits_from_pq(lmspq_out.r), + nits_from_pq(lmspq_out.g), + nits_from_pq(lmspq_out.b) + ); + return float3( + dot(uniforms.ipt_matrix_rows[6].xyz, lms_out), + dot(uniforms.ipt_matrix_rows[7].xyz, lms_out), + dot(uniforms.ipt_matrix_rows[8].xyz, lms_out) + ); +} + +// Hue-preserving gamut mapping: the linear gamut matrix can push highly +// saturated wide-gamut colors outside the target gamut (negative +// components). Blending those towards luma shifts hue — BT.2020 primary +// red picks up blue and turns pink. Instead blend towards the naive clip +// by an out-of-gamut smoothstep factor: slightly-out colors stay nearly +// intact, strongly-out primaries land on the pure target primary with +// their hue intact, matching mpv's perceptual gamut handling. Mirrors the +// WGSL/HLSL `gamut_compress` and the Rust reference in pipeline.rs tests. +// Brightness overshoot (> 1) is left for the tone map. +float3 gamut_compress(float3 rgb) { + float lo = min(rgb.r, min(rgb.g, rgb.b)); + float outness = max(-lo, 0.0); + float k = smoothstep(0.0, 1.0, outness); + return mix(rgb, clamp(rgb, 0.0, 1.0), k); +} + float3 target_nits_to_reference_linear(float3 nits, constant VideoUniforms& uniforms) { - return max(nits, float3(0.0)) / target_reference_white_nits(uniforms); + // libplacebo's encode maps [target black, target peak] onto [0, 1] where + // 1.0 is the target reference white, so the tone-map black-point + // compensation lands back on true black instead of lifting it. + float black = uniforms.tone_map_extra.z; + float peak = target_peak_nits(uniforms); + float range = max(peak - black, 0.0001); + return max(nits - float3(black), float3(0.0)) / range + * (range / target_reference_white_nits(uniforms)); } float3 target_reference_linear_to_output(float3 rgb, constant VideoUniforms& uniforms) { @@ -2949,17 +3704,25 @@ float4 final_output(float3 rgb, float alpha, constant VideoUniforms& uniforms) { return float4(premultiplied, alpha); } -float3 sdr_ui_color_to_target_output(float3 rgb, uint target_transfer, float reference_white_nits) { +// SDR composites the UI in the output transfer. HDR10 (target_transfer == 3) +// PQ-encodes it against ui_nits.x. An Apple EDR / extended-linear drawable +// holds linear light (ui_nits.y != 0), so the sRGB-encoded UI color must be +// linearized and scaled to the same reference white as the video pass. +float3 sdr_ui_color_to_target_output(float3 rgb, uint target_transfer, float4 ui_nits) { if (target_transfer == 3) { constexpr float pq_absolute_peak_nits = 10000.0; float3 linear = pow(max(rgb, float3(0.0)), float3(2.2)); - float3 nits = linear * max(reference_white_nits, 1.0); + float3 nits = linear * max(ui_nits.x, 1.0); return float3( pq_inverse_eotf(nits.r / pq_absolute_peak_nits), pq_inverse_eotf(nits.g / pq_absolute_peak_nits), pq_inverse_eotf(nits.b / pq_absolute_peak_nits) ); } + if (ui_nits.y > 0.0) { + float3 linear_rgb = pow(max(rgb, float3(0.0)), float3(2.2)); + return linear_rgb * (max(ui_nits.x, 1.0) / max(ui_nits.y, 1.0)); + } return rgb; } @@ -2969,6 +3732,12 @@ struct RangeExpandedYCbCr { }; RangeExpandedYCbCr expand_ycbcr_range(float y, float2 cbcr, constant VideoUniforms& uniforms) { + if (uniforms.is_p010 != 0) { + // P010 stores 10-bit codes as code << 6 in a 16-bit UNORM texture. + constexpr float p010_scale = 65535.0 / 65472.0; + y *= p010_scale; + cbcr *= p010_scale; + } if (uniforms.full_range != 0) { return RangeExpandedYCbCr { y, cbcr - float2(0.5) }; } @@ -2984,6 +3753,83 @@ RangeExpandedYCbCr expand_ycbcr_range(float y, float2 cbcr, constant VideoUnifor return RangeExpandedYCbCr { y, cbcr }; } +// Dolby Vision RPU reshaping, ported from libplacebo's `pl_shader_dovi_reshape` +// (the renderer behind mpv's Dolby Vision mapping). The base-layer signal is +// reshaped per component through piecewise polynomial/MMR curves selected by +// pivot comparison, where MMR coefficients mix all three raw components. +float3 dovi_reshaped_signal(float3 sig_in, constant VideoUniforms& uniforms) { + float3 sig = clamp(sig_in, 0.0, 1.0); + float result[3] = { sig.r, sig.g, sig.b }; + float4 flags = uniforms.dovi_flags; + for (uint c = 0u; c < 3u; c = c + 1u) { + uint segments = uint(flags[1u + c]); + if (segments == 0u) { + continue; + } + float s = result[c]; + uint index = 0u; + for (uint i = 0u; i < 7u; i = i + 1u) { + float4 pivot_row = uniforms.dovi_pivots[2u * c + i / 4u]; + float pivot = pivot_row[i % 4u]; + if (s >= pivot) { + index = index + 1u; + } + } + float4 coeff = uniforms.dovi_coefficients[8u * c + index]; + if (coeff.w < 0.5) { + s = (coeff.z * s + coeff.y) * s + coeff.x; + } else { + uint base = 48u * c + uint(coeff.y); + uint order = uint(coeff.w); + float4 sig_x = float4( + sig.x * sig.y, + sig.x * sig.z, + sig.y * sig.z, + sig.x * sig.y * sig.z + ); + s = coeff.x; + s = s + dot(uniforms.dovi_mmr[base].xyz, sig); + s = s + dot(uniforms.dovi_mmr[base + 1u], sig_x); + if (order >= 2u) { + float3 sig2 = sig * sig; + float4 sig_x2 = sig_x * sig_x; + s = s + dot(uniforms.dovi_mmr[base + 2u].xyz, sig2); + s = s + dot(uniforms.dovi_mmr[base + 3u], sig_x2); + if (order >= 3u) { + s = s + dot(uniforms.dovi_mmr[base + 4u].xyz, sig2 * sig); + s = s + dot(uniforms.dovi_mmr[base + 5u], sig_x2 * sig_x); + } + } + } + float4 bounds = uniforms.dovi_bounds[c]; + result[c] = clamp(s, bounds.x, bounds.y); + } + return float3(result[0], result[1], result[2]); +} + +// Reshaped nonlinear signal to PQ-encoded IPT via the RPU's ycc_to_rgb matrix +// and signal offsets. Applying the RPU offsets keeps integer offset codes +// exactly on sample codes (2^bits/(2^bits-1) folded in on the CPU). +float3 dovi_signal_to_pq_rgb(float3 sig, constant VideoUniforms& uniforms) { + float3 reshaped = dovi_reshaped_signal(sig, uniforms) - uniforms.dovi_nonlinear_offset.xyz; + return float3( + dot(uniforms.dovi_nonlinear_matrix[0].xyz, reshaped), + dot(uniforms.dovi_nonlinear_matrix[1].xyz, reshaped), + dot(uniforms.dovi_nonlinear_matrix[2].xyz, reshaped) + ); +} + +// Linearized BT.2020-referred HPE LMS back to linear RGB, using the composite +// of the fixed HPE inverse with the RPU's rgb_to_lms matrix (premultiplied on +// the CPU, matching libplacebo's dovi_lms2rgb). +float3 dovi_lms_to_rgb(float3 linear, constant VideoUniforms& uniforms) { + return float3( + dot(uniforms.dovi_lms_matrix[0].xyz, linear), + dot(uniforms.dovi_lms_matrix[1].xyz, linear), + dot(uniforms.dovi_lms_matrix[2].xyz, linear) + ); +} + vertex VertexOut erika_video_vertex( uint vertex_id [[vertex_id]], constant VideoUniforms& uniforms [[buffer(0)]]) { @@ -3014,6 +3860,7 @@ fragment float4 erika_video_fragment( VertexOut in [[stage_in]], texture2d luma_texture [[texture(0)]], texture2d chroma_texture [[texture(1)]], + texture3d gamut_lut [[texture(2)]], sampler video_sampler [[sampler(0)]], constant VideoUniforms& uniforms [[buffer(0)]]) { bool packed_alpha = uniforms.video_alpha_mode == 1; @@ -3021,24 +3868,39 @@ fragment float4 erika_video_fragment( ? float2(in.tex_coord.x * 0.5, in.tex_coord.y) : in.tex_coord; float2 alpha_coord = float2(0.5 + in.tex_coord.x * 0.5, in.tex_coord.y); - float y = luma_texture.sample(video_sampler, color_coord).r; - float2 cbcr = chroma_texture.sample(video_sampler, color_coord).rg; - RangeExpandedYCbCr expanded = expand_ycbcr_range(y, cbcr, uniforms); - y = expanded.y; - cbcr = expanded.cbcr; - - float kr = uniforms.luma_coefficients.x; - float kg = max(uniforms.luma_coefficients.y, 0.000001); - float kb = uniforms.luma_coefficients.z; + float y_sample = luma_texture.sample(video_sampler, color_coord).r; + float2 cbcr_sample = chroma_texture.sample(video_sampler, color_coord).rg; + bool dovi_enabled = uniforms.dovi_flags.x != 0.0; float3 rgb; - rgb.r = y + 2.0 * (1.0 - kr) * cbcr.y; - rgb.b = y + 2.0 * (1.0 - kb) * cbcr.x; - rgb.g = (y - kr * rgb.r - kb * rgb.b) / kg; + if (dovi_enabled) { + // The base layer carries the raw 12-bit DV signal (10-bit container, + // full range); range expansion and the YCbCr matrix are replaced by + // the RPU reshaping + ycc_to_rgb path. + float3 sig = float3(y_sample, cbcr_sample.x, cbcr_sample.y); + if (uniforms.is_p010 != 0) { + sig *= 65535.0 / 65472.0; + } + rgb = dovi_signal_to_pq_rgb(sig, uniforms); + } else { + RangeExpandedYCbCr expanded = expand_ycbcr_range(y_sample, cbcr_sample, uniforms); + float y = expanded.y; + float2 cbcr = expanded.cbcr; + + float kr = uniforms.luma_coefficients.x; + float kg = max(uniforms.luma_coefficients.y, 0.000001); + float kb = uniforms.luma_coefficients.z; + rgb.r = y + 2.0 * (1.0 - kr) * cbcr.y; + rgb.b = y + 2.0 * (1.0 - kb) * cbcr.x; + rgb.g = (y - kr * rgb.r - kb * rgb.b) / kg; + } rgb = transfer_to_source_reference_linear(rgb, uniforms); - rgb = apply_gamut_map(rgb, uniforms); + if (dovi_enabled) { + rgb = dovi_lms_to_rgb(rgb, uniforms); + } rgb = source_reference_to_nits(rgb, uniforms); - rgb = tone_map_nits(rgb, uniforms); + rgb = tone_map_nits(rgb, gamut_lut, video_sampler, uniforms); rgb = target_nits_to_reference_linear(rgb, uniforms); + rgb = gamut_compress(rgb); rgb = target_reference_linear_to_output(rgb, uniforms); float alpha = 1.0; if (packed_alpha) { @@ -3096,14 +3958,14 @@ fragment float4 erika_overlay_fragment( float3 rgb = sdr_ui_color_to_target_output( uniforms.color.rgb, uniforms.target_transfer, - uniforms.ui_nits.x + uniforms.ui_nits ); return float4(rgb, uniforms.color.a * sampled.r); } sampled.rgb = sdr_ui_color_to_target_output( sampled.rgb, uniforms.target_transfer, - uniforms.ui_nits.x + uniforms.ui_nits ); return sampled; } @@ -3173,7 +4035,7 @@ fragment float4 erika_danmaku_batch_fragment( float3 rgb = sdr_ui_color_to_target_output( in.color.rgb, uniforms.target_transfer, - uniforms.ui_nits.x + uniforms.ui_nits ); return float4(rgb, in.color.a * mask); } @@ -3305,10 +4167,12 @@ fn create_plane_texture( mod tests { use super::{ DANMAKU_FILL_ATLAS_TEXTURE, DANMAKU_OUTLINE_ATLAS_TEXTURE, DanmakuBatchInstance, - VIDEO_SHADER_SOURCE, for_each_ordered_danmaku_instance, metal_pixel_format, + VIDEO_SHADER_SOURCE, for_each_ordered_danmaku_instance, metal_pixel_format, ui_output_nits, }; + use crate::core::{ColorPrimaries, TransferFunction}; use crate::danmaku::DanmakuGlyphInstance; use crate::renderer::metal::MetalDrawablePixelFormat; + use crate::renderer::pipeline::TargetColorState; use objc2_metal::MTLPixelFormat; #[test] @@ -3348,6 +4212,153 @@ mod tests { assert!(VIDEO_SHADER_SOURCE.contains("return final_output(rgb, alpha, uniforms)")); } + #[test] + fn overlay_shader_linearizes_the_ui_for_extended_linear_output() { + // The EDR/extended-linear drawable holds linear light, so the overlay + // and danmaku passes must not composite the sRGB-encoded UI color + // directly (ui_nits.y carries the scene-linear reference white). + assert!(VIDEO_SHADER_SOURCE.contains("if (ui_nits.y > 0.0)")); + assert!( + VIDEO_SHADER_SOURCE + .contains("float3 linear_rgb = pow(max(rgb, float3(0.0)), float3(2.2))") + ); + assert!(VIDEO_SHADER_SOURCE.contains("max(ui_nits.x, 1.0) / max(ui_nits.y, 1.0)")); + } + + #[test] + fn gamut_lut_shader_samples_the_target_black_to_peak_axis() { + // libplacebo's LUT I axis is [target black, target peak]; sampling + // `ipt.x / peak` again would diverge from the generated LUT. The + // lattice position must also be remapped to the texel-center + // coordinate (libplacebo's `texel_scale`), or the zero-chroma texel at + // the low end of the C axis crushes saturation. + assert!(VIDEO_SHADER_SOURCE.contains("float lut_min = pq_code(uniforms.tone_map_extra.z)")); + assert!(VIDEO_SHADER_SOURCE.contains("clamp((ipt.x - lut_min) / lut_range, 0.0, 1.0)")); + assert!(VIDEO_SHADER_SOURCE.contains("pos.y * (31.0 / 32.0) + 0.5 / 32.0")); + assert!(VIDEO_SHADER_SOURCE.contains("pos.x * (47.0 / 48.0) + 0.5 / 48.0")); + } + + #[test] + fn ui_output_nits_flags_only_extended_linear_output() { + let sdr = TargetColorState::sdr(ColorPrimaries::Bt709); + assert_eq!(ui_output_nits(sdr, false), [100.0, 0.0, 0.0, 0.0]); + + let hdr10 = TargetColorState::hdr10(ColorPrimaries::Bt2020); + assert_eq!(ui_output_nits(hdr10, false), [203.0, 0.0, 0.0, 0.0]); + + // `metal_target_color` shapes a non-PQ EDR target with a 100-nit + // reference white; the UI is then linearized against it. + let edr = TargetColorState { + primaries: ColorPrimaries::Bt709, + transfer: TransferFunction::Srgb, + peak_nits: 400.0, + reference_white_nits: 100.0, + edr_headroom: 4.0, + }; + assert_eq!(ui_output_nits(edr, true), [100.0, 100.0, 0.0, 0.0]); + assert_eq!(ui_output_nits(edr, false), [100.0, 0.0, 0.0, 0.0]); + + // An explicit EDR request clamped to headroom 1.0 still renders the + // linear drawable, so the flag must come from the output mode rather + // than from the headroom. + let edr_unit = TargetColorState { + edr_headroom: 1.0, + peak_nits: 100.0, + ..edr + }; + assert_eq!(ui_output_nits(edr_unit, true), [100.0, 100.0, 0.0, 0.0]); + + // A PQ EDR target encodes the UI inside the shader (branch on + // target_transfer), so it must not also set the linear flag. + let edr_pq = TargetColorState { + primaries: ColorPrimaries::Bt2020, + transfer: TransferFunction::Pq, + peak_nits: 10_000.0, + reference_white_nits: 203.0, + edr_headroom: 4.0, + }; + assert_eq!(ui_output_nits(edr_pq, true), [203.0, 0.0, 0.0, 0.0]); + } + + #[test] + fn gamut_lut_key_tracks_only_static_pipeline_state() { + // Regression guard for the placeholder-LUT black frames: the cache may + // only be reused when the key equals the current frame's key, so the + // key must change for anything that changes the LUT (target black and + // peak, source/target primaries) and must not change for per-frame + // content brightness. + use super::{GamutLutKey, quantize_luma_pq}; + use crate::renderer::pipeline::{SourceColorState, ToneMapConfig, VideoRenderPipeline}; + + let source = |peak_nits: f32| { + SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .nominal_peak_nits(peak_nits) + }; + let sdr_target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let key = |source: SourceColorState, target: TargetColorState| { + GamutLutKey::for_pipeline(&VideoRenderPipeline::new(source, target)) + }; + + let base = key(source(1000.0), sdr_target); + // Dolby Vision L1 / measured brightness move `nominal_peak_nits` every + // frame; the LUT must not be regenerated for them. + assert_eq!( + base, + key(source(4000.0), sdr_target), + "a per-frame source peak must not invalidate the LUT" + ); + // A different target peak (EDR headroom) changes the LUT's I axis. + let edr_target = TargetColorState { + primaries: ColorPrimaries::Bt709, + transfer: TransferFunction::Srgb, + peak_nits: 400.0, + reference_white_nits: 100.0, + edr_headroom: 4.0, + }; + assert_ne!( + base, + key(source(1000.0), edr_target), + "a stale cache key must never match the current frame" + ); + // So does the target black (`contrast_ratio`): 1000:1 vs 10000:1 on the + // same 203-nit target must not reuse the LUT. + let contrast_source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + let target_black = |contrast_ratio: f32| VideoRenderPipeline { + tone_map: ToneMapConfig { + contrast_ratio, + ..ToneMapConfig::default() + }, + ..VideoRenderPipeline::new(contrast_source, sdr_target) + }; + assert_ne!( + GamutLutKey::for_pipeline(&target_black(0.0)), + GamutLutKey::for_pipeline(&target_black(10_000.0)), + "a different target black changes the LUT's I axis" + ); + // Primaries on either side change the mapping. + assert_ne!( + base, + key( + source(1000.0), + TargetColorState::sdr_tone_map_target(ColorPrimaries::DisplayP3) + ) + ); + assert_ne!( + base, + key( + SourceColorState::new(ColorPrimaries::DisplayP3, TransferFunction::Pq), + sdr_target + ) + ); + // The luma quantization is monotonic and stable, and keeps sub-1-nit + // target blacks distinguishable. + assert_eq!(quantize_luma_pq(203.0), quantize_luma_pq(203.0)); + assert!(quantize_luma_pq(203.0) < quantize_luma_pq(400.0)); + assert!(quantize_luma_pq(0.1) < quantize_luma_pq(0.203)); + assert_eq!(quantize_luma_pq(10_000.0), 65535); + assert_eq!(quantize_luma_pq(-1.0), 0); + } + #[test] fn video_shader_reconstructs_packed_alpha_as_premultiplied_output() { assert!(VIDEO_SHADER_SOURCE.contains("uniforms.video_alpha_mode == 1")); @@ -3367,7 +4378,6 @@ mod tests { let decode = VIDEO_SHADER_SOURCE .find("rgb = transfer_to_source_reference_linear") .unwrap(); - let gamut = VIDEO_SHADER_SOURCE.find("rgb = apply_gamut_map").unwrap(); let source_nits = VIDEO_SHADER_SOURCE .find("rgb = source_reference_to_nits") .unwrap(); @@ -3378,20 +4388,27 @@ mod tests { let output = VIDEO_SHADER_SOURCE .find("rgb = target_reference_linear_to_output") .unwrap(); - assert!(decode < gamut); - assert!(gamut < source_nits); + assert!(decode < source_nits); assert!(source_nits < tone_map); assert!(tone_map < target_reference); assert!(target_reference < output); } #[test] - fn video_shader_applies_gamut_matrix_before_tone_mapping() { + fn video_shader_runs_the_ipt_tone_map_before_gamut_compression() { assert!(VIDEO_SHADER_SOURCE.contains("gamut_matrix_rows")); assert!(VIDEO_SHADER_SOURCE.contains("apply_gamut_map")); - let gamut = VIDEO_SHADER_SOURCE.find("rgb = apply_gamut_map").unwrap(); + assert!(VIDEO_SHADER_SOURCE.contains("ipt_matrix_rows")); + assert!(VIDEO_SHADER_SOURCE.contains("st2094_pick_knee")); + assert!(VIDEO_SHADER_SOURCE.contains("tone_map_curve_pq")); + // The primaries conversion happens inside the tone map (IPT + // roundtrip), so the fragment only calls tone_map_nits then the + // compression pass. let tone_map = VIDEO_SHADER_SOURCE.find("rgb = tone_map_nits").unwrap(); - assert!(gamut < tone_map); + let compress = VIDEO_SHADER_SOURCE.find("rgb = gamut_compress").unwrap(); + assert!(tone_map < compress); + // The separate matrix call site from the old flow is gone. + assert!(!VIDEO_SHADER_SOURCE.contains("rgb = apply_gamut_map")); } #[test] @@ -3529,6 +4546,7 @@ mod tests { 100, layout, crate::renderer::pipeline::TargetColorState::default(), + false, ); assert_eq!(uniforms.rect, [12.0, 34.0, 56.0, 78.0]); @@ -3626,7 +4644,7 @@ mod tests { #[test] fn video_uniforms_keep_float4_fields_aligned() { - assert_eq!(std::mem::size_of::(), 144); + assert_eq!(std::mem::size_of::(), 3296); assert_eq!(std::mem::offset_of!(super::VideoUniforms, edr_output), 20); assert_eq!(std::mem::offset_of!(super::VideoUniforms, rect), 32); assert_eq!(std::mem::offset_of!(super::VideoUniforms, viewport), 48); @@ -3639,6 +4657,23 @@ mod tests { std::mem::offset_of!(super::VideoUniforms, gamut_matrix_rows), 96 ); + assert_eq!( + std::mem::offset_of!(super::VideoUniforms, ipt_matrix_rows), + 144 + ); + assert_eq!( + std::mem::offset_of!(super::VideoUniforms, tone_map_extra), + 288 + ); + assert_eq!( + std::mem::offset_of!(super::VideoUniforms, tone_map_coeffs), + 304 + ); + assert_eq!( + std::mem::offset_of!(super::VideoUniforms, gamut_lut_enabled), + 320 + ); + assert_eq!(std::mem::offset_of!(super::VideoUniforms, dovi), 336); } #[test] @@ -3669,6 +4704,7 @@ mod tests { 108, layout, crate::renderer::pipeline::TargetColorState::default(), + false, ); assert_eq!(uniforms.viewport, [1000.0, 1000.0]); diff --git a/crates/erika/src/renderer/metal/mod.rs b/crates/erika/src/renderer/metal/mod.rs index c652a555..2fc712d5 100644 --- a/crates/erika/src/renderer/metal/mod.rs +++ b/crates/erika/src/renderer/metal/mod.rs @@ -11,7 +11,8 @@ use crate::ffmpeg::{Frame, PlanarFrame}; use crate::overlay::OverlayFrame; pub use crate::renderer::pipeline::LumaUpscalerMode; use crate::renderer::pipeline::{ - ColorRange, HdrMetadata, MatrixCoefficients, SourceColorState, VideoRenderPipeline, + ColorRange, DoviSourceMetadata, HdrMetadata, MatrixCoefficients, SourceColorState, + VideoRenderPipeline, }; use crate::trace; @@ -142,7 +143,13 @@ pub(crate) fn metal_target_color( ) -> crate::renderer::pipeline::TargetColorState { match mode { MetalOutputMode::Sdr | MetalOutputMode::Auto { .. } => { - crate::renderer::pipeline::TargetColorState::sdr(ColorPrimaries::Bt709) + if source.is_hdr() { + crate::renderer::pipeline::TargetColorState::sdr_tone_map_target( + ColorPrimaries::Bt709, + ) + } else { + crate::renderer::pipeline::TargetColorState::sdr(ColorPrimaries::Bt709) + } } MetalOutputMode::AppleEdr { headroom } | MetalOutputMode::ExtendedLinear { headroom } => { #[cfg(any(target_os = "ios", target_os = "tvos"))] @@ -306,14 +313,28 @@ impl ImportedVideoFrame { range: ColorRange, matrix: MatrixCoefficients, hdr_metadata: Option, + dovi_metadata: Option, ) { self.set_source_color( SourceColorState::new(primaries, transfer) .range(range) .matrix(matrix) - .hdr_metadata(hdr_metadata), + .hdr_metadata(hdr_metadata) + .dovi(dovi_metadata), ); } + + /// Attach a presenter-measured scene-average luminance so the tone map's + /// pivot follows the content like Dolby Vision L1 would. Only meaningful + /// when the source carries no dynamic L1 metadata of its own. + pub fn set_measured_scene_avg(&mut self, scene_avg_nits: Option) { + if scene_avg_nits.is_none() { + return; + } + let mut source = self.source_color(); + source = source.measured_scene_avg_nits(scene_avg_nits); + self.set_source_color(source); + } } pub struct VideoRenderFrame<'a> { @@ -612,6 +633,7 @@ impl MetalRenderer { frame.color_range(), frame.matrix_coefficients(), frame.hdr_metadata(), + frame.dovi_metadata(), ); Ok(imported) } @@ -853,7 +875,8 @@ impl RendererBackend for MetalRenderer { "Metal renderer received a non-VideoToolbox hardware payload".to_string(), ) })?; - let imported = self.import_player_frame(decoded)?; + let mut imported = self.import_player_frame(decoded)?; + imported.set_measured_scene_avg(frame.scene_avg_nits); if !decoded.is_videotoolbox() { self.software_upload_counter = self.software_upload_counter.wrapping_add(1); } @@ -1079,9 +1102,15 @@ impl RendererBackend for MetalRenderer { #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "tvos")))] let active_output_mode = self.output_mode.resolve_for_source(false); let extended = attached && active_output_mode.is_edr(); + #[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos"))] + let is_hdr10_pq = attached && self.inner.is_hdr10_pq(); + #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "tvos")))] + let is_hdr10_pq = false; OutputRuntimeStatus { requested_mode: self.output_mode, - active_encoding: if extended { + active_encoding: if is_hdr10_pq { + ActiveOutputEncoding::Hdr10Pq + } else if extended { ActiveOutputEncoding::AppleEdr } else { ActiveOutputEncoding::SdrSrgb @@ -1093,13 +1122,15 @@ impl RendererBackend for MetalRenderer { }, native_data_space: -1, requested_headroom: self.output_mode.headroom(), - active_headroom: if extended { + active_headroom: if is_hdr10_pq { + 10_000.0 / 203.0 + } else if extended { active_output_mode.headroom() } else { 1.0 }, active_headroom_known: attached, - extended_linear_active: extended, + extended_linear_active: extended && !is_hdr10_pq, fallback_reason: OutputFallbackReason::None, fallback_count: 0, data_space_failures: 0, @@ -1346,6 +1377,7 @@ mod tests { ColorRange::Limited, MatrixCoefficients::Bt709, None, + None, ); assert_eq!(frame.source_color().range, ColorRange::Full); @@ -1372,6 +1404,7 @@ mod tests { ColorRange::Limited, MatrixCoefficients::Bt2020NonConstantLuminance, Some(metadata), + None, ); assert_eq!(frame.source_color().hdr_metadata, Some(metadata)); diff --git a/crates/erika/src/renderer/output.rs b/crates/erika/src/renderer/output.rs index 07ffae9b..6d185a9b 100644 --- a/crates/erika/src/renderer/output.rs +++ b/crates/erika/src/renderer/output.rs @@ -1,5 +1,5 @@ use crate::core::{ColorPrimaries, TransferFunction}; -use crate::renderer::pipeline::TargetColorState; +use crate::renderer::pipeline::{SourceColorState, TargetColorState}; /// Platform-neutral output request selected by the embedder. /// @@ -77,6 +77,57 @@ impl Default for OutputMode { } } +/// Clamp a resolved output mode to what the attached display can actually +/// present. +/// +/// A compositor cannot give an EDR/PQ layer meaningful headroom on a display +/// without EDR support: the HDR signal gets forced into the SDR range and the +/// picture looks blown out. HDR sources therefore tone map to SDR there, and +/// EDR requests are capped at the display's real headroom. +pub fn clamp_output_mode_to_display(mode: OutputMode, display_headroom: f32) -> OutputMode { + match mode { + OutputMode::AppleEdr { headroom } => { + if display_headroom <= 1.05 { + OutputMode::Sdr + } else { + OutputMode::apple_edr(headroom.min(display_headroom)) + } + } + OutputMode::ExtendedLinear { headroom } => { + OutputMode::extended_linear(headroom.min(display_headroom.max(1.0))) + } + other => other, + } +} + +/// Negotiate the output mode for a source against the display the playback +/// window is presented on. +/// +/// `Auto` is the per-screen negotiation path: an HDR source promotes to EDR +/// using the presenting display's *real* headroom when the screen can present +/// EDR at all, and stays on the SDR tone mapping path when it cannot — so the +/// same embedder configuration yields HDR on an HDR display and correctly +/// tone-mapped SDR on a non-HDR one. The embedder's configured headroom (when +/// set above SDR white) caps the negotiated value. Explicit EDR/ExtendedLinear +/// requests are clamped to the display's capability instead. +pub fn negotiate_output_mode( + requested: OutputMode, + source_is_hdr: bool, + display_headroom: f32, +) -> OutputMode { + match requested { + OutputMode::Auto { headroom } => { + if !source_is_hdr || display_headroom <= 1.05 { + OutputMode::Sdr + } else { + let cap = if headroom > 1.0 { headroom } else { f32::MAX }; + OutputMode::apple_edr(display_headroom.min(cap)) + } + } + other => clamp_output_mode_to_display(other, display_headroom), + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OutputColorSpace { Srgb, @@ -295,6 +346,22 @@ impl OutputDescription { pub fn target_transfer(self) -> TransferFunction { self.target.transfer } + + /// The color target an HDR source should render against for this output. + /// + /// An HDR source on a plain SDR output is tone-mapped against the + /// 203-nit HDR reference-white convention (`sdr_tone_map_target`) instead + /// of the 100-nit SDR mastering target. Extended-linear (EDR/scRGB) + /// outputs encode linear light above 1.0 and are *not* SDR outputs even + /// though their transfer reads `Srgb`, so they keep their negotiated + /// target; HDR10 PQ passthrough does its own conversion and is untouched. + pub fn tone_map_target_for(self, source: &SourceColorState) -> TargetColorState { + if source.is_hdr() && self.color_space == OutputColorSpace::Srgb { + TargetColorState::sdr_tone_map_target(self.target.primaries) + } else { + self.target + } + } } fn normalized_headroom(headroom: f32) -> f32 { @@ -327,6 +394,32 @@ mod tests { assert!(description.extended_linear); } + #[test] + fn tone_map_target_swaps_only_on_sdr_outputs() { + let hdr = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + let sdr = SourceColorState::new(ColorPrimaries::Bt709, TransferFunction::Srgb); + + // HDR source on an SDR output: the 203-nit tone-map target. + let swapped = OutputDescription::sdr().tone_map_target_for(&hdr); + assert_eq!(swapped.peak_nits, 203.0); + assert_eq!(swapped.reference_white_nits, 203.0); + + // HDR source on extended-linear/EDR outputs: the negotiated target is + // kept (its transfer is also Srgb, so the check must not key on it). + for mode in [OutputMode::extended_linear(4.0), OutputMode::apple_edr(2.0)] { + let description = OutputDescription::requested(mode); + let kept = description.tone_map_target_for(&hdr); + assert_eq!(kept, description.target, "mode {mode:?}"); + assert!(kept.peak_nits > 203.0); + } + + // HDR10 PQ passthrough and SDR sources are untouched. + let hdr10 = OutputDescription::hdr10().tone_map_target_for(&hdr); + assert_eq!(hdr10, OutputDescription::hdr10().target); + let sdr_target = OutputDescription::sdr().tone_map_target_for(&sdr); + assert_eq!(sdr_target, OutputDescription::sdr().target); + } + #[test] fn output_mode_rejects_invalid_headroom() { assert_eq!(OutputMode::extended_linear(f32::NAN).headroom(), 1.0); @@ -343,10 +436,17 @@ mod tests { automatic.resolve_for_source(true), OutputMode::apple_edr(4.0) ); + // An unconfigured (1.0) headroom keeps HDR sources on the SDR tone + // mapping path; EDR promotion requires the embedder to actually + // request headroom above SDR white. assert_eq!( OutputMode::auto(1.0).resolve_for_source(true), OutputMode::Sdr ); + assert_eq!( + OutputMode::auto(1.0).resolve_for_source(false), + OutputMode::Sdr + ); assert_eq!( OutputDescription::requested(automatic), OutputDescription::sdr() @@ -364,6 +464,71 @@ mod tests { assert_eq!(android.target.edr_headroom, 4.0); } + #[test] + fn auto_negotiates_hdr_per_presenting_display() { + // EDR display: an HDR source promotes to EDR using the display's real + // headroom; an unconfigured host headroom (1.0) defers to the display. + assert_eq!( + negotiate_output_mode(OutputMode::auto(1.0), true, 2.0), + OutputMode::apple_edr(2.0) + ); + // A configured headroom above SDR white caps the negotiated value. + assert_eq!( + negotiate_output_mode(OutputMode::auto(1.5), true, 2.0), + OutputMode::apple_edr(1.5) + ); + // SDR display: HDR sources stay on the tone mapping path, however + // large the embedder's headroom request was. + assert_eq!( + negotiate_output_mode(OutputMode::auto(4.0), true, 1.0), + OutputMode::Sdr + ); + // Non-HDR sources never promote, even on EDR displays. + assert_eq!( + negotiate_output_mode(OutputMode::auto(4.0), false, 2.0), + OutputMode::Sdr + ); + } + + #[test] + fn edr_requests_fall_back_to_sdr_on_displays_without_edr() { + let resolved = OutputMode::auto(4.0).resolve_for_source(true); + + assert_eq!(resolved, OutputMode::apple_edr(4.0)); + assert_eq!(clamp_output_mode_to_display(resolved, 1.0), OutputMode::Sdr); + assert_eq!( + clamp_output_mode_to_display(resolved, 1.04), + OutputMode::Sdr + ); + } + + #[test] + fn edr_requests_are_capped_at_real_display_headroom() { + let resolved = OutputMode::auto(4.0).resolve_for_source(true); + + assert_eq!( + clamp_output_mode_to_display(resolved, 2.0), + OutputMode::apple_edr(2.0) + ); + // A request below the display's capability stays untouched. + assert_eq!( + clamp_output_mode_to_display(OutputMode::apple_edr(1.5), 3.0), + OutputMode::apple_edr(1.5) + ); + } + + #[test] + fn extended_linear_requests_are_capped_but_survive_without_edr() { + assert_eq!( + clamp_output_mode_to_display(OutputMode::extended_linear(2.0), 1.0), + OutputMode::extended_linear(1.0) + ); + assert_eq!( + clamp_output_mode_to_display(OutputMode::Sdr, 1.0), + OutputMode::Sdr + ); + } + #[test] fn fallback_reason_codes_and_labels_are_stable() { let expected = [ diff --git a/crates/erika/src/renderer/pipeline.rs b/crates/erika/src/renderer/pipeline.rs index b7b08463..9e10812c 100644 --- a/crates/erika/src/renderer/pipeline.rs +++ b/crates/erika/src/renderer/pipeline.rs @@ -21,12 +21,27 @@ impl HdrMetadata { } pub fn nominal_peak_nits(self) -> Option { - self.mastering_display - .and_then(|metadata| metadata.max_luminance_nits()) - .or_else(|| { - self.content_light - .and_then(|metadata| metadata.max_content_light_level_nits()) - }) + let mastering = self.mastering_display.and_then(|m| m.max_luminance_nits()); + let cll = self + .content_light + .and_then(|c| c.max_content_light_level_nits()); + match (mastering, cll) { + (Some(m), Some(c)) => { + if c >= 10.0 { + Some(m.min(c)) + } else { + Some(m) + } + } + (Some(m), None) => Some(m), + (None, Some(c)) => Some(c), + (None, None) => None, + } + } + + pub fn max_frame_average_light_level_nits(self) -> Option { + self.content_light + .and_then(|metadata| metadata.max_frame_average_light_level_nits()) } } @@ -58,6 +73,283 @@ impl ContentLightMetadata { Some(self.max_content_light_level_nits as f32) } } + + pub fn max_frame_average_light_level_nits(self) -> Option { + if self.max_frame_average_light_level_nits == 0 { + None + } else { + Some(self.max_frame_average_light_level_nits as f32) + } + } +} + +/// Maximum number of reshaping pieces per component in a Dolby Vision RPU +/// (`AV_DOVI_MAX_PIECES` in FFmpeg, `num_pivots - 1` segments). +pub const DOVI_MAX_PIECES: usize = 8; +/// Maximum number of MMR orders per piece (FFmpeg allows 1..=3). +pub const DOVI_MAX_MMR_ORDER: usize = 3; +/// Number of coefficients per MMR order: 3 linear terms plus the 4 cross +/// products (x·y, x·z, y·z, x·y·z). +pub const DOVI_MMR_COEFFS: usize = 7; + +/// One component's reshaping curve, converted from the RPU's fixed-point +/// representation into shader-ready floats. Pivots are normalized to the +/// base-layer signal range [0, 1] and coefficients by `2^-coef_log2_denom`, +/// exactly like libplacebo's `pl_map_dovi_metadata`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DoviComponentCurve { + /// 0 when this component carries no reshaping, otherwise 2..=9. + pub num_pivots: u8, + /// Sorted ascending, normalized to [0, 1]. Only the first `num_pivots` + /// entries are meaningful. + pub pivots: [f32; DOVI_MAX_PIECES + 1], + /// Polynomial coefficients per segment (x^0, x^1, x^2). Segments above + /// `poly_order` are zero-filled. + pub poly_coeffs: [[f32; 3]; DOVI_MAX_PIECES], + /// Per segment: 0 selects the polynomial, 1..=3 selects MMR of that order. + pub mmr_orders: [u8; DOVI_MAX_PIECES], + pub mmr_constants: [f32; DOVI_MAX_PIECES], + pub mmr_coeffs: [[[f32; DOVI_MMR_COEFFS]; DOVI_MAX_MMR_ORDER]; DOVI_MAX_PIECES], +} + +impl Default for DoviComponentCurve { + fn default() -> Self { + Self { + num_pivots: 0, + pivots: [0.0; DOVI_MAX_PIECES + 1], + poly_coeffs: [[0.0; 3]; DOVI_MAX_PIECES], + mmr_orders: [0; DOVI_MAX_PIECES], + mmr_constants: [0.0; DOVI_MAX_PIECES], + mmr_coeffs: [[[0.0; DOVI_MMR_COEFFS]; DOVI_MAX_MMR_ORDER]; DOVI_MAX_PIECES], + } + } +} + +/// Per-frame Dolby Vision dynamic metadata level 1: 12-bit PQ codes of the +/// frame's black, peak, and average luminance. The frame peak replaces the +/// static mastering peak (`source_max_pq`) in the tone map, matching +/// libplacebo's handling of the RPU's CIE-Y metadata (`pl_map_dovi_metadata` / +/// `pl_hdr_metadata_from_dovi_rpu`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DoviFramePq { + pub min_pq: u16, + pub max_pq: u16, + pub avg_pq: u16, +} + +/// Per-frame Dolby Vision RPU payload copied out of the decoder's +/// `AV_FRAME_DATA_DOVI_METADATA` side data before the frame is retired. +/// +/// The `nonlinear_matrix` is the RPU's "ycc_to_rgb" transform applied to the +/// reshaped (still PQ-encoded) signal; `rgb_to_lms` is the RPU's mastering +/// transform whose inverse converts PQ-linearized LMS back to BT.2020 RGB. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct DoviSourceMetadata { + /// Per-component reshaping curves (luma, Cb, Cr). + pub reshaping: [DoviComponentCurve; 3], + /// RPU's ycc_to_rgb matrix applied to reshaped nonlinear signal. + pub nonlinear_matrix: RgbMatrix, + /// RPU signal offsets applied before the nonlinear matrix. + pub nonlinear_offset: [f32; 3], + /// RPU's rgb_to_lms mastering transform (inverted after PQ linearization). + pub rgb_to_lms: RgbMatrix, + /// 12-bit PQ code of the mastering display's black level (typically 0-100). + pub source_min_pq: u16, + /// 12-bit PQ code of the mastering display's peak level (typically 2000-4000 nits). + pub source_max_pq: u16, + /// Per-frame level 1 brightness metadata (dynamic DM block), or `None` + /// when the RPU carries no usable L1 block. + pub l1: Option, +} + +/// BT.2100 PQ constants shared by the code <-> nits conversions below. +const PQ_M1: f32 = 0.1593017578125; +const PQ_M2: f32 = 78.84375; +const PQ_C1: f32 = 0.8359375; +const PQ_C2: f32 = 18.8515625; +const PQ_C3: f32 = 18.6875; + +/// Decodes a 12-bit PQ code value into absolute nits, matching the PQ EOTF +/// used by the video shaders. +pub fn pq_code_to_nits(code: u16) -> f32 { + if code == 0 { + return 0.0; + } + let encoded = f32::from(code.min(4095)) / 4095.0; + nits_from_pq_code(encoded) +} + +/// PQ OETF: absolute luminance in nits to a PQ code over the 10 000-nit scale +/// (0.0 = black, 1.0 = 10 000 nits). Inverse of [`nits_from_pq_code`]. +pub(crate) fn pq_code_from_nits(nits: f32) -> f32 { + if !nits.is_finite() || nits <= 0.0 { + return 0.0; + } + let p = (nits / 10_000.0).clamp(0.0, 1.0).powf(PQ_M1); + ((PQ_C1 + PQ_C2 * p) / (1.0 + PQ_C3 * p)).powf(PQ_M2) +} + +/// PQ EOTF: a PQ code in [0, 1] over the 10 000-nit scale to absolute nits. +pub(crate) fn nits_from_pq_code(code: f32) -> f32 { + if !code.is_finite() || code <= 0.0 { + return 0.0; + } + let p = code.clamp(0.0, 1.0).powf(1.0 / PQ_M2); + let num = (p - PQ_C1).max(0.0); + let den = (PQ_C2 - PQ_C3 * p).max(0.000001); + 10_000.0 * (num / den).powf(1.0 / PQ_M1) +} + +/// Inverse of the no-crosstalk BT.2020-referred HPE RGB->LMS transform that +/// the RPU's `rgb_to_lms` output is fed into (hard-coded by libplacebo as +/// `dovi_lms2rgb`). +const DOVI_HPE_LMS_TO_RGB: RgbMatrix = RgbMatrix::new([ + [3.06441879, -2.16597676, 0.10155818], + [-0.65612108, 1.78554118, -0.12943749], + [0.01736321, -0.04725154, 1.03004253], +]); + +/// The composite LMS->RGB matrix applied after PQ linearization of a reshaped +/// Dolby Vision signal: the fixed HPE inverse multiplied by the RPU's +/// `rgb_to_lms` matrix, matching libplacebo's `dovi_lms2rgb` composition. +pub fn dovi_lms_to_rgb_matrix(rgb_to_lms: RgbMatrix) -> RgbMatrix { + DOVI_HPE_LMS_TO_RGB.mul(rgb_to_lms) +} + +/// Shader uniform block for Dolby Vision reshaping. All values are +/// vec4-aligned so the block can be appended to the shared video uniform +/// buffer across the WGSL, Metal and HLSL backends without packing tricks. +/// +/// **Size**: ~3KB total (144 vec4s for MMR + overhead). Modern GPUs support +/// this easily, but older mobile devices may have uniform buffer limits around +/// 16KB - this uses ~20% of that budget. +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq)] +#[cfg_attr(feature = "wgpu", derive(bytemuck::Pod, bytemuck::Zeroable))] +pub struct DoviUniforms { + /// x = 1.0 when the source is RPU-mapped; y/z/w = per-component segment + /// counts (`num_pivots - 1`, 0 when the component carries no curve). + pub flags: [f32; 4], + /// Interior pivots per component (two rows each, segments - 1 values, + /// padded with a quasi-infinite sentinel like libplacebo). + pub pivots: [[f32; 4]; 6], + /// Per-component `[first_pivot, last_pivot]` output clamp. + pub bounds: [[f32; 4]; 3], + /// Per-segment payload `[c0, c1, c2, kind]`: kind == 0 is the polynomial + /// `(c2·s + c1)·s + c0`, kind 1..=3 is MMR of that order with constant + /// `c0` and packed rows starting at offset `c1`. + pub coefficients: [[f32; 4]; 3 * DOVI_MAX_PIECES], + /// Packed MMR rows per component (two vec4 rows per order), addressed by + /// the segment's `c1` offset. + pub mmr: [[f32; 4]; 3 * 2 * DOVI_MAX_MMR_ORDER * DOVI_MAX_PIECES], + /// RPU "ycc_to_rgb" rows applied to the reshaped nonlinear signal. + pub nonlinear_matrix: [[f32; 4]; 3], + /// RPU signal offsets, pre-scaled so the shader's normalized 8-bit or + /// P010 samples subtract exactly (libplacebo folds 2^bits/(2^bits-1)). + pub nonlinear_offset: [f32; 4], + /// Composite LMS->RGB rows applied after PQ linearization. + pub lms_matrix: [[f32; 4]; 3], +} + +const DOVI_PIVOT_SENTINEL: f32 = 1e9; +/// RPU offsets are rational /1024-style values while shader samples are +/// normalized by n/(2^bits - 1). `DoviUniforms::of_for_representation` +/// applies the matching +/// 2^bits/(2^bits-1) correction for the actual uploaded representation. +impl DoviUniforms { + pub const fn disabled() -> Self { + Self { + flags: [0.0; 4], + pivots: [[0.0; 4]; 6], + bounds: [[0.0; 4]; 3], + coefficients: [[0.0; 4]; 3 * DOVI_MAX_PIECES], + mmr: [[0.0; 4]; 3 * 2 * DOVI_MAX_MMR_ORDER * DOVI_MAX_PIECES], + nonlinear_matrix: [[0.0; 4]; 3], + nonlinear_offset: [0.0; 4], + lms_matrix: [[0.0; 4]; 3], + } + } + + /// Builds uniforms using the historical 10-bit DV signal representation. + /// New callers with an explicit uploaded format should use + /// [`Self::of_for_representation`]. + pub fn of(source: &SourceColorState) -> Self { + Self::of_for_representation(source, true) + } + + /// Builds uniforms for the concrete plane representation used by the + /// renderer (`P010` when `is_p010` is true, otherwise 8-bit `NV12`). + pub fn of_for_representation(source: &SourceColorState, is_p010: bool) -> Self { + let Some(dovi) = &source.dovi else { + return Self::disabled(); + }; + let mut uniforms = Self::disabled(); + uniforms.flags[0] = 1.0; + for (component, curve) in dovi.reshaping.iter().enumerate() { + if curve.num_pivots < 2 { + continue; + } + let segments = (curve.num_pivots - 1) as usize; + uniforms.flags[1 + component] = segments as f32; + let mut interior = [DOVI_PIVOT_SENTINEL; DOVI_MAX_PIECES]; + interior[..segments - 1].copy_from_slice(&curve.pivots[1..segments]); + uniforms.pivots[2 * component] = [interior[0], interior[1], interior[2], interior[3]]; + uniforms.pivots[2 * component + 1] = + [interior[4], interior[5], interior[6], interior[7]]; + uniforms.bounds[component] = [ + curve.pivots[0].min(curve.pivots[segments]), + curve.pivots[0].max(curve.pivots[segments]), + 0.0, + 0.0, + ]; + + let mut mmr_row = 0usize; + for (segment, &kind) in curve.mmr_orders[..segments].iter().enumerate() { + let slot = DOVI_MAX_PIECES * component + segment; + if kind == 0 { + uniforms.coefficients[slot] = [ + curve.poly_coeffs[segment][0], + curve.poly_coeffs[segment][1], + curve.poly_coeffs[segment][2], + 0.0, + ]; + continue; + } + let order = (kind as usize).min(DOVI_MAX_MMR_ORDER); + let orders = &curve.mmr_coeffs[segment][..order]; + for (index, coefficients) in orders.iter().enumerate() { + let row = + DOVI_MAX_PIECES * 2 * DOVI_MAX_MMR_ORDER * component + mmr_row + 2 * index; + uniforms.mmr[row] = [coefficients[0], coefficients[1], coefficients[2], 0.0]; + uniforms.mmr[row + 1] = [ + coefficients[3], + coefficients[4], + coefficients[5], + coefficients[6], + ]; + } + uniforms.coefficients[slot] = [ + curve.mmr_constants[segment], + mmr_row as f32, + 0.0, + kind as f32, + ]; + mmr_row += 2 * order; + } + } + uniforms.nonlinear_matrix = dovi.nonlinear_matrix.row4s(); + let signal_bits = if is_p010 { 10 } else { 8 }; + let signal_max = ((1_u32 << signal_bits) - 1) as f32; + let signal_offset_scale = (1_u32 << signal_bits) as f32 / signal_max; + uniforms.nonlinear_offset = [ + dovi.nonlinear_offset[0] * signal_offset_scale, + dovi.nonlinear_offset[1] * signal_offset_scale, + dovi.nonlinear_offset[2] * signal_offset_scale, + 0.0, + ]; + uniforms.lms_matrix = dovi_lms_to_rgb_matrix(dovi.rgb_to_lms).row4s(); + uniforms + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -195,7 +487,7 @@ impl RgbMatrix { ] } - fn mul(self, rhs: Self) -> Self { + pub(crate) fn mul(self, rhs: Self) -> Self { let mut rows = [[0.0; 3]; 3]; for (row_index, row) in rows.iter_mut().enumerate() { for (col_index, value) in row.iter_mut().enumerate() { @@ -207,7 +499,7 @@ impl RgbMatrix { Self::new(rows) } - fn mul_vec(self, value: [f32; 3]) -> [f32; 3] { + pub(crate) fn mul_vec(self, value: [f32; 3]) -> [f32; 3] { [ self.rows[0][0] * value[0] + self.rows[0][1] * value[1] + self.rows[0][2] * value[2], self.rows[1][0] * value[0] + self.rows[1][1] * value[1] + self.rows[1][2] * value[2], @@ -215,7 +507,7 @@ impl RgbMatrix { ] } - fn inverse(self) -> Self { + pub(crate) fn inverse(self) -> Self { let m = self.rows; let det = m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) @@ -296,6 +588,36 @@ pub fn source_to_target_rgb_matrix(source: ColorPrimaries, target: ColorPrimarie xyz_to_rgb_matrix(target).mul(rgb_to_xyz_matrix(source)) } +/// RGB (source primaries, absolute D65-referred linear) → HPE-LMS, ported +/// from libplacebo's `pl_ipt_rgb2lms`: a 4% crosstalk mix of HPE XYZ→LMS +/// (D65) applied to the primaries RGB→XYZ matrix. The codebase's primaries +/// are all D65 so the chromatic-adaptation step to D65 is the identity and +/// is omitted. Tone mapping runs in this LMS-PQ-IPT space (see the shader +/// `tone_map_nits`), which converts primaries while mapping the intensity +/// axis instead of running a separate gamut matrix. +pub fn ipt_rgb2lms_matrix(primaries: ColorPrimaries) -> RgbMatrix { + const HPE: [[f32; 3]; 3] = [ + [0.40024, 0.70760, -0.08081], + [-0.22630, 1.16532, 0.04570], + [0.00000, 0.00000, 0.91822], + ]; + let c = 0.04_f32; + let crosstalk = RgbMatrix::new([ + [1.0 - 2.0 * c, c, c], + [c, 1.0 - 2.0 * c, c], + [c, c, 1.0 - 2.0 * c], + ]); + crosstalk + .mul(RgbMatrix::new(HPE)) + .mul(rgb_to_xyz_matrix(primaries)) +} + +/// Inverse of [`ipt_rgb2lms_matrix`] for the *target* primaries; this is +/// what converts the tone-mapped LMS signal back to display RGB. +pub fn ipt_lms2rgb_matrix(primaries: ColorPrimaries) -> RgbMatrix { + ipt_rgb2lms_matrix(primaries).inverse() +} + const D65_WHITE: Chromaticity = Chromaticity::new(0.3127, 0.3290); fn resolve_primaries(primaries: ColorPrimaries) -> ColorPrimaries { @@ -305,6 +627,30 @@ fn resolve_primaries(primaries: ColorPrimaries) -> ColorPrimaries { } } +fn primaries_code(primaries: ColorPrimaries) -> u32 { + match resolve_primaries(primaries) { + ColorPrimaries::Bt709 => 0, + ColorPrimaries::Bt2020 => 1, + ColorPrimaries::DisplayP3 => 2, + ColorPrimaries::Unknown => 0, + } +} + +#[allow(dead_code)] +pub(crate) fn code_to_primaries(code: u32) -> ColorPrimaries { + match code { + 1 => ColorPrimaries::Bt2020, + 2 => ColorPrimaries::DisplayP3, + _ => ColorPrimaries::Bt709, + } +} + +/// Source and target primaries (resolved) for the perceptual gamut LUT key, +/// packed into the uniforms' reserved word. +pub(crate) fn gamut_primaries_code(source: ColorPrimaries, target: ColorPrimaries) -> u32 { + (primaries_code(source) << 8) | primaries_code(target) +} + fn xy_to_xyz(value: Chromaticity) -> [f32; 3] { [value.x / value.y, 1.0, (1.0 - value.x - value.y) / value.y] } @@ -312,13 +658,32 @@ fn xy_to_xyz(value: Chromaticity) -> [f32; 3] { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ToneMapOperator { Clip, + /// Reinhard curve (libplacebo `pl_tone_map_reinhard`, output-relative). Reinhard, + /// Möbius transform with a linear region below the knee (libplacebo + /// `pl_tone_map_mobius`; `curve_param` is the knee, default 0.3). Mobius, + /// ITU-R BT.2390 EETF with black-point compensation (libplacebo + /// `pl_tone_map_bt2390`; `curve_param` is the knee offset, default 1.0). + Bt2390, + /// Perceptually linear single-pivot polynomial, the default in libplacebo + /// and mpv's gpu-next renderer (`curve_param` is the slope contrast, + /// default 0.30; the pivot follows the scene average luminance). + Spline, + /// ITU-R BT.2446 method A (log-domain Weber compression), described by + /// mpv as the recommended curve for well-mastered content. + Bt2446a, + /// SMPTE ST 2094-10 Annex B.2, the DolbyVision dynamic-metadata curve; + /// coefficients are solved per frame from the scene pivot. + St209410, } impl Default for ToneMapOperator { fn default() -> Self { - Self::Mobius + // Aligns with broadcast standard ITU-R BT.2390 and mpv's default + // tone-mapping curve: BT.2390 preserves 1:1 luminance for diffuse content + // below the knee without underexposing midtones, and rolls off highlights smoothly. + Self::Bt2390 } } @@ -364,6 +729,11 @@ pub struct SourceColorState { pub matrix: MatrixCoefficients, pub range: ColorRange, pub hdr_metadata: Option, + pub dovi: Option, + /// Measured per-frame scene-average luminance (nits) from CPU frame + /// statistics (HDR10 without Dolby Vision L1). Precedes the L1 average + /// when driving the tone-map pivot. + pub measured_scene_avg_nits: Option, pub nominal_peak_nits: f32, pub reference_white_nits: f32, } @@ -376,6 +746,8 @@ impl SourceColorState { matrix: MatrixCoefficients::default(), range: ColorRange::default(), hdr_metadata: None, + dovi: None, + measured_scene_avg_nits: None, nominal_peak_nits: nominal_peak_for_transfer(transfer), reference_white_nits: reference_white_for_transfer(transfer), } @@ -404,6 +776,85 @@ impl SourceColorState { self } + /// Attaches per-frame Dolby Vision RPU metadata. The RPU describes an + /// IPT/LMS signal referred to BT.2020 with a PQ transfer, so the primaries + /// and transfer are forced to BT.2020/PQ (matching libplacebo's + /// `pl_map_avdovi_metadata`). The RPU's `source_min_pq`/`source_max_pq` + /// replace static mastering luminance when present, while ordinary display + /// primaries and content-light metadata are retained. When the frame's + /// dynamic L1 block is present its frame peak replaces `source_max_pq` + /// for tone mapping (libplacebo uses the RPU's CIE-Y metadata the same + /// way); the static mastering display peak stays on the L0 value so + /// output-mode negotiation never reacts to per-frame brightness. Forcing + /// the transfer also repairs streams whose VUI tags are missing entirely. + pub fn dovi(mut self, metadata: Option) -> Self { + if let Some(dovi) = metadata { + self.primaries = ColorPrimaries::Bt2020; + self.transfer = TransferFunction::Pq; + self.reference_white_nits = reference_white_for_transfer(self.transfer).max(1.0); + let min_luminance = (dovi.source_min_pq != 0) + .then(|| pq_code_to_nits(dovi.source_min_pq)) + .filter(|value| value.is_finite() && *value >= 0.0); + let frame_peak = dovi + .l1 + .filter(|l1| l1.max_pq != 0) + .map(|l1| pq_code_to_nits(l1.max_pq)); + let mastering_peak = pq_code_to_nits(dovi.source_max_pq); + let peak = frame_peak.unwrap_or(mastering_peak); + if peak > 0.0 { + self.nominal_peak_nits = peak.max(1.0); + } else if self.nominal_peak_nits <= self.reference_white_nits { + self.nominal_peak_nits = nominal_peak_for_transfer(self.transfer); + } + // Keep ordinary mastering primaries/content-light metadata, but + // prefer the RPU's source luminance bounds when present. This lets + // native HDR10 outputs carry Dolby Vision black-level metadata too. + // The mastering peak stays on the static L0 value (never the + // per-frame L1 peak), so output-mode negotiation does not react + // to frame-by-frame brightness. + if min_luminance.is_some() + || (peak.is_finite() && peak > 0.0) + || self.hdr_metadata.is_some() + { + let mut hdr = self + .hdr_metadata + .unwrap_or_else(|| HdrMetadata::new(None, None)); + let mut mastering = hdr.mastering_display.unwrap_or(MasteringDisplayMetadata { + display_primaries: None, + white_point: None, + min_luminance_nits: None, + max_luminance_nits: None, + }); + if let Some(min_luminance) = min_luminance { + mastering.min_luminance_nits = Some(min_luminance); + } + if mastering_peak.is_finite() && mastering_peak > 0.0 { + mastering.max_luminance_nits = Some(mastering_peak); + } + hdr.mastering_display = Some(mastering); + self.hdr_metadata = Some(hdr); + } + self.dovi = Some(dovi); + } else { + self.dovi = None; + } + self + } + + /// Attach a measured scene-average luminance from frame statistics. Only + /// meaningful for HDR10 (PQ) sources without Dolby Vision L1 metadata; + /// the value drives the tone-map pivot exactly like L1's avg would. + pub fn measured_scene_avg_nits(mut self, scene_avg_nits: Option) -> Self { + if let Some(value) = scene_avg_nits { + if value.is_finite() && value > 0.0 { + self.measured_scene_avg_nits = Some(value); + return self; + } + } + self.measured_scene_avg_nits = None; + self + } + pub fn reference_white_nits(mut self, white: f32) -> Self { self.reference_white_nits = white.max(1.0); self @@ -441,6 +892,22 @@ impl TargetColorState { } } + /// SDR target for sources that require tone mapping (HDR → SDR). The tone + /// map peak and the encode reference white follow the HDR reference white + /// convention (BT.2408: 203 nits), matching libplacebo/mpv — NOT the + /// 100-nit SDR mastering value, which pushes the whole picture into the + /// top of the display range (measured: same frame, our render p50=89 vs + /// mpv 67 with identical source content). + pub fn sdr_tone_map_target(primaries: ColorPrimaries) -> Self { + Self { + primaries, + transfer: TransferFunction::Srgb, + peak_nits: 203.0, + reference_white_nits: 203.0, + edr_headroom: 1.0, + } + } + pub fn apple_edr(primaries: ColorPrimaries, headroom: f32) -> Self { Self::extended_linear(primaries, 203.0, headroom) } @@ -481,20 +948,59 @@ impl Default for TargetColorState { #[derive(Debug, Clone, Copy, PartialEq)] pub struct ToneMapConfig { pub operator: ToneMapOperator, - pub knee_start: f32, - pub desaturate: f32, + /// Per-operator curve parameter that mirrors libplacebo's + /// `pl_tone_map_constants` field the operator uses: spline contrast, + /// ST 2094-10 knee adaptation, BT.2390 knee offset, Mobius knee, Reinhard + /// contrast. A value of 0 selects the operator's default (see + /// `default_curve_param`). The shaders read it from `tone_map_extra.x`; + /// ST 2094-10 instead folds it into the coefficients solved on the CPU. + pub curve_param: f32, + /// Target display contrast used for black-point compensation + /// (`--target-contrast` in mpv). 0 selects the automatic value: 1000:1 for + /// SDR targets, infinite (0 black) for HDR/EDR targets. + pub contrast_ratio: f32, } impl Default for ToneMapConfig { fn default() -> Self { Self { - operator: ToneMapOperator::Mobius, - knee_start: 0.75, - desaturate: 0.0, + // Follow the operator enum's default so a default-operator change + // actually reaches the pipeline (a hardcoded value here silently + // overrides it). + operator: ToneMapOperator::default(), + curve_param: 0.0, + contrast_ratio: 0.0, + } + } +} + +impl ToneMapConfig { + /// The effective per-operator curve parameter, substituting the operator + /// default for 0. Mirrors libplacebo's per-function `param_def` where + /// mpv overrides it (spline contrast 0.30, mobius knee 0.30). + pub fn effective_curve_param(self) -> f32 { + if self.curve_param > 0.0 { + self.curve_param + } else { + default_curve_param(self.operator) } } } +fn default_curve_param(operator: ToneMapOperator) -> f32 { + match operator { + ToneMapOperator::Clip => 1.0, + ToneMapOperator::Reinhard => 0.5, + ToneMapOperator::Mobius => 0.3, + ToneMapOperator::Bt2390 => 1.0, + ToneMapOperator::Spline => 0.30, + ToneMapOperator::Bt2446a => 0.0, + // libplacebo's `pl_tone_map_st2094_10.param_def` (knee adaptation). + // mpv's manual documents 1.0 for the same knob; follow libplacebo. + ToneMapOperator::St209410 => 0.7, + } +} + #[derive(Debug, Clone, Copy, PartialEq)] pub struct ScalerConfig { pub kernel: ScalerKernel, @@ -516,6 +1022,7 @@ pub enum RenderPassKind { NeuralUpscale, PlaneSampling, ChromaReconstruction, + DoviReshape, TransferDecode, GamutMap, ToneMap, @@ -617,6 +1124,63 @@ impl VideoRenderPipeline { pub fn gamut_matrix(&self) -> RgbMatrix { source_to_target_rgb_matrix(self.source.primaries, self.target.primaries) } + + /// Rows 0-2: source-primaries RGB→LMS; rows 3-5: source-primaries + /// LMS→RGB; rows 6-8: target-primaries LMS→RGB. Tone mapping and + /// gamut mapping run in a unified single IPT pass (rows 0-2 in, optional + /// 3D LUT in IPT, rows 6-8 out to target primaries) — mirroring libplacebo. + pub fn ipt_matrix_rows(&self) -> [[f32; 4]; 9] { + let mut rows = [[0.0; 4]; 9]; + let source = ipt_rgb2lms_matrix(self.source.primaries); + for (index, row) in source + .rows() + .iter() + .chain(source.inverse().rows().iter()) + .chain(ipt_lms2rgb_matrix(self.target.primaries).rows().iter()) + .enumerate() + { + rows[index] = [row[0], row[1], row[2], 0.0]; + } + rows + } + + /// [curve parameter, scene average nits, target black nits, 0] for the + /// shaders' `tone_map_extra` uniform. + pub fn tone_map_extra(&self) -> [f32; 4] { + [ + self.tone_map.effective_curve_param(), + source_scene_avg_nits(&self.source), + // Black-point compensation only applies when the tone map is + // actually active; applying it to SDR->SDR would crush near-black + // content (mpv skips BPC when `pl_tone_map_params_noop`). + if requires_tone_mapping(self.source, self.target) { + target_black_nits(self.target, self.tone_map.contrast_ratio) + } else { + 0.0 + }, + 0.0, + ] + } + + /// SMPTE ST 2094-10 coefficients for the shaders' `tone_map_coeffs` + /// uniform; zeros when the operator is inactive. + /// Packed (source << 8 | target) resolved-primaries codes used to key + /// the perceptual gamut LUT cache and stored in `_gamut_reserved`. + pub fn gamut_primaries_code(&self) -> u32 { + gamut_primaries_code(self.source.primaries, self.target.primaries) + } + + pub fn tone_map_coeffs(&self) -> [f32; 4] { + st2094_10_coefficients_for(self) + } + + /// Whether the perceptual gamut-mapping LUT is needed: HDR sources + /// tone-mapped to a smaller gamut get the LUT; SDR passthrough and + /// same-gamut rendering keep the fast path (gamut_compress only). + pub fn gamut_lut_active(&self) -> bool { + requires_tone_mapping(self.source, self.target) + && resolve_primaries(self.source.primaries) != resolve_primaries(self.target.primaries) + } } impl Default for VideoRenderPipeline { @@ -653,6 +1217,30 @@ pub struct VideoUniforms { pub nits: [f32; 4], pub luma_coefficients: [f32; 4], pub gamut_matrix_rows: [[f32; 4]; 3], + /// Rows 0-2: source-primaries RGB → HPE-LMS; rows 3-5: target-primaries + /// LMS → RGB (see [`ipt_rgb2lms_matrix`]/[`ipt_lms2rgb_matrix`]). The + /// shader runs tone mapping in LMS-PQ-IPT space so primaries convert as + /// part of the map instead of a separate gamut matrix. + pub ipt_matrix_rows: [[f32; 4]; 9], + /// x: per-operator curve parameter (see `ToneMapConfig::curve_param`; + /// unused by ST 2094-10, which solves its curve on the CPU); + /// y: per-frame scene average luminance in nits (Dolby Vision L1 avg, + /// 0 = unknown); z: target black point in nits + /// (`ToneMapConfig::contrast_ratio`); w: reserved. + pub tone_map_extra: [f32; 4], + /// SMPTE ST 2094-10 tone-map coefficients (c1, c2, c3) solved per frame + /// on the CPU; zero unless the ST2094-10 operator is active. + pub tone_map_coeffs: [f32; 4], + /// 1 when the perceptual gamut-mapping 3D LUT is bound and the shader + /// must sample it after tone mapping; 0 keeps the fast gamut_compress. + pub gamut_lut_enabled: u32, + /// Packed (source << 8 | target) resolved primaries for the LUT cache. + pub _gamut_primaries: u32, + /// Reserved for future per-LUT scaling; keeps the structure padded. + pub _gamut_reserved0: u32, + pub _gamut_reserved1: u32, + /// Dolby Vision reshaping payload; inert unless `flags[0]` is set. + pub dovi: DoviUniforms, } impl VideoUniforms { @@ -675,14 +1263,245 @@ impl VideoUniforms { ], luma_coefficients: [luma.kr, luma.kg, luma.kb, 0.0], gamut_matrix_rows: pipeline.gamut_matrix().row4s(), + ipt_matrix_rows: pipeline.ipt_matrix_rows(), + tone_map_extra: pipeline.tone_map_extra(), + tone_map_coeffs: pipeline.tone_map_coeffs(), + gamut_lut_enabled: u32::from(pipeline.gamut_lut_active()), + _gamut_primaries: gamut_primaries_code( + pipeline.source.primaries, + pipeline.target.primaries, + ), + _gamut_reserved0: 0, + _gamut_reserved1: 0, + dovi: DoviUniforms::of_for_representation(&pipeline.source, is_p010), } } +} + +/// Per-frame average luminance from the Dolby Vision L1 block in nits; 0 when +/// absent (libplacebo then falls back to the default knee fraction). +fn dovi_frame_average_nits(dovi: &DoviSourceMetadata) -> Option { + let l1 = dovi.l1?; + if l1.avg_pq == 0 { + return None; + } + let nits = pq_code_to_nits(l1.avg_pq); + (nits.is_finite() && nits > 0.0).then_some(nits) +} + +/// The scene-average luminance (nits) driving the tone-map pivot: the +/// presenter's measured average first, then the Dolby Vision L1 average, +/// then static MaxFALL, 0 when nothing is known. +fn source_scene_avg_nits(source: &SourceColorState) -> f32 { + source + .measured_scene_avg_nits + .or_else(|| source.dovi.as_ref().and_then(dovi_frame_average_nits)) + .or_else(|| { + source + .hdr_metadata + .and_then(|hdr| hdr.max_frame_average_light_level_nits()) + }) + .unwrap_or(0.0) +} + +/// Target black point for black-point compensation: mpv's `--target-contrast` +/// auto value is 1000:1 for SDR targets and infinite (0 black) for HDR/EDR +/// targets, which the encode stage maps back onto code 0. +fn target_black_nits(target: TargetColorState, contrast_ratio: f32) -> f32 { + if target.edr_headroom > 1.0 || target.transfer == TransferFunction::Pq { + return 0.0; + } + let ratio = if contrast_ratio > 0.0 { + contrast_ratio + } else { + 1000.0 + }; + target.peak_nits / ratio +} + +/// Per-frame ST 2094-10 coefficients for the current source/target +/// luminance envelope, following libplacebo's `pl_tone_map_st2094_10`: the +/// rational Möbius curve passes through (input min, output min), the scene +/// knee and (input max, output max), all in absolute nits. The knee itself is +/// picked in the PQ domain (libplacebo rescales internally even though +/// ST 2094-10 is a NITS-scaled function), and `ToneMapConfig::curve_param` +/// tunes libplacebo's `knee_adaptation` for this operator. +fn st2094_10_coefficients_for(pipeline: &VideoRenderPipeline) -> [f32; 4] { + if pipeline.tone_map.operator != ToneMapOperator::St209410 { + return [0.0; 4]; + } + let input_avg = source_scene_avg_nits(&pipeline.source); + let output_min = target_black_nits(pipeline.target, pipeline.tone_map.contrast_ratio); + let (src_knee, dst_knee) = st2094_pick_knee_nits( + 0.0, + pipeline.source.nominal_peak_nits, + input_avg, + output_min, + pipeline.target.peak_nits, + pipeline.tone_map.effective_curve_param(), + ); + solve_st2094_10( + 0.0, + src_knee, + pipeline.source.nominal_peak_nits, + output_min, + dst_knee, + pipeline.target.peak_nits, + ) +} + +/// The exact libplacebo `st2094_pick_knee` in the **PQ-code domain** (0.0 = +/// black, 1.0 = 10 000 nits). libplacebo always evaluates this in PQ, whatever +/// the tone-map function's own scaling is; callers holding absolute nits must +/// use [`st2094_pick_knee_nits`]. Constants: knee_adaptation 0.4 (the +/// `pl_tone_map_constants` default), knee_min 0.1, knee_max 0.8, knee_default +/// 0.4. Returns the source pivot and the adapted destination pivot. +pub fn st2094_pick_knee( + input_min: f32, + input_max: f32, + input_avg: f32, + output_min: f32, + output_max: f32, +) -> (f32, f32) { + st2094_pick_knee_impl( + input_min, + input_max, + input_avg, + output_min, + output_max, + KNEE_ADAPTATION_DEFAULT, + ) +} +/// [`st2094_pick_knee`] for callers holding absolute nits: the knee is picked +/// in the PQ domain and rescaled back, mirroring libplacebo's +/// `pl_hdr_rescale(input_scaling, PL_HDR_PQ, ..)` round trip inside +/// `st2094_pick_knee`. `knee_adaptation` is libplacebo's +/// `pl_tone_map_constants.knee_adaptation`, which for ST 2094-10 is what +/// `--tone-mapping-param` tunes. +pub fn st2094_pick_knee_nits( + input_min: f32, + input_max: f32, + input_avg: f32, + output_min: f32, + output_max: f32, + knee_adaptation: f32, +) -> (f32, f32) { + let (src_knee, dst_knee) = st2094_pick_knee_impl( + pq_code_from_nits(input_min), + pq_code_from_nits(input_max), + pq_code_from_nits(input_avg), + pq_code_from_nits(output_min), + pq_code_from_nits(output_max), + knee_adaptation, + ); + (nits_from_pq_code(src_knee), nits_from_pq_code(dst_knee)) +} + +const KNEE_ADAPTATION_DEFAULT: f32 = 0.4; +const KNEE_MIN: f32 = 0.1; +const KNEE_MAX: f32 = 0.8; +const KNEE_DEFAULT: f32 = 0.4; + +fn st2094_pick_knee_impl( + input_min: f32, + input_max: f32, + input_avg: f32, + output_min: f32, + output_max: f32, + knee_adaptation: f32, +) -> (f32, f32) { + let mix = |a: f32, b: f32, x: f32| x * b + (1.0 - x) * a; + let src_knee_min = mix(input_min, input_max, KNEE_MIN); + let src_knee_max = mix(input_min, input_max, KNEE_MAX); + let dst_knee_min = mix(output_min, output_max, KNEE_MIN); + let dst_knee_max = mix(output_min, output_max, KNEE_MAX); + let fallback = mix(input_min, input_max, KNEE_DEFAULT); + let src_knee = + if input_avg > 0.0 { input_avg } else { fallback }.clamp(src_knee_min, src_knee_max); + let target = (src_knee - input_min) / (input_max - input_min).max(1e-6); + let adapted = mix(output_min, output_max, target); + let smooth = |edge0: f32, edge1: f32, x: f32| { + let t = ((x - edge0) / (edge1 - edge0)).clamp(0.0, 1.0); + t * t * (3.0 - 2.0 * t) + }; + let tuning = + 1.0 - smooth(KNEE_MAX, KNEE_DEFAULT, target) * smooth(KNEE_MIN, KNEE_DEFAULT, target); + let adaptation = mix(knee_adaptation.clamp(0.0, 1.0), 1.0, tuning); + let dst_knee = mix(src_knee, adapted, adaptation).clamp(dst_knee_min, dst_knee_max); + (src_knee, dst_knee) +} + +/// Solve the ST 2094-10 rational curve y = (c1 + c2 x) / (1 + c3 x) through +/// (x1,y1), (x2,y2) and (x3,y3) with Cramer's rule on the linear system +/// `c1 + xi*c2 - yi*xi*c3 = yi`. +/// +/// A degenerate anchor set (duplicated/collinear points) has no unique +/// solution; libplacebo divides by the zero determinant and produces +/// non-finite coefficients, which the shader would render as a black or NaN +/// frame. Fall back to the identity curve (c1 = 0, c2 = 1, c3 = 0 — the `Clip` +/// operator) so a malformed envelope degrades to no tone mapping instead. +fn solve_st2094_10(x1: f32, x2: f32, x3: f32, y1: f32, y2: f32, y3: f32) -> [f32; 4] { + const IDENTITY: [f32; 4] = [0.0, 1.0, 0.0, 0.0]; + let base = [ + [1.0, x1, -y1 * x1], + [1.0, x2, -y2 * x2], + [1.0, x3, -y3 * x3], + ]; + let det = |a: [[f32; 3]; 3]| { + a[0][0] * (a[1][1] * a[2][2] - a[1][2] * a[2][1]) + - a[0][1] * (a[1][0] * a[2][2] - a[1][2] * a[2][0]) + + a[0][2] * (a[1][0] * a[2][1] - a[1][1] * a[2][0]) + }; + let with_column = |column: usize, replacement: [f32; 3]| { + let mut m = base; + for row in 0..3 { + m[row][column] = replacement[row]; + } + m + }; + let den = det(base); + if !den.is_finite() || den.abs() < 1e-12 { + return IDENTITY; + } + let rhs = [y1, y2, y3]; + let coefficients = [ + det(with_column(0, rhs)) / den, + det(with_column(1, rhs)) / den, + det(with_column(2, rhs)) / den, + 0.0, + ]; + if coefficients[..3].iter().all(|value| value.is_finite()) { + coefficients + } else { + IDENTITY + } +} + +impl VideoUniforms { pub fn rgb_texture_input(mut self) -> Self { self.input_mode = (self.input_mode & !VIDEO_INPUT_MODE_MASK) | 1; self } + /// Updates the decoded sample representation while keeping Dolby Vision + /// signal offsets in the same normalized domain as the texture samples. + /// This matters when a 10-bit P010 frame is down-converted to 8-bit NV12. + pub fn with_p010_representation(mut self, is_p010: bool) -> Self { + let old_bits = if self.is_p010 != 0 { 10 } else { 8 }; + let new_bits = if is_p010 { 10 } else { 8 }; + if old_bits != new_bits { + let old_scale = (1_u32 << old_bits) as f32 / ((1_u32 << old_bits) - 1) as f32; + let new_scale = (1_u32 << new_bits) as f32 / ((1_u32 << new_bits) - 1) as f32; + let ratio = new_scale / old_scale; + for offset in &mut self.dovi.nonlinear_offset[..3] { + *offset *= ratio; + } + } + self.is_p010 = u32::from(is_p010); + self + } + pub fn packed_d2s_luma_input(mut self) -> Self { self.input_mode = (self.input_mode & !VIDEO_INPUT_MODE_MASK) | 2; self @@ -727,6 +1546,10 @@ fn tone_map_code(operator: ToneMapOperator) -> u32 { ToneMapOperator::Clip => 0, ToneMapOperator::Reinhard => 1, ToneMapOperator::Mobius => 2, + ToneMapOperator::Bt2390 => 3, + ToneMapOperator::Spline => 4, + ToneMapOperator::Bt2446a => 5, + ToneMapOperator::St209410 => 6, } } @@ -752,6 +1575,12 @@ fn build_graph( RenderPassKind::ChromaReconstruction, "reconstruct chroma", )); + if source.dovi.is_some() { + graph.push(RenderPass::new( + RenderPassKind::DoviReshape, + "reshape dolby vision signal", + )); + } graph.push(RenderPass::new( RenderPassKind::TransferDecode, "decode transfer function", @@ -777,11 +1606,26 @@ fn build_graph( graph } +/// Peak luminance that is stable frame to frame: the mastering display (L0) +/// peak when the container/RPU provides one, otherwise the source's nominal +/// peak. Pass selection and output-mode decisions must use this instead of +/// `nominal_peak_nits`, which Dolby Vision replaces with the per-frame L1 peak +/// — a dark scene must not silently disable the tone map, the black-point +/// compensation, or the perceptual gamut LUT for one frame. +fn static_source_peak_nits(source: &SourceColorState) -> f32 { + source + .hdr_metadata + .and_then(|hdr| hdr.mastering_display) + .and_then(|mastering| mastering.max_luminance_nits) + .filter(|peak| peak.is_finite() && *peak > 0.0) + .unwrap_or(source.nominal_peak_nits) +} + fn requires_tone_mapping(source: SourceColorState, target: TargetColorState) -> bool { if !source.is_hdr() { return false; } - source.nominal_peak_nits > target.peak_nits * 1.05 + static_source_peak_nits(&source) > target.peak_nits * 1.05 } fn requires_gamut_mapping(source: SourceColorState, target: TargetColorState) -> bool { @@ -928,6 +1772,746 @@ mod tests { } } + /// PQ-code helpers matching the shaders; see `wgpu_video.wgsl`. + fn pq_code(nits: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = (nits / 10000.0).clamp(0.0, 1.0).powf(m1); + ((c1 + c2 * p) / (1.0 + c3 * p).max(0.000_001)).powf(m2) + } + + fn nits_from_pq(code: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = code.clamp(0.0, 1.0).powf(1.0 / m2); + let num = (p - c1).max(0.0); + let den = (c2 - c3 * p).max(0.000_001); + 10000.0 * (num / den).powf(1.0 / m1) + } + + /// Reference implementation of the shaders' `tone_map_curve_pq` spline + /// branch (tone_map code 4): the single-pivot polynomial from libplacebo's + /// `pl_tone_map_spline`, where the pivot follows the scene average. An + /// unknown average (0) takes the shaders' fallback of + /// `clamp(0.4 * src_peak, 100, 400)` nits before the knee pick. + fn spline_pq( + x: f32, + src_peak_nits: f32, + src_avg_nits: f32, + dst_peak_nits: f32, + dst_black_nits: f32, + contrast: f32, + ) -> f32 { + let in_min = 0.0; + let in_max = pq_code(src_peak_nits).max(0.000_001); + let out_min = pq_code(dst_black_nits); + let out_max = pq_code(dst_peak_nits).max(0.000_001); + let fallback_avg = (0.4 * src_peak_nits).clamp(100.0, 400.0); + let effective_src_avg = if src_avg_nits > 0.0 { + src_avg_nits + } else { + fallback_avg + }; + let (src_pivot, dst_pivot) = + st2094_pick_knee(in_min, in_max, pq_code(effective_src_avg), out_min, out_max); + let slope0 = (dst_pivot - out_min) / (src_pivot - in_min).max(0.000_001); + let ratio = (1.5 * (in_max / out_max - 1.0)).clamp(0.2, 1.2); + let slope = slope0.powf((1.0 - contrast) * ratio); + let (in_min0, in_max0) = (in_min - src_pivot, in_max - src_pivot); + let (out_min0, out_max0) = (out_min - dst_pivot, out_max - dst_pivot); + let pa = (out_min0 - slope * in_min0) / (in_min0 * in_min0); + let qa = (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0 * in_max0); + let qb = -3.0 * (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0); + let xr = x.clamp(in_min, in_max) - src_pivot; + let mapped = if xr > 0.0 { + ((qa * xr + qb) * xr + slope) * xr + } else { + (pa * xr + slope) * xr + }; + mapped + dst_pivot + } + + #[test] + fn spline_curve_anchors_and_monotonicity() { + let (src_peak, dst_peak, black) = (1000.0_f32, 100.0_f32, 0.0_f32); + let contrast = 0.3_f32; + // Endpoints are exact by construction: the quadratic anchors the lower + // endpoint, the cubic anchors the source peak on the target peak. + let lo = spline_pq(0.0, src_peak, 0.0, dst_peak, black, contrast); + assert!(nits_from_pq(lo).abs() < 1e-3, "black maps to {lo} PQ"); + let hi = spline_pq(pq_code(src_peak), src_peak, 0.0, dst_peak, black, contrast); + assert!(nits_from_pq(hi).abs() - dst_peak < 0.05, "peak = {hi} PQ"); + // Monotonic and bounded. + let mut previous = -1.0_f32; + for step in 0..=100 { + let x = pq_code(src_peak) * step as f32 / 100.0; + let mapped = spline_pq(x, src_peak, 0.0, dst_peak, black, contrast); + assert!(mapped >= previous, "not monotonic at {step}"); + assert!(mapped <= pq_code(dst_peak) + 1e-3); + previous = mapped; + } + // 10:1 compression puts 100-nit diffuse white below half of the + // mastered value. Without metadata the shaders fall back to a + // 400-nit average (0.4 * 1000 clamped), so the unknown-metadata + // curve is the darker one. + let diffuse = spline_pq(pq_code(100.0), src_peak, 0.0, dst_peak, black, contrast); + let diffuse_nits = nits_from_pq(diffuse); + let diffuse_known = spline_pq(pq_code(100.0), src_peak, 100.0, dst_peak, black, contrast); + let diffuse_known_nits = nits_from_pq(diffuse_known); + assert!( + diffuse_nits > 10.0 && diffuse_nits < 60.0, + "diffuse = {diffuse_nits}" + ); + assert!( + diffuse_nits < diffuse_known_nits, + "fallback avg {diffuse_nits} must sit below the 100-nit-avg curve {diffuse_known_nits}" + ); + } + + #[test] + fn spline_knee_follows_scene_average_brightness() { + // With a known scene average the pivot tracks the content, so a dark + // scene gets a lower source knee than a bright one (both stay within + // the [10%, 80%] of range clamp). + let (src_peak, dst_peak, black) = (1000.0_f32, 100.0_f32, 0.0_f32); + let dark_avg = 100.0_f32; + let bright_avg = 400.0_f32; + let dark_pivot = st2094_pick_knee( + 0.0, + pq_code(src_peak), + pq_code(dark_avg), + pq_code(black), + pq_code(dst_peak), + ) + .0; + let bright_pivot = st2094_pick_knee( + 0.0, + pq_code(src_peak), + pq_code(bright_avg), + pq_code(black), + pq_code(dst_peak), + ) + .0; + assert!( + dark_pivot < bright_pivot, + "dark {dark_pivot} vs bright {bright_pivot}" + ); + // Without metadata the pivot is the 40% default mix. + let default_pivot = st2094_pick_knee( + 0.0, + pq_code(src_peak), + 0.0, + pq_code(black), + pq_code(dst_peak), + ) + .0; + assert!((default_pivot - pq_code(src_peak) * 0.4).abs() < 1e-4); + } + + #[test] + fn pick_knee_clamps_stay_inside_the_fraction_range() { + // The pivot selection happens in the space the curve operates in + // (PQ for the shaders' spline, nits for ST 2094-10), so a mid-average + // input never escapes the [10%, 80%] of range clamp. + let (src_pq, dst_pq) = st2094_pick_knee( + 0.0, + pq_code(1000.0), + pq_code(500.0), + pq_code(0.0), + pq_code(100.0), + ); + assert!(src_pq >= pq_code(1000.0) * 0.1 - 1e-4); + assert!(src_pq <= pq_code(1000.0) * 0.8 + 1e-4); + assert!(dst_pq >= pq_code(100.0) * 0.1 - 1e-4); + assert!(dst_pq <= pq_code(100.0) * 0.8 + 1e-4); + // An out-of-range average clamps to the same fraction window. + let clamped = st2094_pick_knee( + 0.0, + pq_code(1000.0), + pq_code(9999.0), + pq_code(0.0), + pq_code(100.0), + ); + assert!((clamped.0 - pq_code(1000.0) * 0.8).abs() < 1e-4); + } + + /// Reference implementation of the shaders' BT.2390 branch with the + /// libplacebo black-point compensation (tone_map code 3). + fn bt2390_pq( + x: f32, + src_peak_nits: f32, + dst_peak_nits: f32, + dst_black_nits: f32, + knee_offset: f32, + ) -> f32 { + let in_max = pq_code(src_peak_nits).max(0.000_001); + let out_min = pq_code(dst_black_nits); + let out_max = pq_code(dst_peak_nits).max(0.000_001); + let max_lum = (out_max / in_max).clamp(0.0, 1.0); + let min_lum = out_min / in_max; + let ks = (1.0 + knee_offset) * max_lum - knee_offset; + let bp = (1.0 / min_lum.max(0.000_001)).min(4.0); + let mut u = x.clamp(0.0, in_max) / in_max; + if ks < 1.0 && u > ks { + let tb = (u - ks) / (1.0 - ks); + let tb2 = tb * tb; + let tb3 = tb2 * tb; + u = (2.0 * tb3 - 3.0 * tb2 + 1.0) * ks + + (tb3 - 2.0 * tb2 + tb) * (1.0 - ks) + + (-2.0 * tb3 + 3.0 * tb2) * max_lum; + } + if u < 1.0 { + u = u + min_lum * (1.0 - u).powf(bp); + let gain = if max_lum < 1.0 { + 1.0 / (1.0 + min_lum / max_lum * (1.0 - max_lum).powf(bp)) + } else { + 1.0 + }; + u = gain * (u - min_lum) + min_lum; + } + u * in_max + } + + #[test] + fn bt2390_curve_anchors_and_monotonicity() { + let (source_peak, target_peak, black) = (1000.0_f32, 100.0_f32, 0.203_f32); + // Black maps to the compensated target black (0.203 nit), which the + // encode stage maps back onto code 0. + let black_mapped = nits_from_pq(bt2390_pq(0.0, source_peak, target_peak, black, 1.0)); + assert!( + (black_mapped - black).abs() < 0.05, + "black = {black_mapped}" + ); + let peak = nits_from_pq(bt2390_pq( + pq_code(source_peak), + source_peak, + target_peak, + black, + 1.0, + )); + // Black-point compensation perturbs the exact peak slightly; the + // endpoint stays within half a nit of the target. + assert!((peak - target_peak).abs() < 0.5, "peak = {peak}"); + let mut previous = -1.0_f32; + for step in 0..=100 { + let x = pq_code(source_peak) * step as f32 / 100.0; + let mapped = nits_from_pq(bt2390_pq(x, source_peak, target_peak, black, 1.0)); + assert!(mapped >= previous); + previous = mapped; + } + // BPC compensates: for a target with 0 black the curve would be + // unchanged, with 0.203 it lifts dark steps slightly. + let with_bpc = bt2390_pq(pq_code(1.0), source_peak, target_peak, 0.203, 1.0); + let without_bpc = bt2390_pq(pq_code(1.0), source_peak, target_peak, 0.0, 1.0); + assert!(with_bpc > without_bpc); + } + + #[test] + fn bt2390_hdr10_to_sdr_matches_broadcast_diffuse_white() { + let (source_peak, target_peak, black) = (1000.0_f32, 203.0_f32, 0.0_f32); + let diffuse_white_mapped = nits_from_pq(bt2390_pq( + pq_code(203.0), + source_peak, + target_peak, + black, + 1.0, + )); + // On a 203-nit SDR tone-map target, 203 nits diffuse white must map + // to >= 130 nits (sRGB value >= 210, matching Infuse/MPV and well above 188.1). + assert!( + diffuse_white_mapped >= 130.0, + "mapped diffuse white: {diffuse_white_mapped} nits" + ); + } + + /// Reference implementation of the shaders' BT.2446 method A branch + /// (tone_map code 5), evaluated in nits. + fn bt2446a_nits( + x_nits: f32, + src_peak_nits: f32, + dst_peak_nits: f32, + dst_black_nits: f32, + ) -> f32 { + let phdr = 1.0 + 32.0 * (src_peak_nits / 10000.0).powf(1.0 / 2.4); + let psdr = 1.0 + 32.0 * (dst_peak_nits / 10000.0).powf(1.0 / 2.4); + let mut t = (x_nits.clamp(0.0, src_peak_nits) / src_peak_nits).powf(1.0 / 2.4); + t = (1.0 + (phdr - 1.0) * t).ln() / phdr.ln(); + t = if t <= 0.7399 { + 1.0770 * t + } else if t < 0.9909 { + (-1.1510 * t + 2.7811) * t - 0.6302 + } else { + 0.5 * t + 0.5 + }; + t = (psdr.powf(t) - 1.0) / (psdr - 1.0); + let lb = dst_black_nits.max(0.0).powf(1.0 / 2.4); + let lw = dst_peak_nits.max(0.0).powf(1.0 / 2.4); + ((lw - lb) * t + lb).powf(2.4) + } + + #[test] + fn bt2446a_curve_anchors_and_monotonicity() { + let (src_peak, dst_peak, black) = (1000.0_f32, 100.0_f32, 0.203_f32); + let lo = bt2446a_nits(0.0, src_peak, dst_peak, black); + assert!((lo - black).abs() < 0.05, "black = {lo}"); + let hi = bt2446a_nits(src_peak, src_peak, dst_peak, black); + assert!((hi - dst_peak).abs() < 0.05, "peak = {hi}"); + let mut previous = -1.0_f32; + for step in 0..=100 { + let mapped = bt2446a_nits(src_peak * step as f32 / 100.0, src_peak, dst_peak, black); + assert!(mapped >= previous); + previous = mapped; + } + } + + #[test] + fn st2094_10_coefficients_interpolate_the_three_anchors() { + let (src_peak, dst_peak, black) = (1000.0_f32, 100.0_f32, 0.203_f32); + let (src_knee, dst_knee) = + st2094_pick_knee_nits(0.0, src_peak, 250.0, black, dst_peak, 0.4); + let [c1, c2, c3, _] = solve_st2094_10(0.0, src_knee, src_peak, black, dst_knee, dst_peak); + let eval = |x_nits: f32| (c1 + c2 * x_nits) / (1.0 + c3 * x_nits); + assert!((eval(0.0) - black).abs() < 1e-3); + assert!((eval(src_knee) - dst_knee).abs() < 1e-2); + assert!((eval(src_peak) - dst_peak).abs() < 1e-2); + let mut previous = -1.0_f32; + for step in 0..=100 { + let mapped = eval(src_peak * step as f32 / 100.0); + assert!(mapped >= previous); + previous = mapped; + } + } + + #[test] + fn st2094_10_degenerate_anchors_fall_back_to_the_identity_curve() { + // Duplicated or non-finite anchors have no unique solution. The curve + // must degrade to Clip (y = x); returning zero coefficients would make + // the shader map every pixel to PQ 0, i.e. a black frame. + let identity = [0.0, 1.0, 0.0, 0.0]; + assert_eq!(solve_st2094_10(0.0, 0.0, 0.0, 0.0, 1.0, 1.0), identity); + assert_eq!(solve_st2094_10(0.0, 1.0, 1.0, 0.0, 1.0, 1.0), identity); + assert_eq!(solve_st2094_10(f32::NAN, 1.0, 2.0, 0.0, 1.0, 2.0), identity); + assert_eq!( + solve_st2094_10(0.0, 1.0, 2.0, f32::INFINITY, 1.0, 2.0), + identity + ); + let [c1, c2, c3, _] = identity; + let eval = |x: f32| (c1 + c2 * x) / (1.0 + c3 * x); + assert!((eval(0.25) - 0.25).abs() < 1e-6); + assert!((eval(1.0) - 1.0).abs() < 1e-6); + } + + #[test] + fn st2094_10_knee_follows_measured_scene_average_for_hdr10() { + // ST 2094-10 must use the same scene-average fallback chain as the + // spline pivot (measured average, then DV L1, then MaxFALL), so an + // HDR10 stream with measured luma gets a content-following knee + // instead of the 40% default. + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let mut config = ToneMapConfig::default(); + config.operator = ToneMapOperator::St209410; + + let dark = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .measured_scene_avg_nits(Some(60.0)); + let bright = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .measured_scene_avg_nits(Some(400.0)); + let dark = VideoRenderPipeline { + tone_map: config, + ..VideoRenderPipeline::new(dark, target) + }; + let bright = VideoRenderPipeline { + tone_map: config, + ..VideoRenderPipeline::new(bright, target) + }; + let dark_knee = st2094_pick_knee_nits( + 0.0, + dark.source.nominal_peak_nits, + source_scene_avg_nits(&dark.source), + 0.203, + dark.target.peak_nits, + 0.4, + ) + .0; + let bright_knee = st2094_pick_knee_nits( + 0.0, + bright.source.nominal_peak_nits, + source_scene_avg_nits(&bright.source), + 0.203, + bright.target.peak_nits, + 0.4, + ) + .0; + assert!( + dark_knee < bright_knee, + "dark {dark_knee} vs bright {bright_knee}" + ); + + // MaxFALL is the third rung of the fallback chain. + let metadata = HdrMetadata::new( + None, + Some(ContentLightMetadata { + max_content_light_level_nits: 1000, + max_frame_average_light_level_nits: 239, + }), + ); + let fall = VideoRenderPipeline { + tone_map: config, + ..VideoRenderPipeline::new( + SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .hdr_metadata(Some(metadata)), + target, + ) + }; + assert!((source_scene_avg_nits(&fall.source) - 239.0).abs() < 1e-3); + } + + #[test] + fn st2094_pick_knee_nits_picks_the_knee_in_pq_space() { + // libplacebo always picks the ST 2094 knee in the PQ domain, even for + // the NITS-scaled functions. A 50-nit average on a 4000-nit source must + // land near 50 nits, not on the linear-space 10% floor (400 nits) that + // a nits-domain evaluation would clamp to. + let (src_knee, _) = st2094_pick_knee_nits(0.0, 4000.0, 50.0, 0.0, 203.0, 0.4); + assert!( + (src_knee - 50.0).abs() < 2.0, + "knee should follow the scene average in nits: {src_knee}" + ); + // The nits wrapper is exactly the PQ-domain call with a rescale. + let (pq_src, pq_dst) = st2094_pick_knee( + 0.0, + pq_code_from_nits(4000.0), + pq_code_from_nits(50.0), + 0.0, + pq_code_from_nits(203.0), + ); + let (nits_src, nits_dst) = st2094_pick_knee_nits(0.0, 4000.0, 50.0, 0.0, 203.0, 0.4); + assert!((nits_from_pq_code(pq_src) - nits_src).abs() < 1e-3); + assert!((nits_from_pq_code(pq_dst) - nits_dst).abs() < 1e-3); + } + + #[test] + fn st2094_10_curve_param_tunes_the_knee_adaptation() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .measured_scene_avg_nits(Some(120.0)); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let coefficients = |curve_param: f32| { + let config = ToneMapConfig { + operator: ToneMapOperator::St209410, + curve_param, + ..ToneMapConfig::default() + }; + st2094_10_coefficients_for(&VideoRenderPipeline { + tone_map: config, + ..VideoRenderPipeline::new(source, target) + }) + }; + let low = coefficients(0.1); + let high = coefficients(1.0); + assert_ne!( + low, high, + "curve_param must reach the ST 2094-10 knee adaptation" + ); + // 0 resolves to libplacebo's `pl_tone_map_st2094_10.param_def`. + assert_eq!(coefficients(0.0), coefficients(0.7)); + } + + #[test] + fn dovi_l1_peak_does_not_toggle_the_tone_map_or_gamut_lut() { + // `nominal_peak_nits` carries the per-frame L1 peak for Dolby Vision + // content. Pass selection must stay on the static L0 peak so a dark + // scene cannot drop the tone map, the black-point compensation, or the + // perceptual gamut LUT for a single frame. + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let dark = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq).dovi(Some( + sample_dovi_metadata_with_l1(0, pq_code_12(150.0), pq_code_12(80.0)), + )); + let bright = + SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq).dovi(Some( + sample_dovi_metadata_with_l1(0, pq_code_12(2200.0), pq_code_12(1800.0)), + )); + assert!( + dark.nominal_peak_nits < 200.0, + "dark frame L1 peak should sit below the SDR target peak" + ); + let dark = VideoRenderPipeline::new(dark, target); + let bright = VideoRenderPipeline::new(bright, target); + assert!(dark.requires_tone_mapping()); + assert!(bright.requires_tone_mapping()); + assert!(dark.gamut_lut_active()); + assert!(bright.gamut_lut_active()); + assert_eq!(dark.tone_map_extra()[2], bright.tone_map_extra()[2]); + } + + #[test] + fn tone_map_operator_defaults_and_codes() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .reference_white_nits(203.0); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + assert_eq!(pipeline.tone_map.operator, ToneMapOperator::Bt2390); + assert_eq!(tone_map_code(pipeline.tone_map.operator), 3); + // The curve parameter 0 resolves to the operator default. + assert!((pipeline.tone_map.effective_curve_param() - 1.0).abs() < 1e-6); + // Default contrast is 1000:1 for SDR targets -> 203 / 1000 black. + let extra = pipeline.tone_map_extra(); + assert!((extra[2] - 0.203).abs() < 1e-4, "black = {}", extra[2]); + assert_eq!(extra[0], 1.0); + } + + #[test] + fn hdr_output_target_black_is_zero() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + let target = TargetColorState::hdr10(ColorPrimaries::Bt2020); + let extra = VideoRenderPipeline::new(source, target).tone_map_extra(); + assert_eq!(extra[2], 0.0); + let edr = VideoRenderPipeline::new( + source, + TargetColorState::apple_edr(ColorPrimaries::Bt2020, 2.0), + ) + .tone_map_extra(); + assert_eq!(edr[2], 0.0); + // An explicit contrast ratio overrides the automatic default. + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let mut config = ToneMapConfig::default(); + config.contrast_ratio = 500.0; + let pipeline = VideoRenderPipeline::new(source, target); + let custom = VideoRenderPipeline { + tone_map: config, + ..pipeline + }; + assert!((custom.tone_map_extra()[2] - 203.0 / 500.0).abs() < 1e-4); + } + + #[test] + fn sdr_to_sdr_black_point_is_zero() { + // Black-point compensation must never run on SDR->SDR (near-black + // content would crush); the tone-map-only guard zeroes the black. + let source = SourceColorState::new(ColorPrimaries::Bt709, TransferFunction::Srgb); + let target = TargetColorState::sdr(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + assert!(!pipeline.requires_tone_mapping()); + assert_eq!(pipeline.tone_map_extra()[2], 0.0); + // An explicit contrast ratio is still ignored on the inactive path. + let mut config = ToneMapConfig::default(); + config.contrast_ratio = 500.0; + let custom = VideoRenderPipeline { + tone_map: config, + ..pipeline + }; + assert_eq!(custom.tone_map_extra()[2], 0.0); + } + + #[test] + fn ipt_matrices_match_libplacebo_values_and_white_is_invariant() { + // Values computed independently from libplacebo's pl_ipt_rgb2lms + // (4% crosstalk mix of HPE XYZ->LMS times the primaries RGB->XYZ). + let bt709 = ipt_rgb2lms_matrix(ColorPrimaries::Bt709); + let expected709 = [ + [0.2957641, 0.6230725, 0.0811667], + [0.1561920, 0.7272516, 0.1165579], + [0.0351023, 0.1565899, 0.8083030], + ]; + for (row, expected) in bt709.rows().iter().zip(expected709) { + for (value, expected) in row.iter().zip(expected) { + assert!((value - expected).abs() < 1e-5, "{value} != {expected}"); + } + } + // D65 white maps to equal LMS across primaries and inverts back. + for primaries in [ + ColorPrimaries::Bt709, + ColorPrimaries::Bt2020, + ColorPrimaries::DisplayP3, + ] { + let matrix = ipt_rgb2lms_matrix(primaries); + let lms = matrix.mul_vec([1.0, 1.0, 1.0]); + for value in lms { + assert!((value - 1.0).abs() < 1e-3, "white -> {lms:?}"); + } + let inverse = ipt_lms2rgb_matrix(primaries); + let back = inverse.mul_vec(lms); + for value in back { + assert!((value - 1.0).abs() < 1e-3, "roundtrip -> {back:?}"); + } + } + } + + #[test] + fn tone_map_pipeline_is_present_across_video_shaders() { + let shaders = [ + include_str!("wgpu_video.wgsl"), + include_str!("metal/apple.rs"), + include_str!("d3d11.rs"), + ]; + for shader in shaders { + assert!(shader.contains("st2094_pick_knee")); + assert!(shader.contains("tone_map_curve_pq")); + assert!(shader.contains("ipt_matrix_rows")); + assert!(shader.contains("tone_map_extra")); + assert!(shader.contains("tone_map_coeffs")); + assert!(shader.contains("0.0975689")); + assert!(shader.contains("tone_map == 4")); + assert!(shader.contains("tone_map == 5")); + assert!(shader.contains("SMPTE ST 2094-10")); + assert!(shader.contains("0.7399")); + // The IPT path applies the primaries conversion inside the tone + // map, so the old separate matrix call is gone. + assert!(!shader.contains("rgb = apply_gamut_map(rgb)")); + } + } + + #[test] + fn measured_scene_avg_precedes_dovi_l1_and_drives_the_pivot() { + let mut source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + // HDR10 without L1: attach a measured average, tone_map_extra.y follows. + source = source.measured_scene_avg_nits(Some(120.0)); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + let extra = pipeline.tone_map_extra(); + assert!((extra[1] - 120.0).abs() < 1e-3, "scene avg {}", extra[1]); + // A non-finite / zero value clears the channel. + let cleared_source = source.measured_scene_avg_nits(Some(0.0)); + assert_eq!(cleared_source.measured_scene_avg_nits, None); + let cleared = VideoRenderPipeline::new(cleared_source, target); + assert_eq!(cleared.tone_map_extra()[1], 0.0); + } + + #[test] + fn gamut_lut_active_only_for_tone_mapped_wide_gamut_sources() { + // BT.2020 PQ -> BT.709 SDR needs the perceptual LUT. + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + assert!(pipeline.gamut_lut_active()); + assert_eq!( + VideoUniforms::from_pipeline(&pipeline, false, false).gamut_lut_enabled, + 1 + ); + // Same-gamut HDR tone map keeps the fast path. + let same = VideoRenderPipeline::new( + source, + TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt2020), + ); + assert!(!same.gamut_lut_active()); + // SDR -> SDR passthrough never maps. + let sdr = VideoRenderPipeline::new( + SourceColorState::new(ColorPrimaries::Bt709, TransferFunction::Srgb), + TargetColorState::sdr(ColorPrimaries::Bt709), + ); + assert!(!sdr.gamut_lut_active()); + // HDR10 native output (PQ target, display does the mapping) needs no + // gamut LUT either. + let hdr10 = + VideoRenderPipeline::new(source, TargetColorState::hdr10(ColorPrimaries::Bt2020)); + assert!(!hdr10.gamut_lut_active()); + } + + #[test] + fn gamut_lut_primaries_code_round_trips() { + let code = gamut_primaries_code(ColorPrimaries::Bt2020, ColorPrimaries::Bt709); + assert_eq!(code >> 8, 1); + assert_eq!(code & 0xff, 0); + let p3 = gamut_primaries_code(ColorPrimaries::Bt709, ColorPrimaries::DisplayP3); + assert_eq!(p3 >> 8, 0); + assert_eq!(p3 & 0xff, 2); + } + + #[test] + fn gamut_lut_sampling_is_present_across_video_shaders() { + let shaders = [ + include_str!("wgpu_video.wgsl"), + include_str!("metal/apple.rs"), + include_str!("d3d11.rs"), + ]; + for shader in shaders { + assert!(shader.contains("gamut_lut_enabled")); + assert!(shader.contains("0.5 + 0.5 * atan2")); + assert!(shader.contains("ipt_matrix_rows[6].xyz")); + assert!(shader.contains("sampled.y - 0.5")); + // The LUT path replaces the fast gamut_compress. + assert!(shader.contains("gamut_compress")); + } + } + + #[cfg(feature = "wgpu")] + #[test] + fn wgsl_video_shader_parses_with_naga() { + // The wgpu backend compiles the WGSL only at render time; parse it + // here so syntax regressions fail in tests instead of on-device. + let source = include_str!("wgpu_video.wgsl"); + wgpu::naga::front::wgsl::parse_str(source) + .unwrap_or_else(|error| panic!("invalid WGSL: {error}")); + } + + /// Reference implementation of the shaders' `gamut_compress`: colors the + /// linear gamut matrix pushes out of the target gamut are blended + /// towards their naively-clipped version by an out-of-gamut smoothstep. + /// Slightly-out colors stay nearly intact; strongly-out BT.2020 + /// primaries land on the pure target primary with their hue intact — + /// luma blending would instead pull primary red towards grey and turn + /// it pink. Matches mpv's perceptual gamut mapping behavior. + fn gamut_compress(rgb: [f32; 3]) -> [f32; 3] { + let lo = rgb[0].min(rgb[1]).min(rgb[2]); + let outness = (-lo).max(0.0); + let x = (outness / 1.0).clamp(0.0, 1.0); + let k = x * x * (3.0 - 2.0 * x); + let mix = |a: f32, b: f32| a + (b - a) * k; + [ + mix(rgb[0], rgb[0].clamp(0.0, 1.0)), + mix(rgb[1], rgb[1].clamp(0.0, 1.0)), + mix(rgb[2], rgb[2].clamp(0.0, 1.0)), + ] + } + + #[test] + fn gamut_compress_preserves_hue_of_out_of_gamut_primaries() { + // In-gamut colors pass through untouched. + assert_eq!(gamut_compress([0.2, 0.7, 0.3]), [0.2, 0.7, 0.3]); + // A saturated BT.2020 teal-green that the gamut matrix pushes out of + // gamut: the compression never pushes a channel further out, and the + // channel ordering (hue) is preserved — hard clipping would have + // zeroed red/blue and turned it neon. (Residual negative components + // are clamped at encode time, as libplacebo clamps to its gamut + // floor.) + let out_of_gamut = [-0.08_f32, 0.9, -0.04]; + let mapped = gamut_compress(out_of_gamut); + assert!(mapped[0] > out_of_gamut[0] && mapped[2] > out_of_gamut[2]); + assert!(mapped[1] > mapped[2] && mapped[2] > mapped[0]); + // A strongly-out primary red (negative green and blue) maps to a + // hue-pure red: green and blue are compressed to ~0 together, and + // crucially blue is not lifted towards the luma grey — that is what + // turned wide-gamut red pink under luma blending. + let primary_red = [1.1_f32, -0.22, -0.07]; + let red = gamut_compress(primary_red); + assert!(red[1] < 0.01 && red[2] < 0.01, "red = {red:?}"); + assert!(red[0] > 0.9, "red = {red:?}"); + // Compression is monotonic: an out-of-gamut excursion of 1.0 maps fully + // onto the clip (k = 1), while mild excursions keep most of their + // range. + assert_eq!(gamut_compress([-1.0_f32, 0.9, -0.5]), [0.0, 0.9, 0.0]); + let mild = gamut_compress([-0.1_f32, 0.9, -0.05]); + assert!(mild[0] > -0.1 && mild[2] > -0.05); + } + + #[test] + fn gamut_compress_formula_is_present_across_video_shaders() { + let shaders = [ + include_str!("wgpu_video.wgsl"), + include_str!("metal/apple.rs"), + include_str!("d3d11.rs"), + ]; + for shader in shaders { + assert!(shader.contains("gamut_compress")); + assert!(shader.contains("smoothstep(0.0, 1.0, outness)")); + assert!(shader.contains("rgb = gamut_compress(rgb)")); + } + } + #[test] fn overlay_shaders_handle_sdr_ui_for_hdr_targets() { let metal = include_str!("metal/apple.rs"); @@ -1106,6 +2690,51 @@ mod tests { assert_eq!(source.nominal_peak_nits, 1000.0); } + #[test] + fn hdr_metadata_nominal_peak_bounds_by_max_cll_when_lower_than_mastering() { + let metadata = HdrMetadata::new( + Some(MasteringDisplayMetadata { + display_primaries: None, + white_point: None, + min_luminance_nits: Some(0.005), + max_luminance_nits: Some(1000.0), + }), + Some(ContentLightMetadata { + max_content_light_level_nits: 528, + max_frame_average_light_level_nits: 239, + }), + ); + + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .hdr_metadata(Some(metadata)); + + assert_eq!(metadata.nominal_peak_nits(), Some(528.0)); + assert_eq!(source.nominal_peak_nits, 528.0); + } + + #[test] + fn tone_map_extra_falls_back_to_max_fall_when_unmeasured() { + let metadata = HdrMetadata::new( + Some(MasteringDisplayMetadata { + display_primaries: None, + white_point: None, + min_luminance_nits: Some(0.005), + max_luminance_nits: Some(1000.0), + }), + Some(ContentLightMetadata { + max_content_light_level_nits: 528, + max_frame_average_light_level_nits: 239, + }), + ); + + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .hdr_metadata(Some(metadata)); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + let extra = pipeline.tone_map_extra(); + assert_eq!(extra[1], 239.0); + } + #[test] fn bt709_to_bt709_gamut_matrix_is_identity() { let matrix = source_to_target_rgb_matrix(ColorPrimaries::Bt709, ColorPrimaries::Bt709); @@ -1175,6 +2804,307 @@ mod tests { assert!(!uniforms.packed_alpha_right(false).has_packed_alpha_right()); } + /// A curve shaped like the RPU's default luma mapping: two polynomial + /// segments split at pivot 0.25, then one MMR segment of order 2. + fn sample_dovi_metadata() -> DoviSourceMetadata { + let mut reshaping = [DoviComponentCurve::default(); 3]; + reshaping[0].num_pivots = 4; + reshaping[0].pivots = [0.0, 0.25, 0.5, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + reshaping[0].poly_coeffs[0] = [0.0, 0.5, 0.0]; + reshaping[0].poly_coeffs[1] = [1.0, 1.0, 0.5]; + reshaping[0].mmr_orders[2] = 2; + reshaping[0].mmr_constants[2] = 0.25; + reshaping[0].mmr_coeffs[2][0] = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]; + reshaping[0].mmr_coeffs[2][1] = [0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4]; + DoviSourceMetadata { + reshaping, + nonlinear_matrix: RgbMatrix::new([ + [1.0, 0.5, 0.25], + [0.75, 1.0, 0.125], + [0.0625, 0.03125, 1.0], + ]), + nonlinear_offset: [0.25, 0.5, 0.5], + rgb_to_lms: RgbMatrix::new([ + [0.356742, 0.592257, 0.051081], + [0.156705, 0.747860, 0.095435], + [0.0, 0.041455, 0.958545], + ]), + source_min_pq: 62, + source_max_pq: 3079, + l1: None, + } + } + + #[test] + fn dovi_source_uses_per_frame_l1_peak_when_present() { + let mut source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + source = source.dovi(Some(sample_dovi_metadata_with_l1(1500, 2200, 1800))); + + assert_eq!( + source.nominal_peak_nits, + pq_code_to_nits(2200).max(1.0), + "per-frame L1 peak must replace the static RPU peak for tone mapping" + ); + // The static mastering display metadata keeps the L0 peak so + // output-mode negotiation stays stable frame to frame. + let mastering = source.hdr_metadata.unwrap().mastering_display.unwrap(); + assert_eq!(mastering.max_luminance_nits, Some(pq_code_to_nits(3079))); + } + + #[test] + fn dovi_source_falls_back_to_static_peak_when_l1_is_absent() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(sample_dovi_metadata_with_l1(0, 0, 0))); + assert_eq!( + source.nominal_peak_nits, + pq_code_to_nits(3079).max(1.0), + "an all-zero L1 block must not replace the static RPU peak" + ); + + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(sample_dovi_metadata())); + assert_eq!(source.nominal_peak_nits, pq_code_to_nits(3079).max(1.0)); + } + + fn sample_dovi_metadata_with_l1(min_pq: u16, max_pq: u16, avg_pq: u16) -> DoviSourceMetadata { + let mut metadata = sample_dovi_metadata(); + metadata.l1 = Some(DoviFramePq { + min_pq, + max_pq, + avg_pq, + }); + metadata + } + + /// 12-bit PQ code for an absolute luminance, for building RPU L1 blocks. + fn pq_code_12(nits: f32) -> u16 { + (pq_code_from_nits(nits) * 4095.0) + .round() + .clamp(0.0, 4095.0) as u16 + } + + #[test] + fn dovi_uniforms_are_disabled_without_metadata() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + assert_eq!( + DoviUniforms::of_for_representation(&source, false), + DoviUniforms::disabled() + ); + assert_eq!(DoviUniforms::disabled().flags[0], 0.0); + + let target = TargetColorState::sdr(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + let uniforms = VideoUniforms::from_pipeline(&pipeline, false, false).dovi; + + assert_eq!(uniforms, DoviUniforms::disabled()); + assert_eq!(uniforms.flags[0], 0.0); + } + + #[test] + fn dovi_uniforms_pack_pivots_poly_and_mmr() { + let metadata = sample_dovi_metadata(); + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(metadata)); + let uniforms = DoviUniforms::of_for_representation(&source, true); + + assert_eq!(uniforms.flags, [1.0, 3.0, 0.0, 0.0]); + // Interior pivots skip the endpoints; padding gets the sentinel. + assert_eq!( + uniforms.pivots[0], + [0.25, 0.5, DOVI_PIVOT_SENTINEL, DOVI_PIVOT_SENTINEL] + ); + assert_eq!(uniforms.pivots[1], [DOVI_PIVOT_SENTINEL; 4]); + assert_eq!(uniforms.bounds[0], [0.0, 1.0, 0.0, 0.0]); + assert_eq!(uniforms.coefficients[0], [0.0, 0.5, 0.0, 0.0]); + assert_eq!(uniforms.coefficients[1], [1.0, 1.0, 0.5, 0.0]); + // MMR rows start after two polynomial segments; the order rides in w. + assert_eq!(uniforms.coefficients[2], [0.25, 0.0, 0.0, 2.0]); + assert_eq!(uniforms.mmr[0], [0.1, 0.2, 0.3, 0.0]); + assert_eq!(uniforms.mmr[1], [0.4, 0.5, 0.6, 0.7]); + assert_eq!(uniforms.mmr[2], [0.8, 0.9, 1.0, 0.0]); + assert_eq!(uniforms.mmr[3], [1.1, 1.2, 1.3, 1.4]); + assert_eq!(uniforms.mmr[4], [0.0, 0.0, 0.0, 0.0]); + } + + #[test] + fn dovi_uniforms_apply_signal_offset_correction() { + let metadata = sample_dovi_metadata(); + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(metadata)); + let uniforms = DoviUniforms::of_for_representation(&source, true); + let correction = 1024.0_f32 / 1023.0; + + assert!((uniforms.nonlinear_offset[0] - 0.25 * correction).abs() < 1e-6); + assert!((uniforms.nonlinear_offset[1] - 0.5 * correction).abs() < 1e-6); + assert_eq!(uniforms.nonlinear_matrix[0], [1.0, 0.5, 0.25, 0.0]); + } + + #[test] + fn dovi_uniforms_use_the_uploaded_sample_depth_for_offsets() { + let metadata = sample_dovi_metadata(); + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(metadata)); + let p010 = DoviUniforms::of_for_representation(&source, true); + let nv12 = DoviUniforms::of_for_representation(&source, false); + assert!((p010.nonlinear_offset[0] - 0.25 * 1024.0 / 1023.0).abs() < 1e-6); + assert!((nv12.nonlinear_offset[0] - 0.25 * 256.0 / 255.0).abs() < 1e-6); + + let uniforms = VideoUniforms::from_pipeline( + &VideoRenderPipeline::new(source, TargetColorState::sdr(ColorPrimaries::Bt709)), + true, + false, + ); + let converted = uniforms.with_p010_representation(false); + assert_eq!(converted.is_p010, 0); + assert!((converted.dovi.nonlinear_offset[0] - nv12.nonlinear_offset[0]).abs() < 1e-6); + } + + #[test] + fn dovi_lms_matrix_composite_matches_libplacebo_default() { + // libplacebo composites its hard-coded HPE LMS->RGB matrix with the + // RPU's rgb_to_lms rows; for the RPU default matrix the product is + // this near-diagonal, white-preserving transform. + let matrix = dovi_lms_to_rgb_matrix(RgbMatrix::new([ + [5845.0 / 16384.0, 9702.0 / 16384.0, 837.0 / 16384.0], + [2568.0 / 16384.0, 12256.0 / 16384.0, 1561.0 / 16384.0], + [0.0, 679.0 / 16384.0, 15705.0 / 16384.0], + ])); + + let expected = [ + [0.753741425, 0.198592403, 0.047534181], + [0.045791140, 0.941773555, 0.012526896], + [-0.001211792, 0.017623405, 0.983739703], + ]; + for (row, expected_row) in matrix.rows().iter().zip(expected) { + for (value, expected_value) in row.iter().zip(expected_row) { + assert!((value - expected_value).abs() < 1e-5); + } + } + } + + #[test] + fn dovi_source_uses_rpu_peak_and_bt2020_primaries() { + let metadata = sample_dovi_metadata(); + let source = SourceColorState::new(ColorPrimaries::DisplayP3, TransferFunction::Pq) + .hdr_metadata(Some(HdrMetadata::new( + Some(MasteringDisplayMetadata { + display_primaries: None, + white_point: None, + min_luminance_nits: Some(0.005), + max_luminance_nits: Some(4000.0), + }), + None, + ))) + .dovi(Some(metadata)); + + // PQ code 3079 is the 12-bit encoding of ~1000 nits. + assert!((source.nominal_peak_nits - 1000.0).abs() < 5.0); + assert!( + (source + .hdr_metadata + .unwrap() + .mastering_display + .unwrap() + .min_luminance_nits + .unwrap() + - 0.005) + .abs() + < 0.0001 + ); + assert_eq!(source.primaries, ColorPrimaries::Bt2020); + assert!(source.is_hdr()); + assert_eq!(pq_code_to_nits(0), 0.0); + } + + #[test] + fn dovi_source_peak_falls_back_to_pq_default_when_rpu_max_pq_is_zero() { + let mut metadata = sample_dovi_metadata(); + metadata.source_max_pq = 0; + let source = SourceColorState::new(ColorPrimaries::Unknown, TransferFunction::Unknown) + .dovi(Some(metadata)); + + assert_eq!(source.transfer, TransferFunction::Pq); + assert_eq!(source.reference_white_nits, 203.0); + assert_eq!(source.nominal_peak_nits, 1000.0); + assert!(source.nominal_peak_nits > source.reference_white_nits); + } + + #[test] + fn dovi_source_forces_pq_when_stream_tags_are_missing() { + // libplacebo forces BT.2020/PQ from the RPU because P5/P8 VUI tags are + // unreliable; without this an unspecified trc would decode the + // reshaped PQ signal with an sRGB gamma. + let source = SourceColorState::new(ColorPrimaries::Unknown, TransferFunction::Unknown) + .dovi(Some(sample_dovi_metadata())); + + assert_eq!(source.transfer, TransferFunction::Pq); + assert_eq!(source.primaries, ColorPrimaries::Bt2020); + assert_eq!(source.reference_white_nits, 203.0); + assert_eq!(transfer_code(source.transfer), 3); + assert!(source.is_hdr()); + } + + #[test] + fn dovi_source_adds_reshape_pass_and_tone_maps_to_sdr() { + let source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq) + .dovi(Some(sample_dovi_metadata())); + let pipeline = + VideoRenderPipeline::new(source, TargetColorState::sdr(ColorPrimaries::Bt709)); + + assert!(pipeline.graph.contains(RenderPassKind::DoviReshape)); + assert!(pipeline.requires_tone_mapping()); + assert!(pipeline.requires_gamut_mapping()); + + let pipeline = + VideoRenderPipeline::new(source, TargetColorState::hdr10(ColorPrimaries::Bt2020)); + assert!(!pipeline.requires_tone_mapping()); + } + + #[test] + fn dovi_formulas_are_present_across_video_shaders() { + let shaders = [ + include_str!("wgpu_video.wgsl"), + include_str!("metal/apple.rs"), + include_str!("d3d11.rs"), + ]; + for shader in shaders { + assert!(shader.contains("dovi_flags")); + assert!(shader.contains("dovi_pivots")); + assert!(shader.contains("dovi_bounds")); + assert!(shader.contains("dovi_coefficients")); + assert!(shader.contains("dovi_mmr")); + assert!(shader.contains("dovi_nonlinear_matrix")); + assert!(shader.contains("dovi_nonlinear_offset")); + assert!(shader.contains("dovi_lms_matrix")); + assert!(shader.contains("dovi_reshaped_signal")); + assert!(shader.contains("dovi_signal_to_pq_rgb")); + assert!(shader.contains("dovi_lms_to_rgb")); + } + } + + #[test] + fn dovi_sdr_target_configures_bt709_output_matrix() { + let metadata = sample_dovi_metadata(); + let source = SourceColorState::default().dovi(Some(metadata)); + let target = TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709); + let pipeline = VideoRenderPipeline::new(source, target); + let uniforms = VideoUniforms::from_pipeline(&pipeline, false, false); + + // Verify that ipt_matrix_rows[6..8] are properly configured to convert + // LMS directly to the display target primaries (BT.709), matching libplacebo. + let lms_to_target_r = [ + uniforms.ipt_matrix_rows[6][0], + uniforms.ipt_matrix_rows[6][1], + uniforms.ipt_matrix_rows[6][2], + ]; + // Target is BT.709: row 6 dot [1, 1, 1] should equal 1.0 (white point preservation) + let white_sum = lms_to_target_r[0] + lms_to_target_r[1] + lms_to_target_r[2]; + assert!( + (white_sum - 1.0).abs() < 1e-3, + "White point sum was {white_sum}" + ); + } + fn assert_matrix_close(actual: [[f32; 3]; 3], expected: [[f32; 3]; 3], epsilon: f32) { for row in 0..3 { for col in 0..3 { diff --git a/crates/erika/src/renderer/wgpu.rs b/crates/erika/src/renderer/wgpu.rs index 4a0a331b..cb740515 100644 --- a/crates/erika/src/renderer/wgpu.rs +++ b/crates/erika/src/renderer/wgpu.rs @@ -19,7 +19,6 @@ use wgpu::util::DeviceExt; #[cfg(target_os = "android")] use crate::android::{AndroidDataSpaceErrorKind, AndroidNativeWindow}; -#[cfg(target_os = "android")] use crate::core::ColorPrimaries; #[cfg(any( target_os = "android", @@ -31,7 +30,8 @@ use crate::core::ColorPrimaries; use crate::core::WgpuSurfaceKind; use crate::core::{ LumaUpscalerBackendStatus, PlatformSurface, PlayerError, PlayerVideoFrame, RenderFrameContext, - RendererBackend, RendererRuntimeStats, Result, SurfaceOutputCapabilities, WgpuSurfaceHandle, + RendererBackend, RendererRuntimeStats, Result, SurfaceOutputCapabilities, TransferFunction, + WgpuSurfaceHandle, }; use crate::danmaku::{ DanmakuAtlasUpdate, DanmakuGlyphAtlas, DanmakuGlyphInstance, DanmakuRenderPlan, @@ -43,6 +43,9 @@ use crate::renderer::android_vulkan::{ AndroidAhbConversionError, AndroidAhbCrop, AndroidAhbFrameDescription, AndroidAhbIntermediateFormat, AndroidVulkanInterop, retire_ahb_conversion_after_submission, }; +use crate::renderer::gamut::{ + GamutLut, GamutLutJob, GamutLutParams, LUT_SIZE_C, LUT_SIZE_H, LUT_SIZE_I, pack_rgba16f, +}; use crate::renderer::metal::{MetalRendererConfig, VideoAlphaMode}; #[cfg(target_env = "ohos")] use crate::renderer::ohos_vulkan::{ @@ -279,6 +282,38 @@ impl OverlayUniforms { } } +/// Decode the packed (source << 8 | target) primaries codes from +/// `VideoUniforms::_gamut_reserved`, used to key the perceptual LUT cache. +fn gamut_lut_key_of(uniforms: VideoUniforms) -> Option<(ColorPrimaries, ColorPrimaries, u32, u32)> { + if uniforms.gamut_lut_enabled == 0 { + return None; + } + let packed = uniforms._gamut_primaries; + let source = match packed >> 8 { + 1 => ColorPrimaries::Bt2020, + 2 => ColorPrimaries::DisplayP3, + _ => ColorPrimaries::Bt709, + }; + let target = match packed & 0xff { + 1 => ColorPrimaries::Bt2020, + 2 => ColorPrimaries::DisplayP3, + _ => ColorPrimaries::Bt709, + }; + // The LUT's I axis spans [target black, target peak] in PQ codes; any + // change to either endpoint changes the sampled range, so include both + // (`tone_map_extra.z` is the target black in nits). + let target_black_pq = quantize_luma_pq(uniforms.tone_map_extra[2]); + let target_peak_pq = quantize_luma_pq(uniforms.nits[1]); + Some((source, target, target_black_pq, target_peak_pq)) +} + +/// Quantize a luminance (nits) to its PQ code for the LUT cache key. The key +/// only has to change when the LUT's I axis changes, so 16-bit PQ resolution +/// is ample (and keeps the sub-1-nit target blacks of SDR targets distinct). +fn quantize_luma_pq(nits: f32) -> u32 { + (pq_code_for_lut(nits) * 65535.0) as u32 +} + /// Lazily-built GPU objects for the NV12/P010 video pipeline, tied to the color /// target format the pipeline was compiled for. struct VideoPipeline { @@ -365,7 +400,7 @@ fn prepare_planar_upload( } else { (frame, PlanarUploadPath::Native) }; - uniforms.is_p010 = u32::from(frame.format == PlanarPixelFormat::P010); + uniforms = uniforms.with_p010_representation(frame.format == PlanarPixelFormat::P010); Ok(PreparedPlanarUpload { frame, uniforms, @@ -373,6 +408,16 @@ fn prepare_planar_upload( }) } +fn pq_code_for_lut(nits: f32) -> f32 { + let m1 = 0.1593017578125_f32; + let m2 = 78.84375_f32; + let c1 = 0.8359375_f32; + let c2 = 18.8515625_f32; + let c3 = 18.6875_f32; + let p = (nits / 10000.0).clamp(0.0, 1.0).powf(m1); + ((c1 + c2 * p) / (1.0 + c3 * p).max(0.000_001)).powf(m2) +} + fn source_color_for_player_frame(frame: &PlayerVideoFrame) -> SourceColorState { SourceColorState::new( frame.frame.color_primaries(), @@ -381,6 +426,8 @@ fn source_color_for_player_frame(frame: &PlayerVideoFrame) -> SourceColorState { .range(frame.frame.color_range()) .matrix(frame.frame.matrix_coefficients()) .hdr_metadata(frame.frame.hdr_metadata()) + .dovi(frame.frame.dovi_metadata()) + .measured_scene_avg_nits(frame.scene_avg_nits) } #[cfg(target_os = "android")] @@ -423,7 +470,7 @@ impl UploadedVideoFrame { let Some(source) = self.source_color else { return self.uniforms; }; - let pipeline = VideoRenderPipeline::new(source, output.target); + let pipeline = VideoRenderPipeline::new(source, output.tone_map_target_for(&source)); let uniforms = VideoUniforms::from_pipeline( &pipeline, self.uniforms.is_p010 != 0, @@ -499,6 +546,19 @@ pub struct WgpuRenderer { surface: Option, video_pipeline: Option, overlay_pipeline: Option, + /// Perceptual gamut-mapping 3D LUT cache, keyed by the source/target + /// primaries plus the target black/peak (PQ codes) that parametrize the + /// LUT's I axis. + gamut_lut: Option<( + (ColorPrimaries, ColorPrimaries, u32, u32), + wgpu::Texture, + wgpu::TextureView, + )>, + /// Background generation for a cache miss; the fast `gamut_compress` + /// path renders until the LUT lands. + gamut_lut_job: Option, + /// 1x1x1 fallback view so binding 4 always has a valid resource. + dummy_lut: Option, current_video: Option, current_video_visible: bool, upload_serial: u64, @@ -1425,6 +1485,9 @@ impl WgpuRenderer { output_status: OutputRuntimeStatus::requested(output_mode), output_headroom: OutputHeadroomState::default(), upscaler_mode: LumaUpscalerMode::Off, + gamut_lut: None, + gamut_lut_job: None, + dummy_lut: None, upscaler, upscaler_failed_frame_token: None, upscaler_active_frame_reported: false, @@ -1891,7 +1954,7 @@ impl WgpuRenderer { .surface .as_ref() .map_or_else(OutputDescription::sdr, |surface| surface.output); - let pipeline = VideoRenderPipeline::new(source, output.target); + let pipeline = VideoRenderPipeline::new(source, output.tone_map_target_for(&source)); if source.is_hdr() { self.stats.hdr_source_frames += 1; if !output.extended_linear && pipeline.requires_tone_mapping() { @@ -2345,6 +2408,127 @@ impl WgpuRenderer { })) } + /// Lazily create (and cache) the perceptual gamut LUT texture described + /// by the current uniforms. Returns `None` while the fast path is active + /// or the background generation has not finished yet; callers must then + /// mask `gamut_lut_enabled` off so the shader keeps the fast path. + fn gamut_lut_view(&mut self, uniforms: VideoUniforms) -> Option { + let (source, target, target_black_pq, target_peak_pq) = gamut_lut_key_of(uniforms)?; + let cached = self.gamut_lut.as_ref().is_some_and(|(key, _, _)| { + key.0 == source + && key.1 == target + && key.2 == target_black_pq + && key.3 == target_peak_pq + }); + if !cached { + let peak_nits = uniforms.nits[1].max(1.0); + let params = GamutLutParams { + source, + target, + // Same target black the shader derives from tone_map_extra.z. + min_luma: pq_code_for_lut(uniforms.tone_map_extra[2]), + max_luma: pq_code_for_lut(peak_nits), + }; + let job_params = self + .gamut_lut_job + .as_ref() + .map(GamutLutJob::params) + .filter(|job_params| *job_params == params); + match job_params { + // A matching job is running: take its result once it lands. + Some(_) => { + let lut = self.gamut_lut_job.as_ref().and_then(GamutLutJob::poll)?; + self.gamut_lut_job = None; + self.upload_gamut_lut(lut, source, target, target_black_pq, target_peak_pq); + } + None => { + // First request (or the key changed): spawn generation + // and keep the fast path for this frame. + self.gamut_lut_job = Some(GamutLutJob::spawn(params)); + return None; + } + } + } + Some( + self.gamut_lut + .as_ref() + .expect("gamut lut after upload") + .2 + .clone(), + ) + } + + fn upload_gamut_lut( + &mut self, + lut: GamutLut, + source: ColorPrimaries, + target: ColorPrimaries, + target_black_pq: u32, + target_peak_pq: u32, + ) { + let size = wgpu::Extent3d { + width: LUT_SIZE_I as u32, + height: LUT_SIZE_C as u32, + depth_or_array_layers: LUT_SIZE_H as u32, + }; + let texture = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("erika-wgpu-gamut-lut"), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D3, + format: wgpu::TextureFormat::Rgba16Float, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + // The texels are packed RGB (I, P+0.5, T+0.5); pad to RGBA16F. + let rgba16 = pack_rgba16f(&lut.texels, 1.0); + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &rgba16, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some((LUT_SIZE_I * 8) as u32), + // One "row" is a texel; one image (slice) is the C axis. + rows_per_image: Some(LUT_SIZE_C as u32), + }, + size, + ); + let view = texture.create_view(&wgpu::TextureViewDescriptor::default()); + self.gamut_lut = Some(( + (source, target, target_black_pq, target_peak_pq), + texture, + view, + )); + } + + /// A tiny 1x1x1 view for binding 4 when no LUT is in use. + fn dummy_lut_view(&mut self) -> Option { + if self.dummy_lut.is_none() { + let texture = self.device.create_texture(&wgpu::TextureDescriptor { + label: Some("erika-wgpu-gamut-lut-dummy"), + size: wgpu::Extent3d { + width: 1, + height: 1, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D3, + format: wgpu::TextureFormat::Rgba16Float, + usage: wgpu::TextureUsages::TEXTURE_BINDING, + view_formats: &[], + }); + self.dummy_lut = Some(texture.create_view(&wgpu::TextureViewDescriptor::default())); + } + self.dummy_lut.clone() + } + /// Encode and submit a render pass drawing the current video frame into /// `target_view`. The caller must have uploaded a frame and the video pipeline /// must be initialized. @@ -2525,6 +2709,19 @@ impl WgpuRenderer { .as_ref() .map_or(&native_luma_view, |output| &output.view); let chroma_view = &native_chroma_view; + // Create/cache the gamut LUT first: it needs `&mut self`, while the + // pipeline borrow below is immutable and would otherwise conflict. + // While the background generation is pending the uniform keeps the + // fast gamut_compress path and the dummy LUT holds binding 4. + let gamut_lut_view = self.gamut_lut_view(video_uniforms); + if gamut_lut_view.is_none() { + video_uniforms.gamut_lut_enabled = 0; + } + let dummy_lut_view = self.dummy_lut_view(); + let gamut_binding = match &gamut_lut_view { + Some(view) => wgpu::BindingResource::TextureView(view), + None => wgpu::BindingResource::TextureView(&dummy_lut_view.expect("dummy LUT view")), + }; let pipeline = self .video_pipeline .as_ref() @@ -2556,6 +2753,10 @@ impl WgpuRenderer { binding: 3, resource: wgpu::BindingResource::Sampler(&pipeline.sampler), }, + wgpu::BindGroupEntry { + binding: 4, + resource: gamut_binding, + }, ], }); @@ -3170,6 +3371,16 @@ impl WgpuRenderer { ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, + wgpu::BindGroupLayoutEntry { + binding: 4, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + view_dimension: wgpu::TextureViewDimension::D3, + multisampled: false, + }, + count: None, + }, ], }); let layout = self @@ -4837,6 +5048,7 @@ fn retain_player_video_frame(frame: &PlayerVideoFrame) -> Result, luma_coefficients: vec4, gamut_matrix_rows: array, 3>, + ipt_matrix_rows: array, 9>, + tone_map_extra: vec4, + tone_map_coeffs: vec4, + gamut_lut_enabled: u32, + _gamut_primaries: u32, + _gamut_reserved0: u32, + _gamut_reserved1: u32, + dovi_flags: vec4, + dovi_pivots: array, 6>, + dovi_bounds: array, 3>, + dovi_coefficients: array, 24>, + dovi_mmr: array, 144>, + dovi_nonlinear_matrix: array, 3>, + dovi_nonlinear_offset: vec4, + dovi_lms_matrix: array, 3>, }; @group(0) @binding(0) var uniforms: VideoUniforms; @group(0) @binding(1) var luma_texture: texture_2d; @group(0) @binding(2) var chroma_texture: texture_2d; @group(0) @binding(3) var video_sampler: sampler; +@group(0) @binding(4) var gamut_lut: texture_3d; struct VertexOut { @builtin(position) position: vec4, @@ -113,23 +129,242 @@ fn source_reference_to_nits(rgb: vec3) -> vec3 { return max(rgb, vec3(0.0)) * source_reference_white_nits(); } -fn tone_map_nits(nits: vec3) -> vec3 { - let source_peak = source_peak_nits(); - let target_peak = target_peak_nits(); - let x = max(nits, vec3(0.0)) / target_peak; - let white = max(source_peak / target_peak, 1.0); +fn pq_code(nits: f32) -> f32 { + return pq_inverse_eotf(clamp(nits, 0.0, 10000.0) / 10000.0); +} + +fn nits_from_pq(code: f32) -> f32 { + return 10000.0 * pq_eotf(clamp(code, 0.0, 1.0)); +} + +// libplacebo pl_smoothstep with arbitrary edge order (WGSL `smoothstep` has +// undefined results when edge0 >= edge1, and libplacebo's knee tuning term +// deliberately uses reversed edges). +fn sstep(edge0: f32, edge1: f32, x: f32) -> f32 { + let t = clamp((x - edge0) / (edge1 - edge0), 0.0, 1.0); + return t * t * (3.0 - 2.0 * t); +} + +// libplacebo st2094_pick_knee evaluated on absolute PQ codes. The source +// pivot follows the scene average luminance when known and stays within +// [10%, 80%] of the range; the destination pivot rescales it into the output +// range and then adapts towards the 1:1 line (knee_adaptation 0.4). +fn st2094_pick_knee(src_min: f32, src_max: f32, src_avg: f32, dst_min: f32, dst_max: f32) -> vec2 { + let knee_adaptation = 0.4; + let min_knee = 0.1; + let max_knee = 0.8; + let def_knee = 0.4; + let src_knee_min = mix(src_min, src_max, min_knee); + let src_knee_max = mix(src_min, src_max, max_knee); + let dst_knee_min = mix(dst_min, dst_max, min_knee); + let dst_knee_max = mix(dst_min, dst_max, max_knee); + let fallback = mix(src_min, src_max, def_knee); + let src_knee = clamp(select(fallback, src_avg, src_avg > 0.0), src_knee_min, src_knee_max); + let knee_t = (src_knee - src_min) / max(src_max - src_min, 0.000001); + let adapted = mix(dst_min, dst_max, knee_t); + let tuning = 1.0 - sstep(max_knee, def_knee, knee_t) * sstep(min_knee, def_knee, knee_t); + let adaptation = mix(knee_adaptation, 1.0, tuning); + let dst_knee = clamp(mix(src_knee, adapted, adaptation), dst_knee_min, dst_knee_max); + return vec2(src_knee, clamp(dst_knee, dst_knee_min, dst_knee_max)); +} + +// The tone-map curve evaluated on the IPT intensity axis (PQ codes), +// mirroring libplacebo's tone-map functions. `param` is the per-operator +// curve parameter from ToneMapConfig::curve_param (0 = operator default). +fn tone_map_curve_pq(x_in: f32, param: f32) -> f32 { + let src_peak = source_peak_nits(); + let dst_peak = target_peak_nits(); + let src_avg = uniforms.tone_map_extra.y; + let dst_black = uniforms.tone_map_extra.z; + let in_min = 0.0; + let in_max = max(pq_code(src_peak), 0.000001); + let out_min = pq_code(dst_black); + let out_max = max(pq_code(dst_peak), 0.000001); + let out_range = max(out_max - out_min, 0.000001); + let x = clamp(x_in, in_min, in_max); + if (uniforms.tone_map == 0u) { + // Clip: values within the source range pass through untouched. + return x; + } if (uniforms.tone_map == 1u) { - let white2 = white * white; - return target_peak * clamp((x * (vec3(1.0) + x / white2)) / (vec3(1.0) + x), vec3(0.0), vec3(1.0)); + // Reinhard (output-relative, libplacebo pl_tone_map_reinhard). + let peak = in_max / out_range; + let contrast = select(0.5, param, param > 0.0); + let offset = (1.0 - contrast) / max(contrast, 0.000001); + let scale = (peak + offset) / peak; + let t = x / out_range; + let mapped = t / (t + offset) * scale; + return mapped * out_range + out_min; } if (uniforms.tone_map == 2u) { - let knee = 0.75; - let denom = max(white - knee, 0.0001); - let t = clamp((x - vec3(knee)) / denom, vec3(0.0), vec3(1.0)); - let shoulder = knee + (1.0 - knee) * (vec3(1.0) - pow(vec3(1.0) - t, vec3(2.0))); - return target_peak * mix(x, shoulder, step(vec3(knee), x)); + // Mobius: Möbius transform with a 1:1 linear region below the knee. + let peak = in_max / out_range; + let j = select(0.3, param, param > 0.0); + let a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + let b = (j * j - 2.0 * j * peak + peak) / max(peak - 1.0, 0.000001); + let scale = (b * b + 2.0 * b * j + j * j) / (b - a); + let t = x / out_range; + let mapped = select(t, scale * (t + a) / (t + b), t > j); + return mapped * out_range + out_min; + } + if (uniforms.tone_map == 3u) { + // ITU-R BT.2390 EETF with black-point compensation (the libplacebo + // version also compensates target black; the earlier port skipped it). + let knee_offset = select(1.0, param, param > 0.0); + let max_lum = clamp(out_max / in_max, 0.0, 1.0); + let min_lum = out_min / in_max; + let ks = (1.0 + knee_offset) * max_lum - knee_offset; + let bp = min(max(1.0 / max(min_lum, 0.000001), 0.0), 4.0); + var u = x / in_max; + if (ks < 1.0 && u > ks) { + let tb = (u - ks) / (1.0 - ks); + let tb2 = tb * tb; + let tb3 = tb2 * tb; + u = (2.0 * tb3 - 3.0 * tb2 + 1.0) * ks + + (tb3 - 2.0 * tb2 + tb) * (1.0 - ks) + + (-2.0 * tb3 + 3.0 * tb2) * max_lum; + } + if (u < 1.0) { + u = u + min_lum * pow(1.0 - u, bp); + let gain = select(1.0, 1.0 / (1.0 + min_lum / max_lum * pow(1.0 - max_lum, bp)), max_lum < 1.0); + u = gain * (u - min_lum) + min_lum; + } + return u * in_max; + } + if (uniforms.tone_map == 4u) { + // Spline: perceptually linear single-pivot polynomial, the default + // tone map of libplacebo and mpv's gpu-next renderer. + let contrast = select(0.3, param, param > 0.0); + let fallback_avg = clamp(0.4 * src_peak, 100.0, 400.0); + let effective_src_avg = select(fallback_avg, src_avg, src_avg > 0.0); + let knee = st2094_pick_knee( + in_min, + in_max, + pq_code(effective_src_avg), + out_min, + out_max + ); + let src_pivot = knee.x; + let dst_pivot = knee.y; + let slope0 = (dst_pivot - out_min) / max(src_pivot - in_min, 0.000001); + let ratio = clamp(1.5 * (in_max / out_max - 1.0), 0.2, 1.2); + let slope = pow(slope0, (1.0 - contrast) * ratio); + let in_min0 = in_min - src_pivot; + let in_max0 = in_max - src_pivot; + let out_min0 = out_min - dst_pivot; + let out_max0 = out_max - dst_pivot; + let pa = (out_min0 - slope * in_min0) / (in_min0 * in_min0); + let qa = (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0 * in_max0); + let qb = -3.0 * (slope * in_max0 - out_max0) / (2.0 * in_max0 * in_max0); + let xr = x - src_pivot; + let mapped = select((pa * xr + slope) * xr, ((qa * xr + qb) * xr + slope) * xr, xr > 0.0); + return mapped + dst_pivot; + } + if (uniforms.tone_map == 5u) { + // ITU-R BT.2446 method A: Weber-law log compression from the source + // peak envelope and a standardized S-curve (mpv's recommended curve + // for well-mastered content). + let phdr = 1.0 + 32.0 * pow(src_peak / 10000.0, 1.0 / 2.4); + let psdr = 1.0 + 32.0 * pow(dst_peak / 10000.0, 1.0 / 2.4); + var t = pow(nits_from_pq(x) / max(src_peak, 0.000001), 1.0 / 2.4); + t = log(1.0 + (phdr - 1.0) * t) / log(phdr); + if (t <= 0.7399) { + t = 1.0770 * t; + } else if (t < 0.9909) { + t = (-1.1510 * t + 2.7811) * t - 0.6302; + } else { + t = 0.5 * t + 0.5; + } + t = (pow(psdr, t) - 1.0) / (psdr - 1.0); + // BT.1886 EOTF from the target black point and peak. + let lb = pow(max(dst_black, 0.0), 1.0 / 2.4); + let lw = pow(max(dst_peak, 0.0), 1.0 / 2.4); + return pq_code(pow((lw - lb) * t + lb, 2.4)); + } + // SMPTE ST 2094-10 (DolbyVision's dynamic-metadata curve): rational + // Möbius interpolation in absolute nits; coefficients are solved per + // frame on the CPU from the same scene pivot. + let c1 = uniforms.tone_map_coeffs.x; + let c2 = uniforms.tone_map_coeffs.y; + let c3 = uniforms.tone_map_coeffs.z; + let x_nits = nits_from_pq(x); + let y_nits = (c1 + c2 * x_nits) / max(1.0 + c3 * x_nits, 0.000001); + return pq_code(clamp(y_nits, 0.0, 10000.0)); +} + +fn tone_map_nits(nits: vec3) -> vec3 { + if (uniforms.target_transfer == 3u) { + // HDR10 output: convert primaries by the gamut matrix and clamp to + // the PQ range (no tone mapping; the display does the HDR mapping). + return clamp( + apply_gamut_map(max(nits, vec3(0.0)) / source_reference_white_nits()) + * source_reference_white_nits(), + vec3(0.0), + vec3(10000.0) + ); + } + // libplacebo tone map: RGB in source primaries (absolute nits) to + // HPE-LMS, PQ-encode, IPT, map the intensity axis and apply the + // hue-preserving chroma rule, then decode back to RGB in the *source* + // primaries. The gamut LUT (below) performs the single source->target + // primaries conversion. + let rgb = max(nits, vec3(0.0)); + let lms = vec3( + dot(uniforms.ipt_matrix_rows[0].xyz, rgb), + dot(uniforms.ipt_matrix_rows[1].xyz, rgb), + dot(uniforms.ipt_matrix_rows[2].xyz, rgb) + ); + let lmspq = vec3(pq_code(lms.r), pq_code(lms.g), pq_code(lms.b)); + var ipt = vec3( + dot(vec3(0.4, 0.4, 0.2), lmspq), + dot(vec3(4.455, -4.851, 0.396), lmspq), + dot(vec3(0.8056, 0.3572, -1.1628), lmspq) + ); + let i_orig = ipt.x; + ipt.x = tone_map_curve_pq(ipt.x, uniforms.tone_map_extra.x); + // Libplacebo's chroma rule: clamp the saturation boost when brightening + // and desaturate (by the cubic hull term) when the mapping darkens. + let hull = vec2(i_orig, ipt.x); + let hull_c = ((hull - vec2(6.0)) * hull + vec2(9.0)) * hull; + let ratio = min(i_orig / max(ipt.x, 0.000001), hull_c.y / max(hull_c.x, 0.000001)); + ipt = vec3(ipt.x, ipt.y * ratio, ipt.z * ratio); + + if (uniforms.gamut_lut_enabled != 0u) { + // I axis spans the target's [black, peak] in PQ codes, matching + // libplacebo's gamut.min_luma/max_luma (tone_map_extra.z is the + // target black in nits, the same value the LUT was generated for). + let lut_min = pq_code(uniforms.tone_map_extra.z); + let lut_max = max(pq_code(target_peak_nits()), 0.000001); + let lut_range = max(lut_max - lut_min, 0.000001); + let pos = vec3( + clamp((ipt.x - lut_min) / lut_range, 0.0, 1.0), + clamp(2.0 * length(ipt.yz), 0.0, 1.0), + 0.5 + 0.5 * atan2(ipt.z, ipt.y) / 3.14159265 + ); + // libplacebo's texel_scale: lattice position -> texel-center coordinate. + let idx = vec3( + pos.x * (47.0 / 48.0) + 0.5 / 48.0, + pos.y * (31.0 / 32.0) + 0.5 / 32.0, + pos.z * (255.0 / 256.0) + 0.5 / 256.0 + ); + let sampled = textureSample(gamut_lut, video_sampler, idx).xyz; + ipt = vec3(sampled.x, sampled.y - 0.5, sampled.z - 0.5); } - return target_peak * clamp(x, vec3(0.0), vec3(1.0)); + let lmspq_out = vec3( + dot(vec3(1.0, 0.0975689, 0.205226), ipt), + dot(vec3(1.0, -0.113876, 0.133217), ipt), + dot(vec3(1.0, 0.0326151, -0.676887), ipt) + ); + let lms_out = vec3( + nits_from_pq(lmspq_out.r), + nits_from_pq(lmspq_out.g), + nits_from_pq(lmspq_out.b) + ); + return vec3( + dot(uniforms.ipt_matrix_rows[6].xyz, lms_out), + dot(uniforms.ipt_matrix_rows[7].xyz, lms_out), + dot(uniforms.ipt_matrix_rows[8].xyz, lms_out) + ); } fn apply_gamut_map(rgb: vec3) -> vec3 { @@ -140,8 +375,30 @@ fn apply_gamut_map(rgb: vec3) -> vec3 { ); } +// Hue-preserving gamut mapping: the linear gamut matrix can push highly +// saturated wide-gamut colors outside the target gamut (negative +// components). Blending those towards luma shifts hue — BT.2020 primary +// red picks up blue and turns pink. Instead blend towards the naive clip +// by an out-of-gamut smoothstep factor: slightly-out colors stay nearly +// intact, strongly-out primaries land on the pure target primary with +// their hue intact, matching mpv's perceptual gamut handling. Brightness +// overshoot (> 1) is left for the tone map. +fn gamut_compress(rgb: vec3) -> vec3 { + let lo = min(rgb.r, min(rgb.g, rgb.b)); + let outness = max(-lo, 0.0); + let k = smoothstep(0.0, 1.0, outness); + return mix(rgb, clamp(rgb, vec3(0.0), vec3(1.0)), k); +} + fn target_nits_to_reference_linear(nits: vec3) -> vec3 { - return max(nits, vec3(0.0)) / target_reference_white_nits(); + // libplacebo's encode maps [target black, target peak] onto [0, 1] where + // 1.0 is the target reference white, so the tone-map black-point + // compensation lands back on true black instead of lifting it. + let black = uniforms.tone_map_extra.z; + let peak = target_peak_nits(); + let range = max(peak - black, 0.0001); + return max(nits - vec3(black), vec3(0.0)) / range + * (range / target_reference_white_nits()); } fn target_reference_linear_to_output(rgb: vec3) -> vec3 { @@ -194,22 +451,110 @@ struct RangeExpandedYCbCr { }; fn expand_ycbcr_range(y_in: f32, cbcr_in: vec2) -> RangeExpandedYCbCr { + var y = y_in; + var cbcr = cbcr_in; + if (uniforms.is_p010 != 0u) { + // P010 stores 10-bit codes as code << 6 in a 16-bit UNORM texture. + let p010_scale = 65535.0 / 65472.0; + y *= p010_scale; + cbcr *= p010_scale; + } var out: RangeExpandedYCbCr; if (uniforms.full_range != 0u) { - out.y = y_in; - out.cbcr = cbcr_in - vec2(0.5); + out.y = y; + out.cbcr = cbcr - vec2(0.5); return out; } if (uniforms.is_p010 != 0u) { - out.y = (y_in - (64.0 / 1023.0)) * (1023.0 / 876.0); - out.cbcr = (cbcr_in - vec2(512.0 / 1023.0)) * (1023.0 / 896.0); + out.y = (y - (64.0 / 1023.0)) * (1023.0 / 876.0); + out.cbcr = (cbcr - vec2(512.0 / 1023.0)) * (1023.0 / 896.0); return out; } - out.y = (y_in - (16.0 / 255.0)) * (255.0 / 219.0); - out.cbcr = (cbcr_in - vec2(128.0 / 255.0)) * (255.0 / 224.0); + out.y = (y - (16.0 / 255.0)) * (255.0 / 219.0); + out.cbcr = (cbcr - vec2(128.0 / 255.0)) * (255.0 / 224.0); return out; } +// Dolby Vision RPU reshaping, ported from libplacebo's `pl_shader_dovi_reshape` +// (the renderer behind mpv's Dolby Vision mapping). The base-layer signal is +// reshaped per component through piecewise polynomial/MMR curves selected by +// pivot comparison, where MMR coefficients mix all three raw components. +fn dovi_reshaped_signal(sig_in: vec3) -> vec3 { + let sig = clamp(sig_in, vec3(0.0), vec3(1.0)); + var result: array; + result[0] = sig.r; + result[1] = sig.g; + result[2] = sig.b; + let flags = uniforms.dovi_flags; + for (var c = 0u; c < 3u; c = c + 1u) { + let segments = u32(flags[1u + c]); + if (segments == 0u) { + continue; + } + var s = result[c]; + var index = 0u; + for (var i = 0u; i < 7u; i = i + 1u) { + let pivot_row = uniforms.dovi_pivots[2u * c + i / 4u]; + let pivot = pivot_row[i % 4u]; + if (s >= pivot) { + index = index + 1u; + } + } + let coeff = uniforms.dovi_coefficients[8u * c + index]; + if (coeff.w < 0.5) { + s = (coeff.z * s + coeff.y) * s + coeff.x; + } else { + let base = 48u * c + u32(coeff.y); + let order = u32(coeff.w); + let sig_x = vec4( + sig.x * sig.y, + sig.x * sig.z, + sig.y * sig.z, + sig.x * sig.y * sig.z + ); + s = coeff.x; + s = s + dot(uniforms.dovi_mmr[base].xyz, sig); + s = s + dot(uniforms.dovi_mmr[base + 1u], sig_x); + if (order >= 2u) { + let sig2 = sig * sig; + let sig_x2 = sig_x * sig_x; + s = s + dot(uniforms.dovi_mmr[base + 2u].xyz, sig2); + s = s + dot(uniforms.dovi_mmr[base + 3u], sig_x2); + if (order >= 3u) { + s = s + dot(uniforms.dovi_mmr[base + 4u].xyz, sig2 * sig); + s = s + dot(uniforms.dovi_mmr[base + 5u], sig_x2 * sig_x); + } + } + } + let bounds = uniforms.dovi_bounds[c]; + result[c] = clamp(s, bounds.x, bounds.y); + } + return vec3(result[0], result[1], result[2]); +} + +// Reshaped nonlinear signal to PQ-encoded IPT via the RPU's ycc_to_rgb matrix +// and signal offsets. Applying the RPU offsets keeps integer offset codes +// exactly on sample codes (2^bits/(2^bits-1) folded in on the CPU). +fn dovi_signal_to_pq_rgb(sig: vec3) -> vec3 { + let reshaped = dovi_reshaped_signal(sig) - uniforms.dovi_nonlinear_offset.xyz; + return vec3( + dot(uniforms.dovi_nonlinear_matrix[0].xyz, reshaped), + dot(uniforms.dovi_nonlinear_matrix[1].xyz, reshaped), + dot(uniforms.dovi_nonlinear_matrix[2].xyz, reshaped) + ); +} + +// Linearized BT.2020-referred HPE LMS back to linear RGB, using the composite +// of the fixed HPE inverse with the RPU's rgb_to_lms matrix (premultiplied on +// the CPU, matching libplacebo's dovi_lms2rgb). +fn dovi_lms_to_rgb(linear: vec3) -> vec3 { + return vec3( + dot(uniforms.dovi_lms_matrix[0].xyz, linear), + dot(uniforms.dovi_lms_matrix[1].xyz, linear), + dot(uniforms.dovi_lms_matrix[2].xyz, linear) + ); +} + fn packed_luma_texel(virtual_coord_in: vec2, virtual_size: vec2) -> f32 { let virtual_coord = clamp(virtual_coord_in, vec2(0), virtual_size - vec2(1)); let packed_coord = virtual_coord / vec2(2); @@ -275,8 +620,24 @@ fn erika_video_fragment(in: VertexOut) -> @location(0) vec4 { color_coord.x *= 0.5; } let alpha_coord = vec2(0.5 + in.tex_coord.x * 0.5, in.tex_coord.y); + var y_sample = textureSample(luma_texture, video_sampler, color_coord).r; + if (input_mode == 2u) { + y_sample = sample_packed_luma(color_coord); + } + let cbcr_sample = textureSample(chroma_texture, video_sampler, color_coord).rg; var rgb: vec3; - if (input_mode == 1u) { + let dovi_enabled = uniforms.dovi_flags.x != 0.0; + let dovi_ycbcr_input = dovi_enabled && (input_mode == 0u || input_mode == 2u); + if (dovi_ycbcr_input) { + // The base layer carries the raw 12-bit DV signal (10-bit container, + // full range); range expansion and the YCbCr matrix are replaced by + // the RPU reshaping + ycc_to_rgb path. + var sig = vec3(y_sample, cbcr_sample.x, cbcr_sample.y); + if (uniforms.is_p010 != 0u) { + sig *= 65535.0 / 65472.0; + } + rgb = dovi_signal_to_pq_rgb(sig); + } else if (input_mode == 1u) { rgb = textureSample(luma_texture, video_sampler, color_coord).rgb; } else if (input_mode == 3u) { let original_rgb = textureSample(chroma_texture, video_sampler, color_coord).rgb; @@ -284,13 +645,6 @@ fn erika_video_fragment(in: VertexOut) -> @location(0) vec4 { let enhanced_luma = sample_packed_luma(color_coord); rgb = original_rgb + vec3(enhanced_luma - original_luma); } else { - var y_sample: f32; - if (input_mode == 2u) { - y_sample = sample_packed_luma(color_coord); - } else { - y_sample = textureSample(luma_texture, video_sampler, color_coord).r; - } - let cbcr_sample = textureSample(chroma_texture, video_sampler, color_coord).rg; let expanded = expand_ycbcr_range(y_sample, cbcr_sample); let y = expanded.y; let cbcr = expanded.cbcr; @@ -303,10 +657,13 @@ fn erika_video_fragment(in: VertexOut) -> @location(0) vec4 { rgb.g = (y - kr * rgb.r - kb * rgb.b) / kg; } rgb = transfer_to_source_reference_linear(rgb); - rgb = apply_gamut_map(rgb); + if (dovi_ycbcr_input) { + rgb = dovi_lms_to_rgb(rgb); + } rgb = source_reference_to_nits(rgb); rgb = tone_map_nits(rgb); rgb = target_nits_to_reference_linear(rgb); + rgb = gamut_compress(rgb); rgb = target_reference_linear_to_output(rgb); var alpha = 1.0; if (packed_alpha) { diff --git a/crates/erika_ffmpeg_sys/wrapper.h b/crates/erika_ffmpeg_sys/wrapper.h index 620d9533..71c8096f 100644 --- a/crates/erika_ffmpeg_sys/wrapper.h +++ b/crates/erika_ffmpeg_sys/wrapper.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include diff --git a/docs/architecture.md b/docs/architecture.md index 5da0bc62..af01c5f3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -167,7 +167,9 @@ The primary renderer for Apple platforms: - Zero-copy CVPixelBuffer → MTLTexture import via `CVMetalTextureCache`. - YCbCr sampling, transfer decode, gamut mapping (BT.2020→BT.709, Display P3→BT.709). -- Tone mapping: Mobius, Reinhard, clip operators with absolute nits. +- Tone mapping: libplacebo-style IPT-domain color map with BT.2390 EETF + default (plus spline, BT.2446 method A, ST 2094-10, Mobius, Reinhard, clip) + over absolute nits. - SDR output (`BGRA8Unorm`) and Apple EDR output (`RGBA16Float` with EDR headroom). - Neural luma upscaler (`LumaUpscalerMode`): ArtCNN C4F16/C4F16 DS/C4F32 2x doublers diff --git a/docs/dolby-vision.md b/docs/dolby-vision.md new file mode 100644 index 00000000..255bbfed --- /dev/null +++ b/docs/dolby-vision.md @@ -0,0 +1,286 @@ +# Dolby Vision HDR Mapping + +This document describes Erika's implementation of Dolby Vision RPU (Reference Processing Unit) metadata processing and color mapping. + +## Overview + +Dolby Vision is an HDR format that enhances video quality through per-frame metadata called RPU. The implementation follows [libplacebo](https://github.com/haasn/libplacebo)'s approach (the renderer behind mpv's Dolby Vision support) to ensure compatibility and correctness. + +## Supported Profiles + +| Profile | Description | Decode Strategy | RPU Available | +|---------|-------------|-----------------|---------------| +| **5** | Single layer, non-backward compatible (IPTPQc2) | Hardware decode on VideoToolbox & D3D11VA; software fallback on mobile backends | ✅ Yes | +| **8** | Single layer with backward-compatible base (e.g. 8.1 HDR10, 8.4 HLG) | Hardware decode allowed | ✅ Yes | + +> **Note on Profile Architecture**: Profile 8 is a single-layer profile where the base layer carries standard signaling (such as HDR10 PQ for 8.1 or HLG for 8.4) with Dolby Vision RPU metadata interleaved as NAL units. Profile 7 is the dual-layer profile (Base Layer + Enhancement Layer / FEL / MEL) primarily used on Ultra HD Blu-ray discs. + +### Decode Strategy and Metadata Extraction + +On desktop platforms: +- **macOS (VideoToolbox)** and **Windows (D3D11VA)** decoders preserve frame side data (`AV_FRAME_DATA_DOVI_METADATA`) alongside hardware texture surfaces (CVPixelBuffer / D3D11 texture). This enables hardware-accelerated decoding while feeding RPU uniforms directly into GPU shaders for per-frame reshaping and color mapping. + +On mobile/embedded backends: +- Hardware decoders like **MediaCodec** (Android) and generic **AvCodec** backends may not expose RPU side data attached to output frames. +- For **Profile 5** on those mobile backends, because the stream uses IPTPQc2 rather than standard YCbCr and lacks backward compatibility, playback falls back to software decode via FFmpeg's `avcodec` to reliably access `AV_FRAME_DATA_DOVI_METADATA`. Desktop VideoToolbox and D3D11VA keep hardware decode for Profile 5 (see the table above). +- For **Profile 8**, hardware decode can safely be preserved: if RPU side data is unavailable, the video still displays with correct HDR10/HLG colors because the base layer is backward compatible. + +See `dolby_vision_decode_fallback()` in `crates/erika/src/playback.rs`. + +## Pipeline Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ 1. Container (MP4/MKV) │ +│ - dvcC/dvvC box → Dolby Vision profile (5 or 8) │ +└────────────────────┬────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ 2. FFmpeg Decoder (hardware on desktop, software on mobile) │ +│ - Decodes compressed HEVC stream │ +│ - Parses RPU from NAL units │ +│ - Emits AV_FRAME_DATA_DOVI_METADATA side data │ +└────────────────────┬────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ 3. Metadata Extraction (ffmpeg.rs, `frame_dovi_metadata_result`) │ +│ - Reads AVDOVIMetadata via pointer arithmetic │ +│ - Normalizes pivots by base layer bit depth │ +│ - Scales coefficients by 2^(-coef_log2_denom) │ +│ - Produces DoviSourceMetadata │ +└────────────────────┬────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ 4. Uniform Packing (pipeline.rs, `DoviUniforms`) │ +│ - Converts to vec4-aligned DoviUniforms (~3KB) │ +│ - Packs polynomial and MMR coefficients │ +│ - Applies signal offset correction for the uploaded bit depth │ +└────────────────────┬────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────────┐ +│ 5. GPU Shader (wgsl/metal/hlsl) │ +│ a. Reshaping: piecewise polynomial/MMR per component │ +│ b. Nonlinear matrix: RPU's ycc_to_rgb │ +│ c. PQ linearization: EOTF │ +│ d. LMS→RGB: composite HPE inverse × rgb_to_lms │ +│ e. Tone mapping to display │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Reshaping Algorithm + +The core of Dolby Vision mapping is **per-component piecewise reshaping**. Each component (Y, Cb, Cr) is transformed through curves defined by pivots and coefficients. + +### Polynomial Segments + +For a segment between pivots `p[i]` and `p[i+1]`, if the input signal `s` falls in that range: + +``` +output = (c2 · s + c1) · s + c0 +``` + +### MMR (Multivariate Polynomial Regression) + +For higher-order mapping, MMR mixes all three input components: + +```rust +// Order 1 +output = constant + dot([a, b, c], [R, G, B]) + dot([d, e, f, g], [R·G, R·B, G·B, R·G·B]) + +// Order 2: adds R², G², B² and squared cross terms +// Order 3: adds R³, G³, B³ and cubed cross terms +``` + +See `dovi_reshaped_signal()` in shaders for implementation. + +## Color Transform Flow + +After reshaping, the signal goes through: + +1. **Offset subtraction**: `reshaped - nonlinear_offset` +2. **Nonlinear matrix**: RPU's `ycc_to_rgb` (still PQ-encoded) +3. **PQ linearization**: Convert PQ code to linear light +4. **LMS to RGB**: `(HPE⁻¹ × rgb_to_lms) × linearized` +5. **Gamut/tone mapping**: Standard HDR pipeline continues + +## Tone Mapping (libplacebo color map) + +The tone map mirrors libplacebo's color map (`pl_shader_color_map`, the +engine behind mpv's `--vo=gpu-next`): RGB in source primaries (absolute +nits) — HPE-LMS — PQ encode — IPT. The operator's curve runs on the IPT +intensity axis, libplacebo's chroma rule (`hull` cubic on the pre/post +intensity pair) protects saturation, and the decode back to RGB lands in +the target primaries, so the primaries conversion happens inside the +roundtrip instead of a separate gamut matrix. + +Operators (`ToneMapOperator`, default `Bt2390`): + +| code | operator | curve parameter (`ToneMapConfig::curve_param`) | +|---|---|---| +| 0 | Clip | — | +| 1 | Reinhard | contrast (default 0.5) | +| 2 | Mobius | linear knee (default 0.3) | +| 3 | **BT.2390** (default) | knee offset (default 1.0) | +| 4 | Spline | slope contrast (default 0.30) | +| 5 | BT.2446 method A | — | +| 6 | SMPTE ST 2094-10 | knee adaptation (default 0.70; coefficients solved per frame on the CPU) | + +Black-point compensation uses `target black = target peak / contrast` +(`contrast_ratio`; auto 1000:1 for SDR, 0 for HDR/EDR targets) and is gated +so SDR → SDR rendering is untouched. The encode maps `[black, peak]` onto +`[0, 1]`, so the compensated floor lands back on code 0. + +Whether the tone map (and therefore the black-point compensation and the +perceptual gamut LUT) is active at all is decided from the **static** +mastering-display peak (L0), never the per-frame L1 peak: a dark scene must +not silently drop the tone map or the LUT for one frame. The per-frame L1 +peak still sets the curve's source peak (`tone_map_extra`/`nits.x`), which is +what libplacebo does with dynamic metadata. ST 2094-10 picks its knee in the +PQ domain even though the curve itself is solved in absolute nits, matching +libplacebo's internal `pl_hdr_rescale` round trip. + +## Scene-Adaptive HDR10 (measured luma) + +HDR10 streams carry only static ST.2086 mastering metadata, so without +further input the tone-map pivot would sit at the fixed 40% knee for every +scene. `crates/erika/src/luma_stats.rs` gives HDR10 the same treatment +Dolby Vision L1 gives Profile 5/8: for software-decoded PQ frames the +presenter samples a sparse grid of the luma plane (NV12/P010), linearizes +each sample, re-encodes to PQ and averages in that perceptual domain, then +smooths the running estimate with libplacebo's IIR filter +(`coeff = 1 - exp(-1/20)`). The smoothed scene average (nits) is attached to +the frame (`PlayerVideoFrame::scene_avg_nits`) and folded into +`SourceColorState.measured_scene_avg_nits`, which `tone_map_extra.y` +prefers over the L1 average — so the spline pivot follows the content. + +The samples are normalized with the frame's own color range: limited (TV) +range planes are expanded over the legal 16..235 / 64..940 span exactly like +the shaders' `expand_ycbcr_range`. HDR10 is normally limited range, so +measuring it as full range would lift black and clip the highlights before +the PQ re-encode. + +Hardware-decoded frames (VideoToolbox/D3D11VA/MediaCodec) have no CPU luma +plane and keep the static-metadata path. The estimator resets on generation +changes so seeks cannot carry the previous scene's brightness forward. + +## Perceptual Gamut Mapping (IPT 3D LUT) + +After the tone map, when an HDR source is tone-mapped into a smaller gamut +(BT.2020 → BT.709), the renderer applies a perceptual gamut map via a +CPU-generated 48 × 32 × 256 IPT-space LUT (`renderer::gamut`) instead of the +fast `gamut_compress`. Texels hold the perceptually mapped +`(I, P + 0.5, T + 0.5)` color — the I axis spans the target's +`[black, peak]` in PQ codes (libplacebo's `gamut.min_luma`/`max_luma`, i.e. +the tone map's output range). Generation follows libplacebo's IPT LUT layout +and index mapping so the shaders sample the same lattice; the chroma rolloff +is currently a simplified dead-zone blend plus Möbius soft clip rather than +libplacebo's full per-hue boundary search, so highly saturated BT.2020 colors +can diverge slightly from mpv. The shaders rebuild RGB in the source +primaries, run the LMS-PQ-IPT roundtrip, sample the LUT in ICh space and +decode back to the target primaries; the LUT texture is cached per (source, +target, target black, target peak) and bound as binding/slot 4 (WGSL), +texture 2 (Metal) or t2 (D3D11), always with a dummy 1×1×1 fallback so the +fast path keeps a valid layout. + +## Per-Frame L1 Brightness Metadata + +The RPU's dynamic DM extension blocks carry **level 1** per-frame brightness +metadata: `min_pq` / `max_pq` / `avg_pq` in 12-bit PQ codes. FFmpeg's RPU +decoder copies these blocks into the frame side data (`AVDOVIMetadata` +`ext_block_offset` region); `frame_dovi_level1` validates that region and +extracts the level 1 block. + +The frame's `max_pq` is used as the tone-map source peak when L1 metadata is present, and the frame's `avg_pq` becomes the scene average that drives the tone-map pivot. Static mastering-display metadata remains responsible for output-mode negotiation and tone-map/gamut-LUT enablement decisions. This makes the default BT.2390 curve +scene-adaptive: a dark scene is not compressed against a 4000-nit mastering +peak, so its highlights stay distinct. The **static mastering display peak +is untouched** — output-mode negotiation (SDR/EDR per display) and the +decision whether to tone map / bind the perceptual gamut LUT both read the +static L0 peak, so per-frame brightness can never toggle them. + +An absent, all-zero, or inverted level 1 block falls back to the static +`source_max_pq` (and the fixed 40% knee when no average is known); the RPU +itself is never rejected over L1. + +## Forced BT.2020/PQ + +Dolby Vision Profile 5/8 VUI tags are **unreliable**. The implementation forces: + +```rust +self.primaries = ColorPrimaries::Bt2020; +self.transfer = TransferFunction::Pq; +``` + +Without this, a stream tagged as "unspecified transfer" would decode PQ-encoded samples with sRGB gamma, producing completely wrong brightness. + +See `SourceColorState::dovi()` in `pipeline.rs`. + +## Testing Strategy + +### Unit Tests + +- `frame_reads_dovi_side_data`: Verifies FFmpeg side data parsing +- `dovi_uniforms_pack_pivots_poly_and_mmr`: Validates uniform packing +- `dovi_source_forces_pq_when_stream_tags_are_missing`: Confirms PQ forcing +- `dovi_l1_brightness_metadata_is_parsed`: Verifies level 1 ext-block parsing and invalid-L1 fallback +- `dovi_source_uses_per_frame_l1_peak_when_present`: Confirms L1 max_pq drives the tone-map source peak while mastering metadata remains static for output-mode decisions +- `fel_residual_keeps_the_base_layer_mapping`: Confirms a Profile 7 FEL RPU keeps its base-layer mapping and reports `DoviElStatus` instead of being rejected +- `dolby_vision_profile_5_stays_on_hardware_for_videotoolbox_and_d3d11va`: Verifies desktop hardware decoders stay on hardware for Profile 5 +- `dolby_vision_profile_5_falls_back_to_software_on_mobile_backends`: Verifies mobile backends fall back to software decode for Profile 5 +- `dolby_vision_profile_8_stays_on_hardware_decode`: Profile 8 hardware decode preservation + +### Integration Tests (require samples) + +Set environment variables to enable: + +- `ERIKA_DV_SAMPLE`: Profile 5 sample (RPU mapping verification) +- `ERIKA_DV_PROFILE_8_SAMPLE`: Profile 8 sample (hardware decode test) + +## Enhancement-layer residual (Profile 7 FEL / MEL) + +Profile 7 keeps its enhancement layer in a second HEVC layer that this renderer +never decodes, so an RPU with `disable_residual_flag == 0` describes a residual +that has nothing to be added to. Dropping the whole RPU in that case would also +discard the base-layer reshaping curves, color matrices, and L1 trims, which +still apply to the base layer on their own — so the RPU is mapped as usual and +only the un-composable part is reported. + +`Frame::dovi_el_status()` returns a `DoviElStatus` (`residual_requested`, +`nlq_nontrivial`), and playback emits one throttled `dovi_el_not_composed` +diagnostic per stream. This mirrors libplacebo, which exposes `nlq_active` but +documents that "consumers that have not bound an enhancement layer must not +look at these fields": the NLQ fields are parsed for reporting only and never +feed the shaders. + +## Known Limitations + +1. **Dual-layer FEL / MEL residual composition not supported**: Profile 7 + enhancement layers are never decoded, so the NLQ residual an RPU asks for + cannot be composed. The base layer still gets the full Dolby Vision mapping + (curves, matrices, L1 trims) and the gap is reported through the + `dovi_el_not_composed` diagnostic — see "Enhancement-layer residual" above. + Real Profile 7 content therefore renders like mpv/libplacebo rather than + like a plain HDR10 base layer, but without the extra detail the + enhancement layer carries. +2. **Mobile hardware decode RPU extraction**: On mobile backends (MediaCodec), hardware decoders do not expose RPU side data, requiring software decode for Profile 5. +3. **Uniform buffer size**: ~3KB may exceed limits on very old mobile GPUs (pre-2015) +4. **CPU plane upload on software decode**: When software decode fallback is used, software planes incur CPU-to-GPU texture upload overhead. + +## References + +- [Dolby Vision Specification](https://professional.dolby.com/dolby-vision/) +- [libplacebo dovi_reshape implementation](https://github.com/haasn/libplacebo/blob/master/src/shaders/dovi.c) +- [FFmpeg AVDOVIMetadata](https://ffmpeg.org/doxygen/trunk/structAVDOVIMetadata.html) +- [BT.2100 PQ EOTF](https://www.itu.int/rec/R-REC-BT.2100) + +## Implementation Files + +| File | Purpose | +|------|---------| +| `crates/erika/src/ffmpeg.rs` | Container and frame metadata extraction (`frame_dovi_metadata_result`) | +| `crates/erika/src/playback.rs` | Profile-based decode fallback (`dolby_vision_decode_fallback`) | +| `crates/erika/src/renderer/pipeline.rs` | Data structures and uniform packing (`DoviUniforms`) | +| `crates/erika/src/renderer/gamut.rs` | Perceptual gamut LUT generation | +| `crates/erika/src/luma_stats.rs` | Scene-adaptive HDR10 luma measurement | +| `crates/erika/src/renderer/wgpu_video.wgsl` | WGSL shader implementation | +| `crates/erika/src/renderer/metal/apple.rs` | Metal shader implementation | +| `crates/erika/src/renderer/d3d11.rs` | HLSL shader implementation | diff --git a/examples/danmaku_perf_lab/native/DanmakuPerfLab.m b/examples/danmaku_perf_lab/native/DanmakuPerfLab.m index f2911770..a7fa5eda 100644 --- a/examples/danmaku_perf_lab/native/DanmakuPerfLab.m +++ b/examples/danmaku_perf_lab/native/DanmakuPerfLab.m @@ -63,6 +63,7 @@ - (instancetype)initWithFrame:(NSRect)frameRect { if ([self.metalLayer respondsToSelector:@selector(setDisplaySyncEnabled:)]) { self.metalLayer.displaySyncEnabled = !erika_perf_lab_uncapped(); } + self.metalLayer.delegate = (id)self; self.layer = self.metalLayer; self.startTime = CACurrentMediaTime(); } diff --git a/examples/macos_native_demo/native/ErikaMetalDemo.m b/examples/macos_native_demo/native/ErikaMetalDemo.m index 229a1aef..2b7a861b 100644 --- a/examples/macos_native_demo/native/ErikaMetalDemo.m +++ b/examples/macos_native_demo/native/ErikaMetalDemo.m @@ -42,6 +42,7 @@ - (instancetype)initWithFrame:(NSRect)frameRect { self.metalLayer.pixelFormat = MTLPixelFormatBGRA8Unorm; self.metalLayer.framebufferOnly = YES; self.metalLayer.opaque = YES; + self.metalLayer.delegate = (id)self; self.layer = self.metalLayer; self.startTime = CACurrentMediaTime(); } @@ -282,6 +283,9 @@ - (void)applicationDidFinishLaunching:(NSNotification *)notification { [self.window center]; [self.window makeKeyAndOrderFront:nil]; double smokeSeconds = erika_demo_smoke_seconds(); + // Always activate: an inactive app gets App-Napped by macOS, which + // throttles the render timer and stalls the presentation-driven decode. + [NSApp activateIgnoringOtherApps:YES]; if (smokeSeconds > 0.0) { self.smokeTimer = [NSTimer scheduledTimerWithTimeInterval:smokeSeconds target:self @@ -289,8 +293,6 @@ - (void)applicationDidFinishLaunching:(NSNotification *)notification { userInfo:nil repeats:NO]; [[NSRunLoop mainRunLoop] addTimer:self.smokeTimer forMode:NSRunLoopCommonModes]; - } else { - [NSApp activateIgnoringOtherApps:YES]; } } diff --git a/examples/macos_native_demo/src/main.rs b/examples/macos_native_demo/src/main.rs index 0c271fea..cbf13bc1 100644 --- a/examples/macos_native_demo/src/main.rs +++ b/examples/macos_native_demo/src/main.rs @@ -17,6 +17,7 @@ static SUBTITLE_PATH: OnceLock = OnceLock::new(); static DANMAKU_PATH: OnceLock = OnceLock::new(); static SMOKE_SECONDS: OnceLock = OnceLock::new(); static EDR_HEADROOM: OnceLock = OnceLock::new(); +static SEEK_SEQUENCE: OnceLock> = OnceLock::new(); unsafe extern "C" { fn erika_demo_run_app(); @@ -30,6 +31,7 @@ struct DemoState { presenter: PresenterRuntime, load_attempted: bool, overlay_logged: bool, + seek_cursor: usize, } impl DemoState { @@ -44,10 +46,32 @@ impl DemoState { })?, load_attempted: false, overlay_logged: false, + seek_cursor: 0, }) } fn render(&mut self, time_seconds: f64) { + if let Some(sequence) = SEEK_SEQUENCE.get() { + while self.seek_cursor < sequence.len() && time_seconds >= sequence[self.seek_cursor].0 + { + let (trigger, position) = sequence[self.seek_cursor]; + match self + .presenter + .seek(Duration::from_secs_f64(position.max(0.0))) + { + Ok(()) => eprintln!( + "Erika demo seek #{}(at {trigger:.1}s) -> {position:.1}s ok", + self.seek_cursor + 1 + ), + Err(error) => eprintln!( + "Erika demo seek #{} -> {:.1}s failed: {error}", + self.seek_cursor + 1, + position + ), + } + self.seek_cursor += 1; + } + } if !self.load_attempted { self.load_attempted = true; if let Some(uri) = MEDIA_URI.get() { @@ -82,6 +106,16 @@ impl DemoState { eprintln!("Erika demo overlay active through presenter runtime"); self.overlay_logged = true; } + if SEEK_SEQUENCE.get().is_some() { + eprintln!( + "Erika demo seek stats: decoded={} rendered={} import_failures={} render_failures={} backpressure_drops={}", + stats.decoded_video_frames, + stats.rendered_video_frames, + stats.import_failures, + stats.render_failures, + stats.video_frame_backpressure_drops + ); + } } Err(error) => eprintln!("Erika demo render failed: {error}"), } @@ -223,7 +257,7 @@ fn main() { let options = parse_args(&args).unwrap_or_else(|error| { eprintln!("{error}"); eprintln!( - "usage: cargo run -p macos_native_demo -- [--edr [HEADROOM]] [--smoke-seconds N] [--subtitle PATH] [--ass-subtitle PATH] [--danmaku PATH] [media-path-or-uri]" + "usage: cargo run -p macos_native_demo -- [--edr [HEADROOM]] [--smoke-seconds N] [--subtitle PATH] [--ass-subtitle PATH] [--danmaku PATH] [--seek-sequence t:pos,t:pos] [media-path-or-uri]" ); process::exit(2); }); @@ -234,6 +268,21 @@ fn main() { DANMAKU_PATH.set(path).expect("danmaku path is set once"); } + if let Some(sequence) = options.seek_sequence { + let parsed: Vec<(f64, f64)> = sequence + .split(',') + .filter(|entry| !entry.trim().is_empty()) + .map(|entry| { + let (trigger, position) = entry.split_once(':').unwrap_or((entry, "0")); + ( + trigger.trim().parse::().unwrap_or(0.0), + position.trim().parse::().unwrap_or(0.0), + ) + }) + .collect(); + eprintln!("Erika demo seek sequence: {parsed:?}"); + SEEK_SEQUENCE.set(parsed).expect("seek sequence set once"); + } if let Some(headroom) = options.edr_headroom { EDR_HEADROOM .set(headroom) @@ -257,6 +306,7 @@ struct DemoOptions { media_uri: Option, smoke_seconds: Option, edr_headroom: Option, + seek_sequence: Option, subtitle_path: Option, danmaku_path: Option, } @@ -265,11 +315,20 @@ fn parse_args(args: &[String]) -> Result { let mut media_uri = None; let mut smoke_seconds = None; let mut edr_headroom = None; + let mut seek_sequence = None; let mut subtitle_path = None; let mut danmaku_path = None; let mut index = 0; while index < args.len() { match args[index].as_str() { + "--seek-sequence" => { + index += 1; + seek_sequence = Some( + args.get(index) + .cloned() + .ok_or("--seek-sequence requires a value like 3:14,5:16")?, + ); + } "--edr" => { let mut headroom = 4.0; if let Some(value) = args.get(index + 1) { @@ -341,6 +400,7 @@ fn parse_args(args: &[String]) -> Result { media_uri, smoke_seconds, edr_headroom, + seek_sequence, subtitle_path, danmaku_path, }) diff --git a/examples/wgpu_decode_png/src/main.rs b/examples/wgpu_decode_png/src/main.rs index b421cf85..f8b8d5fd 100644 --- a/examples/wgpu_decode_png/src/main.rs +++ b/examples/wgpu_decode_png/src/main.rs @@ -102,14 +102,34 @@ fn render_frame(renderer: &mut WgpuRenderer, frame: Frame, out: &str) { media_time: pts.unwrap_or_default(), late_by: None, generation: 1, + scene_avg_nits: std::env::var("ERIKA_TEST_SCENE_AVG_NITS") + .ok() + .and_then(|value| value.parse::().ok()), }; renderer .upload_player_frame(&player_frame) .expect("upload decoded frame"); - let readback = renderer + // The perceptual gamut LUT is generated on a background thread and only + // bound once ready, so the first render uses the fast path. Repeat the + // render (ERIKA_TEST_RENDER_PASSES, default 1) to exercise the LUT path + // the player reaches after the first frames. + let passes = std::env::var("ERIKA_TEST_RENDER_PASSES") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(1); + let mut readback = renderer .render_current_offscreen(None) .expect("render current frame") .expect("a frame was uploaded"); + for _ in 1..passes { + // TODO: replace fixed-delay waiting with explicit LUT-ready synchronization. + std::thread::sleep(Duration::from_millis(200)); + readback = renderer + .render_current_offscreen(None) + .expect("render current frame") + .expect("a frame was uploaded"); + } write_png(out, readback.width, readback.height, &readback.rgba); println!("wrote {out} ({}x{})", readback.width, readback.height); } diff --git a/examples/wgpu_overlay_png/src/main.rs b/examples/wgpu_overlay_png/src/main.rs index 14dbb1ae..ff04d864 100644 --- a/examples/wgpu_overlay_png/src/main.rs +++ b/examples/wgpu_overlay_png/src/main.rs @@ -3,6 +3,7 @@ //! the wgpu overlay/subtitle compositing path (alpha-blended over the video). use erika::overlay::{OverlayFrame, OverlayViewport}; +use erika::renderer::pipeline::VideoRenderPipeline; use erika::renderer::wgpu::{VideoUniforms, WgpuRenderer}; use erika::subtitle::{SubtitleAlphaBitmap, SubtitleBitmapPlacement, SubtitleBitmapPlane}; @@ -153,21 +154,6 @@ fn color_bars_nv12() -> (Vec, Vec) { } fn bars_uniforms() -> VideoUniforms { - VideoUniforms { - is_p010: 0, - full_range: 0, - source_transfer: 0, - target_transfer: 0, - tone_map: 0, - edr_output: 0, - input_mode: 0, - scene_linear: 0, - nits: [100.0, 100.0, 100.0, 100.0], - luma_coefficients: [0.2126, 0.7152, 0.0722, 0.0], - gamut_matrix_rows: [ - [1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - ], - } + let pipeline = VideoRenderPipeline::sdr_default(); + VideoUniforms::from_pipeline(&pipeline, false, false) } diff --git a/examples/wgpu_video_png/src/main.rs b/examples/wgpu_video_png/src/main.rs index e93a414f..dfb16b2c 100644 --- a/examples/wgpu_video_png/src/main.rs +++ b/examples/wgpu_video_png/src/main.rs @@ -2,6 +2,7 @@ //! pipeline and writes the result to a PNG. A visual smoke test for the wgpu //! YCbCr->RGB path: the output PNG should show clean SMPTE-style color bars. +use erika::renderer::pipeline::VideoRenderPipeline; use erika::renderer::wgpu::{VideoUniforms, WgpuRenderer}; const WIDTH: u32 = 256; @@ -63,23 +64,8 @@ fn build_color_bars_nv12() -> (Vec, Vec) { /// A faithful BT.709 limited-range round-trip: linear in/out, clip tone map, /// matched nits, identity gamut. The decoded RGB should match the source bars. fn bt709_limited_uniforms() -> VideoUniforms { - VideoUniforms { - is_p010: 0, - full_range: 0, - source_transfer: 0, - target_transfer: 0, - tone_map: 0, - edr_output: 0, - input_mode: 0, - scene_linear: 0, - nits: [100.0, 100.0, 100.0, 100.0], - luma_coefficients: [0.2126, 0.7152, 0.0722, 0.0], - gamut_matrix_rows: [ - [1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - ], - } + let pipeline = VideoRenderPipeline::sdr_default(); + VideoUniforms::from_pipeline(&pipeline, false, false) } fn main() { diff --git a/examples/wgpu_window_check/src/main.rs b/examples/wgpu_window_check/src/main.rs index 06e8dffd..45711374 100644 --- a/examples/wgpu_window_check/src/main.rs +++ b/examples/wgpu_window_check/src/main.rs @@ -8,6 +8,7 @@ use std::ffi::c_void; use std::process; use std::time::Duration; +use erika::renderer::pipeline::VideoRenderPipeline; use erika::renderer::wgpu::{VideoUniforms, WgpuRenderer}; use erika::{ PlatformSurface, RenderFrameContext, RendererBackend, WgpuSurfaceHandle, WgpuSurfaceKind, @@ -139,21 +140,6 @@ fn color_bars_nv12() -> (Vec, Vec) { } fn bars_uniforms() -> VideoUniforms { - VideoUniforms { - is_p010: 0, - full_range: 0, - source_transfer: 0, - target_transfer: 0, - tone_map: 0, - edr_output: 0, - input_mode: 0, - scene_linear: 0, - nits: [100.0, 100.0, 100.0, 100.0], - luma_coefficients: [0.2126, 0.7152, 0.0722, 0.0], - gamut_matrix_rows: [ - [1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - ], - } + let pipeline = VideoRenderPipeline::sdr_default(); + VideoUniforms::from_pipeline(&pipeline, false, false) } From 044cb1379c1bf75451533d09ed265319620fe708 Mon Sep 17 00:00:00 2001 From: jumusu <1824239290@qq.com> Date: Sun, 20 Sep 2026 23:11:11 +0800 Subject: [PATCH 2/6] fix(renderer): evaluate the Reinhard and Mobius tone curves in linear light libplacebo runs these two curves in the linear PL_HDR_NORM domain; only the PQ-domain operators (BT.2390, spline, ST 2094-10) work on encoded values. Applying them to PQ codes bent the curves' meaning: against a 1000->203 nit target the Mobius knee landed far below its intended position, and a 50-nit patch that should pass through untouched (the 1:1 region ends at 0.3 * 203 = 60.9 nits) came out at ~31 nits. The formulas were already libplacebo's; only the domain was wrong. Convert the IPT intensity axis to linear nits before the curve and encode back after, in all three shaders (WGSL, Metal, HLSL). The 203-nit normalization cancels: the curves only use ratios and differences of same-unit linear values, so absolute nits are equivalent to PL_HDR_NORM. The reference model in `wgpu` no longer approximates Reinhard/Mobius on plain nits -- it never mirrored the IPT path, so a test built on it would have silently agreed with a wrong shader. It now asserts it is only asked for the clip and BT.2390 cases its configurations use. Verified by: - CPU reference tests for both curves (below-knee passthrough, peak anchor, monotonicity) in `renderer::pipeline`. - A wgpu readback test comparing the rendered sample against an independent linear-domain reference for both curves. - The HLSL source-shape test plus the new `video_shader_compiles` test, which compiles the Metal shader on a real device. --- crates/erika/src/renderer/d3d11.rs | 58 +++++++-- crates/erika/src/renderer/metal/apple.rs | 72 +++++++++-- crates/erika/src/renderer/pipeline.rs | 129 ++++++++++++++++++++ crates/erika/src/renderer/wgpu.rs | 139 +++++++++++++++++++--- crates/erika/src/renderer/wgpu_video.wgsl | 30 +++-- 5 files changed, 388 insertions(+), 40 deletions(-) diff --git a/crates/erika/src/renderer/d3d11.rs b/crates/erika/src/renderer/d3d11.rs index 7b8851dd..ad48093b 100644 --- a/crates/erika/src/renderer/d3d11.rs +++ b/crates/erika/src/renderer/d3d11.rs @@ -276,24 +276,40 @@ float tone_map_curve_pq(float x_in, float param) { } if (tone_map == 1u) { // Reinhard (output-relative, libplacebo pl_tone_map_reinhard). - float peak = in_max / out_range; + // + // libplacebo evaluates this curve in the linear PL_HDR_NORM domain + // and only the PQ-domain operators (BT.2390, spline, ST 2094-10) + // work on encoded values. Running it on PQ codes bends the curve's + // meaning: against a 1000->203 nit target mid-tones darken badly, and + // the Mobius knee lands far below its intended position. Rescale to + // linear nits here and encode back at the end. + float in_max_nits = max(src_peak, 0.000001); + float out_min_nits = dst_black; + float out_range_nits = max(dst_peak - dst_black, 0.000001); + float peak = in_max_nits / out_range_nits; float contrast = param > 0.0 ? param : 0.5; float offset = (1.0 - contrast) / max(contrast, 0.000001); float scale = (peak + offset) / peak; - float t = x / out_range; + float t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; float mapped = t / (t + offset) * scale; - return mapped * out_range + out_min; + return pq_code(mapped * out_range_nits + out_min_nits); } if (tone_map == 2u) { - // Mobius: Mobius transform with a 1:1 linear region below the knee. - float peak = in_max / out_range; + // Mobius: Mobius transform with a 1:1 linear region below the knee, + // also evaluated in linear nits (see the Reinhard note above). The + // knee j is relative to the output range, so the linear region ends + // at dst_black + j * (dst_peak - dst_black). + float in_max_nits = max(src_peak, 0.000001); + float out_min_nits = dst_black; + float out_range_nits = max(dst_peak - dst_black, 0.000001); + float peak = in_max_nits / out_range_nits; float j = param > 0.0 ? param : 0.3; float a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); float b = (j * j - 2.0 * j * peak + peak) / max(peak - 1.0, 0.000001); float scale = (b * b + 2.0 * b * j + j * j) / (b - a); - float t = x / out_range; + float t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; float mapped = t > j ? scale * (t + a) / (t + b) : t; - return mapped * out_range + out_min; + return pq_code(mapped * out_range_nits + out_min_nits); } if (tone_map == 3u) { // ITU-R BT.2390 EETF with black-point compensation (the libplacebo @@ -4347,6 +4363,34 @@ mod tests { assert_eq!(sdr.scene_linear, 0); } + #[test] + fn hlsl_evaluates_reinhard_and_mobius_in_linear_nits() { + // libplacebo runs these two curves in the linear PL_HDR_NORM domain + // and only the PQ-domain operators work on encoded values; evaluating + // them on PQ codes moved the Mobius knee far below its intended + // position. The HLSL is compiled by the D3D11 runtime, not by the Rust + // build, so this pins the source shape rather than executing it. + let source = std::str::from_utf8(SHADER_SOURCE).unwrap(); + for branch in ["if (tone_map == 1u)", "if (tone_map == 2u)"] { + let start = source.find(branch).expect("branch"); + let rest = &source[start..]; + let end = rest.find("\n }").expect("branch end"); + let body = &rest[..end]; + assert!( + body.contains("float out_range_nits = max(dst_peak - dst_black, 0.000001);"), + "{branch} must scale in linear nits" + ); + assert!( + body.contains("nits_from_pq(x)"), + "{branch} must decode the PQ code before the curve" + ); + assert!( + body.contains("return pq_code("), + "{branch} must re-encode the result" + ); + } + } + #[test] fn hlsl_gamut_lut_samples_the_target_black_to_peak_axis() { // libplacebo's LUT I axis is [target black, target peak]; a backend diff --git a/crates/erika/src/renderer/metal/apple.rs b/crates/erika/src/renderer/metal/apple.rs index fb04e42e..80006508 100644 --- a/crates/erika/src/renderer/metal/apple.rs +++ b/crates/erika/src/renderer/metal/apple.rs @@ -3452,24 +3452,40 @@ float tone_map_curve_pq(float x_in, float param, constant VideoUniforms& uniform } if (uniforms.tone_map == 1) { // Reinhard (output-relative, libplacebo pl_tone_map_reinhard). - float peak = in_max / out_range; + // + // libplacebo evaluates this curve in the linear PL_HDR_NORM domain + // and only the PQ-domain operators (BT.2390, spline, ST 2094-10) + // work on encoded values. Running it on PQ codes bends the curve's + // meaning: against a 1000->203 nit target mid-tones darken badly, and + // the Mobius knee lands far below its intended position. Rescale to + // linear nits here and encode back at the end. + float in_max_nits = max(src_peak, 0.000001); + float out_min_nits = dst_black; + float out_range_nits = max(dst_peak - dst_black, 0.000001); + float peak = in_max_nits / out_range_nits; float contrast = param > 0.0 ? param : 0.5; float offset = (1.0 - contrast) / max(contrast, 0.000001); float scale = (peak + offset) / peak; - float t = x / out_range; + float t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; float mapped = t / (t + offset) * scale; - return mapped * out_range + out_min; + return pq_code(mapped * out_range_nits + out_min_nits); } if (uniforms.tone_map == 2) { - // Mobius: Mobius transform with a 1:1 linear region below the knee. - float peak = in_max / out_range; + // Mobius: Mobius transform with a 1:1 linear region below the knee, + // also evaluated in linear nits (see the Reinhard note above). The + // knee j is relative to the output range, so the linear region ends + // at dst_black + j * (dst_peak - dst_black). + float in_max_nits = max(src_peak, 0.000001); + float out_min_nits = dst_black; + float out_range_nits = max(dst_peak - dst_black, 0.000001); + float peak = in_max_nits / out_range_nits; float j = param > 0.0 ? param : 0.3; float a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); float b = (j * j - 2.0 * j * peak + peak) / max(peak - 1.0, 0.000001); float scale = (b * b + 2.0 * b * j + j * j) / (b - a); - float t = x / out_range; + float t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; float mapped = t > j ? scale * (t + a) / (t + b) : t; - return mapped * out_range + out_min; + return pq_code(mapped * out_range_nits + out_min_nits); } if (uniforms.tone_map == 3) { // ITU-R BT.2390 EETF with black-point compensation (the libplacebo @@ -4524,6 +4540,48 @@ mod tests { .expect("dual-atlas danmaku pipeline"); } + #[test] + fn video_shader_compiles() { + // The video shader is the one carrying the tone-map curves, and it is + // only compiled at runtime, so a syntax error there would reach users. + // Needs a real Metal device; skip rather than fail where there is none. + let Ok(mut renderer) = + super::MetalRendererImpl::new(crate::renderer::metal::MetalRendererConfig::default()) + else { + eprintln!("skipping: no Metal device available"); + return; + }; + renderer.video_pipeline_state().expect("video pipeline"); + } + + #[test] + fn video_shader_evaluates_reinhard_and_mobius_in_linear_nits() { + // libplacebo runs these two curves in the linear PL_HDR_NORM domain + // and only the PQ-domain operators work on encoded values; evaluating + // them on PQ codes moved the Mobius knee far below its intended + // position and produced the mid-tone darkening the PR review measured. + for branch in ["uniforms.tone_map == 1", "uniforms.tone_map == 2"] { + let start = VIDEO_SHADER_SOURCE + .find(branch) + .unwrap_or_else(|| panic!("{branch} branch")); + let rest = &VIDEO_SHADER_SOURCE[start..]; + let end = rest.find("\n }").expect("branch end"); + let body = &rest[..end]; + assert!( + body.contains("float out_range_nits = max(dst_peak - dst_black, 0.000001);"), + "{branch} must scale in linear nits" + ); + assert!( + body.contains("nits_from_pq(x)"), + "{branch} must decode the PQ code before the curve" + ); + assert!( + body.contains("return pq_code("), + "{branch} must re-encode the result" + ); + } + } + #[test] fn overlay_uniforms_decode_libass_color() { let bitmap = crate::subtitle::SubtitleAlphaBitmap::new( diff --git a/crates/erika/src/renderer/pipeline.rs b/crates/erika/src/renderer/pipeline.rs index 9e10812c..0485dc20 100644 --- a/crates/erika/src/renderer/pipeline.rs +++ b/crates/erika/src/renderer/pipeline.rs @@ -1978,6 +1978,135 @@ mod tests { u * in_max } + /// Reference implementation of the shaders' `tone_map_curve_pq` Reinhard + /// branch (tone_map code 1). + /// + /// libplacebo evaluates this curve in the linear PL_HDR_NORM domain while + /// the PQ-domain operators (BT.2390, spline, ST 2094-10) work on encoded + /// values, so the reference rescales out of PQ codes before the curve and + /// re-encodes the result. Normalization cancels: `pl_tone_map_reinhard` + /// uses ratios and differences of same-unit linear values, so absolute + /// nits are equivalent to PL_HDR_NORM here. + fn reinhard_nits( + x: f32, + src_peak_nits: f32, + dst_peak_nits: f32, + dst_black_nits: f32, + contrast: f32, + ) -> f32 { + let in_max = src_peak_nits.max(0.000_001); + let out_min = dst_black_nits; + let out_range = (dst_peak_nits - dst_black_nits).max(0.000_001); + let peak = in_max / out_range; + let contrast = if contrast > 0.0 { contrast } else { 0.5 }; + let offset = (1.0 - contrast) / contrast.max(0.000_001); + let scale = (peak + offset) / peak; + let t = nits_from_pq(x).clamp(0.0, in_max) / out_range; + let mapped = t / (t + offset) * scale; + pq_code(mapped * out_range + out_min) + } + + /// Reference implementation of the shaders' `tone_map_curve_pq` Mobius + /// branch (tone_map code 2); linear domain as in [`reinhard_nits`]. The + /// knee `knee` is relative to the output range, so the 1:1 region ends at + /// `dst_black + knee * (dst_peak - dst_black)` nits. + fn mobius_nits( + x: f32, + src_peak_nits: f32, + dst_peak_nits: f32, + dst_black_nits: f32, + knee: f32, + ) -> f32 { + let in_max = src_peak_nits.max(0.000_001); + let out_min = dst_black_nits; + let out_range = (dst_peak_nits - dst_black_nits).max(0.000_001); + let peak = in_max / out_range; + let j = if knee > 0.0 { knee } else { 0.3 }; + let a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + let b = (j * j - 2.0 * j * peak + peak) / (peak - 1.0).max(0.000_001); + let scale = (b * b + 2.0 * b * j + j * j) / (b - a); + let t = nits_from_pq(x).clamp(0.0, in_max) / out_range; + let mapped = if t > j { scale * (t + a) / (t + b) } else { t }; + pq_code(mapped * out_range + out_min) + } + + #[test] + fn reinhard_and_mobius_run_in_linear_luminance() { + // The PR review measured a 1000->203 nit target with a near-zero + // black and the Mobius knee at 0.3: a 50-nit patch must pass through + // untouched, because the 1:1 region ends at 0.3 * 203 = 60.9 nits. + // Evaluating the curve on PQ codes instead pushed the knee far lower + // and darkened that patch to ~31.3 nits. + let (source_peak, target_peak, black) = (1000.0_f32, 203.0_f32, 0.0_f32); + let knee_nits = 0.3 * (target_peak - black); + assert!((knee_nits - 60.9).abs() < 0.05, "knee = {knee_nits}"); + + let below_knee = nits_from_pq(mobius_nits( + pq_code(50.0), + source_peak, + target_peak, + black, + 0.3, + )); + assert!( + (below_knee - 50.0).abs() < 0.5, + "50-nit patch below the knee = {below_knee} nits" + ); + + // Both curves anchor the source peak on the target peak. + for mapped in [ + nits_from_pq(reinhard_nits( + pq_code(source_peak), + source_peak, + target_peak, + black, + 0.0, + )), + nits_from_pq(mobius_nits( + pq_code(source_peak), + source_peak, + target_peak, + black, + 0.3, + )), + ] { + assert!( + (mapped - target_peak).abs() < 0.5, + "peak maps to {mapped} nits" + ); + } + + // Reinhard compresses the whole range, so it must darken the 50-nit + // patch slightly and stay monotonic. + let reinhard_50 = nits_from_pq(reinhard_nits( + pq_code(50.0), + source_peak, + target_peak, + black, + 0.0, + )); + assert!( + reinhard_50 > 40.0 && reinhard_50 < 50.0, + "reinhard 50-nit patch = {reinhard_50} nits" + ); + + for curve in [0, 1] { + let mut previous = -1.0_f32; + for step in 0..=100 { + let x = pq_code(source_peak) * step as f32 / 100.0; + let mapped = if curve == 0 { + reinhard_nits(x, source_peak, target_peak, black, 0.0) + } else { + mobius_nits(x, source_peak, target_peak, black, 0.3) + }; + assert!( + mapped >= previous, + "curve {curve} step {step} is not monotonic" + ); + previous = mapped; + } + } + } #[test] fn bt2390_curve_anchors_and_monotonicity() { let (source_peak, target_peak, black) = (1000.0_f32, 100.0_f32, 0.203_f32); diff --git a/crates/erika/src/renderer/wgpu.rs b/crates/erika/src/renderer/wgpu.rs index cb740515..c51f23fd 100644 --- a/crates/erika/src/renderer/wgpu.rs +++ b/crates/erika/src/renderer/wgpu.rs @@ -5895,27 +5895,21 @@ mod tests { } fn ref_tone_map(nits: [f32; 3], u: &VideoUniforms) -> [f32; 3] { - let source_peak = u.nits[0].max(1.0); + // Modeled for the configurations this suite renders: Clip (code 0) and + // BT.2390 (code 3), which is the identity once the source peak already + // fits the target. Reinhard (1) and Mobius (2) are deliberately not + // approximated here — the shader evaluates them in IPT space with the + // curve in linear light, so a nits-domain stand-in would silently + // disagree with it. `wgpu_reinhard_and_mobius_run_in_linear_luminance` + // covers those against an independent reference instead. + assert!( + matches!(u.tone_map, 0 | 3), + "ref_tone_map models only clip and BT.2390, not code {}", + u.tone_map + ); let target_peak = u.nits[1].max(1.0); - let white = (source_peak / target_peak).max(1.0); let x = nits.map(|n| n.max(0.0) / target_peak); - match u.tone_map { - 1 => { - let white2 = white * white; - x.map(|xi| target_peak * (xi * (1.0 + xi / white2) / (1.0 + xi)).clamp(0.0, 1.0)) - } - 2 => { - let knee = 0.75; - let denom = (white - knee).max(0.0001); - x.map(|xi| { - let t = ((xi - knee) / denom).clamp(0.0, 1.0); - let shoulder = knee + (1.0 - knee) * (1.0 - (1.0 - t).powf(2.0)); - let s = if xi >= knee { shoulder } else { xi }; - target_peak * s - }) - } - _ => x.map(|xi| target_peak * xi.clamp(0.0, 1.0)), - } + x.map(|xi| target_peak * xi.clamp(0.0, 1.0)) } fn ref_output(rgb: [f32; 3], u: &VideoUniforms) -> [f32; 3] { @@ -5996,6 +5990,113 @@ mod tests { (luma, chroma) } + /// Renders a solid grey NV12 sample and returns the readback's red + /// channel. The uniforms below keep the shader output in + /// target-reference-linear space, so the channel value is + /// `255 * nits / target_reference_white`. + fn render_grey_sample( + renderer: &mut WgpuRenderer, + uniforms: &VideoUniforms, + tone_map: u32, + y: u8, + ) -> u8 { + let mut uniforms = *uniforms; + uniforms.tone_map = tone_map; + let (luma, chroma) = build_solid_nv12(4, 4, y, 128, 128); + let out = renderer + .render_nv12_offscreen(4, 4, &luma, &chroma, uniforms) + .unwrap(); + out.pixel(1, 1)[0] + } + + #[test] + fn wgpu_reinhard_and_mobius_run_in_linear_luminance() { + // The PR review's readback case: a 1000-nit source peak mapped onto a + // 203-nit target, so the Mobius knee sits at 0.3 * 203 = 60.9 nits. + // libplacebo evaluates Reinhard and Mobius in the linear PL_HDR_NORM + // domain; running them on PQ codes instead moved the knee far lower + // and darkened a 50-nit patch from ~50 to ~31 nits. + let mut renderer = WgpuRenderer::new().unwrap(); + let sdr = VideoUniforms::from_pipeline(&VideoRenderPipeline::sdr_default(), false, false); + + let mut uniforms = sdr; + uniforms.full_range = 1; + uniforms.source_transfer = 0; + uniforms.target_transfer = 0; + uniforms.scene_linear = 1; + uniforms.nits = [1000.0, 203.0, 203.0, 203.0]; + uniforms.tone_map_extra = [0.0, 0.0, 0.0, 0.0]; + uniforms.gamut_lut_enabled = 0; + uniforms.luma_coefficients = [0.2126, 0.7152, 0.0722, 0.0]; + uniforms.gamut_matrix_rows = [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + ]; + + // Y = 63 is 50.15 nits in this configuration, below the knee. + let below_knee_y = 63u8; + let clip = render_grey_sample(&mut renderer, &uniforms, 0, below_knee_y); + let mobius = render_grey_sample(&mut renderer, &uniforms, 2, below_knee_y); + let reinhard = render_grey_sample(&mut renderer, &uniforms, 1, below_knee_y); + + assert!( + (clip as i16 - 63).abs() <= 2, + "clip should reproduce the 50.15-nit input, got {clip}" + ); + // Mobius' 1:1 region covers everything below the knee, so this sample + // must come back untouched. A PQ-domain curve would return ~39 here. + assert!( + (mobius as i16 - clip as i16).abs() <= 1, + "mobius below the knee = {mobius}, clip = {clip}" + ); + // Reinhard compresses the whole range, so this sample darkens a + // little, but only a little. + assert!( + (clip as i16 - reinhard as i16) >= 1 && (clip as i16 - reinhard as i16) <= 3, + "reinhard below the knee = {reinhard}, clip = {clip}" + ); + + // Independent reference for the readback. libplacebo evaluates both + // curves in linear light, so the sample's nits go through the curve + // and come back as `255 * mapped_nits / target_reference_white`; the + // CPU tests in `renderer::pipeline` pin the same formulas. The + // PQ-domain port this replaces would miss these by tens of LSBs. + let expected = |nits: f32, tone_map: u32| -> i16 { + let (src_peak, dst_peak, dst_black) = (1000.0_f32, 203.0_f32, 0.0_f32); + let out_range = dst_peak - dst_black; + let peak = src_peak / out_range; + let t = nits / out_range; + let mapped = match tone_map { + 1 => { + // Reinhard, contrast 0.5 -> offset 1. + let offset = 1.0; + let scale = (peak + offset) / peak; + t / (t + offset) * scale + } + 2 => { + let j = 0.3; + let a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); + let b = (j * j - 2.0 * j * peak + peak) / (peak - 1.0); + let scale = (b * b + 2.0 * b * j + j * j) / (b - a); + if t > j { scale * (t + a) / (t + b) } else { t } + } + _ => t, + }; + (255.0 * mapped).round() as i16 + }; + + for (y, tone_map) in [(below_knee_y, 1u32), (below_knee_y, 2), (200, 1), (200, 2)] { + let nits = f32::from(y) / 255.0 * 203.0; + let want = expected(nits, tone_map); + let got = i16::from(render_grey_sample(&mut renderer, &uniforms, tone_map, y)); + assert!( + (got - want).abs() <= 2, + "y={y} tone_map={tone_map} ({nits} nits) = {got}, expected {want}" + ); + } + } + #[test] fn wgpu_video_nv12_matches_cpu_reference() { let mut renderer = WgpuRenderer::new().unwrap(); diff --git a/crates/erika/src/renderer/wgpu_video.wgsl b/crates/erika/src/renderer/wgpu_video.wgsl index 524dc06f..94ad7237 100644 --- a/crates/erika/src/renderer/wgpu_video.wgsl +++ b/crates/erika/src/renderer/wgpu_video.wgsl @@ -188,24 +188,40 @@ fn tone_map_curve_pq(x_in: f32, param: f32) -> f32 { } if (uniforms.tone_map == 1u) { // Reinhard (output-relative, libplacebo pl_tone_map_reinhard). - let peak = in_max / out_range; + // + // libplacebo evaluates this curve in the linear PL_HDR_NORM domain + // and only the PQ-domain operators (BT.2390, spline, ST 2094-10) + // work on encoded values. Running it on PQ codes bends the curve's + // meaning: against a 1000->203 nit target mid-tones darken badly, and + // the Mobius knee lands far below its intended position. Rescale to + // linear nits here and encode back at the end. + let in_max_nits = max(src_peak, 0.000001); + let out_min_nits = dst_black; + let out_range_nits = max(dst_peak - dst_black, 0.000001); + let peak = in_max_nits / out_range_nits; let contrast = select(0.5, param, param > 0.0); let offset = (1.0 - contrast) / max(contrast, 0.000001); let scale = (peak + offset) / peak; - let t = x / out_range; + let t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; let mapped = t / (t + offset) * scale; - return mapped * out_range + out_min; + return pq_code(mapped * out_range_nits + out_min_nits); } if (uniforms.tone_map == 2u) { - // Mobius: Möbius transform with a 1:1 linear region below the knee. - let peak = in_max / out_range; + // Mobius: Möbius transform with a 1:1 linear region below the knee, + // also evaluated in linear nits (see the Reinhard note above). The + // knee j is relative to the output range, so the linear region ends + // at dst_black + j * (dst_peak - dst_black). + let in_max_nits = max(src_peak, 0.000001); + let out_min_nits = dst_black; + let out_range_nits = max(dst_peak - dst_black, 0.000001); + let peak = in_max_nits / out_range_nits; let j = select(0.3, param, param > 0.0); let a = -j * j * (peak - 1.0) / (j * j - 2.0 * j + peak); let b = (j * j - 2.0 * j * peak + peak) / max(peak - 1.0, 0.000001); let scale = (b * b + 2.0 * b * j + j * j) / (b - a); - let t = x / out_range; + let t = clamp(nits_from_pq(x), 0.0, in_max_nits) / out_range_nits; let mapped = select(t, scale * (t + a) / (t + b), t > j); - return mapped * out_range + out_min; + return pq_code(mapped * out_range_nits + out_min_nits); } if (uniforms.tone_map == 3u) { // ITU-R BT.2390 EETF with black-point compensation (the libplacebo From 74008a783bdb609bcc39aac3f51b359f96744c09 Mon Sep 17 00:00:00 2001 From: jumusu <1824239290@qq.com> Date: Sun, 20 Sep 2026 23:12:18 +0800 Subject: [PATCH 3/6] fix(renderer): skip the tone-map curve for content that needs no mapping `requires_tone_mapping()` gates the target black point and the perceptual gamut LUT, but not the shader's curve: the uniform still carried whichever operator the embedder selected, and every backend executes it unconditionally. Selecting Reinhard therefore brightened plain SDR->SDR playback -- the review measured a 100-nit grey at Y=128 going from ~130 to ~175 -- because Reinhard is not the identity where Clip is. Send the identity operator (Clip, code 0) whenever the pipeline does not tone map, so the decision that already gates the black point also gates the curve. The recalculation now lives in `VideoRenderPipeline::tone_map_uniform_code`, used by all three backends; Metal built its uniforms by hand and needed both of its construction sites updated. --- crates/erika/src/renderer/metal/apple.rs | 18 ++------ crates/erika/src/renderer/pipeline.rs | 54 +++++++++++++++++++++++- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/crates/erika/src/renderer/metal/apple.rs b/crates/erika/src/renderer/metal/apple.rs index 80006508..da41707d 100644 --- a/crates/erika/src/renderer/metal/apple.rs +++ b/crates/erika/src/renderer/metal/apple.rs @@ -61,7 +61,7 @@ use crate::renderer::metal::{ metal_target_color, }; use crate::renderer::output::negotiate_output_mode; -use crate::renderer::pipeline::{ColorRange, DoviUniforms, LumaUpscalerMode, ToneMapOperator}; +use crate::renderer::pipeline::{ColorRange, DoviUniforms, LumaUpscalerMode}; use crate::renderer::pipeline::{SourceColorState, TargetColorState, VideoRenderPipeline}; use crate::renderer::presentation::PresentationLayout as VideoPresentationLayout; use crate::subtitle::{AssColor, SubtitleAlphaBitmap}; @@ -1182,7 +1182,7 @@ impl MetalRendererImpl { full_range: matches!(frame.pipeline.source.range, ColorRange::Full) as u32, source_transfer: transfer_code(frame.pipeline.source.transfer), target_transfer: transfer_code(frame.pipeline.target.transfer), - tone_map: tone_map_code(frame.pipeline.tone_map.operator), + tone_map: frame.pipeline.tone_map_uniform_code(), edr_output: self.output_mode.is_edr() as u32, _reserved0: self.video_alpha_mode as u32, _reserved1: 0, @@ -1403,7 +1403,7 @@ impl MetalRendererImpl { full_range: matches!(frame.pipeline.source.range, ColorRange::Full) as u32, source_transfer: transfer_code(frame.pipeline.source.transfer), target_transfer: transfer_code(frame.pipeline.target.transfer), - tone_map: tone_map_code(frame.pipeline.tone_map.operator), + tone_map: frame.pipeline.tone_map_uniform_code(), edr_output: self.output_mode.is_edr() as u32, _reserved0: 0, _reserved1: 0, @@ -2745,18 +2745,6 @@ fn ui_output_nits(target: TargetColorState, edr_output: bool) -> [f32; 4] { ] } -fn tone_map_code(operator: ToneMapOperator) -> u32 { - match operator { - ToneMapOperator::Clip => 0, - ToneMapOperator::Reinhard => 1, - ToneMapOperator::Mobius => 2, - ToneMapOperator::Bt2390 => 3, - ToneMapOperator::Spline => 4, - ToneMapOperator::Bt2446a => 5, - ToneMapOperator::St209410 => 6, - } -} - fn luma_coefficients(coeffs: crate::renderer::pipeline::LumaCoefficients) -> [f32; 4] { [coeffs.kr, coeffs.kg, coeffs.kb, 0.0] } diff --git a/crates/erika/src/renderer/pipeline.rs b/crates/erika/src/renderer/pipeline.rs index 0485dc20..c057c1e0 100644 --- a/crates/erika/src/renderer/pipeline.rs +++ b/crates/erika/src/renderer/pipeline.rs @@ -1181,6 +1181,21 @@ impl VideoRenderPipeline { requires_tone_mapping(self.source, self.target) && resolve_primaries(self.source.primaries) != resolve_primaries(self.target.primaries) } + + /// Operator code for the shaders' `tone_map` uniform. + /// + /// Zero (Clip, an identity curve) whenever this pipeline does not tone + /// map, so a Reinhard/Mobius selection cannot alter SDR->SDR content. + /// The configured operator would otherwise run unconditionally: the + /// shaders only branch on the code, and `tone_map_extra`'s black point + /// is already gated the same way. + pub fn tone_map_uniform_code(&self) -> u32 { + if requires_tone_mapping(self.source, self.target) { + tone_map_code(self.tone_map.operator) + } else { + tone_map_code(ToneMapOperator::Clip) + } + } } impl Default for VideoRenderPipeline { @@ -1251,7 +1266,7 @@ impl VideoUniforms { full_range: u32::from(matches!(pipeline.source.range, ColorRange::Full)), source_transfer: transfer_code(pipeline.source.transfer), target_transfer: transfer_code(pipeline.target.transfer), - tone_map: tone_map_code(pipeline.tone_map.operator), + tone_map: pipeline.tone_map_uniform_code(), edr_output: u32::from(edr_output), input_mode: 0, scene_linear: 0, @@ -2107,6 +2122,43 @@ mod tests { } } } + + #[test] + fn tone_map_uniform_code_skips_the_curve_for_unmapped_content() { + // SDR -> SDR needs no tone mapping, so the shader must receive Clip + // (an identity curve) rather than the selected operator: running + // Reinhard here brightened an SDR 100-nit grey from ~130 to ~175 in + // the review's readback. + let source = SourceColorState::new(ColorPrimaries::Bt709, TransferFunction::Srgb); + let target = TargetColorState::sdr(ColorPrimaries::Bt709); + let mut pipeline = VideoRenderPipeline::new(source, target); + assert!(!pipeline.requires_tone_mapping()); + pipeline.tone_map.operator = ToneMapOperator::Reinhard; + assert_eq!( + pipeline.tone_map_uniform_code(), + tone_map_code(ToneMapOperator::Clip) + ); + assert_eq!( + VideoUniforms::from_pipeline(&pipeline, false, false).tone_map, + 0, + "the shader must see the identity curve" + ); + + // An HDR source that overshoots the target keeps the operator so the + // curve actually runs. + let hdr_source = SourceColorState::new(ColorPrimaries::Bt2020, TransferFunction::Pq); + let mut hdr_pipeline = VideoRenderPipeline::new( + hdr_source, + TargetColorState::sdr_tone_map_target(ColorPrimaries::Bt709), + ); + assert!(hdr_pipeline.requires_tone_mapping()); + hdr_pipeline.tone_map.operator = ToneMapOperator::Reinhard; + assert_eq!( + hdr_pipeline.tone_map_uniform_code(), + tone_map_code(ToneMapOperator::Reinhard) + ); + } + #[test] fn bt2390_curve_anchors_and_monotonicity() { let (source_peak, target_peak, black) = (1000.0_f32, 100.0_f32, 0.203_f32); From 355b74e009e1fd2e2fd185aecdffe9c7ed437876 Mon Sep 17 00:00:00 2001 From: jumusu <1824239290@qq.com> Date: Sun, 20 Sep 2026 23:14:01 +0800 Subject: [PATCH 4/6] fix(metal): consume the host-reported display headroom instead of reading AppKit `select_output_mode_for_source` runs on the render thread (the CVDisplayLink callback or the host's render queue) and probed `NSView.window`, `NSWindow.screen`, `NSApplication.windows` and `NSView.subviews` from there every frame to negotiate the output mode. AppKit requires the main thread, and the review reproduced the violation with Main Thread Checker. The host already resolves this value on the main thread -- it owns the view and knows which screen the window is on -- and the C ABI already has `erika_presenter_set_output_headroom`, which the renderer previously ignored. Implement it for Metal: the renderer caches the reported capability and never touches AppKit. `known = false` (and "never reported") negotiates as SDR, which is the fallback the AppKit probe used when it could not answer. Hosts now publish the value from the main thread: the macOS plugin on attach and from the screen-change observer it already installs (a screen parameter change can move EDR capability without changing the display ID), and the macOS native demo from its attach/resize path. A demo that requests EDR explicitly needs this, because an explicit request is otherwise clamped to the unreported SDR capability. --- crates/erika/src/renderer/metal/apple.rs | 210 ++++++------------ crates/erika/src/renderer/metal/mod.rs | 20 +- .../macos_native_demo/native/ErikaMetalDemo.m | 23 ++ examples/macos_native_demo/src/main.rs | 14 ++ .../macos/Classes/ErikaFlutterPlugin.swift | 66 +++++- 5 files changed, 184 insertions(+), 149 deletions(-) diff --git a/crates/erika/src/renderer/metal/apple.rs b/crates/erika/src/renderer/metal/apple.rs index da41707d..ebe12394 100644 --- a/crates/erika/src/renderer/metal/apple.rs +++ b/crates/erika/src/renderer/metal/apple.rs @@ -194,6 +194,16 @@ pub struct MetalRendererImpl { gamut_lut_job: Option, dummy_gamut_lut: Option>>, logged_first_video_frame: bool, + /// EDR headroom of the display the layer is presented on, published by + /// the host through `RendererBackend::set_output_headroom`. + /// + /// Read here instead of querying AppKit: `select_output_mode_for_source` + /// runs on the render thread (CVDisplayLink or the host's render queue), + /// where touching `NSView`/`NSWindow`/`NSApplication` violates AppKit's + /// main-thread requirement. `None` means no host has reported a value + /// yet, which negotiates as SDR — the same fallback this used when AppKit + /// could not answer. + reported_display_headroom: Option, } fn hdr_debug_enabled() -> bool { @@ -268,6 +278,7 @@ impl MetalRendererImpl { gamut_lut_job: None, dummy_gamut_lut: None, logged_first_video_frame: false, + reported_display_headroom: None, }) } @@ -450,67 +461,28 @@ impl MetalRendererImpl { ) } - /// EDR headroom of the display the player window is presented on. + /// EDR headroom of the display the player window is presented on, as + /// published by the host through `RendererBackend::set_output_headroom`. /// - /// The *potential* value is used deliberately: it reports what the display - /// can do regardless of the current brightness setting, so playback does - /// not flip between SDR and EDR while the brightness slider moves. Falls - /// back to 1.0 (no EDR) when AppKit cannot answer. Resolved through the - /// layer's hosting window: `NSScreen.mainScreen` tracks the systemwide - /// key window, which belongs to a *different* app whenever this one is - /// inactive — negotiating from it then enables PQ passthrough while the - /// layer sits on an SDR display, rendering washed-out colors. AppKit - /// makes the hosting NSView the delegate of a view-assigned backing - /// layer, so prefer delegate→window→screen and fall back to mainScreen - /// when that chain is unavailable (e.g. detached layers). - #[cfg(target_os = "macos")] + /// The host resolves this on the main thread (it owns the `NSView` and + /// knows which screen the window is on) and pushes it here; querying + /// AppKit from this renderer would run on the render thread and violate + /// AppKit's main-thread requirement. No report yet negotiates as SDR. fn display_edr_headroom(&self) -> f32 { - use objc2::msg_send; - use objc2::runtime::{AnyClass, AnyObject}; - use objc2::sel; + self.reported_display_headroom.unwrap_or(1.0) + } - unsafe { - let screen: Option> = self - .layer - .as_ref() - .and_then(|layer| { - let layer_obj: &AnyObject = layer; - if let Some(screen) = screen_from_layer_delegate(layer_obj) { - return Some(screen); - } - let mut curr: Option> = msg_send![layer_obj, superlayer]; - while let Some(parent) = curr { - if let Some(screen) = screen_from_layer_delegate(&parent) { - return Some(screen); - } - curr = msg_send![&parent, superlayer]; - } - if let Some(screen) = screen_from_app_windows(layer_obj) { - return Some(screen); - } - None - }) - .or_else(|| { - let class = AnyClass::get(c"NSScreen")?; - msg_send![class, mainScreen] - }); - let Some(screen) = screen else { - return 1.0; - }; - let selector = sel!(maximumPotentialExtendedDynamicRangeColorComponentValue); - let responds: bool = msg_send![&screen, respondsToSelector: selector]; - if !responds { - return 1.0; - } - let potential: f64 = msg_send![ - &screen, - maximumPotentialExtendedDynamicRangeColorComponentValue - ]; - if potential.is_finite() && potential > 0.0 { - potential as f32 - } else { - 1.0 - } + /// Records a host-reported display headroom. `known == false` clears the + /// cache so negotiation falls back to SDR until a real value arrives. + pub fn set_output_headroom(&mut self, headroom: f32, known: bool) { + let resolved = if known && headroom.is_finite() && headroom > 0.0 { + Some(headroom) + } else { + None + }; + if resolved != self.reported_display_headroom { + self.reported_display_headroom = resolved; + self.stats.headroom_updates = self.stats.headroom_updates.saturating_add(1); } } @@ -2515,86 +2487,6 @@ fn configure_layer_dynamic_range(layer: &CAMetalLayer, enabled: bool) { } } -#[cfg(target_os = "macos")] -unsafe fn screen_from_layer_delegate( - layer: &objc2::runtime::AnyObject, -) -> Option> { - use objc2::msg_send; - use objc2::rc::Retained; - use objc2::runtime::{AnyClass, AnyObject}; - let delegate: Option> = msg_send![layer, delegate]; - let delegate = delegate?; - let view_class = AnyClass::get(c"NSView")?; - let is_view: bool = msg_send![&delegate, isKindOfClass: view_class]; - if !is_view { - return None; - } - let window: Option> = msg_send![&delegate, window]; - let window = window?; - let screen: Option> = msg_send![&window, screen]; - screen -} - -#[cfg(target_os = "macos")] -unsafe fn screen_from_app_windows( - target_layer: &objc2::runtime::AnyObject, -) -> Option> { - use objc2::msg_send; - use objc2::rc::Retained; - use objc2::runtime::{AnyClass, AnyObject}; - let app_class = AnyClass::get(c"NSApplication")?; - let app: Option> = msg_send![app_class, sharedApplication]; - let app = app?; - let windows: Option> = msg_send![&app, windows]; - let windows = windows?; - let count: usize = msg_send![&windows, count]; - for i in 0..count { - let window: Retained = msg_send![&windows, objectAtIndex: i]; - let content_view: Option> = msg_send![&window, contentView]; - if let Some(content_view) = content_view { - if unsafe { view_contains_layer(&content_view, target_layer) } { - let screen: Option> = msg_send![&window, screen]; - return screen; - } - } - } - None -} - -#[cfg(target_os = "macos")] -unsafe fn view_contains_layer( - view: &objc2::runtime::AnyObject, - target_layer: &objc2::runtime::AnyObject, -) -> bool { - use objc2::msg_send; - use objc2::rc::Retained; - use objc2::runtime::AnyObject; - let view_layer: Option> = msg_send![view, layer]; - if let Some(vl) = view_layer { - if Retained::as_ptr(&vl) == target_layer as *const AnyObject { - return true; - } - let mut curr: Option> = msg_send![target_layer, superlayer]; - while let Some(parent) = curr { - if Retained::as_ptr(&parent) == Retained::as_ptr(&vl) { - return true; - } - curr = msg_send![&parent, superlayer]; - } - } - let subviews: Option> = msg_send![view, subviews]; - if let Some(subviews) = subviews { - let count: usize = msg_send![&subviews, count]; - for i in 0..count { - let subview: Retained = msg_send![&subviews, objectAtIndex: i]; - if unsafe { view_contains_layer(&subview, target_layer) } { - return true; - } - } - } - false -} - #[repr(C)] #[derive(Debug, Clone, Copy)] struct VideoUniforms { @@ -4528,6 +4420,48 @@ mod tests { .expect("dual-atlas danmaku pipeline"); } + #[test] + fn reported_display_headroom_drives_output_mode_negotiation() { + // The renderer negotiates on the render thread, where AppKit must not + // be touched, so it consumes whatever the host published from the main + // thread. A missing report negotiates as SDR, exactly like the AppKit + // fallback it replaced. + let config = crate::renderer::metal::MetalRendererConfig { + output_mode: crate::renderer::metal::MetalOutputMode::auto(1.0), + ..crate::renderer::metal::MetalRendererConfig::default() + }; + // Needs a real Metal device; skip rather than fail where there is none. + let Ok(mut renderer) = super::MetalRendererImpl::new(config) else { + eprintln!("skipping: no Metal device available"); + return; + }; + let hdr = crate::renderer::pipeline::SourceColorState::new( + ColorPrimaries::Bt2020, + TransferFunction::Pq, + ); + + assert_eq!(renderer.display_edr_headroom(), 1.0); + renderer.select_output_mode_for_source(hdr); + assert!( + !renderer.active_output_mode().is_edr(), + "an unreported display must not promote to EDR" + ); + + renderer.set_output_headroom(4.0, true); + assert_eq!(renderer.display_edr_headroom(), 4.0); + renderer.select_output_mode_for_source(hdr); + assert_eq!( + renderer.active_output_mode(), + crate::renderer::metal::MetalOutputMode::apple_edr(4.0) + ); + + // An unreported value clears the cache and returns to SDR. + renderer.set_output_headroom(4.0, false); + assert_eq!(renderer.display_edr_headroom(), 1.0); + renderer.select_output_mode_for_source(hdr); + assert!(!renderer.active_output_mode().is_edr()); + } + #[test] fn video_shader_compiles() { // The video shader is the one carrying the tone-map curves, and it is diff --git a/crates/erika/src/renderer/metal/mod.rs b/crates/erika/src/renderer/metal/mod.rs index 2fc712d5..63ee07ac 100644 --- a/crates/erika/src/renderer/metal/mod.rs +++ b/crates/erika/src/renderer/metal/mod.rs @@ -221,6 +221,8 @@ pub struct MetalRendererStats { pub edr_rendered_frames: u64, pub sdr_tonemap_frames: u64, pub output_mode_switches: u64, + /// Host headroom reports accepted by `set_output_headroom`. + pub headroom_updates: u64, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1134,7 +1136,7 @@ impl RendererBackend for MetalRenderer { fallback_reason: OutputFallbackReason::None, fallback_count: 0, data_space_failures: 0, - headroom_updates: 0, + headroom_updates: stats.headroom_updates, extended_linear_frames: stats.edr_rendered_frames, } } @@ -1149,6 +1151,22 @@ impl RendererBackend for MetalRenderer { let _ = mode; } } + + /// Caches the display headroom the host resolved on the main thread. + /// + /// `MetalRendererImpl::select_output_mode_for_source` runs on the render + /// thread and must not query AppKit, so the host owns the `NSView` / + /// `NSScreen` lookup and pushes the result here. + fn set_output_headroom(&mut self, headroom: f32, known: bool) { + #[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos"))] + { + self.inner.set_output_headroom(headroom, known); + } + #[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "tvos")))] + { + let _ = (headroom, known); + } + } } #[cfg(test)] diff --git a/examples/macos_native_demo/native/ErikaMetalDemo.m b/examples/macos_native_demo/native/ErikaMetalDemo.m index 2b7a861b..5a2b0b1c 100644 --- a/examples/macos_native_demo/native/ErikaMetalDemo.m +++ b/examples/macos_native_demo/native/ErikaMetalDemo.m @@ -4,6 +4,7 @@ extern void erika_demo_attach_layer(void *layer, unsigned int width, unsigned int height, double scale); extern void erika_demo_resize_layer(unsigned int width, unsigned int height, double scale); +extern void erika_demo_set_display_headroom(float headroom, bool known); extern void erika_demo_render_frame(double time_seconds); extern void erika_demo_toggle_play_pause(void); extern void erika_demo_seek_seconds(double seconds); @@ -12,6 +13,23 @@ extern bool erika_demo_is_playing(void); extern double erika_demo_smoke_seconds(void); +/// Potential EDR headroom of a screen: what it can do regardless of the +/// current brightness setting. 1.0 means no EDR. Main thread only. +static float ErikaPotentialEdrHeadroom(NSScreen *screen) { + if (screen == nil) { + return 1.0f; + } + SEL selector = NSSelectorFromString(@"maximumPotentialExtendedDynamicRangeColorComponentValue"); + if (![screen respondsToSelector:selector]) { + return 1.0f; + } + NSNumber *value = [screen valueForKey:@"maximumPotentialExtendedDynamicRangeColorComponentValue"]; + if (![value isKindOfClass:[NSNumber class]]) { + return 1.0f; + } + return MAX(1.0f, value.floatValue); +} + static NSString *ErikaFormatTime(double seconds) { if (!isfinite(seconds) || seconds < 0.0) { seconds = 0.0; @@ -91,6 +109,11 @@ - (void)updateDrawableSizeAndAttach:(BOOL)attach { self.metalLayer.frame = self.bounds; unsigned int pixelWidth = (unsigned int)MAX(1.0, round(drawableSize.width)); unsigned int pixelHeight = (unsigned int)MAX(1.0, round(drawableSize.height)); + // The renderer negotiates its output mode off the main thread and no longer + // reads AppKit itself, so publish the presenting screen's EDR capability + // from here: on attach, and again whenever the window moves to a screen with + // different backing properties. + erika_demo_set_display_headroom(ErikaPotentialEdrHeadroom(self.window.screen), true); if (attach) { erika_demo_attach_layer((__bridge void *)self.metalLayer, pixelWidth, pixelHeight, scale); } else { diff --git a/examples/macos_native_demo/src/main.rs b/examples/macos_native_demo/src/main.rs index cbf13bc1..4044b3f8 100644 --- a/examples/macos_native_demo/src/main.rs +++ b/examples/macos_native_demo/src/main.rs @@ -198,6 +198,20 @@ pub extern "C" fn erika_demo_attach_layer(layer: *mut c_void, width: u32, height }); } +/// Publishes the presenting display's EDR headroom. +/// +/// The renderer negotiates its output mode on the render thread and no longer +/// probes AppKit itself, so the AppKit side (the demo view) resolves the value +/// on the main thread and pushes it here. +#[unsafe(no_mangle)] +pub extern "C" fn erika_demo_set_display_headroom(headroom: f32, known: bool) { + DEMO.with(|demo| { + demo.borrow_mut() + .presenter + .set_output_headroom(headroom, known); + }); +} + #[unsafe(no_mangle)] pub extern "C" fn erika_demo_resize_layer(width: u32, height: u32, scale: f64) { DEMO.with(|demo| { diff --git a/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift b/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift index 5ba49d6e..5e2d596e 100644 --- a/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift +++ b/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift @@ -647,6 +647,7 @@ private final class ErikaNativeLibrary { ) -> Int32 typealias ResizeSurfaceFn = @convention(c) (UnsafeMutableRawPointer?, UInt32, UInt32, Double) -> Int32 typealias RenderTickFn = @convention(c) (UnsafeMutableRawPointer?, Double, UnsafeMutableRawPointer?) -> Int32 + typealias SetOutputHeadroomFn = @convention(c) (UnsafeMutableRawPointer?, Float, Bool) -> Int32 typealias CaptureFrameRgbaFn = @convention(c) (UnsafeMutableRawPointer?, UInt32, UInt32, UnsafeMutableRawPointer?, Int) -> Int32 typealias PollEventFn = @convention(c) (UnsafeMutableRawPointer?, UnsafeMutableRawPointer?) -> Int32 typealias LastErrorMessageFn = @convention(c) () -> UnsafeMutablePointer? @@ -712,6 +713,7 @@ private final class ErikaNativeLibrary { let renderTick: RenderTickFn let captureFrameRgba: CaptureFrameRgbaFn? let pollEvent: PollEventFn + let setOutputHeadroom: SetOutputHeadroomFn? let lastErrorMessage: LastErrorMessageFn let stringFree: StringFreeFn @@ -783,6 +785,7 @@ private final class ErikaNativeLibrary { renderTick = try Self.load("erika_presenter_render_tick", from: libraryHandle, as: RenderTickFn.self) captureFrameRgba = Self.loadOptional("erika_presenter_capture_frame_rgba", from: libraryHandle, as: CaptureFrameRgbaFn.self) pollEvent = try Self.load("erika_presenter_poll_event", from: libraryHandle, as: PollEventFn.self) + setOutputHeadroom = Self.loadOptional("erika_presenter_set_output_headroom", from: libraryHandle, as: SetOutputHeadroomFn.self) lastErrorMessage = try Self.load("erika_last_error_message", from: libraryHandle, as: LastErrorMessageFn.self) stringFree = try Self.load("erika_string_free", from: libraryHandle, as: StringFreeFn.self) } @@ -1822,6 +1825,7 @@ private final class ErikaPlayerHost { operation: "attach_metal_layer" ) } + reportDisplayHeadroom() } else { try withNativeCall { try check( @@ -1832,6 +1836,23 @@ private final class ErikaPlayerHost { } } + /// Publishes the presenting display's EDR headroom to the renderer. + /// + /// The renderer negotiates its output mode on the render thread (the + /// CVDisplayLink callback), where AppKit must not be touched, so the value + /// is resolved here and pushed through the C ABI instead. Main thread only: + /// it reads the attached `NSView`'s screen. + private func reportDisplayHeadroom() { + guard let setOutputHeadroom = library.setOutputHeadroom, + let view = attachedView else { + return + } + let headroom = view.displayEdrHeadroom() + withNativeCall { + _ = setOutputHeadroom(handle, headroom, true) + } + } + private func startDisplayDriverIfNeeded(resetClock: Bool) { if resetClock { startTimeSeconds = CACurrentMediaTime() @@ -1884,6 +1905,10 @@ private final class ErikaPlayerHost { object: nil, queue: .main ) { [weak self] _ in + // Screen parameters change without the display ID changing (the EDR + // capability of the same screen can move), so republish the headroom + // before the display-link retarget short-circuits. + self?.reportDisplayHeadroom() self?.retargetDisplayDriverIfScreenChanged() } displayConfigurationObservers.append(observer) @@ -2170,6 +2195,23 @@ private func withOptionalCString(_ value: String?, _ body: (UnsafePointer Float { + guard let screen else { + return 1.0 + } + let key = "maximumPotentialExtendedDynamicRangeColorComponentValue" + guard screen.responds(to: Selector((key))), + let number = screen.value(forKey: key) as? NSNumber else { + return 1.0 + } + return max(1.0, number.floatValue) +} + private protocol ErikaMetalSurfaceView: AnyObject { var platformViewId: Int64 { get } var metalLayer: CAMetalLayer { get } @@ -2179,6 +2221,11 @@ private protocol ErikaMetalSurfaceView: AnyObject { func updateDrawableSize() func pngSnapshotData() -> Data? + + /// EDR headroom of the display this view is presented on. Main thread only: + /// the renderer consumes the reported value on its own thread and must not + /// query AppKit itself. + func displayEdrHeadroom() -> Float } private final class WeakErikaVideoPlatformViewBox { @@ -2279,6 +2326,10 @@ final class ErikaVideoPlatformView: NSView, ErikaMetalSurfaceView { func pngSnapshotData() -> Data? { snapshotPngData(of: self) } + + func displayEdrHeadroom() -> Float { + screenPotentialEdrHeadroom(window?.screen) + } } final class ErikaWindowOverlayView: NSView, ErikaMetalSurfaceView { @@ -2408,6 +2459,10 @@ final class ErikaWindowOverlayView: NSView, ErikaMetalSurfaceView { func pngSnapshotData() -> Data? { snapshotPngData(of: self) } + + func displayEdrHeadroom() -> Float { + screenPotentialEdrHeadroom(window?.screen) + } } private func snapshotPngData(of view: NSView) -> Data? { @@ -3451,16 +3506,7 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan NSApp.keyWindow?.screen ?? NSApp.mainWindow?.screen ?? NSScreen.main - guard let screen else { - return 1.0 - } - - let key = "maximumPotentialExtendedDynamicRangeColorComponentValue" - guard screen.responds(to: Selector((key))), - let number = screen.value(forKey: key) as? NSNumber else { - return 1.0 - } - return max(1.0, number.floatValue) + return screenPotentialEdrHeadroom(screen) } private func boolEnvironmentFlag( From c1dbacd5833c76b6bc84a5ffb92fda47575c9903 Mon Sep 17 00:00:00 2001 From: jumusu <1824239290@qq.com> Date: Sun, 20 Sep 2026 23:14:45 +0800 Subject: [PATCH 5/6] fix(d3d11): invalidate the gamut LUT when the device changes `ensure_device_for_texture` drops the retired video and the danmaku atlas cache when a decoder hands over a new D3D11 device, but the cached gamut LUT SRV stayed: it is keyed on (source, target, target black, target peak) only, so the same uniforms returned the previous device's SRV and `PSSetShaderResources` bound it to the new context. D3D11 does not support cross-device binding. Drop the SRV in `set_device`, the single choke point for both the switch path and the first-device path. Only the GPU resource is device-bound, so the CPU-side `gamut_lut_job` stays and the next frame reuses it to rebuild the texture on the new device. The D3D11 module is Windows-only and its CI job builds rather than runs tests, so this one is verified by the call chain, not by an executed test; the HLSL source-shape test added alongside the tone-map fix is the closest guard. --- crates/erika/src/renderer/d3d11.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/erika/src/renderer/d3d11.rs b/crates/erika/src/renderer/d3d11.rs index ad48093b..b51f55c2 100644 --- a/crates/erika/src/renderer/d3d11.rs +++ b/crates/erika/src/renderer/d3d11.rs @@ -1351,6 +1351,11 @@ impl D3d11Renderer { surface.output_texture = None; } } + // The cached LUT SRV belongs to the previous device, and D3D11 cannot + // bind a resource across devices. Drop it even when the key is + // unchanged: the CPU-side `gamut_lut_job` only holds parameters, so a + // later frame regenerates the texture on the new device and reuses it. + self.gamut_lut = None; self.state = Some(state); self.recreate_surface_targets()?; Ok(()) From 19dddf071b7c9078ebf0c41427eca232f1ef6ac0 Mon Sep 17 00:00:00 2001 From: jumusu <1824239290@qq.com> Date: Sun, 20 Sep 2026 23:15:35 +0800 Subject: [PATCH 6/6] fix(flutter): honour ERIKA_DISABLE_EDR with SDR output instead of Auto(1.0) `resolvedEdrHeadroom()` returned 1.0 for the disable flag and the caller still built `auto(headroom: 1.0)`. Under the per-presenting-display Auto negotiation a headroom of 1.0 means "no embedder cap, defer to the display", so an EDR screen promoted to EDR again and the explicit disable was silently ignored -- the review reproduced it as Sdr turning into AppleEdr(4.0) on a 4x display. Make the flag unrepresentable as a headroom: `resolvedEdrHeadroom()` now returns `nil` when EDR is disabled, and both the default path and an explicit `outputMode = auto` request become `.sdr`. The `Auto(1.0) == unconfigured` contract in `negotiate_output_mode` is unchanged, since every other embedder relies on it. Applied to the macOS, iOS and tvOS plugins; only macOS reaches the per-display negotiation today, but the other two construct the same config and would hit the same trap. --- .../ios/Classes/ErikaFlutterPlugin.swift | 21 +++++++++++++---- .../macos/Classes/ErikaFlutterPlugin.swift | 23 +++++++++++++++---- .../tvos/Classes/ErikaFlutterPlugin.swift | 21 +++++++++++++---- 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/packages/erika_flutter/ios/Classes/ErikaFlutterPlugin.swift b/packages/erika_flutter/ios/Classes/ErikaFlutterPlugin.swift index 9385b537..e71a84fc 100644 --- a/packages/erika_flutter/ios/Classes/ErikaFlutterPlugin.swift +++ b/packages/erika_flutter/ios/Classes/ErikaFlutterPlugin.swift @@ -2872,6 +2872,14 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan } private func presenterConfigForNewPlayer(arguments: Any?, hdrDebug: Bool) -> ErikaPresenterConfigC { + // An explicit disable must become SDR, not `auto(headroom: 1.0)`: under + // the per-presenting-display Auto negotiation a headroom of 1.0 means "no + // embedder cap, defer to the display", so an EDR display would promote + // again and ignore ERIKA_DISABLE_EDR. + let edrDisabled = boolEnvironmentFlag( + "ERIKA_DISABLE_EDR", + environment: ProcessInfo.processInfo.environment + ) if let args = arguments as? [String: Any], let explicitMode = int32Value(args["outputMode"]) { let headroom = floatValue(args["edrHeadroom"]) ?? 4.0 let config: ErikaPresenterConfigC @@ -2881,7 +2889,7 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan case 2: config = ErikaPresenterConfigC(outputMode: 2, edrHeadroom: max(1.0, headroom)) case 3: - config = .auto(headroom: headroom) + config = edrDisabled ? .sdr : .auto(headroom: headroom) default: config = .sdr } @@ -2891,7 +2899,10 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan ) return config } - let headroom = resolvedEdrHeadroom(hdrDebug: hdrDebug) + guard let headroom = resolvedEdrHeadroom(hdrDebug: hdrDebug) else { + erikaHdrLog(hdrDebug, "ERIKA_DISABLE_EDR is set; using SDR output") + return .sdr + } let config = ErikaPresenterConfigC.auto(headroom: headroom) erikaHdrLog( hdrDebug, @@ -2900,11 +2911,13 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan return config } - private func resolvedEdrHeadroom(hdrDebug: Bool) -> Float { + /// EDR headroom to request for a new player, or `nil` when + /// `ERIKA_DISABLE_EDR` disables EDR and the player must use SDR. + private func resolvedEdrHeadroom(hdrDebug: Bool) -> Float? { let environment = ProcessInfo.processInfo.environment if boolEnvironmentFlag("ERIKA_DISABLE_EDR", environment: environment) { erikaHdrLog(hdrDebug, "EDR disabled by ERIKA_DISABLE_EDR") - return 1.0 + return nil } if let override = floatEnvironmentValue("ERIKA_EDR_HEADROOM", environment: environment), override > 1.0 { erikaHdrLog(hdrDebug, "EDR headroom override ERIKA_EDR_HEADROOM=\(String(format: "%.3f", override))") diff --git a/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift b/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift index 5e2d596e..1b33d8a4 100644 --- a/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift +++ b/packages/erika_flutter/macos/Classes/ErikaFlutterPlugin.swift @@ -3454,6 +3454,14 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan private func presenterConfigForNewPlayer(arguments: Any?) throws -> ErikaPresenterConfigC { let alphaMode = (arguments as? [String: Any]) .flatMap { int32Value($0["videoAlphaMode"]) } ?? 0 + // An explicit disable must become SDR, not `auto(headroom: 1.0)`: under + // the per-presenting-display Auto negotiation a headroom of 1.0 means "no + // embedder cap, defer to the display", so an EDR display would promote + // again and ignore ERIKA_DISABLE_EDR. + let edrDisabled = boolEnvironmentFlag( + "ERIKA_DISABLE_EDR", + environment: ProcessInfo.processInfo.environment + ) if let args = arguments as? [String: Any], let explicitMode = int32Value(args["outputMode"]) { let headroom = floatValue(args["edrHeadroom"]) ?? 4.0 @@ -3464,7 +3472,7 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan case 2: config = ErikaPresenterConfigC(outputMode: 2, edrHeadroom: max(1.0, headroom)) case 3: - config = .auto(headroom: headroom) + config = edrDisabled ? .sdr : .auto(headroom: headroom) default: config = .sdr } @@ -3472,7 +3480,12 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan return config } - let headroom = resolvedEdrHeadroom() + guard let headroom = resolvedEdrHeadroom() else { + NSLog("ErikaFlutterPlugin: ERIKA_DISABLE_EDR is set; using SDR output") + var config = ErikaPresenterConfigC.sdr + config.videoAlphaMode = alphaMode + return config + } NSLog("ErikaFlutterPlugin: using automatic Apple output, headroom \(headroom)x") let config = ErikaPresenterConfigC.auto(headroom: headroom) var alphaConfig = config @@ -3480,10 +3493,12 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan return alphaConfig } - private func resolvedEdrHeadroom() -> Float { + /// EDR headroom to request for a new player, or `nil` when + /// `ERIKA_DISABLE_EDR` disables EDR and the player must use SDR. + private func resolvedEdrHeadroom() -> Float? { let environment = ProcessInfo.processInfo.environment if boolEnvironmentFlag("ERIKA_DISABLE_EDR", environment: environment) { - return 1.0 + return nil } if let override = floatEnvironmentValue("ERIKA_EDR_HEADROOM", environment: environment), override > 1.0 { diff --git a/packages/erika_flutter/tvos/Classes/ErikaFlutterPlugin.swift b/packages/erika_flutter/tvos/Classes/ErikaFlutterPlugin.swift index fd38a3ec..912dad82 100644 --- a/packages/erika_flutter/tvos/Classes/ErikaFlutterPlugin.swift +++ b/packages/erika_flutter/tvos/Classes/ErikaFlutterPlugin.swift @@ -2656,6 +2656,14 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan } private func presenterConfigForNewPlayer(arguments: Any?, hdrDebug: Bool) -> ErikaPresenterConfigC { + // An explicit disable must become SDR, not `auto(headroom: 1.0)`: under + // the per-presenting-display Auto negotiation a headroom of 1.0 means "no + // embedder cap, defer to the display", so an EDR display would promote + // again and ignore ERIKA_DISABLE_EDR. + let edrDisabled = boolEnvironmentFlag( + "ERIKA_DISABLE_EDR", + environment: ProcessInfo.processInfo.environment + ) if let args = arguments as? [String: Any], let explicitMode = int32Value(args["outputMode"]) { let headroom = floatValue(args["edrHeadroom"]) ?? 4.0 let config: ErikaPresenterConfigC @@ -2665,7 +2673,7 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan case 2: config = ErikaPresenterConfigC(outputMode: 2, edrHeadroom: max(1.0, headroom)) case 3: - config = .auto(headroom: headroom) + config = edrDisabled ? .sdr : .auto(headroom: headroom) default: config = .sdr } @@ -2675,7 +2683,10 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan ) return config } - let headroom = resolvedEdrHeadroom(hdrDebug: hdrDebug) + guard let headroom = resolvedEdrHeadroom(hdrDebug: hdrDebug) else { + erikaHdrLog(hdrDebug, "ERIKA_DISABLE_EDR is set; using SDR output") + return .sdr + } let config = ErikaPresenterConfigC.auto(headroom: headroom) erikaHdrLog( hdrDebug, @@ -2684,11 +2695,13 @@ public final class ErikaFlutterPlugin: NSObject, FlutterPlugin, FlutterStreamHan return config } - private func resolvedEdrHeadroom(hdrDebug: Bool) -> Float { + /// EDR headroom to request for a new player, or `nil` when + /// `ERIKA_DISABLE_EDR` disables EDR and the player must use SDR. + private func resolvedEdrHeadroom(hdrDebug: Bool) -> Float? { let environment = ProcessInfo.processInfo.environment if boolEnvironmentFlag("ERIKA_DISABLE_EDR", environment: environment) { erikaHdrLog(hdrDebug, "EDR disabled by ERIKA_DISABLE_EDR") - return 1.0 + return nil } if let override = floatEnvironmentValue("ERIKA_EDR_HEADROOM", environment: environment), override > 1.0 { erikaHdrLog(hdrDebug, "EDR headroom override ERIKA_EDR_HEADROOM=\(String(format: "%.3f", override))")