diff --git a/app/src/pipeline.rs b/app/src/pipeline.rs index 9359551..96fe1ba 100644 --- a/app/src/pipeline.rs +++ b/app/src/pipeline.rs @@ -1,5 +1,5 @@ use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering}; use anyhow::{Context, Result}; use gstreamer as gst; @@ -25,6 +25,39 @@ pub struct VideoConfig { pub fps: u32, } +/// H.264 profiles mpph264enc can produce. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum H264Profile { + /// Emitted as Constrained Baseline (constraint_set0 and set1 both set). + Baseline, + Main, + High, +} + +impl H264Profile { + #[inline] + pub const fn as_gst_str(self) -> &'static str { + match self { + Self::Baseline => "baseline", + Self::Main => "main", + Self::High => "high", + } + } + + /// Profile named by an SDP `profile-level-id` (RFC 6184: profile_idc, + /// profile-iop and level_idc as hex). Profiles the encoder cannot produce, + /// such as High 4:4:4, yield `None`. + pub fn from_profile_level_id(profile_level_id: &str) -> Option { + let profile_idc = u8::from_str_radix(profile_level_id.get(0..2)?, 16).ok()?; + match profile_idc { + 0x42 => Some(Self::Baseline), + 0x4d => Some(Self::Main), + 0x64 => Some(Self::High), + _ => None, + } + } +} + impl Default for VideoConfig { fn default() -> Self { Self { @@ -186,6 +219,7 @@ impl VideoPipeline { Arc::default(); let probe_requested_at = keyframe_requested_at.clone(); let is_h264 = matches!(config.encoder, crate::cli::VideoEncoder::H264); + let last_profile_idc = AtomicU8::new(0); encoder.static_pad("src").context("Encoder has no src pad")?.add_probe( gst::PadProbeType::BUFFER, move |_pad, info| { @@ -196,6 +230,11 @@ impl VideoPipeline { && let Ok(map) = buffer.map_readable() && contains_h264_idr(map.as_slice()) { + if let Some(idc) = h264_sps_profile_idc(map.as_slice()) + && last_profile_idc.swap(idc, Ordering::Relaxed) != idc + { + info!(profile_idc = idc, "encoder SPS profile"); + } let delta = buffer.flags().contains(gst::BufferFlags::DELTA_UNIT); match probe_requested_at.lock().take() { Some(at) => info!( @@ -835,6 +874,14 @@ impl PipelineManager { } } + /// Switches the H.264 encoder's profile; mpph264enc applies it from the + /// next frame. A no-op when the pipeline encodes another codec. + pub fn set_h264_profile(&self, profile: H264Profile) { + if let Some(encoder) = self.video.pipeline.by_name("mpph264enc") { + encoder.set_property_from_str("profile", profile.as_gst_str()); + } + } + pub fn request_raw_frame(&self) -> tokio::sync::oneshot::Receiver { self.video.request_raw_frame() } @@ -902,6 +949,21 @@ fn contains_h264_idr(data: &[u8]) -> bool { false } +/// The profile_idc of the first SPS in an Annex B access unit, if any comes +/// before the first slice. +fn h264_sps_profile_idc(data: &[u8]) -> Option { + for (i, w) in data.windows(4).enumerate() { + if w[0] == 0 && w[1] == 0 && w[2] == 1 { + match w[3] & 0x1f { + 7 => return data.get(i + 4).copied(), + 1 | 5 => return None, + _ => {} + } + } + } + None +} + pub fn encode_jpeg(sample: &gst::Sample) -> Result { let caps = sample.caps_owned().context("Raw frame has no caps")?; let encoder = gst::ElementFactory::make("mppjpegenc") diff --git a/app/src/video.rs b/app/src/video.rs index 1f2d385..f17a02e 100644 --- a/app/src/video.rs +++ b/app/src/video.rs @@ -10,7 +10,7 @@ use webrtc::media::Sample; use webrtc::track::track_local::track_local_static_sample::TrackLocalStaticSample; use crate::api::{JsonRpcProcessor, default_registry}; -use crate::pipeline::{AudioConfig, PipelineManager, VideoConfig}; +use crate::pipeline::{AudioConfig, H264Profile, PipelineManager, VideoConfig}; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct VideoInputState { @@ -54,6 +54,11 @@ static VIDEO_PIPELINE_STARTED: tokio::sync::OnceCell<()> = tokio::sync::OnceCell static PIPELINE_MANAGER: tokio::sync::Mutex> = tokio::sync::Mutex::const_new(None); +/// The H.264 profile the attached viewer negotiated. It overrides the +/// configured profile on the running encoder and on every rebuild, so the +/// bitstream always matches the payload type the browser decodes it as. +static H264_PROFILE: parking_lot::Mutex> = parking_lot::Mutex::new(None); + static PIPELINE_CONFIG: OnceLock<(VideoConfig, Option)> = OnceLock::new(); static SUPERVISOR_STOPPED: AtomicBool = AtomicBool::new(false); @@ -128,6 +133,19 @@ pub async fn ensure_video_pipeline_started() -> anyhow::Result<()> { .map(|_| ()) } +/// Makes the encoder emit `profile`, the H.264 profile a new viewer +/// negotiated. Called before the viewer's keyframe is requested, so its first +/// IDR already carries the new SPS. +pub async fn set_h264_profile(profile: H264Profile) { + if H264_PROFILE.lock().replace(profile) == Some(profile) { + return; + } + if let Some(manager) = PIPELINE_MANAGER.lock().await.as_ref() { + manager.set_h264_profile(profile); + } + info!(profile = profile.as_gst_str(), "H.264 encoder profile set to match the viewer"); +} + /// Switches video output to a new session's track. The keyframe for the new /// viewer is requested once its connection is fully up (see webrtc.rs). pub async fn attach_webrtc_sink(track: Arc) { @@ -190,9 +208,14 @@ pub async fn start_native_video_with_cli(cli: &crate::cli::Cli) -> anyhow::Resul } async fn start_pipeline( - video_config: VideoConfig, + mut video_config: VideoConfig, audio_config: Option, ) -> anyhow::Result<()> { + if matches!(video_config.encoder, crate::cli::VideoEncoder::H264) + && let Some(profile) = *H264_PROFILE.lock() + { + video_config.profile = Some(profile.as_gst_str().to_owned()); + } let manager = PipelineManager::new(video_config, audio_config)?; manager.set_video_callback(move |data, pts_us| { diff --git a/app/src/webrtc.rs b/app/src/webrtc.rs index 7700013..1d0d8b4 100644 --- a/app/src/webrtc.rs +++ b/app/src/webrtc.rs @@ -25,6 +25,7 @@ use webrtc::track::track_local::track_local_static_sample::TrackLocalStaticSampl use crate::api::{JsonRpcProcessor, default_registry}; use crate::data_channel::DataChannelManager; use crate::hardware::usb::storage as storage_mod; +use crate::pipeline::H264Profile; use crate::session::Session; use crate::state::AppState; use crate::video; @@ -52,6 +53,31 @@ impl Default for SessionConfig { const H264_HIGH_FMTP: &str = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640034"; +/// The H.264 profile the video track sends with. Binding picks the first +/// negotiated codec matching [`H264_HIGH_FMTP`] (packetization-mode 1 and a +/// profile-level-id starting `6400`, level ignored), else the first negotiated +/// H.264 codec; this mirrors that choice so the encoder can follow it. +pub fn negotiated_h264_profile(codecs: &[RTCRtpCodecParameters]) -> Option { + let h264: Vec<_> = codecs + .iter() + .filter(|c| c.capability.mime_type.eq_ignore_ascii_case(MIME_TYPE_H264)) + .collect(); + let fmtp = |c: &RTCRtpCodecParameters, key: &str| { + c.capability.sdp_fmtp_line.split(';').find_map(|kv| { + let (k, v) = kv.split_once('=')?; + (k.trim() == key).then(|| v.trim().to_owned()) + }) + }; + let is_high = |c: &&&RTCRtpCodecParameters| { + fmtp(c, "packetization-mode").as_deref() == Some("1") + && fmtp(c, "profile-level-id") + .and_then(|p| p.get(0..4).map(|p| p.eq_ignore_ascii_case("6400"))) + .unwrap_or(false) + }; + let bound = h264.iter().find(is_high).or_else(|| h264.first())?; + H264Profile::from_profile_level_id(&fmtp(bound, "profile-level-id")?) +} + pub struct WebRTCApi; impl WebRTCApi { @@ -209,6 +235,7 @@ impl WebRTCApi { ) .await?; let video_sender = video_transceiver.sender().await; + let video_sender_for_profile = Arc::clone(&video_sender); let audio_transceiver = peer_connection .add_transceiver_from_track( @@ -272,6 +299,7 @@ impl WebRTCApi { let session_id = session_id_clone.clone(); let video_track = video_track_clone.clone(); let audio_track = audio_track_clone.clone(); + let video_sender = video_sender_for_profile.clone(); let is_connected = is_connected_clone.clone(); let app_state = app_state_for_close.clone(); Box::pin(async move { @@ -288,6 +316,14 @@ impl WebRTCApi { drop(connected); info!("WebRTC session {} connected", session_id); increment_session_counter().await; + let codecs = video_sender.get_parameters().await.rtp_parameters.codecs; + match negotiated_h264_profile(&codecs) { + Some(profile) => video::set_h264_profile(profile).await, + None => warn!( + "session {} negotiated no H.264 profile the encoder can produce", + session_id + ), + } if let Some(track) = video_track.clone() { video::attach_webrtc_sink(track).await; video::set_encoding_active(true).await; diff --git a/app/tests/h264_profile.rs b/app/tests/h264_profile.rs new file mode 100644 index 0000000..92629b4 --- /dev/null +++ b/app/tests/h264_profile.rs @@ -0,0 +1,64 @@ +use rustkvm::pipeline::H264Profile; +use rustkvm::webrtc::negotiated_h264_profile; +use webrtc::rtp_transceiver::rtp_codec::{RTCRtpCodecCapability, RTCRtpCodecParameters}; + +fn codec(mime_type: &str, fmtp: &str) -> RTCRtpCodecParameters { + RTCRtpCodecParameters { + capability: RTCRtpCodecCapability { + mime_type: mime_type.to_owned(), + clock_rate: 90_000, + sdp_fmtp_line: fmtp.to_owned(), + ..Default::default() + }, + ..Default::default() + } +} + +fn h264(profile_level_id: &str) -> RTCRtpCodecParameters { + codec( + "video/H264", + &format!( + "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id={profile_level_id}" + ), + ) +} + +#[test] +fn profile_level_id_maps_to_encoder_profiles() { + assert_eq!(H264Profile::from_profile_level_id("42e01f"), Some(H264Profile::Baseline)); + assert_eq!(H264Profile::from_profile_level_id("4d001f"), Some(H264Profile::Main)); + assert_eq!(H264Profile::from_profile_level_id("640034"), Some(H264Profile::High)); + assert_eq!(H264Profile::from_profile_level_id("f4001f"), None); + assert_eq!(H264Profile::from_profile_level_id("x"), None); +} + +#[test] +fn baseline_only_offer_negotiates_baseline() { + // Chrome without a High-capable decoder, as captured from a device session. + assert_eq!(negotiated_h264_profile(&[h264("42e01f")]), Some(H264Profile::Baseline)); +} + +#[test] +fn high_is_preferred_whenever_negotiated() { + let codecs = [h264("42e01f"), h264("64001f")]; + assert_eq!(negotiated_h264_profile(&codecs), Some(H264Profile::High)); +} + +#[test] +fn constrained_high_falls_back_to_first_codec() { + // 640c doesn't match the track's 6400, so binding takes the first codec. + let codecs = [h264("42e01f"), h264("640c1f")]; + assert_eq!(negotiated_h264_profile(&codecs), Some(H264Profile::Baseline)); +} + +#[test] +fn high_needs_packetization_mode_1() { + let codecs = + [h264("42e01f"), codec("video/H264", "packetization-mode=0;profile-level-id=64001f")]; + assert_eq!(negotiated_h264_profile(&codecs), Some(H264Profile::Baseline)); +} + +#[test] +fn no_h264_negotiated() { + assert_eq!(negotiated_h264_profile(&[codec("video/VP8", "")]), None); +}