From 0bfe2f00ed1dcd459d537653916648613c665c6e Mon Sep 17 00:00:00 2001 From: Andrei Volykhin Date: Fri, 19 Dec 2025 09:57:34 +0300 Subject: [PATCH] gstreamer: Enhance containers/codecs registration and parsing Adjusted media type parsing to more stricter parsing rules of the `mime` crate. Make registration of the audio/video containers (mp4, webm) and the associated codecs (opus, aac, h264, vp8, ..) more fine-granted. Fixes: https://github.com/servo/media/issues/450 Fixes: https://github.com/servo/media/issues/452 Signed-off-by: Andrei Volykhin --- backends/gstreamer/lib.rs | 83 +++++++---- backends/gstreamer/registry_scanner.rs | 198 +++++++++++++++++-------- 2 files changed, 191 insertions(+), 90 deletions(-) diff --git a/backends/gstreamer/lib.rs b/backends/gstreamer/lib.rs index 37cf63ef..a2c2b976 100644 --- a/backends/gstreamer/lib.rs +++ b/backends/gstreamer/lib.rs @@ -245,37 +245,66 @@ impl Backend for GStreamerBackend { media_capture::create_videoinput_stream(set) } - fn can_play_type(&self, media_type: &str) -> SupportsMediaType { - if let Ok(mime) = media_type.parse::() { - // XXX GStreamer is currently not very reliable playing OGG and most of - // the media related WPTs uses OGG if we report that we are able to - // play this type. So we report that we are unable to play it to force - // the usage of other types. - // https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/520 - if mime.subtype() == mime::OGG { - return SupportsMediaType::No; - } + fn can_play_type(&self, type_: &str) -> SupportsMediaType { + // Remove all whitespace from the input string to conform to the + // stricter parsing rules of the `mime` crate. + // + // + let stripped_type: String = type_.chars().filter(|c| !c.is_whitespace()).collect(); + + match stripped_type.parse::() { + Ok(mime) => { + // XXX GStreamer is currently not very reliable playing OGG and most of + // the media related WPTs uses OGG if we report that we are able to + // play this type. So we report that we are unable to play it to force + // the usage of other types. + // https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/520 + if mime.subtype() == mime::OGG { + return SupportsMediaType::No; + } + + let mime_type = mime.type_().as_str().to_owned() + "/" + mime.subtype().as_str(); - let mime_type = mime.type_().as_str().to_owned() + "/" + mime.subtype().as_str(); - let codecs = match mime.get_param("codecs") { - Some(codecs) => codecs - .as_str() - .split(',') - .map(|codec| codec.trim()) - .collect(), - None => vec![], - }; - - if GSTREAMER_REGISTRY_SCANNER.is_container_type_supported(&mime_type) { - if codecs.is_empty() { + let codecs = match mime.get_param("codecs") { + Some(codecs) if !codecs.as_str().is_empty() => codecs + .as_str() + .split(',') + .map(|codec| codec.trim()) + .collect(), + _ => vec![], + }; + + if GSTREAMER_REGISTRY_SCANNER.is_container_type_supported(&mime_type) { + if codecs.is_empty() { + return SupportsMediaType::Maybe; + } else if GSTREAMER_REGISTRY_SCANNER.are_all_codecs_supported(&codecs) { + return SupportsMediaType::Probably; + } + } + }, + Err(_) if type_.contains(";codecs") => { + // The HTML specification says that a media resource type + // may be present and if present, the value must be a valid MIME + // type string. However there are still a lot of sites using incomplete + // `codecs` parameter string that don't work with the stricter + // parsing rules (e.g. "audio/webm;codecs"). + // + // + // + + // Let's use fallback mode with MIME type essence to match the + // behavior of other browsers. + let mime_type = &type_[0..type_.find(';').unwrap_or(type_.len())]; + + if GSTREAMER_REGISTRY_SCANNER.is_container_type_supported(&mime_type) { return SupportsMediaType::Maybe; - } else if GSTREAMER_REGISTRY_SCANNER.are_all_codecs_supported(&codecs) { - return SupportsMediaType::Probably; - } else { - return SupportsMediaType::No; } - } + }, + Err(error) => { + log::debug!("Failed to parse MIME type ({type_:?}): {error:?}"); + }, } + SupportsMediaType::No } diff --git a/backends/gstreamer/registry_scanner.rs b/backends/gstreamer/registry_scanner.rs index 443fa60e..44588ad0 100644 --- a/backends/gstreamer/registry_scanner.rs +++ b/backends/gstreamer/registry_scanner.rs @@ -28,7 +28,12 @@ impl GStreamerRegistryScanner { } fn is_codec_supported(&self, codec: &str) -> bool { - self.supported_codecs.contains(codec) + for supported_codec in &self.supported_codecs { + if codec.contains(supported_codec) { + return true; + } + } + false } pub fn are_all_codecs_supported(&self, codecs: &Vec<&str>) -> bool { @@ -57,71 +62,154 @@ impl GStreamerRegistryScanner { gst::Rank::MARGINAL, ); - if has_element_for_media_type(&audio_decoder_factories, "audio/mpeg, mpegversion=(int)4") { - self.supported_mime_types.insert("audio/aac"); - self.supported_mime_types.insert("audio/mp4"); - self.supported_mime_types.insert("audio/x-m4a"); - self.supported_codecs.insert("mpeg"); - self.supported_codecs.insert("mp4a*"); - } - - let is_opus_supported = - has_element_for_media_type(&audio_decoder_factories, "audio/x-opus"); - if is_opus_supported && has_element_for_media_type(&audio_parser_factories, "audio/x-opus") - { + let is_opus_supported = has_element_for_media_type(&audio_parser_factories, "audio/x-opus") + && has_element_for_media_type( + &audio_decoder_factories, + "audio/x-opus, channel-mapping-family=(int)0", + ); + if is_opus_supported { self.supported_mime_types.insert("audio/opus"); self.supported_codecs.insert("opus"); self.supported_codecs.insert("x-opus"); } let is_vorbis_supported = - has_element_for_media_type(&audio_decoder_factories, "audio/x-vorbis"); - if is_vorbis_supported - && has_element_for_media_type(&audio_parser_factories, "audio/x-vorbis") - { + has_element_for_media_type(&audio_parser_factories, "audio/x-vorbis") + && has_element_for_media_type(&audio_decoder_factories, "audio/x-vorbis"); + if is_vorbis_supported { self.supported_codecs.insert("vorbis"); self.supported_codecs.insert("x-vorbis"); } - if has_element_for_media_type(&demux_factories, "video/x-matroska") { - let is_vp8_decoder_available = - has_element_for_media_type(&video_decoder_factories, "video/x-vp8"); - let is_vp9_decoder_available = - has_element_for_media_type(&video_decoder_factories, "video/x-vp9"); + // + if has_element_for_media_type(&demux_factories, "audio/webm") { + if is_opus_supported || is_vorbis_supported { + self.supported_mime_types.insert("audio/webm"); + } + } + + let is_vp8_supported = has_element_for_media_type(&video_decoder_factories, "video/x-vp8"); + if is_vp8_supported { + self.supported_codecs.insert("vp8"); + self.supported_codecs.insert("x-vp8"); + self.supported_codecs.insert("vp8.0"); + self.supported_codecs.insert("vp08"); + } - if is_vp8_decoder_available || is_vp9_decoder_available { + let is_vp9_supported = has_element_for_media_type(&video_parser_factories, "video/x-vp9") + && has_element_for_media_type(&video_decoder_factories, "video/x-vp9"); + if is_vp9_supported { + self.supported_codecs.insert("vp9"); + self.supported_codecs.insert("x-vp9"); + self.supported_codecs.insert("vp9.0"); + self.supported_codecs.insert("vp09"); + } + + let is_av1_supported = has_element_for_media_type(&video_parser_factories, "video/x-av1") + && has_element_for_media_type(&video_decoder_factories, "video/x-av1"); + if is_av1_supported { + self.supported_codecs.insert("av1"); + self.supported_codecs.insert("x-av1"); + self.supported_codecs.insert("av01"); + } + + // + if has_element_for_media_type(&demux_factories, "video/webm") { + if is_vp8_supported || is_vp9_supported || is_av1_supported { self.supported_mime_types.insert("video/webm"); } + } + + let is_aac_supported = + has_element_for_media_type(&audio_parser_factories, "audio/mpeg, mpegversion=(int)4") + && has_element_for_media_type( + &audio_decoder_factories, + "audio/mpeg, mpegversion=(int)4, stream-format=(string)adts", + ); + if is_aac_supported { + self.supported_mime_types.insert("audio/aac"); + self.supported_codecs.insert("mpeg"); + self.supported_codecs.insert("mp4a"); + } + + let is_mpeg4v_supported = has_element_for_media_type( + &video_parser_factories, + "video/mpeg, mpegversion=(int)4, systemstream=(boolean)false", + ) && has_element_for_media_type( + &video_decoder_factories, + "video/mpeg, mpegversion=(int)4, systemstream=(boolean)false", + ); + if is_mpeg4v_supported { + self.supported_codecs.insert("mp4v"); + } + + let mut is_h264_supported = false; + if has_element_for_media_type(&video_parser_factories, "video/x-h264") { + let is_h264_avc1_supported = has_element_for_media_type( + &video_decoder_factories, + "video/x-h264, stream-format=(string)avc, alignment=(string)au", + ); + + let is_h264_avc3_supported = has_element_for_media_type( + &video_decoder_factories, + "video/x-h264, stream-format=(string)avc3, alignment=(string)au", + ); + + if is_h264_avc1_supported { + self.supported_codecs.insert("avc1"); + } - if is_vp8_decoder_available { - self.supported_codecs.insert("vp8"); - self.supported_codecs.insert("x-vp8"); - self.supported_codecs.insert("vp8.0"); + if is_h264_avc3_supported { + self.supported_codecs.insert("avc3"); } - if is_vp9_decoder_available { - self.supported_codecs.insert("vp9"); - self.supported_codecs.insert("x-vp9"); - self.supported_codecs.insert("vp9.0"); + if is_h264_avc1_supported || is_h264_avc3_supported { + self.supported_codecs.insert("x-h264"); + is_h264_supported = true; } + }; - if is_opus_supported { - self.supported_mime_types.insert("audio/webm"); + let mut is_h265_supported = false; + if has_element_for_media_type(&video_parser_factories, "video/x-h265") { + let is_h265_hvc1_supported = has_element_for_media_type( + &video_decoder_factories, + "video/x-h265, stream-format=(string)hvc1, alignment=(string)au", + ); + + let is_h265_hev1_supported = has_element_for_media_type( + &video_decoder_factories, + "video/x-h265, stream-format=(string)hev1, alignment=(string)au", + ); + + if is_h265_hvc1_supported { + self.supported_codecs.insert("hvc1"); } - } - let is_h264_decoder_available = has_element_for_media_type( - &video_decoder_factories, - "video/x-h264, profile=(string){ constrained-baseline, baseline, high }", - ); - if is_h264_decoder_available - && has_element_for_media_type(&video_parser_factories, "video/x-h264") - { - self.supported_mime_types.insert("video/mp4"); - self.supported_mime_types.insert("video/x-m4v"); - self.supported_codecs.insert("x-h264"); - self.supported_codecs.insert("avc*"); - self.supported_codecs.insert("mp4v*"); + if is_h265_hev1_supported { + self.supported_codecs.insert("hev1"); + } + + if is_h265_hvc1_supported || is_h265_hev1_supported { + self.supported_codecs.insert("x-h265"); + is_h265_supported = true; + } + }; + + // + if has_element_for_media_type(&demux_factories, "video/quicktime") { + if is_aac_supported || is_opus_supported { + self.supported_mime_types.insert("audio/mp4"); + self.supported_mime_types.insert("audio/x-m4a"); + } + + if is_mpeg4v_supported + || is_h264_supported + || is_h265_supported + || is_av1_supported + || is_vp9_supported + { + self.supported_mime_types.insert("video/mp4"); + } } if has_element_for_media_type(&audio_decoder_factories, "audio/midi") { @@ -179,10 +267,6 @@ impl GStreamerRegistryScanner { self.supported_codecs.insert("1"); } - if has_element_for_media_type(&demux_factories, "video/quicktime, variant=(string)3gpp") { - self.supported_mime_types.insert("video/3gpp"); - } - if has_element_for_media_type(&demux_factories, "application/ogg") { self.supported_mime_types.insert("application/ogg"); @@ -225,20 +309,8 @@ impl GStreamerRegistryScanner { self.supported_mime_types.insert("audio/x-mpeg"); } - let is_matroska_supported = - has_element_for_media_type(&demux_factories, "video/x-matroska"); - if is_matroska_supported { + if has_element_for_media_type(&demux_factories, "video/x-matroska") { self.supported_mime_types.insert("video/x-matroska"); - - if has_element_for_media_type(&video_decoder_factories, "video/x-vp10") { - self.supported_mime_types.insert("video/webm"); - } - } - - if (is_matroska_supported || self.is_container_type_supported("video/mp4")) - && has_element_for_media_type(&video_decoder_factories, "video/x-av1") - { - self.supported_codecs.insert("av01*"); } } }