Skip to content
This repository was archived by the owner on Apr 23, 2026. It is now read-only.

gstreamer: Enhance containers/codecs registration and parsing #483

Open
tharkum wants to merge 1 commit into
base: main
Choose a base branch
from
Open

gstreamer: Enhance containers/codecs registration and parsing #483

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 56 additions & 27 deletions backends/gstreamer/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,37 +245,66 @@ impl Backend for GStreamerBackend {
media_capture::create_videoinput_stream(set)
}

fn can_play_type(&self, media_type: &str) -> SupportsMediaType {
if let Ok(mime) = media_type.parse::<Mime>() {
// XXX GStreamer is currently not very reliable playing OGG and most of
// the media related WPTs uses OGG if we report that we are able to
// play this type. So we report that we are unable to play it to force
// the usage of other types.
// https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/520
if mime.subtype() == mime::OGG {
return SupportsMediaType::No;
}
fn can_play_type(&self, type_: &str) -> SupportsMediaType {
// Remove all whitespace from the input string to conform to the
// stricter parsing rules of the `mime` crate.
// <https://mimesniff.spec.whatwg.org/#parsing-a-mime-type>
// <https://github.com/hyperium/mime/blob/v0.3.17/src/parse.rs#L111>
let stripped_type: String = type_.chars().filter(|c| !c.is_whitespace()).collect();

match stripped_type.parse::<Mime>() {
Ok(mime) => {
// XXX GStreamer is currently not very reliable playing OGG and most of
// the media related WPTs uses OGG if we report that we are able to
// play this type. So we report that we are unable to play it to force
// the usage of other types.
// https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/issues/520
if mime.subtype() == mime::OGG {
return SupportsMediaType::No;
}

let mime_type = mime.type_().as_str().to_owned() + "/" + mime.subtype().as_str();

let mime_type = mime.type_().as_str().to_owned() + "/" + mime.subtype().as_str();
let codecs = match mime.get_param("codecs") {
Some(codecs) => codecs
.as_str()
.split(',')
.map(|codec| codec.trim())
.collect(),
None => vec![],
};

if GSTREAMER_REGISTRY_SCANNER.is_container_type_supported(&mime_type) {
if codecs.is_empty() {
let codecs = match mime.get_param("codecs") {
Some(codecs) if !codecs.as_str().is_empty() => codecs
.as_str()
.split(',')
.map(|codec| codec.trim())
.collect(),
_ => vec![],
};

if GSTREAMER_REGISTRY_SCANNER.is_container_type_supported(&mime_type) {
if codecs.is_empty() {
return SupportsMediaType::Maybe;
} else if GSTREAMER_REGISTRY_SCANNER.are_all_codecs_supported(&codecs) {
return SupportsMediaType::Probably;
}
}
},
Err(_) if type_.contains(";codecs") => {
// The HTML specification says that a media resource type
// may be present and if present, the value must be a valid MIME
// type string. However there are still a lot of sites using incomplete
// `codecs` parameter string that don't work with the stricter
// parsing rules (e.g. "audio/webm;codecs").
// <https://html.spec.whatwg.org/multipage/#mime-types>
// <https://www.rfc-editor.org/rfc/rfc6381#section-3.2>
// <https://github.com/web-platform-tests/wpt/pull/7294>

// Let's use fallback mode with MIME type essence to match the
// behavior of other browsers.
let mime_type = &type_[0..type_.find(';').unwrap_or(type_.len())];

if GSTREAMER_REGISTRY_SCANNER.is_container_type_supported(&mime_type) {
return SupportsMediaType::Maybe;
} else if GSTREAMER_REGISTRY_SCANNER.are_all_codecs_supported(&codecs) {
return SupportsMediaType::Probably;
} else {
return SupportsMediaType::No;
}
}
},
Err(error) => {
log::debug!("Failed to parse MIME type ({type_:?}): {error:?}");
},
}

SupportsMediaType::No
}

Expand Down
198 changes: 135 additions & 63 deletions backends/gstreamer/registry_scanner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,12 @@ impl GStreamerRegistryScanner {
}

fn is_codec_supported(&self, codec: &str) -> bool {
self.supported_codecs.contains(codec)
for supported_codec in &self.supported_codecs {
if codec.contains(supported_codec) {
return true;
}
}
false
}

pub fn are_all_codecs_supported(&self, codecs: &Vec<&str>) -> bool {
Expand Down Expand Up @@ -57,71 +62,154 @@ impl GStreamerRegistryScanner {
gst::Rank::MARGINAL,
);

if has_element_for_media_type(&audio_decoder_factories, "audio/mpeg, mpegversion=(int)4") {
self.supported_mime_types.insert("audio/aac");
self.supported_mime_types.insert("audio/mp4");
self.supported_mime_types.insert("audio/x-m4a");
self.supported_codecs.insert("mpeg");
self.supported_codecs.insert("mp4a*");
}

let is_opus_supported =
has_element_for_media_type(&audio_decoder_factories, "audio/x-opus");
if is_opus_supported && has_element_for_media_type(&audio_parser_factories, "audio/x-opus")
{
let is_opus_supported = has_element_for_media_type(&audio_parser_factories, "audio/x-opus")
&& has_element_for_media_type(
&audio_decoder_factories,
"audio/x-opus, channel-mapping-family=(int)0",
);
if is_opus_supported {
self.supported_mime_types.insert("audio/opus");
self.supported_codecs.insert("opus");
self.supported_codecs.insert("x-opus");
}

let is_vorbis_supported =
has_element_for_media_type(&audio_decoder_factories, "audio/x-vorbis");
if is_vorbis_supported
&& has_element_for_media_type(&audio_parser_factories, "audio/x-vorbis")
{
has_element_for_media_type(&audio_parser_factories, "audio/x-vorbis")
&& has_element_for_media_type(&audio_decoder_factories, "audio/x-vorbis");
if is_vorbis_supported {
self.supported_codecs.insert("vorbis");
self.supported_codecs.insert("x-vorbis");
}

if has_element_for_media_type(&demux_factories, "video/x-matroska") {
let is_vp8_decoder_available =
has_element_for_media_type(&video_decoder_factories, "video/x-vp8");
let is_vp9_decoder_available =
has_element_for_media_type(&video_decoder_factories, "video/x-vp9");
// <https://developer.mozilla.org/en-US/docs/Web/Media/Guides/Formats/Containers#webm>
if has_element_for_media_type(&demux_factories, "audio/webm") {
if is_opus_supported || is_vorbis_supported {
self.supported_mime_types.insert("audio/webm");
}
}

let is_vp8_supported = has_element_for_media_type(&video_decoder_factories, "video/x-vp8");
if is_vp8_supported {
self.supported_codecs.insert("vp8");
self.supported_codecs.insert("x-vp8");
self.supported_codecs.insert("vp8.0");
self.supported_codecs.insert("vp08");
}

if is_vp8_decoder_available || is_vp9_decoder_available {
let is_vp9_supported = has_element_for_media_type(&video_parser_factories, "video/x-vp9")
&& has_element_for_media_type(&video_decoder_factories, "video/x-vp9");
if is_vp9_supported {
self.supported_codecs.insert("vp9");
self.supported_codecs.insert("x-vp9");
self.supported_codecs.insert("vp9.0");
self.supported_codecs.insert("vp09");
}

let is_av1_supported = has_element_for_media_type(&video_parser_factories, "video/x-av1")
&& has_element_for_media_type(&video_decoder_factories, "video/x-av1");
if is_av1_supported {
self.supported_codecs.insert("av1");
self.supported_codecs.insert("x-av1");
self.supported_codecs.insert("av01");
}

// <https://developer.mozilla.org/en-US/docs/Web/Media/Guides/Formats/Containers#webm>
if has_element_for_media_type(&demux_factories, "video/webm") {
if is_vp8_supported || is_vp9_supported || is_av1_supported {
self.supported_mime_types.insert("video/webm");
}
}

let is_aac_supported =
has_element_for_media_type(&audio_parser_factories, "audio/mpeg, mpegversion=(int)4")
&& has_element_for_media_type(
&audio_decoder_factories,
"audio/mpeg, mpegversion=(int)4, stream-format=(string)adts",
);
if is_aac_supported {
self.supported_mime_types.insert("audio/aac");
self.supported_codecs.insert("mpeg");
self.supported_codecs.insert("mp4a");
}

let is_mpeg4v_supported = has_element_for_media_type(
&video_parser_factories,
"video/mpeg, mpegversion=(int)4, systemstream=(boolean)false",
) && has_element_for_media_type(
&video_decoder_factories,
"video/mpeg, mpegversion=(int)4, systemstream=(boolean)false",
);
if is_mpeg4v_supported {
self.supported_codecs.insert("mp4v");
}

let mut is_h264_supported = false;
if has_element_for_media_type(&video_parser_factories, "video/x-h264") {
let is_h264_avc1_supported = has_element_for_media_type(
&video_decoder_factories,
"video/x-h264, stream-format=(string)avc, alignment=(string)au",
);

let is_h264_avc3_supported = has_element_for_media_type(
&video_decoder_factories,
"video/x-h264, stream-format=(string)avc3, alignment=(string)au",
);

if is_h264_avc1_supported {
self.supported_codecs.insert("avc1");
}

if is_vp8_decoder_available {
self.supported_codecs.insert("vp8");
self.supported_codecs.insert("x-vp8");
self.supported_codecs.insert("vp8.0");
if is_h264_avc3_supported {
self.supported_codecs.insert("avc3");
}

if is_vp9_decoder_available {
self.supported_codecs.insert("vp9");
self.supported_codecs.insert("x-vp9");
self.supported_codecs.insert("vp9.0");
if is_h264_avc1_supported || is_h264_avc3_supported {
self.supported_codecs.insert("x-h264");
is_h264_supported = true;
}
};

if is_opus_supported {
self.supported_mime_types.insert("audio/webm");
let mut is_h265_supported = false;
if has_element_for_media_type(&video_parser_factories, "video/x-h265") {
let is_h265_hvc1_supported = has_element_for_media_type(
&video_decoder_factories,
"video/x-h265, stream-format=(string)hvc1, alignment=(string)au",
);

let is_h265_hev1_supported = has_element_for_media_type(
&video_decoder_factories,
"video/x-h265, stream-format=(string)hev1, alignment=(string)au",
);

if is_h265_hvc1_supported {
self.supported_codecs.insert("hvc1");
}
}

let is_h264_decoder_available = has_element_for_media_type(
&video_decoder_factories,
"video/x-h264, profile=(string){ constrained-baseline, baseline, high }",
);
if is_h264_decoder_available
&& has_element_for_media_type(&video_parser_factories, "video/x-h264")
{
self.supported_mime_types.insert("video/mp4");
self.supported_mime_types.insert("video/x-m4v");
self.supported_codecs.insert("x-h264");
self.supported_codecs.insert("avc*");
self.supported_codecs.insert("mp4v*");
if is_h265_hev1_supported {
self.supported_codecs.insert("hev1");
}

if is_h265_hvc1_supported || is_h265_hev1_supported {
self.supported_codecs.insert("x-h265");
is_h265_supported = true;
}
};

// <https://developer.mozilla.org/en-US/docs/Web/Media/Guides/Formats/Containers#mpeg-4_mp4>
if has_element_for_media_type(&demux_factories, "video/quicktime") {
if is_aac_supported || is_opus_supported {
self.supported_mime_types.insert("audio/mp4");
self.supported_mime_types.insert("audio/x-m4a");
}

if is_mpeg4v_supported
|| is_h264_supported
|| is_h265_supported
|| is_av1_supported
|| is_vp9_supported
{
self.supported_mime_types.insert("video/mp4");
}
}

if has_element_for_media_type(&audio_decoder_factories, "audio/midi") {
Expand Down Expand Up @@ -179,10 +267,6 @@ impl GStreamerRegistryScanner {
self.supported_codecs.insert("1");
}

if has_element_for_media_type(&demux_factories, "video/quicktime, variant=(string)3gpp") {
self.supported_mime_types.insert("video/3gpp");
}

if has_element_for_media_type(&demux_factories, "application/ogg") {
self.supported_mime_types.insert("application/ogg");

Expand Down Expand Up @@ -225,20 +309,8 @@ impl GStreamerRegistryScanner {
self.supported_mime_types.insert("audio/x-mpeg");
}

let is_matroska_supported =
has_element_for_media_type(&demux_factories, "video/x-matroska");
if is_matroska_supported {
if has_element_for_media_type(&demux_factories, "video/x-matroska") {
self.supported_mime_types.insert("video/x-matroska");

if has_element_for_media_type(&video_decoder_factories, "video/x-vp10") {
self.supported_mime_types.insert("video/webm");
}
}

if (is_matroska_supported || self.is_container_type_supported("video/mp4"))
&& has_element_for_media_type(&video_decoder_factories, "video/x-av1")
{
self.supported_codecs.insert("av01*");
}
}
}
Expand Down
Loading