From b9bc811b657fd14e39aa44ee0c7499e05e2d8256 Mon Sep 17 00:00:00 2001 From: fma-rc <188743521+fma-rc@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:31:40 +0100 Subject: [PATCH 1/2] fix(share-worker): start one GPU driver at a time NVIDIA's VA-API driver calls cuInit from its constructor, while the loader holds its lock; cuInit holds CUDA's lock and loads libraries. NVENC and VA-API opening at once in one process wait on each other for good, which hung the share worker's tests on a PC with an NVIDIA GPU. NvEncoder::open and VaEncoder::open now hold one lock while they run. The new test opens both at once in a child process: it never ended without the change and passes with it. --- crates/share-worker/src/hw/mod.rs | 55 +++++++++++++++++++++++++++++ crates/share-worker/src/hw/nvenc.rs | 1 + crates/share-worker/src/hw/vaapi.rs | 1 + 3 files changed, 57 insertions(+) diff --git a/crates/share-worker/src/hw/mod.rs b/crates/share-worker/src/hw/mod.rs index 9bf3b60..2281165 100644 --- a/crates/share-worker/src/hw/mod.rs +++ b/crates/share-worker/src/hw/mod.rs @@ -118,6 +118,16 @@ pub fn open_h264( Ok((encoder, "software")) } +/// Held while a GPU driver starts: one at a time in a process. NVIDIA's VA-API driver calls +/// `cuInit` while the loader loads it, and `cuInit` loads libraries itself, so NVENC and VA-API +/// starting at once wait on each other's lock (CUDA's and the loader's) for good. +pub(crate) fn starting_a_driver() -> std::sync::MutexGuard<'static, ()> { + static STARTING: std::sync::Mutex<()> = std::sync::Mutex::new(()); + STARTING + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + /// A small setup that every hardware encoder opens, for the start-up probe. pub const PROBE: EncoderSetup = EncoderSetup { width: 640, @@ -305,4 +315,49 @@ mod tests { assert_eq!(why, "nvenc: no driver; vaapi: no render node"); assert!(open_first(&[], PROBE).is_err()); } + + /// Set in the child process of `nvenc_and_vaapi_open_at_the_same_time`. + #[cfg(target_os = "linux")] + const RACE_CHILD: &str = "ANYPLAY_HW_RACE_CHILD"; + + #[cfg(target_os = "linux")] + #[test] + fn nvenc_and_vaapi_open_at_the_same_time() { + use std::time::{Duration, Instant}; + if std::env::var_os(RACE_CHILD).is_some() { + // Both drivers start at once, as two streams or two tests would start them. + let start = std::sync::Arc::new(std::sync::Barrier::new(2)); + let va = start.clone(); + let vaapi = std::thread::spawn(move || { + va.wait(); + let _ = vaapi::VaEncoder::open(PROBE); + }); + start.wait(); + let _ = nvenc::NvEncoder::open(PROBE); + vaapi.join().unwrap(); + return; + } + // In a child process: a deadlock there can be ended, here it would hang every test. + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "hw::tests::nvenc_and_vaapi_open_at_the_same_time", + ]) + .env(RACE_CHILD, "1") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(30); + while Instant::now() < deadline { + if let Some(status) = child.try_wait().unwrap() { + assert!(status.success(), "{status}"); + return; + } + std::thread::sleep(Duration::from_millis(50)); + } + let _ = child.kill(); + let _ = child.wait(); + panic!("opening NVENC and VA-API at the same time never ended"); + } } diff --git a/crates/share-worker/src/hw/nvenc.rs b/crates/share-worker/src/hw/nvenc.rs index 87caa23..60cd425 100644 --- a/crates/share-worker/src/hw/nvenc.rs +++ b/crates/share-worker/src/hw/nvenc.rs @@ -850,6 +850,7 @@ unsafe impl Send for NvEncoder {} impl NvEncoder { /// Opens an encoder for `setup`, on the first GPU that has one. pub fn open(setup: EncoderSetup) -> Result { + let _one_at_a_time = super::starting_a_driver(); let count = Cuda::load()?.device_count()?; if count == 0 { return Err("no CUDA device".into()); diff --git a/crates/share-worker/src/hw/vaapi.rs b/crates/share-worker/src/hw/vaapi.rs index 619da6b..350a681 100644 --- a/crates/share-worker/src/hw/vaapi.rs +++ b/crates/share-worker/src/hw/vaapi.rs @@ -984,6 +984,7 @@ impl VaEncoder { /// Opens an encoder for `setup` (already lowered to the hardware limits) on the first render /// node and profile that works. pub fn open(setup: EncoderSetup) -> Result { + let _one_at_a_time = super::starting_a_driver(); let api = Arc::new(Api::load()?); let mut nodes = render_nodes(); if nodes.is_empty() { From b1df8f2a4594b51dbafd49b4ed56cfc2f452a21b Mon Sep 17 00:00:00 2001 From: fma-rc <188743521+fma-rc@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:23:26 +0100 Subject: [PATCH 2/2] fix(share-worker): build the GPU driver lock on Linux only Only the Linux backends take it; on Windows and macOS it was dead code and clippy refused it. --- crates/share-worker/src/hw/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/share-worker/src/hw/mod.rs b/crates/share-worker/src/hw/mod.rs index 2281165..471772b 100644 --- a/crates/share-worker/src/hw/mod.rs +++ b/crates/share-worker/src/hw/mod.rs @@ -121,6 +121,7 @@ pub fn open_h264( /// Held while a GPU driver starts: one at a time in a process. NVIDIA's VA-API driver calls /// `cuInit` while the loader loads it, and `cuInit` loads libraries itself, so NVENC and VA-API /// starting at once wait on each other's lock (CUDA's and the loader's) for good. +#[cfg(target_os = "linux")] pub(crate) fn starting_a_driver() -> std::sync::MutexGuard<'static, ()> { static STARTING: std::sync::Mutex<()> = std::sync::Mutex::new(()); STARTING