From 0a32df142d799217205e8744a419f70decf61aef Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:23:21 +0100 Subject: [PATCH] Voice: cancel the echo of the speakers with AEC3 (sonora), on by default ADR 0040. Echo cancellation sits first in the sending chain, the reference is released when the device has taken it off its queue, and the control protocol gets voice_echo. Offline ERLE tests. --- CONTRIBUTING.md | 2 +- Cargo.lock | 88 ++++++ Cargo.toml | 2 +- crates/control/src/lib.rs | 8 + crates/net/src/service.rs | 9 + crates/net/src/voice.rs | 14 + crates/voice/Cargo.toml | 3 +- crates/voice/src/echo.rs | 137 ++++++++++ crates/voice/src/lib.rs | 23 +- crates/voice/src/send.rs | 32 ++- crates/voice/tests/echo.rs | 250 ++++++++++++++++++ ...-voice-is-an-engine-without-a-transport.md | 2 +- ...ice-cancels-echo-with-aec3-in-pure-rust.md | 29 ++ spec/control-protocol.md | 1 + 14 files changed, 592 insertions(+), 8 deletions(-) create mode 100644 crates/voice/src/echo.rs create mode 100644 crates/voice/tests/echo.rs create mode 100644 docs/adr/0040-voice-cancels-echo-with-aec3-in-pure-rust.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8fdb21d..f4e7f34 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ Issues and pull requests from anyone are welcome here. The rules that hold in ev ## Build and test -You need Rust 1.88 or newer. These are the commands CI runs on Windows and on Linux. A pull request needs all of them green: +You need Rust 1.91 or newer. These are the commands CI runs on Windows and on Linux. A pull request needs all of them green: ```sh cargo fmt --all -- --check diff --git a/Cargo.lock b/Cargo.lock index d25550f..6037b34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -313,6 +313,7 @@ dependencies = [ "cpal", "nnnoiseless", "opus", + "sonora", ] [[package]] @@ -789,6 +790,20 @@ name = "bytemuck" version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] [[package]] name = "byteorder" @@ -4796,6 +4811,79 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sonora" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1ce442ab5cb392c4067b8f2cced20be4cf12e479f6dfbf44acbe9b14ac9250" +dependencies = [ + "sonora-aec3", + "sonora-agc2", + "sonora-common-audio", + "sonora-ns", + "sonora-simd", + "tracing", +] + +[[package]] +name = "sonora-aec3" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50280e31256d09f7838aff6715ed4b7640e514dc1092a1e14013c01e552380db" +dependencies = [ + "sonora-common-audio", + "sonora-fft", + "sonora-simd", +] + +[[package]] +name = "sonora-agc2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a93a9fa8373a33e61a7fb441d8a9e725ac6024079d2c77a845cf6abe1790b9f" +dependencies = [ + "bytemuck", + "derive_more", + "sonora-common-audio", + "sonora-fft", + "sonora-simd", + "tracing", +] + +[[package]] +name = "sonora-common-audio" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6635bc61b24807ceac93ad0807b1292b6b0620caa6cd360d3be89fc616b833cc" +dependencies = [ + "derive_more", + "sonora-simd", +] + +[[package]] +name = "sonora-fft" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27e272c70f34363afb5382dd1818c05f8f63ef6785635736a14e9ccbbb5ba012" + +[[package]] +name = "sonora-ns" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d511feef19302c716d245c4555087d161ba91c178735ea11107dcce0d0b7a86f" +dependencies = [ + "sonora-fft", +] + +[[package]] +name = "sonora-simd" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7cf0804e4752a5562262e143014f53043080ead4cd5a98f9278d3ceaf32d868" +dependencies = [ + "cpufeatures 0.3.1", +] + [[package]] name = "sorted-index-buffer" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 8490ea6..d101d4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ version = "0.0.0" edition = "2024" license = "MIT OR Apache-2.0" repository = "https://github.com/anyplaynet/network" -rust-version = "1.88" +rust-version = "1.91" [workspace.dependencies] serde = { version = "1", features = ["derive"] } diff --git a/crates/control/src/lib.rs b/crates/control/src/lib.rs index 3acb292..6385f7b 100644 --- a/crates/control/src/lib.rs +++ b/crates/control/src/lib.rs @@ -288,6 +288,10 @@ pub enum Command { VoiceNoise { enabled: bool, }, + /// Echo cancellation on the microphone: on (the default of a new service) or off. + VoiceEcho { + enabled: bool, + }, VoiceMode { mode: VoiceMode, }, @@ -1764,6 +1768,10 @@ mod tests { parse_command(r#"{"id":7,"do":"voice_noise","enabled":false}"#), Ok((7, Command::VoiceNoise { enabled: false })) ); + assert_eq!( + parse_command(r#"{"id":8,"do":"voice_echo","enabled":false}"#), + Ok((8, Command::VoiceEcho { enabled: false })) + ); assert_eq!( parse_command(r#"{"id":2,"do":"push_to_talk","talking":false}"#), Ok((2, Command::PushToTalk { talking: false })) diff --git a/crates/net/src/service.rs b/crates/net/src/service.rs index b72984d..79e5ef5 100644 --- a/crates/net/src/service.rs +++ b/crates/net/src/service.rs @@ -1479,6 +1479,13 @@ impl Service { } self.ok(id, Done {}); } + Command::VoiceEcho { enabled } => { + self.voice.settings.echo_cancellation = Some(enabled); + if let Some(hub) = &self.voice.hub { + hub.set_echo_cancellation(enabled); + } + self.ok(id, Done {}); + } Command::MicMute { muted } => { self.voice.settings.mic_muted = muted; if let Some(hub) = &self.voice.hub { @@ -3082,6 +3089,7 @@ impl Service { let (input, output) = (self.voice.input.clone(), self.voice.output.clone()); let mode = self.voice.settings.mode; let noise = self.voice.settings.noise_suppression; + let echo = self.voice.settings.echo_cancellation; self.spawn( async move { tokio::task::spawn_blocking(move || { @@ -3089,6 +3097,7 @@ impl Service { let config = anyplay_voice::Config { mode, noise_suppression: noise.unwrap_or(true), + echo_cancellation: echo.unwrap_or(true), ..anyplay_voice::Config::default() }; Engine::new(config, device).map_err(|e| e.to_string()) diff --git a/crates/net/src/voice.rs b/crates/net/src/voice.rs index 04280f6..c72cda7 100644 --- a/crates/net/src/voice.rs +++ b/crates/net/src/voice.rs @@ -164,6 +164,8 @@ pub struct Settings { pub talking: bool, /// Noise suppression on the microphone; `None` leaves what the engine was opened with. pub noise_suppression: Option, + /// Echo cancellation on the microphone; `None` leaves what the engine was opened with. + pub echo_cancellation: Option, /// Volume per account, 0.0 to 2.0; missing is 1.0. pub volumes: HashMap, pub muted: HashSet, @@ -197,6 +199,7 @@ enum Ctl { Talking(bool), Mode(TalkMode), Noise(bool), + Echo(bool), Volume(u64, f32), Muted(u64, bool), MicMuted(bool), @@ -269,6 +272,10 @@ impl Hub { let _ = self.ctl.send(Ctl::Noise(enabled)); } + pub fn set_echo_cancellation(&self, enabled: bool) { + let _ = self.ctl.send(Ctl::Echo(enabled)); + } + /// 0.0 (silent) to 2.0. pub fn set_volume(&self, account: u64, volume: f32) { let _ = self.ctl.send(Ctl::Volume(account, volume)); @@ -335,6 +342,9 @@ impl Task { if let Some(enabled) = self.settings.noise_suppression { self.engine.set_noise_suppression(enabled); } + if let Some(enabled) = self.settings.echo_cancellation { + self.engine.set_echo_cancellation(enabled); + } self.engine.set_talking(self.settings.talking); self.engine.set_mic_muted(self.settings.mic_muted); self.engine.set_deafened(self.settings.deafened); @@ -387,6 +397,10 @@ impl Task { self.settings.noise_suppression = Some(enabled); self.engine.set_noise_suppression(enabled); } + Ctl::Echo(enabled) => { + self.settings.echo_cancellation = Some(enabled); + self.engine.set_echo_cancellation(enabled); + } Ctl::MicMuted(muted) => { self.settings.mic_muted = muted; self.engine.set_mic_muted(muted); diff --git a/crates/voice/Cargo.toml b/crates/voice/Cargo.toml index 9b24c4c..c64782d 100644 --- a/crates/voice/Cargo.toml +++ b/crates/voice/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "anyplay-voice" -description = "Party voice for the AnyPlay network: noise suppression, Opus, jitter buffer and mixer behind a small interface" +description = "Party voice for the AnyPlay network: noise suppression, echo cancellation, Opus, jitter buffer and mixer behind a small interface" version.workspace = true edition.workspace = true license.workspace = true @@ -15,6 +15,7 @@ devices = ["dep:cpal"] [dependencies] nnnoiseless = { version = "0.5.2", default-features = false } opus = "0.4" +sonora = "0.2" cpal = { version = "0.18", optional = true } [[example]] diff --git a/crates/voice/src/echo.rs b/crates/voice/src/echo.rs new file mode 100644 index 0000000..66d37ae --- /dev/null +++ b/crates/voice/src/echo.rs @@ -0,0 +1,137 @@ +//! Acoustic echo cancellation: the far end's voice, played by the speakers, is taken out of what +//! the microphone hears. +//! +//! [`EchoCanceller`] is WebRTC's AEC3 (the pure-Rust port `sonora`) on mono 48 kHz in blocks of +//! 10 ms. It wants two things: every block the speakers play (the reference, "render"), and every +//! block the microphone captured ("capture"), which comes back with the echo removed. +//! +//! The engine writes the reference itself, so it knows it exactly. What it does not know is when +//! the speakers play it: it queues up to 60 ms ahead of the device. [`Path`] holds the reference +//! back until the device has taken it off the queue, so the delay the canceller has to find is +//! only the device's own buffers and the room, which AEC3's delay estimator follows. + +use std::collections::VecDeque; + +use sonora::config::EchoCanceller as AecConfig; +use sonora::{AudioProcessing, Config, StreamConfig}; + +/// Samples in the 10 ms block AEC3 works on, at 48 kHz. +pub const BLOCK: usize = 480; + +/// What the device is guessed to add between the engine and the air, in milliseconds. AEC3 +/// estimates the delay itself; this only tells it where to start. +const DEVICE_DELAY_HINT_MS: i32 = 40; +/// The most reference kept waiting for a device that has stopped taking samples: one second. +const PENDING_LIMIT: usize = 48_000; +/// Capture blocks (10 ms) without any reference released after which the canceller is bypassed: +/// a device that plays nothing (stopped, or not draining) makes no echo, and AEC3 without a +/// reference only colours the microphone. +const STALLED_BLOCKS: u32 = 50; + +/// AEC3 on one mono 48 kHz stream pair. +pub struct EchoCanceller { + apm: AudioProcessing, +} + +impl Default for EchoCanceller { + fn default() -> EchoCanceller { + EchoCanceller::new() + } +} + +impl EchoCanceller { + pub fn new() -> EchoCanceller { + let config = Config { + echo_canceller: Some(AecConfig::default()), + ..Config::default() + }; + let mut apm = AudioProcessing::builder() + .config(config) + .capture_config(StreamConfig::new(48_000, 1)) + .render_config(StreamConfig::new(48_000, 1)) + .build(); + // Out of range is clamped and still processes; nothing to do about it here. + let _ = apm.set_stream_delay_ms(DEVICE_DELAY_HINT_MS); + EchoCanceller { apm } + } + + /// The next 10 ms the speakers play. Anything but [`BLOCK`] samples is ignored. + pub fn render(&mut self, block: &[f32]) { + if block.len() != BLOCK { + return; + } + let mut sink = [0.0f32; BLOCK]; + let _ = self.apm.process_render_f32(&[block], &mut [&mut sink[..]]); + } + + /// The next 10 ms the microphone captured, in place: the echo of what [`render`] was given + /// is taken out. If the canceller refuses the block it stays as it was. + /// + /// [`render`]: EchoCanceller::render + pub fn capture(&mut self, block: &mut [f32]) { + if block.len() != BLOCK { + return; + } + let input: [f32; BLOCK] = std::array::from_fn(|i| block[i]); + let mut output = [0.0f32; BLOCK]; + if self + .apm + .process_capture_f32(&[&input[..]], &mut [&mut output[..]]) + .is_ok() + { + block.copy_from_slice(&output); + } + } +} + +/// An [`EchoCanceller`] plus the reference waiting for the speakers to play it. +pub(crate) struct Path { + canceller: EchoCanceller, + /// Reference at 48 kHz that was written to the device and not yet released to the canceller. + pending: VecDeque, + /// Capture blocks since the last reference block went to the canceller; `None` before the + /// first. + since_reference: Option, +} + +impl Path { + pub(crate) fn new() -> Path { + Path { + canceller: EchoCanceller::new(), + pending: VecDeque::new(), + since_reference: None, + } + } + + /// Samples just written to the device, at 48 kHz. + pub(crate) fn reference(&mut self, samples: &[f32]) { + self.pending.extend(samples); + if self.pending.len() > PENDING_LIMIT { + let excess = self.pending.len() - PENDING_LIMIT; + self.pending.drain(..excess); + } + } + + /// The device still has `queued` samples (at 48 kHz) of what was written to it waiting; the + /// rest has been taken to the speakers, so it goes to the canceller now. + pub(crate) fn release(&mut self, queued: usize) { + while self.pending.len() >= queued + BLOCK { + let block: [f32; BLOCK] = + std::array::from_fn(|_| self.pending.pop_front().unwrap_or(0.0)); + self.canceller.render(&block); + self.since_reference = Some(0); + } + } + + pub(crate) fn capture(&mut self, block: &mut [f32]) { + for half in block.as_chunks_mut::().0 { + match &mut self.since_reference { + Some(n) if *n < STALLED_BLOCKS => { + *n += 1; + self.canceller.capture(half); + } + _ => {} + } + } + } +} diff --git a/crates/voice/src/lib.rs b/crates/voice/src/lib.rs index 27ab9df..8f89c32 100644 --- a/crates/voice/src/lib.rs +++ b/crates/voice/src/lib.rs @@ -45,11 +45,13 @@ //! //! # Echo //! -//! There is no acoustic echo cancellation. Push-to-talk is the default, and with headphones that -//! is all a party needs. With the open microphone ([`TalkMode::VoiceActivity`]) and speakers, the -//! members hear themselves back. See `docs/adr/` for the decision and what would replace it. +//! Echo cancellation is on by default ([`Config::echo_cancellation`]). The engine writes what the +//! speakers play, so it hands that to the canceller as the reference, delayed until the device has +//! taken it off its queue, and runs the microphone through the canceller before anything else +//! (ADR 0040). With headphones there is no echo and the canceller does nothing audible. mod device; +mod echo; mod gate; mod jitter; mod mix; @@ -66,6 +68,7 @@ use std::fmt; use std::time::Instant; pub use device::{AudioIo, MemoryDevice, MemoryHandle}; +pub use echo::EchoCanceller; pub use gate::VadConfig; pub use jitter::PeerStats; @@ -109,6 +112,9 @@ pub struct Config { pub mode: TalkMode, /// RNNoise noise suppression on the microphone. pub noise_suppression: bool, + /// Acoustic echo cancellation: what the speakers play is taken out of the microphone, so + /// the party does not hear itself back without headphones. + pub echo_cancellation: bool, /// Opus bitrate in bits per second, 16000 to 64000. pub bitrate: u32, /// What the encoder should expect to lose, in percent, which sets how much redundancy it adds. @@ -124,6 +130,7 @@ impl Default for Config { Config { mode: TalkMode::PushToTalk, noise_suppression: true, + echo_cancellation: true, bitrate: 28_000, expected_loss_percent: 10, dtx: true, @@ -265,6 +272,11 @@ impl Engine { self.sender.set_noise_suppression(enabled); } + /// Switches the echo cancellation on or off while the engine runs. + pub fn set_echo_cancellation(&mut self, enabled: bool) { + self.sender.set_echo_cancellation(enabled); + } + /// How loud one member is heard, 0.0 (silent) to 2.0 (double). Applies to members not yet /// heard from too. pub fn set_peer_volume(&mut self, peer: PeerId, volume: f32) { @@ -326,6 +338,10 @@ impl Engine { /// Move audio: take what the microphone captured, and fill the speakers' queue. Call every /// 5 to 10 ms. Datagrams to send are then in [`outgoing`](Engine::outgoing). pub fn pump(&mut self) { + // What the device took off its queue since the last call has reached the speakers. + let queued = self.device.playback_queued(); + self.sender + .release_reference((queued as u64 * 48_000 / u64::from(self.playback_rate)) as usize); self.scratch.clear(); self.device.read_capture(&mut self.scratch); self.sender.push(&self.scratch); @@ -342,6 +358,7 @@ impl Engine { break; } let frame = self.mix_frame(); + self.sender.reference(&frame); self.scratch.clear(); self.playback.process(&frame, &mut self.scratch); self.device.write_playback(&self.scratch); diff --git a/crates/voice/src/send.rs b/crates/voice/src/send.rs index 3895bd2..ae15c23 100644 --- a/crates/voice/src/send.rs +++ b/crates/voice/src/send.rs @@ -1,11 +1,12 @@ //! The sending half: microphone samples in, wire frames out. //! -//! capture rate -> 48 kHz mono -> noise suppression -> gate (push-to-talk or voice activity) -> +//! capture rate -> 48 kHz mono -> echo cancellation -> noise suppression -> gate (push-to-talk or voice activity) -> //! Opus -> frame. use nnnoiseless::DenoiseState; use opus::{Application, Bitrate, Channels, Encoder, Signal}; +use crate::echo::Path; use crate::gate::{Vad, rms, to_db}; use crate::jitter::FRAME_SAMPLES; use crate::resample::Resampler; @@ -23,6 +24,7 @@ const DTX_KEEPALIVE: u32 = 20; pub(crate) struct Sender { resampler: Resampler, pending: Vec, + echo: Option, denoise: Option>>, encoder: Encoder, vad: Vad, @@ -58,6 +60,7 @@ impl Sender { Ok(Sender { resampler: Resampler::new(capture_rate, 48_000), pending: Vec::new(), + echo: config.echo_cancellation.then(Path::new), denoise: config.noise_suppression.then(DenoiseState::new), encoder, vad: Vad::new(config.vad), @@ -86,6 +89,30 @@ impl Sender { } } + /// Starts or stops cancelling the echo of the speakers. A canceller that is already running + /// is kept, with what it learned of the room. + pub(crate) fn set_echo_cancellation(&mut self, enabled: bool) { + match (enabled, self.echo.is_some()) { + (true, false) => self.echo = Some(Path::new()), + (false, true) => self.echo = None, + _ => {} + } + } + + /// Samples just written to the speakers' queue, at 48 kHz. + pub(crate) fn reference(&mut self, samples: &[f32]) { + if let Some(echo) = &mut self.echo { + echo.reference(samples); + } + } + + /// How much of what was written is still queued at the device, at 48 kHz. + pub(crate) fn release_reference(&mut self, queued: usize) { + if let Some(echo) = &mut self.echo { + echo.release(queued); + } + } + pub(crate) fn set_talking(&mut self, talking: bool) { self.talking = talking; } @@ -119,6 +146,9 @@ impl Sender { for s in frame.iter_mut() { *s = s.clamp(-1.0, 1.0); } + if let Some(echo) = &mut self.echo { + echo.capture(frame); + } let probability = match &mut self.denoise { Some(dn) => { let mut probability = 0.0f32; diff --git a/crates/voice/tests/echo.rs b/crates/voice/tests/echo.rs new file mode 100644 index 0000000..903dff3 --- /dev/null +++ b/crates/voice/tests/echo.rs @@ -0,0 +1,250 @@ +//! Echo cancellation, measured. No audio devices: a far-end voice is played into a simulated room +//! (a delay and a few reflections), and what the microphone hears of it goes back through the +//! canceller. The figure is ERLE, echo return loss enhancement: how many dB quieter the echo got. +//! Every signal is generated from fixed seeds, so the numbers are the same on every run. + +use std::f32::consts::TAU; + +use anyplay_voice::wire::Frame; +use anyplay_voice::{Config, EchoCanceller, Engine, MemoryDevice, PeerId, TalkMode}; +use opus::{Channels, Decoder}; + +const RATE: usize = 48_000; +const BLOCK: usize = 480; + +struct Lcg(u32); + +impl Lcg { + fn signed(&mut self) -> f32 { + self.0 = self.0.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + ((self.0 >> 8) % 20001) as f32 / 10000.0 - 1.0 + } +} + +/// Something that moves like a voice: a pitch with harmonics shaped by two formants, a syllable +/// rhythm of about 4 per second, and a little breath. `pitch` and `seed` tell two voices apart. +fn voiced(secs: f32, amp: f32, pitch: f32, seed: u32) -> Vec { + let mut rng = Lcg(seed); + (0..(RATE as f32 * secs) as usize) + .map(|i| { + let t = i as f32 / RATE as f32; + let f0 = pitch + 8.0 * (TAU * 0.7 * t).sin(); + let envelope = (0.5 + 0.5 * (TAU * 4.0 * t + seed as f32).sin()).powf(0.7); + let mut v = 0.0; + for h in 1..30 { + let f = f0 * h as f32; + if f > RATE as f32 / 2.2 { + break; + } + let formant = 1.0 / (1.0 + ((f - 700.0) / 250.0).powi(2)) + + 0.6 / (1.0 + ((f - 1800.0) / 350.0).powi(2)); + v += formant / h as f32 * (TAU * f0 * h as f32 * t).sin(); + } + amp * (envelope * v * 0.9 + 0.01 * rng.signed()) + }) + .collect() +} + +/// A room: what the speakers played, delayed and reflected, as the microphone hears it at +/// sample `n`. The taps are (delay in samples, gain). +struct Room { + taps: Vec<(usize, f32)>, +} + +impl Room { + fn living_room() -> Room { + Room { + taps: [(40, 0.45), (48, 0.20), (63, 0.12), (91, 0.06)] + .iter() + .map(|(ms, g)| (ms * RATE / 1000, *g)) + .collect(), + } + } + + fn hear(&self, played: &[f32], n: usize) -> f32 { + self.taps + .iter() + .filter(|(d, _)| n >= *d && n - d < played.len()) + .map(|(d, g)| g * played[n - d]) + .sum() + } +} + +fn energy(x: &[f32]) -> f64 { + x.iter().map(|v| f64::from(*v).powi(2)).sum::() / x.len().max(1) as f64 +} + +fn db(ratio: f64) -> f64 { + 10.0 * ratio.max(1e-30).log10() +} + +/// The canceller on its own: `far` plays, the microphone hears `room(far) + near`. Returns the +/// echo at the microphone and what came out of the canceller. +fn run_canceller(far: &[f32], near: &[f32], room: &Room) -> (Vec, Vec) { + let mut canceller = EchoCanceller::new(); + let mut echo = Vec::new(); + let mut out = Vec::new(); + for (b, block) in far.as_chunks::().0.iter().enumerate() { + // The reference goes in first, as the engine releases it: the speakers play it before + // the microphone can hear it. + canceller.render(block); + let start = b * BLOCK; + let heard: Vec = (0..BLOCK).map(|i| room.hear(far, start + i)).collect(); + let mut mic: Vec = heard + .iter() + .enumerate() + .map(|(i, e)| e + near.get(start + i).copied().unwrap_or(0.0)) + .collect(); + echo.extend(heard); + canceller.capture(&mut mic); + out.extend(mic); + } + (echo, out) +} + +#[test] +fn the_canceller_takes_the_echo_out_of_a_quiet_microphone() { + let far = voiced(12.0, 0.5, 120.0, 5); + let (echo, out) = run_canceller(&far, &[], &Room::living_room()); + // The last four seconds, after it has found the room. + let tail = out.len() - 4 * RATE..; + let erle = db(energy(&echo[tail.clone()]) / energy(&out[tail])); + println!("canceller ERLE {erle:.1} dB"); + assert!(erle > 20.0, "ERLE {erle:.1} dB"); +} + +#[test] +fn the_canceller_is_transparent_when_nothing_plays() { + // A headset: the speakers are silent, the microphone hears only the player. + let near = voiced(6.0, 0.4, 150.0, 9); + let silent_far = vec![0.0f32; near.len()]; + let (_, out) = run_canceller(&silent_far, &near, &Room::living_room()); + let tail = out.len() - 3 * RATE..; + let kept = db(energy(&out[tail.clone()]) / energy(&near[tail])); + println!("voice kept {kept:.1} dB"); + assert!(kept > -1.5, "the voice lost {kept:.1} dB"); +} + +#[test] +fn the_player_is_still_heard_while_the_far_end_talks() { + // Double talk: both voices at once. The echo goes down; the player's voice stays. + let far = voiced(12.0, 0.5, 120.0, 5); + // The player is not periodic like the far voice: shaped noise with a syllable rhythm, so the + // two cannot be confused by their harmonics. + let mut rng = Lcg(77); + let mut low = 0.0f32; + let near: Vec = (0..12 * RATE) + .map(|i| { + low += 0.35 * (rng.signed() - low); + let t = i as f32 / RATE as f32; + 0.5 * low * (0.5 + 0.5 * (TAU * 3.0 * t).sin()) + }) + .collect(); + let (echo, out) = run_canceller(&far, &near, &Room::living_room()); + let tail = out.len() - 4 * RATE..; + // The two voices are unrelated, so the share of the output that follows the echo (its + // projection on the echo) is the echo that is left. The player's voice is followed by its + // correlation with the output; the canceller delays the signal by a few milliseconds, so the + // best lag within 100 ms is taken. + let (o, n, e) = (&out[tail.clone()], &near[tail.clone()], &echo[tail]); + let dot = |a: &[f32], b: &[f32]| a.iter().zip(b).map(|(a, b)| f64::from(a * b)).sum::(); + let erle = db(dot(e, e) / (dot(o, e).powi(2) / dot(e, e))); + let mut corr = 0.0f64; + for lag in (0..4800).step_by(8) { + let (o, n) = (&o[lag..], &n[..n.len() - lag]); + corr = corr.max(dot(o, n) / (dot(o, o) * dot(n, n)).sqrt()); + } + println!("double talk: ERLE {erle:.1} dB, correlation with the player's voice {corr:.2}"); + assert!(erle > 20.0, "ERLE {erle:.1} dB"); + assert!( + corr > 0.2, + "the player's voice is gone: correlation {corr:.2}" + ); +} + +// ---------------------------------------------------------------- the whole engine + +/// A far-end engine talks to a near-end one; the near-end engine's speakers feed its own +/// microphone through the room. Returns the energy the near-end engine would send (decoded) and +/// the energy of the echo at its microphone, both over the last `tail` seconds. +fn engine_run(echo_cancellation: bool, secs: usize, tail: usize) -> (f64, f64) { + let (far_dev, far_hw) = MemoryDevice::new(48_000, 48_000); + let (near_dev, near_hw) = MemoryDevice::new(48_000, 48_000); + let config = Config { + mode: TalkMode::PushToTalk, + noise_suppression: false, + echo_cancellation, + dtx: false, + ..Config::default() + }; + let mut far = Engine::new(config.clone(), Box::new(far_dev)).unwrap(); + let mut near = Engine::new(config, Box::new(near_dev)).unwrap(); + far.set_talking(true); + near.set_talking(true); + + let voice = voiced(secs as f32, 0.5, 120.0, 5); + let room = Room::living_room(); + let mut decoder = Decoder::new(48_000, Channels::Mono).unwrap(); + let mut played: Vec = Vec::new(); + let mut echo: Vec = Vec::new(); + let mut sent: Vec = Vec::new(); + let step = RATE / 50; + for (s, chunk) in voice.chunks_exact(step).enumerate() { + far_hw.feed_capture(chunk); + far.pump(); + for datagram in far.outgoing() { + near.incoming(PeerId(1), &datagram).unwrap(); + } + let mic: Vec = (0..step) + .map(|i| room.hear(&played, s * step + i)) + .collect(); + echo.extend(&mic); + near_hw.feed_capture(&mic); + near.pump(); + for datagram in near.outgoing() { + let frame = Frame::parse(&datagram).unwrap(); + let mut pcm = vec![0.0f32; 960]; + let n = decoder + .decode_float(frame.payload, &mut pcm, false) + .unwrap(); + sent.extend(&pcm[..n]); + } + played.extend(near_hw.play(step)); + let _ = far_hw.play(step); + } + assert!( + sent.len() > (secs - 1) * RATE, + "sent {} samples", + sent.len() + ); + let sent_tail = &sent[sent.len() - tail * RATE..]; + let echo_tail = &echo[echo.len() - tail * RATE..]; + (energy(sent_tail), energy(echo_tail)) +} + +#[test] +fn the_engine_does_not_send_the_echo_of_its_own_speakers() { + let (off_sent, echo) = engine_run(false, 14, 4); + let (on_sent, _) = engine_run(true, 14, 4); + let off = db(echo / off_sent); + let on = db(echo / on_sent); + println!( + "engine ERLE: off {off:.1} dB, on {on:.1} dB (echo {:.1} dBFS)", + db(echo) + ); + assert!(on - off > 15.0, "off {off:.1} dB, on {on:.1} dB"); + assert!(on > 15.0, "ERLE with the canceller on is {on:.1} dB"); +} + +#[test] +fn echo_cancellation_can_be_switched_while_running() { + let (dev, hw) = MemoryDevice::new(48_000, 48_000); + let mut engine = Engine::new(Config::default(), Box::new(dev)).unwrap(); + hw.feed_capture(&[0.0; 960]); + engine.set_echo_cancellation(false); + engine.pump(); + engine.set_echo_cancellation(true); + engine.set_echo_cancellation(true); + engine.pump(); + let _ = hw.play(960); +} diff --git a/docs/adr/0014-voice-is-an-engine-without-a-transport.md b/docs/adr/0014-voice-is-an-engine-without-a-transport.md index 32e1a87..50ce68a 100644 --- a/docs/adr/0014-voice-is-an-engine-without-a-transport.md +++ b/docs/adr/0014-voice-is-an-engine-without-a-transport.md @@ -1,6 +1,6 @@ # Voice is an engine without a transport -Party voice is the crate `anyplay-voice`: bytes in, bytes out, with the sound hardware behind a trait. It encodes with libopus, suppresses noise with RNNoise, runs its own jitter buffer and mixer, and knows nothing about parties, accounts or tunnels. The net service gives it datagrams from the tunnels and sends the datagrams it returns ([ADR 0010](0010-voice-and-shared-play-go-through-the-tunnels.md)). Push-to-talk is the default, and there is no echo cancellation: a player who opens the microphone without headphones is heard by the others as an echo of their own party. The hardware (cpal) is behind the cargo feature `devices`, so builds and tests need no audio headers. +Party voice is the crate `anyplay-voice`: bytes in, bytes out, with the sound hardware behind a trait. It encodes with libopus, suppresses noise with RNNoise, runs its own jitter buffer and mixer, and knows nothing about parties, accounts or tunnels. The net service gives it datagrams from the tunnels and sends the datagrams it returns ([ADR 0010](0010-voice-and-shared-play-go-through-the-tunnels.md)). Push-to-talk is the default, and there was no echo cancellation (since [ADR 0040](0040-voice-cancels-echo-with-aec3-in-pure-rust.md) there is): a player who opens the microphone without headphones is heard by the others as an echo of their own party. The hardware (cpal) is behind the cargo feature `devices`, so builds and tests need no audio headers. ## Considered options diff --git a/docs/adr/0040-voice-cancels-echo-with-aec3-in-pure-rust.md b/docs/adr/0040-voice-cancels-echo-with-aec3-in-pure-rust.md new file mode 100644 index 0000000..7d8b1bf --- /dev/null +++ b/docs/adr/0040-voice-cancels-echo-with-aec3-in-pure-rust.md @@ -0,0 +1,29 @@ +# Voice cancels echo with AEC3, ported to Rust, on by default + +[ADR 0014](0014-voice-is-an-engine-without-a-transport.md) left echo to push-to-talk and headphones and named echo cancellation as the next step if players ask for an open microphone on speakers. A console-style launcher is played from a couch with the TV's or a soundbar's speakers, so that is the normal case, not the exception: the far member's voice leaves the speakers, enters the microphone, and goes back to the whole party. The engine now cancels it. `Config::echo_cancellation` is on by default, the control protocol has `voice_echo` to switch it off, and the canceller sits first in the sending chain: capture, resample to 48 kHz, **echo cancellation**, RNNoise, gate, Opus. + +## The choice: `sonora`, the pure-Rust port of WebRTC's AEC3 + +[`sonora`](https://crates.io/crates/sonora) 0.2 (BSD-3-Clause, [source](https://github.com/dignifiedquire/sonora)) is WebRTC's audio processing module M145 ported to Rust: AEC3 with its own delay estimator and echo-path change detection, a noise suppressor, AGC2, and SIMD paths (SSE2, AVX2, NEON). It is pure Rust with three small dependencies of its own, so it builds the same on Windows (MSVC and GNU), Linux and macOS with the toolchain the workspace already has, and needs no system package, no CMake step, and no meson or ninja. BSD-3-Clause is already on the allow list of `deny.toml` and is compatible with distributing under MIT OR Apache-2.0 (the notice is kept in the licence listing the launcher ships). It needs Rust 1.91, so the workspace `rust-version` goes from 1.88 to 1.91; CI and the launcher build with stable, which is newer. + +What we use of it is the echo canceller. We do not use its noise suppressor (RNNoise stays: its speech probability is the voice activity detector, and it is tuned) and not AGC2. The gain control would be cheap to add, but a microphone whose level the player cannot see moving is a setting of its own, and nobody asked. + +### The delay + +AEC3 needs the exact signal the speakers play and finds the delay to the microphone itself, up to a few hundred milliseconds. The engine knows the first part exactly, because it writes the playback samples itself, from the mixer, in 20 ms frames. It writes up to 60 ms ahead of the device, though, so handing the reference over at once would add that queue to the delay AEC3 must find, and the queue's length changes with every pump. Instead the reference waits in a small line and is released to the canceller when the device has taken it off its queue (`playback_queued` shrank). What AEC3 still has to find is then the device's hardware buffers, the room, and the capture ring, which is its job, and a fixed start hint of 40 ms. The queue length is counted at 48 kHz whatever the device's rate. + +## Considered options + +- **`webrtc-audio-processing` (tonarino), the C++ library behind a Rust wrapper** (2.1, BSD-3-Clause style licence): the original AEC3, with years of field use. Rejected as the first choice only for the build: the `bundled` feature needs meson, ninja and a C++ compiler on every runner and every developer's PC, without it the build links the system library (`libwebrtc-audio-processing-dev`), which the release builds do not have and Windows does not have at all. This is the fallback, behind the same two calls (`render`, `capture`) in `echo.rs`, if `sonora` turns out to behave worse than the original on real hardware: it is a one-file swap. +- **`aec3`** (RubyBit, MIT OR BSD-3-Clause): another port of AEC3, built around a node graph with its own scheduler. More than we need to put one filter in a chain. +- **The speex canceller (`speexdsp` bindings)**: older, a simple adaptive filter, no delay estimation, a C dependency. Clearly behind AEC3 for a room with reflections. +- **The operating system's canceller.** Windows: capturing in the communications category turns on the driver's echo cancellation only where the driver provides one, which many do not, and cpal does not expose the category. PipeWire and PulseAudio: `module-echo-cancel` is a module the user loads, not something an application can count on. macOS: the VoiceProcessingIO audio unit does it well, but it is a different audio API from the one cpal uses and also changes the output (ducks other audio, which would duck the game). Three platform back ends with three behaviours, none testable without a device, against one library that is the same everywhere and can be tested offline. Rejected as the primary; if a platform later offers a reliable one, it can come in front of this as a bypass. +- **Nothing, as before.** Rejected by the reason above. + +## What follows + +- The setting is `Config::echo_cancellation` and `Engine::set_echo_cancellation`; the net service carries `voice_echo { enabled }` ([control protocol](../../spec/control-protocol.md)). An older launcher never sends it and gets the default, on; an older net service that receives it answers it as an unknown command, which a launcher treats as "not supported". +- With headphones there is no echo and AEC3's transparent mode keeps the player's voice as it was; a test checks that a silent reference changes the voice by less than 1.5 dB. +- The cost is a few percent of one core while the engine runs. +- Tests are offline (`crates/voice/tests/echo.rs`): a fixed-seed voice plays into a simulated room (a 40 ms delay and three reflections) and the figure asserted is ERLE, with the canceller alone, with a second voice talking at the same time, and through the whole engine (two engines, Opus in between). +- Not covered by any test, because it needs hardware: how it sounds on real speakers and microphones, with devices whose buffers drift against each other, and on a laptop with the microphone a few centimetres from the speakers. Those are listening tests on real PCs. diff --git a/spec/control-protocol.md b/spec/control-protocol.md index 39489e4..73108b6 100644 --- a/spec/control-protocol.md +++ b/spec/control-protocol.md @@ -133,6 +133,7 @@ When the cloud retires this launcher's version ([cloud API](cloud-api.md#convent | `deafen` | `deafened` | `{}`: nobody is heard, and the microphone is silenced too | | `push_to_talk` | `talking` | `{}` | | `voice_noise` | `enabled` | `{}`: noise suppression (RNNoise) on the microphone; on in a new service | +| `voice_echo` | `enabled` | `{}`: echo cancellation (WebRTC AEC3) on the microphone, so the speakers are not heard back; on in a new service | | `voice_mode` | `mode`: `push_to_talk` or `voice_activity` | `{}` | | `peer_volume` | `account`, `percent` (0 to 200) | `{}`; `bad_request` above 200 | | `peer_mute` | `account`, `muted` | `{}` |