From 1f9d96c4ba9848b3f72485927dfd9cc8f640ef24 Mon Sep 17 00:00:00 2001 From: FacuM Date: Sun, 25 Oct 2026 21:10:02 -0300 Subject: [PATCH 01/12] Negotiate and carry an audio stream between host and client Adds protocol minor 6: the client advertises what it can play (ClientAudio), the host picks PCM or Opus (AudioConfig) and sends AudioPacket frames on the audio channel. The mux gives audio its own short queue (and its own QUIC stream) between cursor and video, dropping the oldest packets when the link stalls. The host session pumps an optional AudioBackend on its own thread; an audio failure stops only the audio. --- crates/ezd-client-core/src/session.rs | 53 +- .../i18n/de/ezd-fake-client.ftl | 3 + .../i18n/en/ezd-fake-client.ftl | 3 + .../i18n/es/ezd-fake-client.ftl | 3 + .../i18n/fr/ezd-fake-client.ftl | 3 + .../i18n/it/ezd-fake-client.ftl | 3 + .../i18n/pt/ezd-fake-client.ftl | 3 + crates/ezd-fake-client/src/lib.rs | 54 +- crates/ezd-fake-client/src/main.rs | 6 +- crates/ezd-fake-client/tests/loopback.rs | 76 ++- crates/ezd-host-core/src/lib.rs | 10 +- crates/ezd-host-core/src/platform.rs | 5 +- crates/ezd-host-core/src/session.rs | 583 +++++++++++++++++- crates/ezd-host-core/src/source.rs | 65 +- crates/ezd-host-core/src/testing.rs | 113 +++- crates/ezd-host-service/src/service.rs | 9 +- crates/ezd-link/src/link.rs | 15 +- crates/ezd-link/src/mux.rs | 168 ++++- crates/ezd-link/src/pair.rs | 21 +- crates/ezd-link/src/quic.rs | 102 ++- crates/ezd-proto/src/frame.rs | 2 +- crates/ezd-proto/src/lib.rs | 7 +- crates/ezd-proto/src/msg.rs | 187 +++++- crates/ezd-proto/src/negotiate.rs | 202 +++++- crates/ezd-proto/src/pair.rs | 2 +- docs/protocol.md | 76 ++- 26 files changed, 1695 insertions(+), 79 deletions(-) diff --git a/crates/ezd-client-core/src/session.rs b/crates/ezd-client-core/src/session.rs index ea47ed4..3f6d08e 100644 --- a/crates/ezd-client-core/src/session.rs +++ b/crates/ezd-client-core/src/session.rs @@ -3,8 +3,8 @@ use crate::{ClientHandle, now_us}; use ezd_link::{Link, MuxError, MuxReceiver, MuxSender}; use ezd_proto::{ - Bye, ClientCaps, ClientHdr, CursorShape, CursorState, FrameAck, Hello, Message, Ping, Pong, - Role, StreamColor, StreamConfig, VideoFrame, + AudioConfig, AudioPacket, Bye, ClientAudio, ClientCaps, ClientHdr, CursorShape, CursorState, + FrameAck, Hello, Message, Ping, Pong, Role, StreamColor, StreamConfig, VideoFrame, }; use std::time::Duration; use tokio::time::{MissedTickBehavior, interval, timeout}; @@ -40,6 +40,17 @@ pub trait ClientHandler: Send { /// [`on_stream_config`]: ClientHandler::on_stream_config fn on_stream_color(&mut self, _color: &StreamColor) {} + /// The host's audio decision (protocol 0.6), called right before [`on_stream_config`] and + /// only when the host sent one: `stream: None` means no audio this session. Create the + /// audio decoder/output here. Not called for a host older than protocol 0.6. + /// + /// [`on_stream_config`]: ClientHandler::on_stream_config + fn on_audio_config(&mut self, _cfg: &AudioConfig) {} + + /// An audio packet. Hand it to the audio thread without blocking; packets only arrive + /// when [`on_audio_config`](Self::on_audio_config) announced a stream. + fn on_audio_packet(&mut self, _pkt: AudioPacket) {} + /// A `CODEC_CONFIG` frame (parameter sets). Never acked. fn on_codec_config(&mut self, frame: VideoFrame) -> HandlerResult; @@ -67,6 +78,10 @@ pub struct ClientConfig { /// `ClientHdr` is sent right after the host `Hello`; `None` sends [`ClientHdr::none`] /// (no HDR10), so the host always gets an answer and never waits for it in vain. pub hdr: Option, + /// Audio playback capabilities. Whenever the host speaks protocol 0.6 a `ClientAudio` is sent + /// right after `ClientHdr`; `None` sends [`ClientAudio::none`] (no audio), so the host + /// always gets an answer and never waits for it in vain. + pub audio: Option, /// Interval of `ClientStats` messages. pub stats_interval: Duration, /// Interval of client-initiated pings (clock sync). @@ -92,6 +107,7 @@ impl ClientConfig { app_version: env!("CARGO_PKG_VERSION").into(), caps, hdr: None, + audio: None, stats_interval: Duration::from_secs(1), ping_interval: Duration::from_millis(500), max_duration: None, @@ -125,6 +141,8 @@ pub struct ClientOutcome { pub stream: StreamConfig, /// Colour description of the stream (BT.709 8-bit when the host sent none). pub color: StreamColor, + /// The host's audio decision; `None` when the host sent none (older than protocol 0.6). + pub audio: Option, /// Why it ended. pub end: ClientEnd, /// Estimated `host_clock - client_clock` in microseconds. @@ -181,8 +199,11 @@ async fn wait_for( t2_us: t, })); } - Err(Message::Pong(_)) | Err(Message::CursorShape(_)) | Err(Message::CursorState(_)) => { - } + Err(Message::Pong(_)) + | Err(Message::CursorShape(_)) + | Err(Message::CursorState(_)) + | Err(Message::AudioConfig(_)) + | Err(Message::AudioPacket(_)) => {} Err(Message::Reject(r)) => { return Err(ClientError::Handshake(format!("rejected: {}", r.reason))); } @@ -217,7 +238,8 @@ pub async fn run_client( if let Some(minor) = cfg.proto_minor { my_hello.proto_minor = minor; } - let my_minor_supports_hdr = my_hello.supports_hdr(); + let (my_minor_supports_hdr, my_minor_supports_audio) = + (my_hello.supports_hdr(), my_hello.supports_audio()); tx.send_control(Message::Hello(my_hello))?; tx.send_control(Message::ClientCaps(cfg.caps.clone()))?; let hello = wait_for(&mut rx, &tx, "Hello", wait, |m| match m { @@ -233,13 +255,20 @@ pub async fn run_client( cfg.hdr.clone().unwrap_or_else(ClientHdr::none), ))?; } + if hello.supports_audio() && my_minor_supports_audio { + tx.send_control(Message::ClientAudio( + cfg.audio.clone().unwrap_or_else(ClientAudio::none), + ))?; + } wait_for(&mut rx, &tx, "HostCaps", wait, |m| match m { Message::HostCaps(h) => Ok(h), o => Err(o), }) .await?; - // `StreamColor` (0.5) may precede `StreamConfig`; without it the stream is BT.709 8-bit. + // `StreamColor` (0.5) and `AudioConfig` (0.6) may precede `StreamConfig`; without the colour + // the stream is BT.709 8-bit. let mut color = StreamColor::sdr_bt709(); + let mut audio: Option = None; let stream = loop { let next = wait_for(&mut rx, &tx, "StreamConfig", wait, |m| match m { Message::StreamConfig(c) => Ok(Some(c)), @@ -247,6 +276,10 @@ pub async fn run_client( color = c; Ok(None) } + Message::AudioConfig(a) => { + audio = Some(a); + Ok(None) + } o => Err(o), }) .await?; @@ -256,6 +289,9 @@ pub async fn run_client( }; handle.store_stream_color(color); handler.on_stream_color(&color); + if let Some(a) = &audio { + handler.on_audio_config(a); + } if let Err(e) = handler.on_stream_config(&stream) { let _ = tx.send_control(Message::Bye(Bye { reason: format!("decoder init failed: {e}"), @@ -335,6 +371,7 @@ pub async fn run_client( let outcome = ClientOutcome { stream, color, + audio, end, clock_offset_us: handle.clock_offset_us(), rtt_us: handle.rtt_us(), @@ -408,6 +445,10 @@ fn on_message( handle.count_received(v.data.len()); flow(handler.on_video_frame(v, received_us)) } + Message::AudioPacket(p) => { + handler.on_audio_packet(p); + Ok(None) + } Message::InputStatus(st) => { handle.store_input_status(st); Ok(None) diff --git a/crates/ezd-fake-client/i18n/de/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/de/ezd-fake-client.ftl index cd9118d..7c8d0ef 100644 --- a/crates/ezd-fake-client/i18n/de/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/de/ezd-fake-client.ftl @@ -16,6 +16,7 @@ arg-connect-timeout = Versucht die TCP-Verbindung so viele Sekunden lang erneut. arg-max-seconds = Bricht nach dieser Anzahl von Sekunden ab. arg-input-script = Spielt einmal eine vorgefertigte Eingabesequenz ab, beginnend mit dem ersten Videobild: tap, drag, circle, touch2, scroll oder keys (Strg+Leertaste während eines Stiftstrichs gehalten; benötigt einen Host mit KEYS). arg-exit-after-script = Beendet kurz nach Ende des Eingabeskripts. +arg-audio = Meldet die Wiedergabe des PC-Audios: Der Host streamt sein Audio, die Pakete werden gezählt (nicht abgespielt). err-dump-frame = --dump-frame N muss eine Zahl sein err-connecting = Verbindung zu { $addr } @@ -28,6 +29,8 @@ summary-clock = Uhr: Versatz { $offset } us, RTT { $rtt } us summary-decode = Dekodierung: p50 { $p50 } us, p95 { $p95 } us summary-e2e = Ende-zu-Ende (Host-Aufnahme -> dekodiert): p50 { $p50 } us, p95 { $p95 } us summary-input = Eingabe: Status { $status }, Optionen { $options }, gesendet { $sent } +summary-audio = Audio: { $stream }, { $packets } Pakete empfangen ({ $bytes } Bytes) +summary-audio-off = aus arg-wifi = Verbindet sich per WLAN (QUIC) mit dem Host unter ADDR statt per TCP. Ein bereits gekoppelter Host braucht nichts weiter; beim ersten Mal gib `--pin` an (oder nutze `--uri`). arg-pin = Koppelt zuerst mit dem Host über die 6-stellige PIN, die er anzeigt. diff --git a/crates/ezd-fake-client/i18n/en/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/en/ezd-fake-client.ftl index b79ba16..0d782c7 100644 --- a/crates/ezd-fake-client/i18n/en/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/en/ezd-fake-client.ftl @@ -16,6 +16,7 @@ arg-connect-timeout = Keep retrying the TCP connection for this many seconds. arg-max-seconds = Abort after this many seconds. arg-input-script = Play a canned input sequence once, starting at the first video frame: tap, drag, circle, touch2, scroll, or keys (Ctrl+Space held during a pen stroke; needs a host with KEYS). arg-exit-after-script = Exit shortly after the input script finished. +arg-audio = Advertise PC audio playback: the host streams its audio and the packets are counted (not played). err-dump-frame = --dump-frame N must be a number err-connecting = connecting to { $addr } @@ -28,6 +29,8 @@ summary-clock = clock: offset { $offset } us, rtt { $rtt } us summary-decode = decode: p50 { $p50 } us, p95 { $p95 } us summary-e2e = e2e (host capture -> decoded): p50 { $p50 } us, p95 { $p95 } us summary-input = input: status { $status }, options { $options }, sent { $sent } +summary-audio = audio: { $stream }, received { $packets } packets ({ $bytes } bytes) +summary-audio-off = off arg-wifi = Connect to the host over Wi-Fi (QUIC) at ADDR instead of TCP. A paired host needs nothing else; the first time add `--pin` (or use `--uri`). arg-pin = Pair with the host first, using the 6-digit PIN it shows. diff --git a/crates/ezd-fake-client/i18n/es/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/es/ezd-fake-client.ftl index 01fc992..1076a26 100644 --- a/crates/ezd-fake-client/i18n/es/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/es/ezd-fake-client.ftl @@ -16,6 +16,7 @@ arg-connect-timeout = Sigue reintentando la conexión TCP durante este número d arg-max-seconds = Aborta tras este número de segundos. arg-input-script = Reproduce una vez una secuencia de entrada predefinida, desde el primer fotograma de video: tap, drag, circle, touch2, scroll o keys (Ctrl+Espacio mantenido durante un trazo del lápiz; requiere un host con KEYS). arg-exit-after-script = Termina poco después de acabar el guion de entrada. +arg-audio = Anuncia la reproducción del audio de la PC: el host envía su audio y los paquetes se cuentan (no se reproducen). err-dump-frame = --dump-frame N debe ser un número err-connecting = conectando con { $addr } @@ -28,6 +29,8 @@ summary-clock = reloj: desfase { $offset } us, rtt { $rtt } us summary-decode = decodificación: p50 { $p50 } us, p95 { $p95 } us summary-e2e = extremo a extremo (captura del host -> decodificado): p50 { $p50 } us, p95 { $p95 } us summary-input = entrada: estado { $status }, opciones { $options }, enviados { $sent } +summary-audio = audio: { $stream }, recibidos { $packets } paquetes ({ $bytes } bytes) +summary-audio-off = desactivado arg-wifi = Se conecta al host por Wi-Fi (QUIC) en ADDR en lugar de TCP. Un host ya vinculado no necesita nada más; la primera vez agregá `--pin` (o usá `--uri`). arg-pin = Se vincula primero con el host usando el PIN de 6 dígitos que muestra. diff --git a/crates/ezd-fake-client/i18n/fr/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/fr/ezd-fake-client.ftl index 52aee3a..4d8a306 100644 --- a/crates/ezd-fake-client/i18n/fr/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/fr/ezd-fake-client.ftl @@ -16,6 +16,7 @@ arg-connect-timeout = Réessaie la connexion TCP pendant ce nombre de secondes. arg-max-seconds = Abandonne après ce nombre de secondes. arg-input-script = Joue une fois une séquence d'entrée prédéfinie, dès la première image vidéo : tap, drag, circle, touch2, scroll ou keys (Ctrl+Espace maintenu pendant un tracé du stylet ; nécessite un hôte avec KEYS). arg-exit-after-script = Quitte peu après la fin du script d'entrée. +arg-audio = Annonce la lecture de l'audio du PC : l'hôte diffuse son audio et les paquets sont comptés (pas lus). err-dump-frame = --dump-frame N doit être un nombre err-connecting = connexion à { $addr } @@ -28,6 +29,8 @@ summary-clock = horloge : décalage { $offset } us, rtt { $rtt } us summary-decode = décodage : p50 { $p50 } us, p95 { $p95 } us summary-e2e = de bout en bout (capture hôte -> décodé) : p50 { $p50 } us, p95 { $p95 } us summary-input = entrée : état { $status }, options { $options }, envoyés { $sent } +summary-audio = audio : { $stream }, { $packets } paquets reçus ({ $bytes } octets) +summary-audio-off = désactivé arg-wifi = Se connecte à l'hôte en Wi-Fi (QUIC) à ADDR au lieu de TCP. Un hôte déjà appairé n'a besoin de rien d'autre ; la première fois, ajoutez `--pin` (ou utilisez `--uri`). arg-pin = Appaire d'abord avec l'hôte grâce au code PIN à 6 chiffres qu'il affiche. diff --git a/crates/ezd-fake-client/i18n/it/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/it/ezd-fake-client.ftl index b34d8f5..8624cef 100644 --- a/crates/ezd-fake-client/i18n/it/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/it/ezd-fake-client.ftl @@ -16,6 +16,7 @@ arg-connect-timeout = Continua a ritentare la connessione TCP per questo numero arg-max-seconds = Interrompe dopo questo numero di secondi. arg-input-script = Riproduce una volta una sequenza di input predefinita, a partire dal primo fotogramma video: tap, drag, circle, touch2, scroll o keys (Ctrl+Spazio tenuto durante un tratto della penna; richiede un host con KEYS). arg-exit-after-script = Esce poco dopo la fine dello script di input. +arg-audio = Annuncia la riproduzione dell'audio del PC: l'host trasmette il suo audio e i pacchetti vengono contati (non riprodotti). err-dump-frame = --dump-frame N deve essere un numero err-connecting = connessione a { $addr } @@ -28,6 +29,8 @@ summary-clock = orologio: scarto { $offset } us, rtt { $rtt } us summary-decode = decodifica: p50 { $p50 } us, p95 { $p95 } us summary-e2e = end-to-end (cattura host -> decodificato): p50 { $p50 } us, p95 { $p95 } us summary-input = input: stato { $status }, opzioni { $options }, inviati { $sent } +summary-audio = audio: { $stream }, ricevuti { $packets } pacchetti ({ $bytes } byte) +summary-audio-off = disattivato arg-wifi = Si connette all'host via Wi-Fi (QUIC) a ADDR invece che con TCP. Un host già associato non richiede altro; la prima volta aggiungi `--pin` (o usa `--uri`). arg-pin = Si associa prima all'host usando il PIN a 6 cifre che mostra. diff --git a/crates/ezd-fake-client/i18n/pt/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/pt/ezd-fake-client.ftl index 590c698..577c4e0 100644 --- a/crates/ezd-fake-client/i18n/pt/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/pt/ezd-fake-client.ftl @@ -16,6 +16,7 @@ arg-connect-timeout = Continua tentando a conexão TCP por este número de segun arg-max-seconds = Aborta após este número de segundos. arg-input-script = Reproduz uma vez uma sequência de entrada predefinida, a partir do primeiro quadro de vídeo: tap, drag, circle, touch2, scroll ou keys (Ctrl+Espaço mantido durante um traço da caneta; requer um host com KEYS). arg-exit-after-script = Sai logo após o término do roteiro de entrada. +arg-audio = Anuncia a reprodução do áudio do PC: o host envia o áudio e os pacotes são contados (não reproduzidos). err-dump-frame = --dump-frame N deve ser um número err-connecting = conectando a { $addr } @@ -28,6 +29,8 @@ summary-clock = relógio: deslocamento { $offset } us, rtt { $rtt } us summary-decode = decodificação: p50 { $p50 } us, p95 { $p95 } us summary-e2e = ponta a ponta (captura do host -> decodificado): p50 { $p50 } us, p95 { $p95 } us summary-input = entrada: estado { $status }, opções { $options }, enviados { $sent } +summary-audio = áudio: { $stream }, { $packets } pacotes recebidos ({ $bytes } bytes) +summary-audio-off = desativado arg-wifi = Conecta ao host por Wi-Fi (QUIC) em ADDR em vez de TCP. Um host já pareado não precisa de mais nada; na primeira vez, adicione `--pin` (ou use `--uri`). arg-pin = Pareia primeiro com o host usando o PIN de 6 dígitos que ele mostra. diff --git a/crates/ezd-fake-client/src/lib.rs b/crates/ezd-fake-client/src/lib.rs index 94c1022..be807bc 100644 --- a/crates/ezd-fake-client/src/lib.rs +++ b/crates/ezd-fake-client/src/lib.rs @@ -19,8 +19,9 @@ use ezd_ffmpeg::{Decoder, FfmpegError}; use ezd_host_core::synthetic::read_frame_number; use ezd_link::{Link, MuxError}; use ezd_proto::{ - ClientCaps, ClientOptions, Codec, CursorShape, CursorState, DecoderCaps, DisplayInfo, - InputStatus, StreamConfig, VideoFrame, + AudioCodec, AudioConfig, AudioPacket, AudioStream, ClientAudio, ClientCaps, ClientOptions, + Codec, CursorShape, CursorState, DecoderCaps, DisplayInfo, InputStatus, SampleFormat, + StreamConfig, VideoFrame, }; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -61,6 +62,9 @@ pub struct FakeClientConfig { pub stop_after_script: bool, /// Protocol minor to advertise in `Hello` instead of ours (to test older clients). pub proto_minor: Option, + /// Advertise PC audio playback (48 kHz stereo PCM or Opus); the packets are counted, not + /// played. Off sends `ClientAudio::none()`. + pub audio: bool, } impl Default for FakeClientConfig { @@ -81,11 +85,26 @@ impl Default for FakeClientConfig { input_script: None, stop_after_script: false, proto_minor: None, + audio: false, } } } impl FakeClientConfig { + /// The `ClientAudio` this client advertises. + pub fn client_audio(&self) -> ClientAudio { + if !self.audio { + return ClientAudio::none(); + } + ClientAudio { + enabled: true, + sample_rates: vec![48_000], + max_channels: 2, + formats: vec![SampleFormat::S16, SampleFormat::F32], + codecs: vec![AudioCodec::Pcm, AudioCodec::Opus], + } + } + /// The `ClientCaps` this client advertises: one software H.264 decoder named "ffmpeg". pub fn caps(&self) -> ClientCaps { ClientCaps { @@ -179,6 +198,12 @@ pub struct ClientReport { pub client_options: Option, /// Input messages sent to the host. pub inputs_sent: u64, + /// The audio stream the host negotiated; `None` when there is no audio in the session. + pub audio: Option, + /// Audio packets received. + pub audio_packets: u64, + /// Audio payload bytes received. + pub audio_bytes: u64, } impl ClientReport { @@ -228,6 +253,18 @@ impl ClientReport { p50 = self.e2e_us_p50.to_string(), p95 = self.e2e_us_p95.to_string() ), + fl!( + "summary-audio", + stream = match &self.audio { + Some(a) => format!( + "{:?} {:?} {} Hz {} ch", + a.codec, a.format, a.sample_rate, a.channels + ), + None => fl!("summary-audio-off"), + }, + packets = self.audio_packets.to_string(), + bytes = self.audio_bytes.to_string() + ), fl!( "summary-input", status = format!("{:?}", self.input_status), @@ -342,6 +379,15 @@ impl ClientHandler for Handler { Ok(Flow::Continue) } + fn on_audio_config(&mut self, cfg: &AudioConfig) { + self.report.audio = cfg.stream.clone(); + } + + fn on_audio_packet(&mut self, pkt: AudioPacket) { + self.report.audio_packets += 1; + self.report.audio_bytes += pkt.data.len() as u64; + } + fn on_codec_config(&mut self, v: VideoFrame) -> HandlerResult { self.report.codec_configs += 1; if self.have_keyframe { @@ -419,6 +465,7 @@ pub async fn run_fake_client( core_cfg.app_version = env!("CARGO_PKG_VERSION").into(); core_cfg.proto_minor = cfg.proto_minor; + core_cfg.audio = Some(cfg.client_audio()); let handle = ClientHandle::new(); let (start_script, script_task) = match cfg.input_script { @@ -479,6 +526,9 @@ pub async fn run_fake_client( input_status: None, client_options: None, inputs_sent: 0, + audio: None, + audio_packets: 0, + audio_bytes: 0, }, all_decode: Vec::new(), all_e2e: Vec::new(), diff --git a/crates/ezd-fake-client/src/main.rs b/crates/ezd-fake-client/src/main.rs index 3e02f44..4a3a37c 100644 --- a/crates/ezd-fake-client/src/main.rs +++ b/crates/ezd-fake-client/src/main.rs @@ -38,6 +38,8 @@ struct Args { input_script: Option, #[arg(long)] exit_after_script: bool, + #[arg(long)] + audio: bool, #[arg(long, hide = true)] proto_minor: Option, #[cfg(feature = "wifi")] @@ -85,7 +87,8 @@ fn command(l: &FluentLanguageLoader) -> Command { .mut_arg("input_script", |a| a.help(fl!(l, "arg-input-script"))) .mut_arg("exit_after_script", |a| { a.help(fl!(l, "arg-exit-after-script")) - }); + }) + .mut_arg("audio", |a| a.help(fl!(l, "arg-audio"))); #[cfg(feature = "wifi")] let cmd = cmd .mut_arg("wifi", |a| a.help(fl!(l, "arg-wifi"))) @@ -136,6 +139,7 @@ async fn main() -> Result<()> { }), stop_after_script: a.exit_after_script, proto_minor: a.proto_minor, + audio: a.audio, ..Default::default() }; diff --git a/crates/ezd-fake-client/tests/loopback.rs b/crates/ezd-fake-client/tests/loopback.rs index d6a43d2..e72e5d7 100644 --- a/crates/ezd-fake-client/tests/loopback.rs +++ b/crates/ezd-fake-client/tests/loopback.rs @@ -2,9 +2,10 @@ use ezd_fake_client::{ClientEnd, FakeClientConfig, run_fake_client}; use ezd_ffmpeg::{SwVideoSource, encoder_available}; -use ezd_host_core::{EndReason, run_host_session}; +use ezd_host_core::testing::ScriptedAudioBackend; +use ezd_host_core::{EndReason, SessionOptions, run_host_session, run_host_session_with}; use ezd_link::Link; -use ezd_proto::Prefs; +use ezd_proto::{AudioCodec, Prefs, SampleFormat}; use std::time::Duration; use tokio::net::TcpListener; use tokio::time::timeout; @@ -123,3 +124,74 @@ async fn host_to_fake_client_over_tcp() { "client stats reached the host" ); } + +/// Runs a short session with the scripted audio backend; `audio` is what the client advertises. +async fn audio_loopback( + audio: bool, +) -> Option<(ezd_fake_client::ClientReport, ezd_host_core::SessionSummary)> { + if !encoder_available("libx264") && !encoder_available("libopenh264") { + eprintln!("no software H.264 encoder, skipping"); + return None; + } + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let host = tokio::spawn(async move { + let (sock, _) = listener.accept().await.unwrap(); + let link = Link::from_tcp(sock).unwrap(); + let opts = SessionOptions { + audio: Some(std::sync::Arc::new(ScriptedAudioBackend::new())), + ..Default::default() + }; + run_host_session_with(link, Box::new(SwVideoSource::new()), Prefs::default(), opts).await + }); + let cfg = FakeClientConfig { + width: 640, + height: 360, + refresh_rates_mhz: vec![60_000], + frames: Some(90), + audio, + ..Default::default() + }; + let link = Link::connect_tcp(addr).await.unwrap(); + let report = timeout(Duration::from_secs(60), run_fake_client(link, cfg)) + .await + .unwrap() + .unwrap(); + let summary = timeout(Duration::from_secs(10), host) + .await + .unwrap() + .unwrap() + .unwrap(); + println!("{}", report.summary()); + Some((report, summary)) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn audio_packets_reach_the_fake_client() { + let Some((report, summary)) = audio_loopback(true).await else { + return; + }; + let a = report.audio.as_ref().expect("audio negotiated"); + // The default link is USB, where a client that plays F32 gets uncompressed F32. + assert_eq!( + (a.codec, a.format, a.channels, a.sample_rate), + (AudioCodec::Pcm, SampleFormat::F32, 2, 48_000) + ); + assert!(report.audio_packets >= 10, "{}", report.audio_packets); + assert_eq!(report.audio_bytes, report.audio_packets * 480 * 2 * 4); + assert!(summary.audio_packets >= report.audio_packets); + assert_eq!(summary.audio, report.audio); + assert_eq!(summary.end, EndReason::ClientBye("client done".into())); + assert!(report.summary().contains("audio:")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn client_without_audio_gets_none() { + let Some((report, summary)) = audio_loopback(false).await else { + return; + }; + assert_eq!(report.audio, None); + assert_eq!(report.audio_packets, 0); + assert_eq!(summary.audio, None); + assert_eq!(summary.audio_packets, 0); +} diff --git a/crates/ezd-host-core/src/lib.rs b/crates/ezd-host-core/src/lib.rs index 473db9e..f2a6312 100644 --- a/crates/ezd-host-core/src/lib.rs +++ b/crates/ezd-host-core/src/lib.rs @@ -1,14 +1,16 @@ //! Platform-independent host core. //! //! Platform crates (Linux KDE, Windows) implement [`VideoSource`] (capture + -//! encode of a virtual display) and [`VirtualDisplay`]; this crate provides the +//! encode of a virtual display), [`VirtualDisplay`] and [`AudioSource`] (PC audio capture); this crate provides the //! session state machine ([`run_host_session`]) that drives them over a //! [`ezd_link::Link`]. //! //! Threading model: the session is async, but the source is owned by one //! dedicated OS thread that loops `wait for mux video slot -> next_frame -> send`. //! `next_frame` therefore always runs *after* the link has signalled it can -//! take a frame and must return the newest content (coalescing). +//! take a frame and must return the newest content (coalescing). Audio has its own thread +//! (`AudioSource::next_samples` -> re-chunk -> PCM or Opus -> mux audio lane); its failure never +//! ends the video session. pub mod input; pub mod platform; @@ -25,8 +27,8 @@ pub use session::{ run_host_session, run_host_session_with, }; pub use source::{ - CursorSink, DisplayMode, EncodedFrame, OutputInfo, SourceError, SourceHdr, VideoSource, - VirtualDisplay, + AudioBackend, AudioBlock, AudioEncoder, AudioSource, CursorSink, DisplayMode, EncodedFrame, + OutputInfo, SourceError, SourceHdr, VideoSource, VirtualDisplay, }; pub use synthetic::{PixelFormat, SyntheticFrames, read_frame_number}; pub use time::now_us; diff --git a/crates/ezd-host-core/src/platform.rs b/crates/ezd-host-core/src/platform.rs index 16374d7..2189e62 100644 --- a/crates/ezd-host-core/src/platform.rs +++ b/crates/ezd-host-core/src/platform.rs @@ -5,9 +5,12 @@ //! ```ignore //! pub fn create_virtual_source(opts: &PlatformOptions) //! -> Result, PlatformError>; +//! pub fn create_audio_source(opts: &PlatformOptions) -> Option>; //! pub fn diag_section() -> serde_json::Value; //! ``` -//! so `ezd-host` can call them without knowing the platform. +//! so `ezd-host` can call them without knowing the platform. `create_audio_source` returns +//! `None` when the PC audio cannot be captured; the host wraps it in an +//! [`AudioBackend`](crate::AudioBackend) for the sessions. /// Options chosen on the command line that platform backends interpret. #[derive(Debug, Clone, Default, PartialEq, Eq)] diff --git a/crates/ezd-host-core/src/session.rs b/crates/ezd-host-core/src/session.rs index 63b8c41..c197f58 100644 --- a/crates/ezd-host-core/src/session.rs +++ b/crates/ezd-host-core/src/session.rs @@ -6,8 +6,10 @@ //! |<------------------- Hello --| //! |-- ClientCaps -------------->| //! |-- ClientHdr (minor >= 5) --->| the host waits for it when the client speaks minor >= 5 +//! |-- ClientAudio (minor >= 6) ->| likewise; the host waits for it //! |<--------------- HostCaps ---| //! |<------------ StreamColor ---| minor >= 5 only +//! |<------------ AudioConfig ---| minor >= 6 only; `stream: None` = no audio //! |<------------- StreamConfig -| negotiate_hdr(ClientCaps, HostCaps, Prefs, ..) //! |-- ConfigAck (decoder ready)>| //! |<== CODEC_CONFIG, keyframe ==| source.start() runs after the ack @@ -16,18 +18,23 @@ //! ``` //! A rejected handshake is answered with `Reject` and returned as an error. When the host shuts //! down ([`SessionOptions::shutdown`]) it sends `Bye` and stops the source before returning. +//! Audio runs on its own thread once the client acked the configuration; if it fails the session +//! carries on without sound. use crate::input::{ InputBackend, InputCmd, InputContext, InputStats, LiveLatency, PenMode, SharedOffset, input_thread, }; -use crate::source::{CursorSink, SourceError, SourceHdr, VideoSource}; +use crate::source::{ + AudioBackend, AudioEncoder, AudioSource, CursorSink, SourceError, SourceHdr, VideoSource, +}; use crate::time::now_us; use ezd_link::{Link, MuxClosed, MuxError, MuxOptions, MuxReceiver, MuxSender, VideoCredit}; use ezd_proto::{ - Bye, ClientCaps, ClientHdr, ClientOptions, ClientStats, ClockSync, Codec, DEFAULT_CHUNK_LEN, - DynamicRange, Hello, HostCaps, InputStatus, Message, Ping, Pong, Prefs, ProtoError, Reject, - Role, StreamColor, StreamConfig, VideoFrame, negotiate_hdr, + AudioCodec, AudioConfig, AudioPacket, AudioStream, Bye, ClientAudio, ClientCaps, ClientHdr, + ClientOptions, ClientStats, ClockSync, Codec, DEFAULT_CHUNK_LEN, DynamicRange, Hello, + HostAudio, HostCaps, InputStatus, Message, Ping, Pong, Prefs, ProtoError, Reject, Role, + SampleFormat, StreamColor, StreamConfig, VideoFrame, negotiate_audio, negotiate_hdr, }; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -64,6 +71,9 @@ pub struct SessionOptions { pub telemetry: bool, /// Receives live [`SessionEvent`]s (default: none). Send errors are ignored. pub events: Option>, + /// Creates the audio source and Opus encoder of each session; `None` means no audio (the + /// client is told when it speaks protocol 0.6). + pub audio: Option>, } /// How often [`SessionEvent::Stats`] is emitted. @@ -82,6 +92,8 @@ pub enum SessionEvent { color: StreamColor, /// Dynamic range decision (HDR10, tone-mapped SDR with the reason, or SDR). dynamic_range: DynamicRange, + /// The audio stream, or `None` when the session has no audio. + audio: Option, }, /// Measurements of the last window, once per second. Stats(LiveStats), @@ -129,6 +141,7 @@ impl Default for SessionOptions { pen_mode: PenMode::default(), telemetry: false, events: None, + audio: None, } } } @@ -161,6 +174,8 @@ pub struct SessionSummary { pub color: StreamColor, /// Dynamic range decision of the session. pub dynamic_range: DynamicRange, + /// The audio stream negotiated for the session, if any. + pub audio: Option, /// Why the session ended. pub end: EndReason, /// Wall time from handshake start to the end. @@ -184,6 +199,12 @@ pub struct SessionSummary { pub transfer_us_p95: Option, /// Video frames whose ack timed out. pub ack_timeouts: u64, + /// Audio packets handed to the mux. + pub audio_packets: u64, + /// Total audio payload bytes handed to the mux. + pub audio_bytes: u64, + /// Audio packets the mux dropped because the link fell behind. + pub audio_dropped: u64, /// Estimated `client_clock - host_clock` in microseconds. pub clock_offset_us: Option, /// Best RTT in microseconds. @@ -235,10 +256,13 @@ struct VideoShared { keyframes: AtomicU64, configs: AtomicU64, bytes: AtomicU64, + audio_packets: AtomicU64, + audio_bytes: AtomicU64, /// `(frame_id, capture_us, encoded_us)` of sent frames awaiting their ack (bounded). sent: std::sync::Mutex>, } +/// `stop` ends the audio thread as well as the video one. struct StopOnDrop(Arc); impl Drop for StopOnDrop { fn drop(&mut self) { @@ -283,13 +307,19 @@ pub async fn run_host_session_with( source_hdr: source.source_hdr(), main10: source.main10_codecs(), }; - let setup = handshake(&tx, &mut rx, &host_caps, &prefs, &opts, &hdr_in).await; + let audio_source = opts.audio.as_ref().and_then(|b| b.create_source()); + let audio_in = AudioInputs { + backend: opts.audio.clone(), + has_source: audio_source.is_some(), + }; + let setup = handshake(&tx, &mut rx, &host_caps, &prefs, &opts, &hdr_in, &audio_in).await; let Negotiated { hello: client_hello, caps: client_caps, cfg, color, range: dynamic_range, + audio, } = match setup { Ok(v) => v, Err(e) => { @@ -299,6 +329,7 @@ pub async fn run_host_session_with( } }; tracing::info!(?cfg, ?dynamic_range, client = %client_hello.device_name, "stream configured"); + tracing::info!(audio = ?audio.as_ref().map(|a| &a.stream), "audio configured"); emit( &opts.events, SessionEvent::Configured { @@ -306,6 +337,7 @@ pub async fn run_host_session_with( stream: cfg.clone(), color, dynamic_range, + audio: audio.as_ref().map(|a| a.stream.clone()), }, ); @@ -381,6 +413,24 @@ pub async fn run_host_session_with( })?; } + // Audio thread: its failure ends the audio only. + let mut audio_join: Option> = None; + let audio_stream = audio.as_ref().map(|a| a.stream.clone()); + if let (Some(a), Some(src)) = (audio, audio_source) { + let (shared, tx) = (shared.clone(), tx.clone()); + let poll = opts.poll_interval; + audio_join = Some(std::thread::Builder::new().name("ezd-audio".into()).spawn( + move || { + let mut src = src; + match audio_loop(src.as_mut(), a.encoder, &a.stream, &tx, &shared, poll) { + Ok(()) => {} + Err(e) => tracing::warn!("audio stopped, video goes on: {e}"), + } + src.stop(); + }, + )?); + } + // Control loop. let mut clock = ClockSync::default(); let mut ping_seq = 0u32; @@ -572,6 +622,16 @@ pub async fn run_host_session_with( if !video_done && timeout(Duration::from_secs(5), &mut done_rx).await.is_err() { tracing::warn!("video source did not stop in time"); } + if let Some(j) = audio_join + && timeout( + Duration::from_secs(2), + tokio::task::spawn_blocking(|| j.join()), + ) + .await + .is_err() + { + tracing::warn!("audio thread did not stop in time"); + } // The input thread releases everything that is down before we return. let input_stats = match input_join { Some(j) => match timeout( @@ -592,7 +652,8 @@ pub async fn run_host_session_with( }, None => InputStats::default(), }; - let ack_timeouts = tx.stats().ack_timeouts; + let mux_stats = tx.stats(); + let ack_timeouts = mux_stats.ack_timeouts; drop(tx); if timeout(Duration::from_secs(2), task.join()).await.is_err() { tracing::warn!("mux writer did not drain in time"); @@ -603,6 +664,7 @@ pub async fn run_host_session_with( stream: cfg, color, dynamic_range, + audio: audio_stream, end, duration: started.elapsed(), frames_sent: shared.frames.load(Ordering::Relaxed), @@ -614,6 +676,9 @@ pub async fn run_host_session_with( transfer_us_p50: (!transfer_all.is_empty()).then(|| percentile(&transfer_all, 50)), transfer_us_p95: (!transfer_all.is_empty()).then(|| percentile(&transfer_all, 95)), ack_timeouts, + audio_packets: shared.audio_packets.load(Ordering::Relaxed), + audio_bytes: shared.audio_bytes.load(Ordering::Relaxed), + audio_dropped: mux_stats.audio_dropped, clock_offset_us: clock.offset_us(), rtt_us: clock.rtt_us(), input_events: input_stats.events, @@ -705,6 +770,20 @@ struct HostHdrInputs { main10: Vec, } +/// What the host offers for audio, queried before the handshake. +struct AudioInputs { + backend: Option>, + /// The backend produced a capture source for this session. + has_source: bool, +} + +/// The negotiated audio of a session. +struct NegotiatedAudio { + stream: AudioStream, + /// The Opus encoder, for [`AudioCodec::Opus`] streams. + encoder: Option>, +} + /// Outcome of a successful handshake. struct Negotiated { hello: Hello, @@ -712,6 +791,7 @@ struct Negotiated { cfg: StreamConfig, color: StreamColor, range: DynamicRange, + audio: Option, } async fn handshake( @@ -721,6 +801,7 @@ async fn handshake( prefs: &Prefs, opts: &SessionOptions, hdr_in: &HostHdrInputs, + audio_in: &AudioInputs, ) -> Result { let wait = opts.handshake_timeout; tx.send_control(Message::Hello(Hello::new( @@ -755,6 +836,18 @@ async fn handshake( } else { None }; + // Likewise a client with minor >= 6 sends `ClientAudio` (possibly `none()`). + let client_audio: Option = if hello.supports_audio() { + Some( + expect(rx, tx, "ClientAudio", wait, |m| match m { + Message::ClientAudio(a) => Ok(a), + o => Err(o), + }) + .await?, + ) + } else { + None + }; tx.send_control(Message::HostCaps(host_caps.clone()))?; let (cfg, range) = match negotiate_hdr( &caps, @@ -777,6 +870,19 @@ async fn handshake( if hello.supports_hdr() { tx.send_control(Message::StreamColor(color))?; } + let audio = match &client_audio { + Some(ca) => { + let ceiling = prefs + .max_bitrate_kbps + .unwrap_or_else(|| prefs.link.default_max_bitrate_kbps()); + let audio = pick_audio(ca, audio_in, prefs, ceiling); + tx.send_control(Message::AudioConfig(AudioConfig { + stream: audio.as_ref().map(|a| a.stream.clone()), + }))?; + audio + } + None => None, + }; tx.send_control(Message::StreamConfig(cfg.clone()))?; expect(rx, tx, "ConfigAck", wait, |m| match m { Message::ConfigAck => Ok(()), @@ -789,9 +895,50 @@ async fn handshake( cfg, color, range, + audio, }) } +/// Negotiates the audio stream and builds its Opus encoder; an encoder that cannot be built +/// falls back to PCM (or to no audio). +fn pick_audio( + client: &ClientAudio, + audio_in: &AudioInputs, + prefs: &Prefs, + ceiling_kbps: u32, +) -> Option { + let mut host = HostAudio { + available: audio_in.has_source, + opus: audio_in + .backend + .as_ref() + .is_some_and(|b| b.opus_available()), + }; + loop { + let mut stream = negotiate_audio(prefs.audio, client, &host, prefs.link, ceiling_kbps)?; + if stream.codec != AudioCodec::Opus { + return Some(NegotiatedAudio { + stream, + encoder: None, + }); + } + let backend = audio_in.backend.as_ref()?; + match backend.create_opus_encoder(stream.channels) { + Ok(enc) => { + stream.codec_config = enc.codec_config(); + return Some(NegotiatedAudio { + stream, + encoder: Some(enc), + }); + } + Err(e) => { + tracing::warn!("cannot create the Opus encoder: {e}"); + host.opus = false; + } + } + } +} + fn reject(tx: &MuxSender, reason: String) { let _ = tx.send_control(Message::Reject(Reject { reason })); } @@ -883,11 +1030,77 @@ fn video_loop( } } +/// Pumps captured audio to the client until stopped; returns the failure of the source or the +/// encoder. Samples are re-chunked into frames of `frame_samples` per channel, stamped with the +/// capture time of their first sample (continuing from the previous frame unless the source +/// had a gap). +fn audio_loop( + source: &mut dyn AudioSource, + mut encoder: Option>, + stream: &AudioStream, + tx: &MuxSender, + sh: &VideoShared, + poll: Duration, +) -> Result<(), SourceError> { + source.start(stream.channels)?; + let frame_len = stream.frame_samples as usize * stream.channels as usize; + let frame_us = stream.frame_samples as u64 * 1_000_000 / stream.sample_rate.max(1) as u64; + let mut pending: Vec = Vec::with_capacity(frame_len * 2); + let mut next_pts = 0u64; + while !sh.stop.load(Ordering::Acquire) { + let Some(block) = source.next_samples(poll)? else { + continue; + }; + if pending.is_empty() { + next_pts = block.pts_us; + } + pending.extend_from_slice(&block.samples); + let mut used = 0; + while pending.len() - used >= frame_len { + let pcm = &pending[used..used + frame_len]; + used += frame_len; + let data = match (stream.codec, encoder.as_mut()) { + (AudioCodec::Opus, Some(enc)) => enc.encode(pcm)?, + (AudioCodec::Opus, None) => return Err(SourceError::msg("no Opus encoder")), + (AudioCodec::Pcm, _) => pcm_bytes(pcm, stream.format), + }; + let pts_us = next_pts; + next_pts += frame_us; + let len = data.len() as u64; + if tx + .send(&Message::AudioPacket(AudioPacket { pts_us, data })) + .is_err() + { + return Ok(()); + } + sh.audio_packets.fetch_add(1, Ordering::Relaxed); + sh.audio_bytes.fetch_add(len, Ordering::Relaxed); + } + pending.drain(..used); + } + Ok(()) +} + +/// Little-endian interleaved bytes of `pcm` in `format` (S16 clamps and rounds). +fn pcm_bytes(pcm: &[f32], format: SampleFormat) -> Vec { + let mut out = Vec::with_capacity(pcm.len() * format.bytes()); + for &v in pcm { + match format { + SampleFormat::F32 => out.extend_from_slice(&v.to_le_bytes()), + SampleFormat::S16 => { + let s = (v.clamp(-1.0, 1.0) * 32767.0).round() as i16; + out.extend_from_slice(&s.to_le_bytes()); + } + } + } + out +} + #[cfg(test)] mod tests { use super::*; use crate::source::EncodedFrame; - use crate::testing::ScriptedSource; + use crate::testing::{ScriptedAudioBackend, ScriptedSource}; use ezd_proto::{Codec, DecoderCaps, DisplayInfo}; fn caps() -> ClientCaps { @@ -944,12 +1157,18 @@ mod tests { tx.send_control(Message::ClientCaps(caps())).unwrap(); tx.send_control(Message::ClientHdr(ClientHdr::none())) .unwrap(); + tx.send_control(Message::ClientAudio(ClientAudio::none())) + .unwrap(); assert!(matches!(recv(&mut rx).await, Message::Hello(h) if h.role == Role::Host)); assert!(matches!(recv(&mut rx).await, Message::HostCaps(_))); assert_eq!( recv(&mut rx).await, Message::StreamColor(StreamColor::sdr_bt709()) ); + assert_eq!( + recv(&mut rx).await, + Message::AudioConfig(AudioConfig { stream: None }) + ); let cfg = match recv(&mut rx).await { Message::StreamConfig(c) => c, m => panic!("{m:?}"), @@ -1089,6 +1308,8 @@ mod tests { tx.send_control(Message::ClientCaps(c)).unwrap(); tx.send_control(Message::ClientHdr(ClientHdr::none())) .unwrap(); + tx.send_control(Message::ClientAudio(ClientAudio::none())) + .unwrap(); assert!(matches!(recv(&mut rx).await, Message::Hello(_))); assert!(matches!(recv(&mut rx).await, Message::HostCaps(_))); assert!(matches!(recv(&mut rx).await, Message::Reject(_))); @@ -1177,6 +1398,10 @@ mod tests { if let Some(h) = client_hdr { tx.send_control(Message::ClientHdr(h)).unwrap(); } + if minor >= ezd_proto::MINOR_AUDIO { + tx.send_control(Message::ClientAudio(ClientAudio::none())) + .unwrap(); + } let mut seen = Vec::new(); loop { let m = recv(&mut rx).await; @@ -1204,7 +1429,8 @@ mod tests { let want = StreamColor::hdr10(None, 1000, 400); assert!(matches!(seen[1], Message::HostCaps(_))); assert_eq!(seen[2], Message::StreamColor(want)); - assert!(matches!(&seen[3], Message::StreamConfig(c) if c.codec == Codec::Hevc)); + assert!(matches!(seen[3], Message::AudioConfig(_))); + assert!(matches!(&seen[4], Message::StreamConfig(c) if c.codec == Codec::Hevc)); assert_eq!( told, Some((DynamicRange::Hdr10 { codec: Codec::Hevc }, want)) @@ -1215,7 +1441,7 @@ mod tests { async fn sdr_display_gets_sdr_color_and_the_reason() { let (seen, told) = hdr_handshake(ezd_proto::PROTOCOL_MINOR, Some(ClientHdr::none())).await; assert_eq!(seen[2], Message::StreamColor(StreamColor::sdr_bt709())); - assert!(matches!(&seen[3], Message::StreamConfig(c) if c.codec == Codec::H264)); + assert!(matches!(&seen[4], Message::StreamConfig(c) if c.codec == Codec::H264)); assert_eq!( told, Some(( @@ -1295,7 +1521,9 @@ mod tests { tx.send_control(Message::ClientCaps(caps())).unwrap(); tx.send_control(Message::ClientHdr(ClientHdr::none())) .unwrap(); - for _ in 0..3 { + tx.send_control(Message::ClientAudio(ClientAudio::none())) + .unwrap(); + for _ in 0..4 { recv(&mut rx).await; } tx.send_control(Message::ConfigAck).unwrap(); @@ -1339,7 +1567,9 @@ mod tests { tx.send_control(Message::ClientCaps(caps())).unwrap(); tx.send_control(Message::ClientHdr(ClientHdr::none())) .unwrap(); - for _ in 0..3 { + tx.send_control(Message::ClientAudio(ClientAudio::none())) + .unwrap(); + for _ in 0..4 { recv(&mut rx).await; } tx.send_control(Message::ConfigAck).unwrap(); @@ -1382,7 +1612,9 @@ mod tests { tx.send_control(Message::ClientCaps(caps())).unwrap(); tx.send_control(Message::ClientHdr(ClientHdr::none())) .unwrap(); - for _ in 0..3 { + tx.send_control(Message::ClientAudio(ClientAudio::none())) + .unwrap(); + for _ in 0..4 { recv(&mut rx).await; } tx.send_control(Message::ConfigAck).unwrap(); @@ -1420,7 +1652,9 @@ mod tests { stream, color, dynamic_range, + audio, } => { + assert_eq!(audio, None); assert_eq!(color, StreamColor::sdr_bt709()); assert_eq!(dynamic_range, DynamicRange::Sdr); assert_eq!(client_hello.device_name, "tablet"); @@ -1452,4 +1686,329 @@ mod tests { .unwrap(); assert_eq!(s.end, EndReason::ClientBye("test over".into())); } + + /// What a client saw of a session with audio. + struct AudioRun { + /// Handshake messages up to and including `StreamConfig`. + seen: Vec, + packets: Vec, + frames: u32, + other_audio_messages: usize, + summary: SessionSummary, + } + + fn pcm_client() -> ClientAudio { + ClientAudio { + enabled: true, + sample_rates: vec![48000], + max_channels: 2, + formats: vec![SampleFormat::S16, SampleFormat::F32], + codecs: vec![AudioCodec::Pcm], + } + } + + /// Runs a session with `backend`, a client of `minor` that sends `client_audio` (if any), + /// and collects at least `want_packets` audio packets and a few video frames. + async fn audio_session( + minor: u16, + client_audio: Option, + prefs: Prefs, + backend: ScriptedAudioBackend, + want_packets: usize, + ) -> AudioRun { + let (host_link, client_link) = Link::duplex_pair(1 << 20); + let host = tokio::spawn(run_host_session_with( + host_link, + Box::new(ScriptedSource::new(Duration::from_millis(5), 3000)), + prefs, + SessionOptions { + audio: Some(Arc::new(backend)), + ..fast_opts() + }, + )); + let (tx, mut rx, _t) = client_link.into_mux(DEFAULT_CHUNK_LEN); + let mut hello = Hello::new(Role::Client, "t", "client"); + hello.proto_minor = minor; + tx.send_control(Message::Hello(hello)).unwrap(); + tx.send_control(Message::ClientCaps(caps())).unwrap(); + if minor >= ezd_proto::MINOR_HDR { + tx.send_control(Message::ClientHdr(ClientHdr::none())) + .unwrap(); + } + if let Some(a) = client_audio { + tx.send_control(Message::ClientAudio(a)).unwrap(); + } + let mut seen = Vec::new(); + loop { + let m = recv(&mut rx).await; + let done = matches!(m, Message::StreamConfig(_)); + seen.push(m); + if done { + break; + } + } + tx.send_control(Message::ConfigAck).unwrap(); + let (mut packets, mut frames, mut other_audio_messages) = (Vec::new(), 0u32, 0usize); + let start = Instant::now(); + while !(frames >= 5 + && packets.len() >= want_packets + && start.elapsed() >= Duration::from_millis(if want_packets == 0 { 400 } else { 0 })) + { + match recv(&mut rx).await { + Message::VideoFrame(v) => { + tx.send_control(Message::FrameAck(ezd_proto::FrameAck { + frame_id: v.frame_id, + received_us: now_us(), + })) + .unwrap(); + frames += 1; + } + Message::AudioPacket(p) => packets.push(p), + Message::AudioConfig(_) | Message::ClientAudio(_) => other_audio_messages += 1, + _ => {} + } + } + tx.send_control(Message::Bye(Bye { reason: "t".into() })) + .unwrap(); + let summary = timeout(Duration::from_secs(5), host) + .await + .unwrap() + .unwrap() + .unwrap(); + AudioRun { + seen, + packets, + frames, + other_audio_messages, + summary, + } + } + + fn audio_config(seen: &[Message]) -> Option { + seen.iter().find_map(|m| match m { + Message::AudioConfig(c) => Some(c.clone()), + _ => None, + }) + } + + #[tokio::test] + async fn pcm_audio_is_configured_before_the_stream_and_packets_have_whole_frames() { + // USB link, client plays F32: 480 samples * 2 channels * 4 bytes. + let r = audio_session( + ezd_proto::PROTOCOL_MINOR, + Some(pcm_client()), + Prefs::default(), + ScriptedAudioBackend::new(), + 12, + ) + .await; + let n = r.seen.len(); + assert!(matches!(r.seen[n - 1], Message::StreamConfig(_))); + let Message::AudioConfig(AudioConfig { stream: Some(a) }) = &r.seen[n - 2] else { + panic!("AudioConfig must precede StreamConfig: {:?}", r.seen); + }; + assert_eq!( + ( + a.codec, + a.format, + a.channels, + a.sample_rate, + a.frame_samples + ), + (AudioCodec::Pcm, SampleFormat::F32, 2, 48000, 480) + ); + assert!(a.codec_config.is_empty()); + assert!(r.packets.len() >= 12); + for p in &r.packets { + assert_eq!(p.data.len(), 480 * 2 * 4); + } + // Re-chunked from 700-sample blocks without losing or reordering samples: the ramp + // continues across packets and the time stamps advance by 10 ms. + let mut expect = 0u64; + for p in &r.packets { + for frame in p.data.chunks(8) { + let l = f32::from_le_bytes(frame[..4].try_into().unwrap()); + let rr = f32::from_le_bytes(frame[4..].try_into().unwrap()); + assert_eq!(l, (expect % 1000) as f32 / 1000.0); + assert_eq!(l, rr); + expect += 1; + } + } + for w in r.packets.windows(2) { + assert_eq!(w[1].pts_us - w[0].pts_us, 10_000); + } + assert_eq!(r.summary.audio.as_ref(), Some(a)); + assert!(r.summary.audio_packets >= 12); + assert_eq!( + r.summary.audio_bytes, + r.summary.audio_packets * (480 * 2 * 4) + ); + } + + #[tokio::test] + async fn network_link_gets_s16_pcm() { + let prefs = Prefs { + link: ezd_proto::LinkKind::Network, + ..Prefs::default() + }; + let r = audio_session( + ezd_proto::PROTOCOL_MINOR, + Some(pcm_client()), + prefs, + ScriptedAudioBackend::new(), + 3, + ) + .await; + let a = audio_config(&r.seen).unwrap().stream.unwrap(); + assert_eq!((a.codec, a.format), (AudioCodec::Pcm, SampleFormat::S16)); + for p in &r.packets { + assert_eq!(p.data.len(), 480 * 2 * 2); + } + // The ramp sample 1 / 1000 as S16. + let s = i16::from_le_bytes(r.packets[0].data[4..6].try_into().unwrap()); + assert_eq!(s, (0.001f32 * 32767.0).round() as i16); + } + + #[tokio::test] + async fn opus_audio_carries_the_encoder_config() { + let mut client = pcm_client(); + client.codecs = vec![AudioCodec::Opus]; + let r = audio_session( + ezd_proto::PROTOCOL_MINOR, + Some(client), + Prefs::default(), + ScriptedAudioBackend::new(), + 5, + ) + .await; + let a = audio_config(&r.seen).unwrap().stream.unwrap(); + assert_eq!(a.codec, AudioCodec::Opus); + assert_eq!(a.codec_config, ScriptedAudioBackend::OPUS_HEAD); + for p in &r.packets { + assert_eq!(p.data.len(), ScriptedAudioBackend::OPUS_PACKET_LEN); + } + } + + #[tokio::test] + async fn failing_opus_encoder_falls_back_to_pcm() { + let mut client = pcm_client(); + client.codecs = vec![AudioCodec::Pcm, AudioCodec::Opus]; + let prefs = Prefs { + audio: ezd_proto::AudioMode::Opus, + ..Prefs::default() + }; + let backend = ScriptedAudioBackend { + fail_opus: true, + ..ScriptedAudioBackend::new() + }; + let r = audio_session(ezd_proto::PROTOCOL_MINOR, Some(client), prefs, backend, 3).await; + let a = audio_config(&r.seen).unwrap().stream.unwrap(); + assert_eq!(a.codec, AudioCodec::Pcm); + } + + #[tokio::test] + async fn disabled_client_gets_no_audio() { + let r = audio_session( + ezd_proto::PROTOCOL_MINOR, + Some(ClientAudio::none()), + Prefs::default(), + ScriptedAudioBackend::new(), + 0, + ) + .await; + assert_eq!(audio_config(&r.seen), Some(AudioConfig { stream: None })); + assert!(r.packets.is_empty()); + assert!(r.frames >= 5); + assert_eq!(r.summary.audio, None); + } + + #[tokio::test] + async fn host_without_audio_says_so() { + let backend = ScriptedAudioBackend { + has_source: false, + ..ScriptedAudioBackend::new() + }; + let r = audio_session( + ezd_proto::PROTOCOL_MINOR, + Some(pcm_client()), + Prefs::default(), + backend, + 0, + ) + .await; + assert_eq!(audio_config(&r.seen), Some(AudioConfig { stream: None })); + assert!(r.packets.is_empty()); + } + + #[tokio::test] + async fn audio_mode_off_gets_no_audio() { + let prefs = Prefs { + audio: ezd_proto::AudioMode::Off, + ..Prefs::default() + }; + let r = audio_session( + ezd_proto::PROTOCOL_MINOR, + Some(pcm_client()), + prefs, + ScriptedAudioBackend::new(), + 0, + ) + .await; + assert_eq!(audio_config(&r.seen), Some(AudioConfig { stream: None })); + assert!(r.packets.is_empty()); + } + + #[tokio::test] + async fn audio_source_failure_keeps_the_video_running() { + let backend = ScriptedAudioBackend { + fail_start: true, + ..ScriptedAudioBackend::new() + }; + let r = audio_session( + ezd_proto::PROTOCOL_MINOR, + Some(pcm_client()), + Prefs::default(), + backend, + 0, + ) + .await; + // Audio was promised in the handshake, but never arrives; the session is unharmed. + assert!(audio_config(&r.seen).unwrap().stream.is_some()); + assert!(r.packets.is_empty()); + assert!(r.frames >= 5); + assert_eq!(r.summary.end, EndReason::ClientBye("t".into())); + assert_eq!(r.summary.audio_packets, 0); + } + + #[tokio::test] + async fn old_client_gets_no_audio_messages() { + let r = audio_session( + ezd_proto::MINOR_AUDIO - 1, + None, + Prefs::default(), + ScriptedAudioBackend::new(), + 0, + ) + .await; + assert_eq!(audio_config(&r.seen), None); + assert!(r.packets.is_empty()); + assert_eq!(r.other_audio_messages, 0); + assert!(r.frames >= 5); + assert_eq!(r.summary.audio, None); + } + + #[test] + fn s16_conversion_clamps_and_rounds() { + let b = pcm_bytes( + &[0.0, 1.0, -1.0, 2.0, -3.0, 0.5, f32::NAN], + SampleFormat::S16, + ); + let v: Vec = b + .chunks(2) + .map(|c| i16::from_le_bytes([c[0], c[1]])) + .collect(); + assert_eq!(v, vec![0, 32767, -32767, 32767, -32767, 16384, 0]); + let f = pcm_bytes(&[0.25], SampleFormat::F32); + assert_eq!(f, 0.25f32.to_le_bytes()); + } } diff --git a/crates/ezd-host-core/src/source.rs b/crates/ezd-host-core/src/source.rs index d3ef480..d1088fb 100644 --- a/crates/ezd-host-core/src/source.rs +++ b/crates/ezd-host-core/src/source.rs @@ -9,7 +9,8 @@ use std::collections::HashSet; use std::sync::{Arc, Mutex}; use std::time::Duration; -/// Error returned by [`VideoSource`] / [`VirtualDisplay`] implementations. +/// Error returned by [`VideoSource`] / [`VirtualDisplay`] / [`AudioSource`] / [`AudioEncoder`] +/// implementations. #[derive(Debug, thiserror::Error)] pub enum SourceError { /// Free-form failure description. @@ -198,6 +199,68 @@ pub trait VideoSource: Send { fn stop(&mut self); } +/// A block of captured audio, see [`AudioSource::next_samples`]. +#[derive(Debug, Clone, PartialEq)] +pub struct AudioBlock { + /// [`now_us`](crate::now_us) of the first sample (capture time). + pub pts_us: u64, + /// Interleaved samples, 48 kHz, in -1.0..=1.0. + pub samples: Vec, +} + +/// Capture of the PC audio that is streamed to the client. +/// +/// Implemented per platform. The session owns the source on a dedicated thread ("ezd-audio") and +/// calls `start`, then repeatedly `next_samples`, and finally `stop`. A failure ends the audio +/// only; the video session goes on. +pub trait AudioSource: Send { + /// Starts capturing 48 kHz audio with `channels` interleaved `f32` channels (the source + /// downmixes or upmixes as needed). + fn start(&mut self, channels: u8) -> Result<(), SourceError>; + + /// Blocks up to `timeout` for the next block of captured samples (any length; the session + /// re-chunks them). `Ok(None)` means nothing captured yet (timeout, or silence on a backend + /// that produces none). + fn next_samples(&mut self, timeout: Duration) -> Result, SourceError>; + + /// Stops capture and releases resources. Also called if `start` failed. + fn stop(&mut self); +} + +/// An Opus encoder for one audio stream (48 kHz, the negotiated channel count, 10 ms frames). +pub trait AudioEncoder: Send { + /// The OpusHead identification header, sent to the client in `AudioStream::codec_config`. + fn codec_config(&self) -> Vec; + + /// Encodes one frame of `frame_samples * channels` interleaved `f32` samples into one packet. + fn encode(&mut self, pcm: &[f32]) -> Result, SourceError>; +} + +/// Creates the audio parts of each session. Shared by all sessions of a host process. +pub trait AudioBackend: Send + Sync { + /// A capture source for one session, or `None` when this host cannot capture audio (the + /// session then has no audio). Must not start capturing: that happens in + /// [`AudioSource::start`] on the audio thread. + fn create_source(&self) -> Option>; + + /// True if [`create_opus_encoder`](Self::create_opus_encoder) can work (default: no). + fn opus_available(&self) -> bool { + false + } + + /// An Opus encoder for `channels` (1 or 2) at 48 kHz with 10 ms frames. If it fails the + /// session falls back to PCM when the client allows it. + fn create_opus_encoder(&self, _channels: u8) -> Result, SourceError> { + Err(SourceError::msg("no Opus encoder on this host")) + } +} + +impl std::fmt::Debug for dyn AudioBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("AudioBackend") + } +} + /// Mode of a virtual display. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DisplayMode { diff --git a/crates/ezd-host-core/src/testing.rs b/crates/ezd-host-core/src/testing.rs index bb3944a..6940753 100644 --- a/crates/ezd-host-core/src/testing.rs +++ b/crates/ezd-host-core/src/testing.rs @@ -2,7 +2,8 @@ use crate::input::{InputBackend, InputSink, InputTarget}; use crate::source::{ - CursorSink, DisplayMode, EncodedFrame, OutputInfo, SourceError, VideoSource, VirtualDisplay, + AudioBackend, AudioBlock, AudioEncoder, AudioSource, CursorSink, DisplayMode, EncodedFrame, + OutputInfo, SourceError, VideoSource, VirtualDisplay, }; use crate::time::now_us; use ezd_proto::{ @@ -159,6 +160,116 @@ impl VideoSource for ScriptedSource { } } +/// An [`AudioBackend`] whose sources produce a ramp (sample `i` of the stream is +/// `(i % 1000) / 1000` on every channel) in blocks that are not a multiple of the frame size, +/// and whose Opus "encoder" returns fixed-size dummy packets. +#[derive(Debug, Clone)] +pub struct ScriptedAudioBackend { + /// `create_source` returns a source (default true). + pub has_source: bool, + /// The sources fail in `start`. + pub fail_start: bool, + /// `opus_available` (default true). + pub opus: bool, + /// The Opus encoder cannot be created. + pub fail_opus: bool, + /// Samples per channel in each block (default 700). + pub block_samples: usize, +} + +impl ScriptedAudioBackend { + /// A backend with a working source and Opus encoder. + pub fn new() -> Self { + ScriptedAudioBackend { + has_source: true, + fail_start: false, + opus: true, + fail_opus: false, + block_samples: 700, + } + } + + /// Size in bytes of every packet of the dummy Opus encoder. + pub const OPUS_PACKET_LEN: usize = 40; + /// `codec_config` of the dummy Opus encoder. + pub const OPUS_HEAD: &'static [u8] = b"OpusHead-test"; +} + +impl Default for ScriptedAudioBackend { + fn default() -> Self { + Self::new() + } +} + +impl AudioBackend for ScriptedAudioBackend { + fn create_source(&self) -> Option> { + self.has_source.then(|| { + Box::new(ScriptedAudio { + channels: 0, + n: 0, + block_samples: self.block_samples, + fail_start: self.fail_start, + }) as Box + }) + } + fn opus_available(&self) -> bool { + self.opus + } + fn create_opus_encoder(&self, channels: u8) -> Result, SourceError> { + if self.fail_opus { + return Err(SourceError::msg("scripted Opus failure")); + } + Ok(Box::new(ScriptedOpus { + frame_len: 480 * channels as usize, + })) + } +} + +struct ScriptedAudio { + channels: usize, + n: u64, + block_samples: usize, + fail_start: bool, +} + +impl AudioSource for ScriptedAudio { + fn start(&mut self, channels: u8) -> Result<(), SourceError> { + if self.fail_start { + return Err(SourceError::msg("scripted audio failure")); + } + self.channels = channels as usize; + Ok(()) + } + fn next_samples(&mut self, timeout: Duration) -> Result, SourceError> { + std::thread::sleep(Duration::from_millis(3).min(timeout)); + let mut samples = Vec::with_capacity(self.block_samples * self.channels); + for _ in 0..self.block_samples { + let v = (self.n % 1000) as f32 / 1000.0; + samples.extend(std::iter::repeat_n(v, self.channels)); + self.n += 1; + } + Ok(Some(AudioBlock { + pts_us: now_us(), + samples, + })) + } + fn stop(&mut self) {} +} + +struct ScriptedOpus { + frame_len: usize, +} + +impl AudioEncoder for ScriptedOpus { + fn codec_config(&self) -> Vec { + ScriptedAudioBackend::OPUS_HEAD.to_vec() + } + fn encode(&mut self, pcm: &[f32]) -> Result, SourceError> { + assert_eq!(pcm.len(), self.frame_len, "the session feeds whole frames"); + Ok(vec![0xFC; ScriptedAudioBackend::OPUS_PACKET_LEN]) + } +} + /// One call received by a [`RecordingSink`]. #[derive(Debug, Clone, PartialEq, Eq)] pub enum InputCall { diff --git a/crates/ezd-host-service/src/service.rs b/crates/ezd-host-service/src/service.rs index b99eefc..d80ab82 100644 --- a/crates/ezd-host-service/src/service.rs +++ b/crates/ezd-host-service/src/service.rs @@ -642,8 +642,8 @@ pub(crate) mod tests { use ezd_link::adb::DeviceState as AdbState; use ezd_link::{MuxReceiver, MuxSender}; use ezd_proto::{ - Bye, ClientCaps, ClientHdr, Codec, DEFAULT_CHUNK_LEN, DecoderCaps, DisplayInfo, Hello, - Message, Role, + Bye, ClientAudio, ClientCaps, ClientHdr, Codec, DEFAULT_CHUNK_LEN, DecoderCaps, + DisplayInfo, Hello, Message, Role, }; use tokio::time::timeout; @@ -722,12 +722,15 @@ pub(crate) mod tests { tx.send_control(Message::ClientCaps(caps())).unwrap(); tx.send_control(Message::ClientHdr(ClientHdr::none())) .unwrap(); + tx.send_control(Message::ClientAudio(ClientAudio::none())) + .unwrap(); let mut seen = 0; - while seen < 4 { + while seen < 5 { match timeout(WAIT, rx.recv()).await.unwrap().unwrap().unwrap() { Message::Hello(_) | Message::HostCaps(_) | Message::StreamColor(_) + | Message::AudioConfig(_) | Message::StreamConfig(_) => seen += 1, m => panic!("{m:?}"), } diff --git a/crates/ezd-link/src/link.rs b/crates/ezd-link/src/link.rs index 3c52313..8881597 100644 --- a/crates/ezd-link/src/link.rs +++ b/crates/ezd-link/src/link.rs @@ -13,13 +13,14 @@ pub type BoxWrite = Box; /// The streams of a link besides the control stream, one per channel that gets its own stream. /// /// A channel without a lane (`None`) travels on the control stream, in the mux's priority order. -/// Audio will be one more field here. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Lanes { /// Input channel (pen, touch, buttons, scroll, keys). pub input: Option, /// Cursor channel (shapes and the latest-wins cursor state). pub cursor: Option, + /// Audio channel (PC audio packets, host to client). + pub audio: Option, /// Video channel. pub video: Option, } @@ -29,6 +30,7 @@ impl Default for Lanes { Lanes { input: None, cursor: None, + audio: None, video: None, } } @@ -37,12 +39,15 @@ impl Default for Lanes { impl Lanes { /// True when no channel has a lane. pub fn is_empty(&self) -> bool { - self.input.is_none() && self.cursor.is_none() && self.video.is_none() + self.input.is_none() + && self.cursor.is_none() + && self.audio.is_none() + && self.video.is_none() } - /// The present lanes in the fixed order input, cursor, video. + /// The present lanes in the fixed order input, cursor, audio, video. pub fn into_vec(self) -> Vec { - [self.input, self.cursor, self.video] + [self.input, self.cursor, self.audio, self.video] .into_iter() .flatten() .collect() @@ -53,7 +58,7 @@ impl Lanes { /// /// Transports (TCP and QUIC now; AOA bulk endpoints later) only have to produce a /// `Link`; everything above it is transport-agnostic. A link may carry further streams for -/// input, cursor and video (see [`Link::with_lanes`]), which the mux then uses so a stalled +/// input, cursor, audio and video (see [`Link::with_lanes`]), which the mux then uses so a stalled /// channel cannot delay the others. pub struct Link { /// Read half. diff --git a/crates/ezd-link/src/mux.rs b/crates/ezd-link/src/mux.rs index a67a82b..654faa7 100644 --- a/crates/ezd-link/src/mux.rs +++ b/crates/ezd-link/src/mux.rs @@ -2,7 +2,7 @@ //! //! [`MuxSender`] (cloneable) queues messages; a single writer task serializes //! them onto the link with strict priority -//! `Control > Input > CursorShape > CursorState > Video`: +//! `Control > Input > CursorShape > CursorState > Audio > Video`: //! //! * control messages and cursor shapes are reliable FIFO; //! * input messages (pen, touch, buttons, scroll) are reliable FIFO too. When @@ -13,6 +13,9 @@ //! frame, replaces it; transitions are never dropped; //! * [`CursorState`] is *latest wins*: a newer state replaces one that has not //! been written yet; +//! * audio packets are a short FIFO: when more than [`AUDIO_BACKLOG`] are queued the oldest is +//! dropped (counted in [`MuxStats::audio_dropped`]), so a stalled link plays at most about +//! 100 ms of stale sound once it recovers; //! * video frames are fragmented into chunks and the writer re-checks the //! higher-priority queues between chunks, so a large keyframe delays control //! or cursor traffic by at most one chunk; @@ -25,7 +28,7 @@ //! pulling the newest frame only when a slot is free. //! //! [`spawn_mux_lanes`] serves the same queues onto several byte streams: a control -//! stream plus, optionally, one *lane* each for input, cursor and video (audio will be one more). +//! stream plus, optionally, one *lane* each for input, cursor, audio and video. //! A stalled or retransmitting lane cannot delay the others (head-of-line blocking on a //! QUIC-like transport). A channel without a lane is written on the control stream in the //! priority order above. Each lane has its own writer loop and wake-up. [`MuxReceiver::new_lanes`] @@ -80,6 +83,10 @@ pub struct MuxStats { pub inputs_sent: u64, /// Input messages dropped by backlog coalescing (replaced by a newer one). pub inputs_coalesced: u64, + /// Audio packets written. + pub audio_sent: u64, + /// Audio packets dropped because more than [`AUDIO_BACKLOG`] were queued. + pub audio_dropped: u64, /// Video frames fully written. pub video_frames_sent: u64, /// Total frames (all channels, fragments counted individually) written. @@ -88,6 +95,9 @@ pub struct MuxStats { pub ack_timeouts: u64, } +/// Queued audio packets beyond which the oldest is dropped (about 100 ms of 10 ms packets). +pub const AUDIO_BACKLOG: usize = 10; + /// When a sent video frame stops holding the video credit. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VideoCredit { @@ -197,6 +207,7 @@ struct Queues { input: VecDeque, cursor_shapes: VecDeque, cursor_state: Option, + audio: VecDeque, video: VecDeque, /// A video frame is being written (popped from `video`). writing: bool, @@ -221,6 +232,7 @@ struct Shared { /// Wakes the writer of each lane that exists; the other channels use `wake`. wake_input: Notify, wake_cursor: Notify, + wake_audio: Notify, wake_video: Notify, /// Which channels have their own stream. lanes: Lanes<()>, @@ -229,6 +241,8 @@ struct Shared { states_sent: AtomicU64, inputs_sent: AtomicU64, inputs_coalesced: AtomicU64, + audio_sent: AtomicU64, + audio_dropped: AtomicU64, videos_sent: AtomicU64, frames_written: AtomicU64, ack_timeouts: AtomicU64, @@ -260,6 +274,7 @@ impl Shared { self.wake.notify_one(); self.wake_input.notify_one(); self.wake_cursor.notify_one(); + self.wake_audio.notify_one(); self.wake_video.notify_one(); } @@ -269,6 +284,7 @@ impl Shared { Lane::Control => (false, &self.wake), Lane::Input => (self.lanes.input.is_some(), &self.wake_input), Lane::Cursor => (self.lanes.cursor.is_some(), &self.wake_cursor), + Lane::Audio => (self.lanes.audio.is_some(), &self.wake_audio), Lane::Video => (self.lanes.video.is_some(), &self.wake_video), }; if present { @@ -285,6 +301,7 @@ enum Lane { Control, Input, Cursor, + Audio, Video, } @@ -315,6 +332,8 @@ impl MuxSender { /// Queues any message on its channel. /// /// * [`CursorState`] replaces a pending one (latest wins); + /// * an [`AudioPacket`](ezd_proto::AudioPacket) is queued FIFO; beyond [`AUDIO_BACKLOG`] + /// queued packets the oldest is dropped; /// * [`VideoFrame`] is queued behind earlier video frames, never dropped /// (the credit is advisory, see [`wait_video_slot`](Self::wait_video_slot)); /// * input messages go to a reliable FIFO served right after control; under @@ -368,6 +387,14 @@ impl MuxSender { q.cursor_shapes.push_back(enc); Lane::Cursor } + Message::AudioPacket(_) => { + q.audio.push_back(enc); + while q.audio.len() > AUDIO_BACKLOG { + q.audio.pop_front(); + self.shared.audio_dropped.fetch_add(1, Ordering::Relaxed); + } + Lane::Audio + } Message::VideoFrame(_) => { q.video.push_back(enc); self.shared.update_slot(&q); @@ -451,6 +478,8 @@ impl MuxSender { cursor_states_sent: self.shared.states_sent.load(Ordering::Relaxed), inputs_sent: self.shared.inputs_sent.load(Ordering::Relaxed), inputs_coalesced: self.shared.inputs_coalesced.load(Ordering::Relaxed), + audio_sent: self.shared.audio_sent.load(Ordering::Relaxed), + audio_dropped: self.shared.audio_dropped.load(Ordering::Relaxed), video_frames_sent: self.shared.videos_sent.load(Ordering::Relaxed), frames_written: self.shared.frames_written.load(Ordering::Relaxed), ack_timeouts: self.shared.ack_timeouts.load(Ordering::Relaxed), @@ -488,6 +517,7 @@ fn new_shared(opts: MuxOptions, lanes: Lanes<()>) -> Arc { wake: Notify::new(), wake_input: Notify::new(), wake_cursor: Notify::new(), + wake_audio: Notify::new(), wake_video: Notify::new(), lanes, slot, @@ -495,6 +525,8 @@ fn new_shared(opts: MuxOptions, lanes: Lanes<()>) -> Arc { states_sent: AtomicU64::new(0), inputs_sent: AtomicU64::new(0), inputs_coalesced: AtomicU64::new(0), + audio_sent: AtomicU64::new(0), + audio_dropped: AtomicU64::new(0), videos_sent: AtomicU64::new(0), frames_written: AtomicU64::new(0), ack_timeouts: AtomicU64::new(0), @@ -530,6 +562,7 @@ pub fn spawn_mux_lanes( let present = Lanes { input: lanes.input.as_ref().map(|_| ()), cursor: lanes.cursor.as_ref().map(|_| ()), + audio: lanes.audio.as_ref().map(|_| ()), video: lanes.video.as_ref().map(|_| ()), }; let shared = new_shared(opts, present); @@ -560,11 +593,11 @@ async fn lanes_task( let serves = Serves { input: lanes.input.is_none(), cursor: lanes.cursor.is_none(), + audio: lanes.audio.is_none(), video: lanes.video.is_none(), }; let res = { // `try_join!` returns at the first error and otherwise waits for every loop to drain. - // Audio: add its lane to `Lanes`, `Serves` and this join. tokio::try_join!( writer_loop(&shared, &mut control, chunk_len, serves), async { @@ -579,6 +612,12 @@ async fn lanes_task( None => Ok(()), } }, + async { + match lanes.audio.as_mut() { + Some(w) => audio_loop(&shared, w, chunk_len).await, + None => Ok(()), + } + }, async { match lanes.video.as_mut() { Some(w) => video_loop(&shared, w, chunk_len).await, @@ -599,6 +638,7 @@ async fn lanes_task( shut(Some(&mut control)), shut(lanes.input.as_mut()), shut(lanes.cursor.as_mut()), + shut(lanes.audio.as_mut()), shut(lanes.video.as_mut()), ); if let Err(e) = &res { @@ -612,11 +652,12 @@ async fn lanes_task( struct Serves { input: bool, cursor: bool, + audio: bool, video: bool, } /// Writer of the control stream: control, then (when they have no lane) input, cursor shapes, -/// cursor state and video chunks, in that order. +/// cursor state, audio and video chunks, in that order. async fn writer_loop( shared: &Shared, w: &mut BoxWrite, @@ -642,6 +683,8 @@ async fn writer_loop( Next::Whole(m, false) } else if let Some(m) = serves.cursor.then(|| q.cursor_state.take()).flatten() { Next::Whole(m, true) + } else if let Some(m) = serves.audio.then(|| q.audio.pop_front()).flatten() { + Next::Whole(m, false) } else if in_flight.is_some() { Next::VideoChunk } else if let Some(v) = serves.video.then(|| q.video.pop_front()).flatten() { @@ -715,6 +758,9 @@ async fn write_whole( if is_state { shared.states_sent.fetch_add(1, Ordering::Relaxed); } + if m.channel == Channel::Audio as u8 { + shared.audio_sent.fetch_add(1, Ordering::Relaxed); + } Ok(()) } @@ -800,6 +846,22 @@ async fn cursor_loop(shared: &Shared, w: &mut BoxWrite, chunk_len: usize) -> io: } } +/// Writer of the audio lane: FIFO, one write and flush per packet. +async fn audio_loop(shared: &Shared, w: &mut BoxWrite, chunk_len: usize) -> io::Result<()> { + let mut buf: Vec = Vec::with_capacity(chunk_len + 16); + loop { + let (next, closing) = { + let mut q = shared.lock(); + (q.audio.pop_front(), q.closing) + }; + match next { + Some(m) => write_whole(shared, w, &mut buf, &m, false, chunk_len).await?, + None if closing => return Ok(()), + None => shared.wake_audio.notified().await, + } + } +} + /// Writer of the video lane; also expires ack deadlines while idle. async fn video_loop(shared: &Shared, w: &mut BoxWrite, chunk_len: usize) -> io::Result<()> { let mut buf: Vec = Vec::with_capacity(chunk_len + 16); @@ -1005,7 +1067,9 @@ pub fn priority(ch: Channel) -> u8 { mod tests { use super::*; use crate::Link; - use ezd_proto::{Bye, CursorShape, Key, PenSample, Ping, PointerButton, Scroll, TouchFrame}; + use ezd_proto::{ + AudioPacket, Bye, CursorShape, Key, PenSample, Ping, PointerButton, Scroll, TouchFrame, + }; use std::time::Duration; use tokio::time::{sleep, timeout}; @@ -1617,6 +1681,7 @@ mod tests { Lanes { input: Some(cap), cursor: Some(cap), + audio: Some(cap), video: Some(cap), } } @@ -1634,6 +1699,7 @@ mod tests { for (cap, w, r) in [ (caps.input, &mut writes.input, &mut reads.input), (caps.cursor, &mut writes.cursor, &mut reads.cursor), + (caps.audio, &mut writes.audio, &mut reads.audio), (caps.video, &mut writes.video, &mut reads.video), ] { if let Some(cap) = cap { @@ -1681,6 +1747,13 @@ mod tests { }) } + fn ap(pts: u64, n: usize) -> Message { + Message::AudioPacket(AudioPacket { + pts_us: pts, + data: vec![pts as u8; n], + }) + } + fn ping(seq: u32) -> Message { Message::Ping(Ping { seq, t0_us: 0 }) } @@ -1726,6 +1799,7 @@ mod tests { let lanes = Lanes { input: Some(Box::new(iw) as BoxWrite), cursor: Some(Box::new(kw) as BoxWrite), + audio: None, video: Some(Box::new(vw) as BoxWrite), }; let (tx, _task) = spawn_mux_lanes(Box::new(cw), lanes, onwrite(4096)); @@ -1746,6 +1820,7 @@ mod tests { let lanes = Lanes { input: Some(Box::new(iw) as BoxWrite), cursor: Some(Box::new(kw) as BoxWrite), + audio: None, video: None, }; let (tx, _task) = spawn_mux_lanes(Box::new(cw), lanes, onwrite(4096)); @@ -1799,10 +1874,82 @@ mod tests { tx.send(&shape(1)).unwrap(); tx.send(&pen(1, PenSample::IN_RANGE)).unwrap(); tx.send_control(ping(1)).unwrap(); + tx.send(&ap(1, 8)).unwrap(); + tx.send_video(vf(1, 100)).unwrap(); assert!(matches!(recv(&mut rx).await, Message::Ping(_))); assert!(matches!(recv(&mut rx).await, Message::Pen(_))); assert!(matches!(recv(&mut rx).await, Message::CursorShape(_))); assert!(matches!(recv(&mut rx).await, Message::CursorState(_))); + assert!(matches!(recv(&mut rx).await, Message::AudioPacket(_))); + assert!(matches!(recv(&mut rx).await, Message::VideoFrame(_))); + } + + #[tokio::test] + async fn audio_overtakes_big_video_on_one_stream() { + let (a, b) = Link::duplex_pair(8 * 1024); + let (tx, _arx, _t) = a.into_mux_with(onwrite(16 * 1024)); + let (_btx, mut rx, _t2) = b.into_mux(16 * 1024); + tx.send_video(vf(1, 2 * 1024 * 1024)).unwrap(); + sleep(Duration::from_millis(50)).await; // writer is now blocked mid-frame + tx.send(&ap(3, 16)).unwrap(); + assert!(matches!(recv(&mut rx).await, Message::AudioPacket(p) if p.pts_us == 3)); + assert!(matches!(recv(&mut rx).await, Message::VideoFrame(_))); + assert_eq!(tx.stats().audio_sent, 1); + } + + #[tokio::test] + async fn audio_drops_the_oldest_beyond_the_backlog() { + for lanes in [false, true] { + let (tx, _task, mut rx) = if lanes { + lanes_pair( + 1 << 20, + Lanes { + audio: Some(1 << 20), + ..Lanes::default() + }, + onwrite(4096), + ) + } else { + lanes_pair(1 << 20, Lanes::default(), onwrite(4096)) + }; + // Everything is queued before the writer can run (single-threaded test runtime). + for i in 0..25u64 { + tx.send(&ap(i, 8)).unwrap(); + } + let mut got = vec![]; + for _ in 0..AUDIO_BACKLOG { + match recv(&mut rx).await { + Message::AudioPacket(p) => got.push(p.pts_us), + m => panic!("{m:?}"), + } + } + let want: Vec = (25 - AUDIO_BACKLOG as u64..25).collect(); + assert_eq!(got, want, "lanes: {lanes}"); + let st = tx.stats(); + assert_eq!( + (st.audio_dropped, st.audio_sent), + (15, 10), + "lanes: {lanes}" + ); + } + } + + #[tokio::test] + async fn lanes_stalled_video_does_not_delay_audio_lane() { + let (cw, _cr) = tokio::io::duplex(64 * 1024); + let (aw, ar) = tokio::io::duplex(64 * 1024); + let (vw, _vr_never_read) = tokio::io::duplex(1024); + let lanes = Lanes { + audio: Some(Box::new(aw) as BoxWrite), + video: Some(Box::new(vw) as BoxWrite), + ..Lanes::default() + }; + let (tx, _task) = spawn_mux_lanes(Box::new(cw), lanes, onwrite(4096)); + let mut rx = MuxReceiver::new(Box::new(ar)); + tx.send_video(vf(1, 1024 * 1024)).unwrap(); + sleep(Duration::from_millis(50)).await; + tx.send(&ap(7, 32)).unwrap(); + assert!(matches!(recv(&mut rx).await, Message::AudioPacket(p) if p.pts_us == 7)); } #[tokio::test] @@ -1810,10 +1957,12 @@ mod tests { let (cw, cr) = tokio::io::duplex(1 << 20); let (iw, ir) = tokio::io::duplex(1 << 20); let (kw, kr) = tokio::io::duplex(1 << 20); + let (aw, ar) = tokio::io::duplex(1 << 20); let (vw, vr) = tokio::io::duplex(1 << 20); let lanes = Lanes { input: Some(Box::new(iw) as BoxWrite), cursor: Some(Box::new(kw) as BoxWrite), + audio: Some(Box::new(aw) as BoxWrite), video: Some(Box::new(vw) as BoxWrite), }; let (tx, _task) = spawn_mux_lanes(Box::new(cw), lanes, onwrite(4096)); @@ -1821,15 +1970,18 @@ mod tests { tx.send(&pen(1, PenSample::IN_RANGE)).unwrap(); tx.send(&shape(1)).unwrap(); tx.send_cursor_state(cs(1)).unwrap(); + tx.send(&ap(1, 4)).unwrap(); tx.send_video(vf(1, 100)).unwrap(); let mut c = MuxReceiver::new(Box::new(cr)); let mut i = MuxReceiver::new(Box::new(ir)); let mut k = MuxReceiver::new(Box::new(kr)); + let mut a = MuxReceiver::new(Box::new(ar)); let mut v = MuxReceiver::new(Box::new(vr)); assert!(matches!(recv(&mut c).await, Message::Ping(_))); assert!(matches!(recv(&mut i).await, Message::Pen(_))); assert!(matches!(recv(&mut k).await, Message::CursorShape(_))); assert!(matches!(recv(&mut k).await, Message::CursorState(_))); + assert!(matches!(recv(&mut a).await, Message::AudioPacket(_))); assert!(matches!(recv(&mut v).await, Message::VideoFrame(_))); } @@ -1946,6 +2098,7 @@ mod tests { let lanes = Lanes { input: Some(Box::new(ir) as BoxRead), cursor: Some(Box::new(kr) as BoxRead), + audio: None, video: Some(Box::new(vr) as BoxRead), }; let mut rx = MuxReceiver::new_lanes(Box::new(cr), lanes); @@ -1988,7 +2141,7 @@ mod tests { #[tokio::test] async fn lanes_write_error_on_any_lane_closes_the_mux() { - for failing in ["input", "cursor", "video"] { + for failing in ["input", "cursor", "audio", "video"] { let (cw, _cr) = tokio::io::duplex(4096); let (ow, _or) = tokio::io::duplex(4096); let bad = || Some(Box::new(FailWriter) as BoxWrite); @@ -1996,17 +2149,20 @@ mod tests { let mut lanes = Lanes { input: ok(), cursor: None, + audio: None, video: None, }; match failing { "input" => lanes.input = bad(), "cursor" => lanes.cursor = bad(), + "audio" => lanes.audio = bad(), _ => lanes.video = bad(), } let (tx, task) = spawn_mux_lanes(Box::new(cw), lanes, onwrite(1024)); match failing { "input" => tx.send(&pen(1, PenSample::IN_RANGE)).unwrap(), "cursor" => tx.send_cursor_state(cs(1)).unwrap(), + "audio" => tx.send(&ap(1, 8)).unwrap(), _ => tx.send_video(vf(1, 100)).unwrap(), } let err = timeout(Duration::from_secs(5), task.join()) diff --git a/crates/ezd-link/src/pair.rs b/crates/ezd-link/src/pair.rs index 2a3b4da..d3e1ce7 100644 --- a/crates/ezd-link/src/pair.rs +++ b/crates/ezd-link/src/pair.rs @@ -660,6 +660,11 @@ mod tests { t0_us: 7, })) .unwrap(); + tx.send(&ezd_proto::Message::AudioPacket(ezd_proto::AudioPacket { + pts_us: 9, + data: vec![1, 2, 3], + })) + .unwrap(); // Waits for the client's Bye so the connection outlives the exchange. let _ = rx.recv().await; } else { @@ -700,7 +705,9 @@ mod tests { let link = conn.open_session().await.unwrap(); assert!(link.has_video_stream()); let (tx, mut rx, _t) = link.into_mux(1024); - let m = rx.recv().await.unwrap().unwrap(); + // The ping travels on the control stream, the audio packet on its own stream. + let mut m = vec![rx.recv().await.unwrap().unwrap()]; + m.push(rx.recv().await.unwrap().unwrap()); tx.send(&ezd_proto::Message::Bye(ezd_proto::Bye { reason: "done".into(), })) @@ -709,9 +716,15 @@ mod tests { }; let (h, m) = tokio::join!(host_once(&f, Some(host.fingerprint)), session); h.unwrap(); - assert_eq!( - m, - ezd_proto::Message::Ping(ezd_proto::Ping { seq: 1, t0_us: 7 }) + assert!(m.contains(&ezd_proto::Message::Ping(ezd_proto::Ping { + seq: 1, + t0_us: 7 + }))); + assert!( + m.contains(&ezd_proto::Message::AudioPacket(ezd_proto::AudioPacket { + pts_us: 9, + data: vec![1, 2, 3], + })) ); } diff --git a/crates/ezd-link/src/quic.rs b/crates/ezd-link/src/quic.rs index 793e3ae..80e7470 100644 --- a/crates/ezd-link/src/quic.rs +++ b/crates/ezd-link/src/quic.rs @@ -9,23 +9,23 @@ //! Streams (see `docs/protocol.md`): //! * control, bidirectional, opened by the client: [`LinkOpen`] / [`LinkReply`], then the framed //! control traffic; -//! * cursor and video, unidirectional, opened by the host after [`LinkReply::Accepted`]; +//! * cursor, video and audio, unidirectional, opened by the host after [`LinkReply::Accepted`] +//! (the audio stream stays empty when the session has no audio); //! * input, unidirectional, opened by the client after it read [`LinkReply::Accepted`]. //! //! Every unidirectional stream starts with one tag byte ([`CURSOR_STREAM_TAG`], -//! [`VIDEO_STREAM_TAG`], [`INPUT_STREAM_TAG`]) and then carries the framed messages of its -//! channel. The receiver dispatches by tag, not by arrival order. Audio will be one more stream -//! ([`AUDIO_STREAM_TAG`] is reserved). +//! [`VIDEO_STREAM_TAG`], [`AUDIO_STREAM_TAG`], [`INPUT_STREAM_TAG`]) and then carries the framed messages of its +//! channel. The receiver dispatches by tag, not by arrival order. //! -//! Send priorities (higher first): control 4, input 3, cursor 2, audio 1 (reserved), video 0. +//! Send priorities (higher first): control 4, input 3, cursor 2, audio 1, video 0. //! Each channel has its own stream, so a lost packet delays only the channel it belongs to. use crate::identity::{Identity, fingerprint_of}; use crate::link::{BoxRead, BoxWrite, Lanes, Link}; use ezd_proto::ProtoError; use ezd_proto::pair::{ - self, ALPN, CURSOR_STREAM_TAG, Fingerprint, INPUT_STREAM_TAG, LinkOpen, LinkReply, - MAX_PREAMBLE_LEN, Refusal, VIDEO_STREAM_TAG, + self, ALPN, AUDIO_STREAM_TAG, CURSOR_STREAM_TAG, Fingerprint, INPUT_STREAM_TAG, LinkOpen, + LinkReply, MAX_PREAMBLE_LEN, Refusal, VIDEO_STREAM_TAG, }; use quinn::{Connection, Endpoint, RecvStream, SendStream, VarInt}; use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; @@ -48,13 +48,15 @@ use tokio::io::{AsyncRead, ReadBuf}; const SERVER_NAME: &str = "ezdisplay"; /// Time allowed for each preamble step (open, reply, lane streams). pub const PREAMBLE_TIMEOUT: Duration = Duration::from_secs(10); +/// How long to wait for the audio stream, which a peer from before audio never opens or allows. +const OPTIONAL_STREAM_GRACE: Duration = Duration::from_millis(500); /// Application close code for a normal end or a refusal. const CLOSE_OK: u32 = 0; -/// Send priorities (higher is served first). 1 is left for audio, which slots between cursor -/// and video. +/// Send priorities (higher is served first); audio slots between cursor and video. const CONTROL_PRIORITY: i32 = 4; const INPUT_PRIORITY: i32 = 3; const CURSOR_PRIORITY: i32 = 2; +const AUDIO_PRIORITY: i32 = 1; const VIDEO_PRIORITY: i32 = 0; /// Congestion controller of a Wi-Fi link. @@ -168,7 +170,7 @@ fn transport(cc: Congestion, side: Side) -> quinn::TransportConfig { .stream_receive_window(VarInt::from_u32(8 << 20)) .receive_window(VarInt::from_u32(16 << 20)) .max_concurrent_bidi_streams(VarInt::from_u32(0)) - .max_concurrent_uni_streams(VarInt::from_u32(2)), + .max_concurrent_uni_streams(VarInt::from_u32(3)), }; if cc == Congestion::Bbr { t.congestion_controller_factory(Arc::new(quinn::congestion::BbrConfig::default())); @@ -366,18 +368,24 @@ impl QuicConn { } } let input = open_tagged(&self.conn, INPUT_STREAM_TAG, INPUT_PRIORITY).await?; - let [cursor, video] = - accept_tagged(&self.conn, [CURSOR_STREAM_TAG, VIDEO_STREAM_TAG]).await?; + let ([cursor, video], audio) = accept_tagged_with_optional( + &self.conn, + [CURSOR_STREAM_TAG, VIDEO_STREAM_TAG], + AUDIO_STREAM_TAG, + ) + .await?; let peer = format!("quic:{}", self.conn.remote_address()); let keep = Arc::new(Lingering(self)); let read = Lanes { input: None, cursor: Some(Box::new(Keep::new(cursor, &keep)) as BoxRead), + audio: audio.map(|a| Box::new(Keep::new(a, &keep)) as BoxRead), video: Some(Box::new(Keep::new(video, &keep)) as BoxRead), }; let write = Lanes { input: Some(Box::new(input) as BoxWrite), cursor: None, + audio: None, video: None, }; Ok( @@ -423,23 +431,39 @@ impl Opened { self.conn.remote_address() } - /// Accepts a session: answers [`LinkReply::Accepted`], opens the cursor and video streams, + /// Accepts a session: answers [`LinkReply::Accepted`], opens the cursor, video and audio streams, /// accepts the client's input stream and returns the link. pub async fn accept_session(mut self) -> Result { write_msg(&mut self.send, &LinkReply::Accepted).await?; let cursor = open_tagged(&self.conn.conn, CURSOR_STREAM_TAG, CURSOR_PRIORITY).await?; let video = open_tagged(&self.conn.conn, VIDEO_STREAM_TAG, VIDEO_PRIORITY).await?; + // A client from before audio allows only two unidirectional streams, so the open would + // wait for credit that never comes: give up after a short grace and go without the lane. + let audio = match tokio::time::timeout( + OPTIONAL_STREAM_GRACE, + open_tagged(&self.conn.conn, AUDIO_STREAM_TAG, AUDIO_PRIORITY), + ) + .await + { + Ok(stream) => Some(stream?), + Err(_) => { + tracing::debug!("client takes no audio stream"); + None + } + }; let [input] = accept_tagged(&self.conn.conn, [INPUT_STREAM_TAG]).await?; let peer = format!("quic:{}", self.conn.remote_address()); let keep = Arc::new(Lingering(self.conn)); let read = Lanes { input: Some(Box::new(Keep::new(input, &keep)) as BoxRead), cursor: None, + audio: None, video: None, }; let write = Lanes { input: None, cursor: Some(Box::new(cursor) as BoxWrite), + audio: audio.map(|a| Box::new(a) as BoxWrite), video: Some(Box::new(video) as BoxWrite), }; Ok(Link::new( @@ -498,6 +522,58 @@ async fn accept_tagged( Ok(found.map(|s| s.expect("every tag was filled once"))) } +/// Like [`accept_tagged`] for `N` required streams plus one that an older peer never opens: after +/// the required ones arrived, waits only [`OPTIONAL_STREAM_GRACE`] more for the `optional` one. +async fn accept_tagged_with_optional( + conn: &Connection, + tags: [u8; N], + optional: u8, +) -> Result<([RecvStream; N], Option), QuicError> { + async fn accept_one( + conn: &Connection, + tags: &[u8; N], + optional: u8, + found: &mut [Option; N], + extra: &mut Option, + ) -> Result<(), QuicError> { + let mut stream = conn.accept_uni().await?; + let mut tag = [0u8; 1]; + stream.read_exact(&mut tag).await?; + let slot = if tag[0] == optional { + extra + } else { + let i = tags + .iter() + .position(|&t| t == tag[0]) + .ok_or(QuicError::Unexpected("unknown stream tag"))?; + &mut found[i] + }; + if slot.replace(stream).is_some() { + return Err(QuicError::Unexpected("duplicate stream tag")); + } + Ok(()) + } + let mut found: [Option; N] = std::array::from_fn(|_| None); + let mut extra: Option = None; + let required = async { + while found.iter().any(Option::is_none) { + accept_one(conn, &tags, optional, &mut found, &mut extra).await?; + } + Ok::<_, QuicError>(()) + }; + tokio::time::timeout(PREAMBLE_TIMEOUT, required) + .await + .map_err(|_| QuicError::Timeout)??; + if extra.is_none() { + let more = accept_one(conn, &tags, optional, &mut found, &mut extra); + match tokio::time::timeout(OPTIONAL_STREAM_GRACE, more).await { + Ok(r) => r?, + Err(_) => tracing::debug!("host opened no audio stream"), + } + } + Ok((found.map(|s| s.expect("every tag was filled once")), extra)) +} + /// Finishes `send`, waits briefly for the peer to read it, then closes the connection. pub(crate) async fn finish_and_close(send: &mut SendStream, conn: &QuicConn, reason: &str) { let _ = send.finish(); diff --git a/crates/ezd-proto/src/frame.rs b/crates/ezd-proto/src/frame.rs index 0d1cc6d..6e5fc2f 100644 --- a/crates/ezd-proto/src/frame.rs +++ b/crates/ezd-proto/src/frame.rs @@ -33,7 +33,7 @@ pub enum Channel { Cursor = 2, /// Encoded video frames. Video = 3, - /// Audio (reserved). + /// Audio packets (protocol 0.6). Audio = 4, } diff --git a/crates/ezd-proto/src/lib.rs b/crates/ezd-proto/src/lib.rs index 1f13ffd..28c9c34 100644 --- a/crates/ezd-proto/src/lib.rs +++ b/crates/ezd-proto/src/lib.rs @@ -26,17 +26,20 @@ pub use frame::{ }; pub use msg::*; pub use negotiate::{ - DynamicRange, HdrMode, LinkKind, Prefs, SdrReason, negotiate, negotiate_hdr, pick_dynamic_range, + AudioMode, DynamicRange, HdrMode, HostAudio, LinkKind, Prefs, SdrReason, negotiate, + negotiate_audio, negotiate_hdr, pick_dynamic_range, }; /// Wire protocol major version. A mismatch is fatal at handshake. pub const PROTOCOL_MAJOR: u16 = 0; /// Wire protocol minor version. Newer minors may add new `(channel, kind)` pairs. -pub const PROTOCOL_MINOR: u16 = 5; +pub const PROTOCOL_MINOR: u16 = 6; /// First protocol minor with the input messages, `InputStatus` and `ClientOptions`. pub const MINOR_INPUT: u16 = 3; /// First protocol minor with `ClientHdr` and `StreamColor` (HDR10 streaming). pub const MINOR_HDR: u16 = 5; +/// First protocol minor with `ClientAudio`, `AudioConfig` and the Audio channel (PC audio streaming). +pub const MINOR_AUDIO: u16 = 6; /// Default TCP port used for the host listener and `adb reverse`. pub const DEFAULT_PORT: u16 = 27183; diff --git a/crates/ezd-proto/src/msg.rs b/crates/ezd-proto/src/msg.rs index 2dbce8b..812ecf4 100644 --- a/crates/ezd-proto/src/msg.rs +++ b/crates/ezd-proto/src/msg.rs @@ -90,6 +90,12 @@ impl Hello { self.proto_minor >= crate::MINOR_HDR } + /// True if the peer speaks at least protocol minor [`MINOR_AUDIO`](crate::MINOR_AUDIO) + /// (`ClientAudio`, `AudioConfig`, the Audio channel). + pub fn supports_audio(&self) -> bool { + self.proto_minor >= crate::MINOR_AUDIO + } + /// Fails with [`ProtoError::VersionMismatch`] if the peer's major differs from ours. pub fn check_compatible(&self) -> Result<()> { if self.proto_major != crate::PROTOCOL_MAJOR { @@ -598,6 +604,113 @@ impl ClientHdr { } } +/// Audio codec of the PC audio stream (protocol 0.6). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum AudioCodec { + /// Uncompressed little-endian interleaved samples. + Pcm, + /// Opus, one packet per frame; always decoded at 48 kHz. + Opus, +} + +/// Sample format of uncompressed audio: little-endian, interleaved. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum SampleFormat { + /// Signed 16-bit integer. + S16, + /// 32-bit IEEE float in -1.0..=1.0. + F32, +} + +impl SampleFormat { + /// Bytes per sample. + pub fn bytes(self) -> usize { + match self { + SampleFormat::S16 => 2, + SampleFormat::F32 => 4, + } + } +} + +/// What the client can play (client to host, protocol 0.6). +/// +/// Sent right after `ClientHdr` when the host `Hello` has minor >= [`MINOR_AUDIO`](crate::MINOR_AUDIO) +/// and the client itself advertised one. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ClientAudio { + /// The user wants PC audio on this device. + pub enabled: bool, + /// Output sample rates in Hz the client can play uncompressed audio at. + pub sample_rates: Vec, + /// Maximum output channel count (0 = no audio output). + pub max_channels: u8, + /// Uncompressed sample formats the client can play. + pub formats: Vec, + /// Codecs the client can decode. + pub codecs: Vec, +} + +impl ClientAudio { + /// A client without audio output (or with audio switched off). + pub fn none() -> Self { + ClientAudio { + enabled: false, + sample_rates: Vec::new(), + max_channels: 0, + formats: Vec::new(), + codecs: Vec::new(), + } + } +} + +/// The negotiated audio stream (host to client, protocol 0.6). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AudioStream { + /// Codec of every [`AudioPacket`]. + pub codec: AudioCodec, + /// Sample rate in Hz (48000). + pub sample_rate: u32, + /// Interleaved channel count. + pub channels: u8, + /// Sample format of PCM packets. Ignored for Opus (set to `S16`): the client picks its own + /// decoder output format. + pub format: SampleFormat, + /// Samples per channel in every packet (480 = 10 ms at 48 kHz). + pub frame_samples: u32, + /// Codec configuration: the OpusHead identification header for Opus, empty for PCM. + #[serde(with = "serde_bytes")] + pub codec_config: Vec, +} + +impl AudioStream { + /// Bytes of one PCM packet; `None` for Opus (variable size). + pub fn pcm_packet_len(&self) -> Option { + (self.codec == AudioCodec::Pcm) + .then(|| self.frame_samples as usize * self.channels as usize * self.format.bytes()) + } +} + +/// Audio decision of the host (host to client, protocol 0.6), sent right after `StreamColor` and +/// before `StreamConfig` when both sides speak minor >= [`MINOR_AUDIO`](crate::MINOR_AUDIO). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AudioConfig { + /// The stream, or `None` when there is no audio in this session. + pub stream: Option, +} + +/// One audio packet (host to client, Audio/1, protocol 0.6). +/// +/// `data` is exactly one frame: `frame_samples * channels` interleaved samples for PCM, one Opus +/// packet for Opus. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AudioPacket { + /// Host monotonic time of the first sample, microseconds. + pub pts_us: u64, + /// Packet payload. + #[serde(with = "serde_bytes")] + pub data: Vec, +} + /// SMPTE ST 2086 mastering display colour volume. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct Mastering { @@ -678,7 +791,7 @@ impl Default for StreamColor { } /// Any message of protocol 0.x (0.2 added `FrameAck`, 0.3 the input messages, -/// `InputStatus` and `ClientOptions`, 0.5 `ClientHdr` and `StreamColor`). +/// `InputStatus` and `ClientOptions`, 0.5 `ClientHdr` and `StreamColor`, 0.6 audio). #[derive(Debug, Clone, PartialEq)] #[allow(clippy::large_enum_variant)] pub enum Message { @@ -714,6 +827,10 @@ pub enum Message { ClientHdr(ClientHdr), /// Control/16 (0.5), host to client. StreamColor(StreamColor), + /// Control/17 (0.6), client to host. + ClientAudio(ClientAudio), + /// Control/18 (0.6), host to client. + AudioConfig(AudioConfig), /// Input/1 (0.3), client to host. Pen(PenSample), /// Input/2 (0.3), client to host. @@ -730,6 +847,8 @@ pub enum Message { CursorState(CursorState), /// Video/1. VideoFrame(VideoFrame), + /// Audio/1 (0.6), host to client. + AudioPacket(AudioPacket), } impl Message { @@ -743,6 +862,7 @@ impl Message { | Message::Key(_) => Channel::Input, Message::CursorShape(_) | Message::CursorState(_) => Channel::Cursor, Message::VideoFrame(_) => Channel::Video, + Message::AudioPacket(_) => Channel::Audio, _ => Channel::Control, } } @@ -766,6 +886,8 @@ impl Message { Message::ClientOptions(_) => 14, Message::ClientHdr(_) => 15, Message::StreamColor(_) => 16, + Message::ClientAudio(_) => 17, + Message::AudioConfig(_) => 18, Message::Pen(_) => 1, Message::Touch(_) => 2, Message::PointerButton(_) => 3, @@ -774,6 +896,7 @@ impl Message { Message::CursorShape(_) => 1, Message::CursorState(_) => 2, Message::VideoFrame(_) => 1, + Message::AudioPacket(_) => 1, } } } @@ -801,6 +924,8 @@ pub fn encode_message(msg: &Message) -> Result<(Channel, u8, Vec)> { Message::ClientOptions(m) => enc(m)?, Message::ClientHdr(m) => enc(m)?, Message::StreamColor(m) => enc(m)?, + Message::ClientAudio(m) => enc(m)?, + Message::AudioConfig(m) => enc(m)?, Message::Pen(m) => enc(m)?, Message::Touch(m) => enc(m)?, Message::PointerButton(m) => enc(m)?, @@ -809,6 +934,7 @@ pub fn encode_message(msg: &Message) -> Result<(Channel, u8, Vec)> { Message::CursorShape(m) => enc(m)?, Message::CursorState(m) => enc(m)?, Message::VideoFrame(m) => enc(m)?, + Message::AudioPacket(m) => enc(m)?, }; Ok((msg.channel(), msg.kind(), payload)) } @@ -846,6 +972,8 @@ pub fn decode_message(channel: u8, kind: u8, payload: &[u8]) -> Result Message::ClientOptions(dec(c, k, payload)?), (Channel::Control, 15) => Message::ClientHdr(dec(c, k, payload)?), (Channel::Control, 16) => Message::StreamColor(dec(c, k, payload)?), + (Channel::Control, 17) => Message::ClientAudio(dec(c, k, payload)?), + (Channel::Control, 18) => Message::AudioConfig(dec(c, k, payload)?), (Channel::Input, 1) => Message::Pen(dec(c, k, payload)?), (Channel::Input, 2) => Message::Touch(dec(c, k, payload)?), (Channel::Input, 3) => Message::PointerButton(dec(c, k, payload)?), @@ -854,6 +982,7 @@ pub fn decode_message(channel: u8, kind: u8, payload: &[u8]) -> Result Message::CursorShape(dec(c, k, payload)?), (Channel::Cursor, 2) => Message::CursorState(dec(c, k, payload)?), (Channel::Video, 1) => Message::VideoFrame(dec(c, k, payload)?), + (Channel::Audio, 1) => Message::AudioPacket(dec(c, k, payload)?), _ => return Ok(None), })) } @@ -963,6 +1092,23 @@ mod tests { 1000, 400, )), + Message::ClientAudio(ClientAudio { + enabled: true, + sample_rates: vec![44100, 48000], + max_channels: 2, + formats: vec![SampleFormat::S16, SampleFormat::F32], + codecs: vec![AudioCodec::Pcm, AudioCodec::Opus], + }), + Message::AudioConfig(AudioConfig { + stream: Some(AudioStream { + codec: AudioCodec::Opus, + sample_rate: 48000, + channels: 2, + format: SampleFormat::S16, + frame_samples: 480, + codec_config: b"OpusHead".to_vec(), + }), + }), Message::Pen(PenSample { t_us: 1 << 40, x: 65535, @@ -1031,6 +1177,10 @@ mod tests { encoded_us: 6, data: vec![0, 0, 0, 1, 0x67, 1, 2, 3], }), + Message::AudioPacket(AudioPacket { + pts_us: 1 << 35, + data: vec![1, 2, 3, 4], + }), ] } @@ -1152,6 +1302,41 @@ mod tests { assert!(h.supports_input()); } + #[test] + fn audio_support_by_minor() { + let mut h = Hello::new(Role::Client, "x", "y"); + assert!(h.supports_audio()); + h.proto_minor = crate::MINOR_AUDIO - 1; + assert!(!h.supports_audio()); + assert!(h.supports_hdr()); + } + + #[test] + fn audio_stream_helpers() { + assert!(!ClientAudio::none().enabled); + let mut s = AudioStream { + codec: AudioCodec::Pcm, + sample_rate: 48000, + channels: 2, + format: SampleFormat::F32, + frame_samples: 480, + codec_config: Vec::new(), + }; + assert_eq!(s.pcm_packet_len(), Some(480 * 2 * 4)); + s.format = SampleFormat::S16; + assert_eq!(s.pcm_packet_len(), Some(480 * 2 * 2)); + s.codec = AudioCodec::Opus; + assert_eq!(s.pcm_packet_len(), None); + let m = Message::AudioPacket(AudioPacket { + pts_us: 0, + data: vec![255; 200], + }); + assert_eq!(encode_message(&m).unwrap().0, Channel::Audio); + let none = Message::AudioConfig(AudioConfig { stream: None }); + let (ch, kind, p) = encode_message(&none).unwrap(); + assert_eq!(decode_message(ch as u8, kind, &p).unwrap(), Some(none)); + } + #[test] fn stream_color_presets() { let sdr = StreamColor::sdr_bt709(); diff --git a/crates/ezd-proto/src/negotiate.rs b/crates/ezd-proto/src/negotiate.rs index d8bbbdb..650f941 100644 --- a/crates/ezd-proto/src/negotiate.rs +++ b/crates/ezd-proto/src/negotiate.rs @@ -1,6 +1,9 @@ //! Capability negotiation: a pure function from caps + preferences to a [`StreamConfig`]. -use crate::msg::{ClientCaps, ClientHdr, Codec, DecoderCaps, EncoderCaps, HostCaps, StreamConfig}; +use crate::msg::{ + AudioCodec, AudioStream, ClientAudio, ClientCaps, ClientHdr, Codec, DecoderCaps, EncoderCaps, + HostCaps, SampleFormat, StreamConfig, +}; use crate::{ProtoError, Result}; /// Kind of transport carrying the link; selects the default bitrate ceiling. @@ -32,6 +35,29 @@ pub enum HdrMode { Off, } +/// User preference for PC audio streaming. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum AudioMode { + /// Uncompressed PCM when the link and the client allow it, else Opus. + #[default] + Auto, + /// Prefer PCM even on a slow link; fall back to Opus when the client cannot play it. + Pcm, + /// Prefer Opus; fall back to PCM as in [`AudioMode::Auto`]. + Opus, + /// No audio. + Off, +} + +/// What the host can do for audio, fed into [`negotiate_audio`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct HostAudio { + /// An audio capture source exists. + pub available: bool, + /// An Opus encoder exists. + pub opus: bool, +} + /// Why an HDR source is streamed as tone-mapped SDR. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SdrReason { @@ -88,6 +114,8 @@ pub struct Prefs { pub scale_permille: u16, /// HDR10 streaming policy, used by [`negotiate_hdr`]. Default [`HdrMode::Auto`]. pub hdr: HdrMode, + /// Audio streaming policy, used by [`negotiate_audio`]. Default [`AudioMode::Auto`]. + pub audio: AudioMode, } impl Default for Prefs { @@ -101,6 +129,7 @@ impl Default for Prefs { encoder: None, scale_permille: 1000, hdr: HdrMode::Auto, + audio: AudioMode::Auto, } } } @@ -286,6 +315,73 @@ pub fn negotiate_hdr( Ok((negotiate(client, host, prefs)?, range)) } +/// Audio sample rate of the stream, Hz. +const AUDIO_RATE: u32 = 48_000; +/// Samples per channel in one audio packet (10 ms). +const AUDIO_FRAME_SAMPLES: u32 = 480; +/// Smallest link ceiling that lets [`AudioMode::Auto`] pick PCM, kbit/s. +const PCM_MIN_LINK_KBPS: u32 = 8_000; + +/// Decides the audio stream, or `None` for no audio. +/// +/// * `None` when the mode is [`AudioMode::Off`], the client has audio disabled or no output +/// channels, or the host has no audio source; +/// * 48 kHz, `min(2, client.max_channels)` channels, 480 samples per packet; +/// * PCM needs client PCM and 48 kHz support, and (unless forced with [`AudioMode::Pcm`]) a +/// link ceiling `max_bitrate_kbps` of at least 8 Mbit/s; the format is F32 on USB when the +/// client plays it, else S16, else F32. Opus needs client Opus and a host encoder; +/// * [`AudioMode::Auto`]/[`AudioMode::Pcm`] try PCM then Opus, [`AudioMode::Opus`] the reverse. +/// +/// `codec_config` is left empty: the host fills in the OpusHead from its encoder. +pub fn negotiate_audio( + mode: AudioMode, + client: &ClientAudio, + host: &HostAudio, + link: LinkKind, + max_bitrate_kbps: u32, +) -> Option { + if mode == AudioMode::Off || !client.enabled || !host.available || client.max_channels == 0 { + return None; + } + let channels = client.max_channels.min(2); + let stream = |codec, format| AudioStream { + codec, + sample_rate: AUDIO_RATE, + channels, + format, + frame_samples: AUDIO_FRAME_SAMPLES, + codec_config: Vec::new(), + }; + let has_fmt = |f| client.formats.contains(&f); + let pcm = || { + if !client.codecs.contains(&AudioCodec::Pcm) || !client.sample_rates.contains(&AUDIO_RATE) { + return None; + } + if mode != AudioMode::Pcm && max_bitrate_kbps < PCM_MIN_LINK_KBPS { + return None; + } + let format = if link == LinkKind::Usb && has_fmt(SampleFormat::F32) { + SampleFormat::F32 + } else if has_fmt(SampleFormat::S16) { + SampleFormat::S16 + } else if has_fmt(SampleFormat::F32) { + SampleFormat::F32 + } else { + return None; + }; + Some(stream(AudioCodec::Pcm, format)) + }; + let opus = || { + (client.codecs.contains(&AudioCodec::Opus) && host.opus) + // The client picks its own decoder output format; `format` is ignored for Opus. + .then(|| stream(AudioCodec::Opus, SampleFormat::S16)) + }; + match mode { + AudioMode::Opus => opus().or_else(pcm), + _ => pcm().or_else(opus), + } +} + /// Computes the stream configuration for a client/host pair. /// /// * codec: `prefs.codec` if both sides support it, else H.264 (then HEVC, AV1); @@ -373,6 +469,110 @@ mod tests { } } + fn client_audio(rates: &[u32], formats: &[SampleFormat], codecs: &[AudioCodec]) -> ClientAudio { + ClientAudio { + enabled: true, + sample_rates: rates.to_vec(), + max_channels: 2, + formats: formats.to_vec(), + codecs: codecs.to_vec(), + } + } + + #[test] + fn audio_negotiation_matrix() { + use AudioCodec::{Opus, Pcm}; + use SampleFormat::{F32, S16}; + let full = client_audio(&[44100, 48000], &[S16, F32], &[Pcm, Opus]); + let both = HostAudio { + available: true, + opus: true, + }; + let pcm_only = HostAudio { + available: true, + opus: false, + }; + let got = |mode, c: &ClientAudio, h: &HostAudio, link, kbps| { + negotiate_audio(mode, c, h, link, kbps).map(|s| (s.codec, s.format, s.channels)) + }; + let (auto, pcm_mode, opus_mode, off) = ( + AudioMode::Auto, + AudioMode::Pcm, + AudioMode::Opus, + AudioMode::Off, + ); + use LinkKind::*; + // Auto: PCM F32 on USB, S16 on the network, Opus on a slow link. + assert_eq!(got(auto, &full, &both, Usb, 80_000), Some((Pcm, F32, 2))); + assert_eq!( + got(auto, &full, &both, Network, 30_000), + Some((Pcm, S16, 2)) + ); + assert_eq!( + got(auto, &full, &both, Network, 4_000), + Some((Opus, S16, 2)) + ); + assert_eq!(got(auto, &full, &pcm_only, Network, 4_000), None); + // Forced PCM ignores the bitrate rule but still needs client support. + assert_eq!( + got(pcm_mode, &full, &both, Network, 4_000), + Some((Pcm, S16, 2)) + ); + let opus_only = client_audio(&[], &[], &[Opus]); + assert_eq!( + got(pcm_mode, &opus_only, &both, Usb, 80_000), + Some((Opus, S16, 2)) + ); + assert_eq!(got(pcm_mode, &opus_only, &pcm_only, Usb, 80_000), None); + // Forced Opus, with PCM as the fallback. + assert_eq!( + got(opus_mode, &full, &both, Usb, 80_000), + Some((Opus, S16, 2)) + ); + assert_eq!( + got(opus_mode, &full, &pcm_only, Usb, 80_000), + Some((Pcm, F32, 2)) + ); + assert_eq!(got(opus_mode, &full, &pcm_only, Usb, 1_000), None); + // No 48 kHz output: PCM is impossible, Opus still works. + let c44 = client_audio(&[44100], &[S16], &[Pcm, Opus]); + assert_eq!(got(auto, &c44, &both, Usb, 80_000), Some((Opus, S16, 2))); + assert_eq!(got(auto, &c44, &pcm_only, Usb, 80_000), None); + // Format choice. + let s16 = client_audio(&[48000], &[S16], &[Pcm]); + assert_eq!(got(auto, &s16, &both, Usb, 80_000), Some((Pcm, S16, 2))); + let f32c = client_audio(&[48000], &[F32], &[Pcm]); + assert_eq!( + got(auto, &f32c, &both, Network, 30_000), + Some((Pcm, F32, 2)) + ); + let nofmt = client_audio(&[48000], &[], &[Pcm]); + assert_eq!(got(auto, &nofmt, &pcm_only, Usb, 80_000), None); + // Off, disabled, no source, no channels. + assert_eq!(got(off, &full, &both, Usb, 80_000), None); + let mut disabled = full.clone(); + disabled.enabled = false; + assert_eq!(got(auto, &disabled, &both, Usb, 80_000), None); + assert_eq!(got(auto, &ClientAudio::none(), &both, Usb, 80_000), None); + assert_eq!(got(auto, &full, &HostAudio::default(), Usb, 80_000), None); + let mut mute = full.clone(); + mute.max_channels = 0; + assert_eq!(got(auto, &mute, &both, Usb, 80_000), None); + // Channels follow the client, capped at stereo. + let mut mono = full.clone(); + mono.max_channels = 1; + assert_eq!(got(auto, &mono, &both, Usb, 80_000), Some((Pcm, F32, 1))); + let mut surround = full.clone(); + surround.max_channels = 8; + assert_eq!( + got(auto, &surround, &both, Usb, 80_000), + Some((Pcm, F32, 2)) + ); + let s = negotiate_audio(auto, &full, &both, Usb, 80_000).unwrap(); + assert_eq!((s.sample_rate, s.frame_samples), (48_000, 480)); + assert!(s.codec_config.is_empty()); + } + #[test] fn dynamic_range_table() { let hdr = |display: bool, codecs: &[Codec]| ClientHdr::new(display, 1000, 600, 50, codecs); diff --git a/crates/ezd-proto/src/pair.rs b/crates/ezd-proto/src/pair.rs index 2818740..cde8ebe 100644 --- a/crates/ezd-proto/src/pair.rs +++ b/crates/ezd-proto/src/pair.rs @@ -21,7 +21,7 @@ pub const INPUT_STREAM_TAG: u8 = b'I'; pub const CURSOR_STREAM_TAG: u8 = b'C'; /// First byte of the host's video stream. pub const VIDEO_STREAM_TAG: u8 = b'V'; -/// First byte reserved for the audio stream (not used yet). +/// First byte of the audio stream (QUIC backend, protocol 0.6). pub const AUDIO_STREAM_TAG: u8 = b'A'; /// Largest preamble message payload. pub const MAX_PREAMBLE_LEN: usize = 1024; diff --git a/docs/protocol.md b/docs/protocol.md index 9eb472a..ea1c04d 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -1,4 +1,4 @@ -# ezdisplay wire protocol (version 0.5) +# ezdisplay wire protocol (version 0.6) All integers are little endian. The protocol runs over any reliable ordered byte stream: TCP through `adb reverse` over USB, or a QUIC connection on Wi-Fi (see [Transports](#transports)). It is implemented in the `ezd-proto` @@ -30,7 +30,7 @@ Every frame is an 8-byte header followed by `payload_len` bytes of payload. | 1 | Input | Pen, touch, pointer buttons, scroll and keys, client to host (0.3; keys 0.4). | | 2 | Cursor | Cursor shape and position. | | 3 | Video | Encoded video frames. | -| 4 | Audio | Reserved. | +| 4 | Audio | PC audio packets, host to client (0.6). | ### Kinds @@ -52,6 +52,8 @@ Every frame is an 8-byte header followed by `payload_len` bytes of payload. | Control | 14 | `ClientOptions` | host to client | postcard (0.3) | | Control | 15 | `ClientHdr` | client to host | postcard (0.5) | | Control | 16 | `StreamColor` | host to client | postcard (0.5) | +| Control | 17 | `ClientAudio` | client to host | postcard (0.6) | +| Control | 18 | `AudioConfig` | host to client | postcard (0.6) | | Input | 1 | `Pen` (`PenSample`) | client to host | postcard (0.3) | | Input | 2 | `Touch` (`TouchFrame`) | client to host | postcard (0.3) | | Input | 3 | `PointerButton` | client to host | postcard (0.3) | @@ -60,10 +62,11 @@ Every frame is an 8-byte header followed by `payload_len` bytes of payload. | Cursor | 1 | `CursorShape` | host to client | postcard | | Cursor | 2 | `CursorState` | host to client | postcard | | Video | 1 | `VideoFrame` | host to client | postcard | +| Audio | 1 | `AudioPacket` | host to client | postcard (0.6) | The payload of a message is the [postcard](https://docs.rs/postcard) encoding (serde, varint integers, no field names) of the struct of that name in `ezd-proto/src/msg.rs`. Byte vectors -(`VideoFrame.data`, `CursorShape.rgba`) use `serde_bytes`, i.e. a varint length followed by +(`VideoFrame.data`, `CursorShape.rgba`, `AudioPacket.data`, `AudioStream.codec_config`) use `serde_bytes`, i.e. a varint length followed by raw bytes. The struct definitions are the normative field list and order. ### Fragmentation and reassembly @@ -83,7 +86,7 @@ per push) and yields complete messages. ### Forward compatibility * A `(channel, kind)` pair the receiver does not know is skipped silently (it is reassembled, - counted and dropped), never an error. This includes the reserved Input and Audio channels. + counted and dropped), never an error. A peer older than a channel simply skips its messages. * Unknown flag bits are ignored. * A known pair with a malformed payload is a protocol error and the connection should be closed. Because postcard is not self-describing, existing structs are never extended in @@ -91,9 +94,9 @@ per push) and yields complete messages. ### Versioning -`Hello` carries `proto_major` and `proto_minor` (currently 0 and 5; 0.2 added `FrameAck`, 0.3 +`Hello` carries `proto_major` and `proto_minor` (currently 0 and 6; 0.2 added `FrameAck`, 0.3 the input messages, `InputStatus` and `ClientOptions`, 0.4 the `Key` message, 0.5 `ClientHdr` and -`StreamColor` for HDR10; the Wi-Fi transport is below the framed protocol and needs no minor +`StreamColor` for HDR10, 0.6 `ClientAudio`, `AudioConfig` and the Audio channel for PC audio; the Wi-Fi transport is below the framed protocol and needs no minor bump). A different major version is fatal: the receiver answers `Reject` and closes. Minor versions only add new `(channel, kind)` pairs, so peers with different minors interoperate. `Hello::supports_input()` is true @@ -103,6 +106,10 @@ Input messages to an older host. `Key` needs no minor gate: a client sends it on host's `InputStatus.caps` has `KEYS`, and an older host would skip the unknown pair anyway. `Hello::supports_hdr()` is true when the peer's minor is at least `MINOR_HDR` = 5: a client sends `ClientHdr` only to such a host, and a host sends `StreamColor` only to such a client. +`Hello::supports_audio()` is true when the peer's minor is at least `MINOR_AUDIO` = 6: a client +sends `ClientAudio` only to such a host (and only if it advertised such a minor itself), and a +host sends `AudioConfig` and `AudioPacket` only to such a client. A peer with an older minor sees +no audio message at all. ## Transports @@ -126,13 +133,16 @@ Streams, one per channel: | Control | bidirectional | client | none (preamble first) | 4 (highest) | | Input | unidirectional | client, after reading `Accepted` | `'I'` | 3 | | Cursor | unidirectional | host, after writing `Accepted` | `'C'` | 2 | -| Audio (reserved) | unidirectional | host | `'A'` | 1 | +| Audio | unidirectional | host, after writing `Accepted` | `'A'` | 1 | | Video | unidirectional | host, after writing `Accepted` | `'V'` | 0 (lowest) | Every unidirectional stream starts with its tag byte, then carries the framed messages of its channel. Control carries the preamble, then the framed protocol for the control channel. The -receiver accepts the expected streams (the host one, the client two) within 10 s and dispatches -them by tag, not by arrival order; an unknown or repeated tag ends the session. The cursor stream +receiver accepts the expected streams (the host one, the client three) within 10 s and dispatches +them by tag, not by arrival order; an unknown or repeated tag ends the session. The audio stream +is optional for compatibility: the host gives up opening it after 0.5 s (a client older than +0.6 allows only two), and the client waits 0.5 s for it after the cursor and video streams (a +host older than 0.6 never opens it); a link without the audio stream carries no audio. The cursor stream carries cursor shapes and the latest-wins cursor state, shapes first. There is no compatibility with earlier layouts. @@ -152,6 +162,7 @@ client host |=== input stream: 'I', Pen/Touch ... ===>| opened after reading Accepted |<== cursor stream: 'C', CursorState ... ==| opened after writing Accepted |<=== video stream: 'V', VideoFrame ... ===| + |<=== audio stream: 'A', AudioPacket ... ===| | Hello ... (as in Handshake and session) | ``` @@ -161,8 +172,8 @@ Transport settings tuned for latency: keep-alive 1 s, idle timeout 5 s, initial The host accepts any client certificate in the handshake, so it treats every peer as untrusted until the preamble: it finishes each handshake in its own task within 10 s, keeps at most 8 connections, allows the client one bidirectional stream and one unidirectional one (input), and uses -256 KiB receive windows (it never receives video). The client allows the host only the cursor -and video streams and uses 8 MiB stream and 16 MiB connection receive windows, room for a large keyframe. +256 KiB receive windows (it never receives video). The client allows the host only the cursor, +video and audio streams and uses 8 MiB stream and 16 MiB connection receive windows, room for a large keyframe. A client only opens a session on a connection whose host certificate it pinned. Hosts advertise themselves over mDNS as `_ezdisplay._udp.local.` with TXT records `fp` (short @@ -223,15 +234,18 @@ client host |<----- Hello(role=Host, version, name) | either order; major mismatch -> Reject | ClientCaps(display, decoders, ...) ---->| | ClientHdr (host minor >= 5) ----------->| host waits for it if the client minor >= 5 + | ClientAudio (host minor >= 6) --------->| host waits for it if the client minor >= 6 |<---------------------- HostCaps(encoders)| | negotiate_hdr(ClientCaps, HostCaps, Prefs, ...) (host, pure function) |<----------------- StreamColor (minor >= 5)| sent before StreamConfig + |<----------------- AudioConfig (minor >= 6)| after StreamColor; stream = None: no audio |<-------------------------- StreamConfig | or Reject{reason} | open decoder | | ConfigAck ----------------------------->| | | host starts capture + encode |<=== VideoFrame(CODEC_CONFIG) ============| |<=== VideoFrame(KEYFRAME) ================| + |<=== AudioPacket (every 10 ms) ===========| only if AudioConfig carried a stream |<=== VideoFrame..., CursorShape/State ====| Ping/Pong both ways, ClientStats ~1 Hz | FrameAck (per frame, before decode) --->| releases the host's video credit | RequestKeyframe (on decode error) ----->| next frame is an IDR @@ -262,6 +276,8 @@ gives up (the host uses 10 s per step). tone-mapped, for one reason of: turned off, old client, display not HDR, no 10-bit decoder, no 10-bit encoder. +* Audio (`negotiate_audio`): see [Audio](#audio-06). + ## Video `VideoFrame { frame_id, flags, capture_us, encoded_us, data }`. `data` is an Annex-B bitstream @@ -305,13 +321,16 @@ which is only meant for tests. ### Priority -The writer serves queues in the order control, input, cursor shapes, cursor state, video. +The writer serves queues in the order control, input, cursor shapes, cursor state, audio, video. Video frames are written in chunks and the higher-priority queues are re-checked between chunks, so a large keyframe delays a control, input or cursor message by at most one chunk (64 KiB by default). `CursorState` is latest-wins: a state that has not been written yet is replaced by a newer one. Everything else is reliable FIFO. Note that control messages overtake video: send `Bye` after the last frame has been written if the peer should see all frames first. +Audio is a short FIFO: when more than 10 audio packets (about 100 ms) are queued, the oldest +is dropped and counted (`audio_dropped`), so a stalled link replays no stale sound. + Input is batched: when the writer takes input it drains every queued input message into one buffer and writes and flushes it once. Under a backlog (128 or more queued input messages) a new `Pen` replaces the last queued message if that is a `Pen` with identical `flags`, and a new @@ -373,6 +392,39 @@ white: [u16; 2], max_luminance: u32, min_luminance: u32 }` holds the red, green chromaticities and the white point in units of 0.00002, and the luminances in units of 0.0001 cd/m2. A client that receives no `StreamColor` assumes 8-bit limited-range BT.709, as before 0.5. +## Audio (0.6) + +The host streams the PC's audio output to the client. + +Client to host, on the Control channel, right after `ClientHdr` when the host minor is at least +6 (and the client minor too): `ClientAudio { enabled: bool, sample_rates: Vec, max_channels: +u8, formats: Vec, codecs: Vec }`. `enabled` is the user's choice, the +rest is what the output can do: `SampleFormat` is `S16` or `F32` (little-endian, interleaved), +`AudioCodec` is `Pcm` or `Opus`. A client without audio sends `enabled = false` and empty lists. +The host waits for the message (10 s per step like the others), so a client of minor 6 must +always send it. + +Host to client, right after `StreamColor` and before `StreamConfig`: `AudioConfig { stream: +Option }`; `None` means no audio in this session. `AudioStream { codec, sample_rate: +u32, channels: u8, format: SampleFormat, frame_samples: u32, codec_config: Vec }`. +`sample_rate` is 48000, `channels` is the smaller of 2 and the client's `max_channels`, +`frame_samples` (samples per channel per packet) is 480, i.e. 10 ms. `codec_config` is the OpusHead +identification header for Opus and empty for PCM; `format` applies to PCM only (it is `S16` and +ignored for Opus, the client chooses its decoder output format). + +Host to client, on the Audio channel: `AudioPacket { pts_us: u64, data: Vec }`, one every 10 ms +while audio is captured. `pts_us` is the host monotonic time of the packet's first sample. `data` +is exactly `frame_samples * channels` interleaved samples for PCM (`frame_samples * channels * 2` +or `* 4` bytes) and one Opus packet for Opus. Audio is not acknowledged; audio packets carry no +credit. If audio capture fails the host stops sending packets and the video session goes on. + +Negotiation (`negotiate_audio`, mode `Auto` / `Pcm` / `Opus` / `Off`): audio is off when the mode +is `Off`, the client is disabled or has no channels, or the host has no capture source. PCM needs +the client to support `Pcm` and 48 kHz (and, in `Auto`, a link ceiling of at least 8 Mbit/s); its +format is `F32` on USB when the client plays it, otherwise `S16`, otherwise `F32`. Opus needs the +client to support `Opus` and the host an Opus encoder. `Auto` and `Pcm` try PCM then Opus; `Opus` +tries Opus then PCM; forced `Pcm` ignores the link ceiling. + ## Cursor `CursorShape { shape_id, width, height, hot_x, hot_y, rgba }` is sent once per `shape_id` per From cec3e60b8f27b4500fc5417446ab5a94e72c26a1 Mon Sep 17 00:00:00 2001 From: FacuM Date: Sun, 25 Oct 2026 21:14:41 -0300 Subject: [PATCH 02/12] Encode audio as Opus through libopus Adds an AudioEncoder in ezd-ffmpeg (48 kHz, 10 ms frames, low-delay application) that the host uses when Opus is negotiated, and builds libopus statically into the AppImage's FFmpeg. --- crates/ezd-ffmpeg/src/audio.rs | 361 ++++++++++++++++++++++++++++++ crates/ezd-ffmpeg/src/lib.rs | 2 + packaging/linux/appimage/build.sh | 26 ++- packaging/linux/appimage/pins.env | 6 + 4 files changed, 390 insertions(+), 5 deletions(-) create mode 100644 crates/ezd-ffmpeg/src/audio.rs diff --git a/crates/ezd-ffmpeg/src/audio.rs b/crates/ezd-ffmpeg/src/audio.rs new file mode 100644 index 0000000..14f91e3 --- /dev/null +++ b/crates/ezd-ffmpeg/src/audio.rs @@ -0,0 +1,361 @@ +//! Opus audio encoder on top of libavcodec's `libopus` wrapper (48 kHz, 10 ms frames). + +use crate::error::{EAGAIN, FfmpegError, init_logging}; +use ezd_host_core::{AudioEncoder, SourceError}; +use ffmpeg_sys_next as ffi; +use std::ffi::CStr; +use std::ptr; +use std::sync::OnceLock; + +/// Sample rate of the audio stream. +const SAMPLE_RATE: i32 = 48_000; +/// Samples per channel in one 10 ms frame. +const FRAME_SAMPLES: usize = 480; + +/// Whether this FFmpeg build has the `libopus` encoder. The experimental native `opus` encoder is +/// deliberately not used. +pub fn opus_available() -> bool { + static AVAILABLE: OnceLock = OnceLock::new(); + *AVAILABLE.get_or_init(|| { + // SAFETY: plain lookup with a valid NUL-terminated name. + unsafe { !ffi::avcodec_find_encoder_by_name(c"libopus".as_ptr()).is_null() } + }) +} + +/// Creates an Opus encoder for `channels` (1 or 2) at 48 kHz with 10 ms frames. +pub fn create_opus_encoder(channels: u8) -> Result, SourceError> { + OpusEncoder::open(channels) + .map(|e| Box::new(e) as Box) + .map_err(SourceError::backend) +} + +/// The 19-byte OpusHead identification header (RFC 7845, channel mapping family 0). +pub fn opus_head(channels: u8, pre_skip: u16, input_sample_rate: u32) -> Vec { + let mut h = Vec::with_capacity(19); + h.extend_from_slice(b"OpusHead"); + h.push(1); // version + h.push(channels); + h.extend_from_slice(&pre_skip.to_le_bytes()); + h.extend_from_slice(&input_sample_rate.to_le_bytes()); + h.extend_from_slice(&0i16.to_le_bytes()); // output gain + h.push(0); // mapping family 0 + h +} + +/// A `libopus` encoder producing one packet per 10 ms frame. +pub struct OpusEncoder { + ctx: *mut ffi::AVCodecContext, + frame: *mut ffi::AVFrame, + pkt: *mut ffi::AVPacket, + channels: usize, + head: Vec, + pts: i64, +} + +// SAFETY: the FFmpeg objects are owned exclusively by this struct and only used through &mut self. +unsafe impl Send for OpusEncoder {} + +impl std::fmt::Debug for OpusEncoder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OpusEncoder") + .field("channels", &self.channels) + .finish_non_exhaustive() + } +} + +fn set_opt(ctx: *mut ffi::AVCodecContext, name: &CStr, value: &CStr) -> Result<(), FfmpegError> { + // SAFETY: ctx is a valid context whose priv_data holds the encoder's options; both strings are + // NUL-terminated. + let r = unsafe { ffi::av_opt_set((*ctx).priv_data, name.as_ptr(), value.as_ptr(), 0) }; + if r < 0 { + return Err(FfmpegError::av( + format!("av_opt_set {}", name.to_string_lossy()), + r, + )); + } + Ok(()) +} + +impl OpusEncoder { + /// Opens the encoder for 1 or 2 channels. + pub fn open(channels: u8) -> Result { + init_logging(); + if !(1..=2).contains(&channels) { + return Err(FfmpegError::msg(format!( + "Opus encoder supports 1 or 2 channels, not {channels}" + ))); + } + // SAFETY: plain lookup with a valid NUL-terminated name. + let codec = unsafe { ffi::avcodec_find_encoder_by_name(c"libopus".as_ptr()) }; + if codec.is_null() { + return Err(FfmpegError::msg( + "libopus encoder not available in this FFmpeg build", + )); + } + // SAFETY: `codec` is a valid encoder. + let ctx = unsafe { ffi::avcodec_alloc_context3(codec) }; + if ctx.is_null() { + return Err(FfmpegError::msg("avcodec_alloc_context3 failed")); + } + // From here on Drop releases everything that has been allocated. + let mut enc = OpusEncoder { + ctx, + frame: ptr::null_mut(), + pkt: ptr::null_mut(), + channels: channels as usize, + head: Vec::new(), + pts: 0, + }; + // SAFETY: ctx is a freshly allocated, exclusively owned AVCodecContext; we only write plain + // fields before opening it. libopus always supports interleaved f32 input. + let r = unsafe { + let c = &mut *ctx; + c.sample_rate = SAMPLE_RATE; + c.sample_fmt = ffi::AVSampleFormat::AV_SAMPLE_FMT_FLT; + c.bit_rate = i64::from(channels) * 96_000; + c.time_base = ffi::AVRational { + num: 1, + den: SAMPLE_RATE, + }; + ffi::av_channel_layout_default(&mut c.ch_layout, i32::from(channels)); + c.ch_layout.nb_channels + }; + if r != i32::from(channels) { + return Err(FfmpegError::msg("could not set the channel layout")); + } + set_opt(ctx, c"frame_duration", c"10")?; + set_opt(ctx, c"application", c"lowdelay")?; + set_opt(ctx, c"vbr", c"on")?; + // SAFETY: ctx is configured; no options dictionary. + let r = unsafe { ffi::avcodec_open2(ctx, codec, ptr::null_mut()) }; + if r < 0 { + return Err(FfmpegError::av("avcodec_open2 (libopus)", r)); + } + // SAFETY: ctx is opened; reading plain fields and the extradata buffer it owns. + let (frame_size, pre_skip, extradata) = unsafe { + let c = &*ctx; + let extra = if c.extradata.is_null() || c.extradata_size <= 0 { + Vec::new() + } else { + std::slice::from_raw_parts(c.extradata, c.extradata_size as usize).to_vec() + }; + (c.frame_size, c.initial_padding, extra) + }; + if frame_size != FRAME_SAMPLES as i32 { + return Err(FfmpegError::msg(format!( + "libopus opened with frame size {frame_size}, expected {FRAME_SAMPLES}" + ))); + } + enc.head = if extradata.len() >= 19 && extradata.starts_with(b"OpusHead") { + extradata + } else { + opus_head( + channels, + pre_skip.clamp(0, i32::from(u16::MAX)) as u16, + SAMPLE_RATE as u32, + ) + }; + // SAFETY: plain allocations; null is checked. + unsafe { + enc.frame = ffi::av_frame_alloc(); + enc.pkt = ffi::av_packet_alloc(); + if enc.frame.is_null() || enc.pkt.is_null() { + return Err(FfmpegError::msg("av_frame_alloc/av_packet_alloc failed")); + } + let f = &mut *enc.frame; + f.nb_samples = FRAME_SAMPLES as i32; + f.format = ffi::AVSampleFormat::AV_SAMPLE_FMT_FLT as i32; + f.sample_rate = SAMPLE_RATE; + let r = ffi::av_channel_layout_copy(&mut f.ch_layout, &(*ctx).ch_layout); + if r < 0 { + return Err(FfmpegError::av("av_channel_layout_copy", r)); + } + let r = ffi::av_frame_get_buffer(enc.frame, 0); + if r < 0 { + return Err(FfmpegError::av("av_frame_get_buffer", r)); + } + } + Ok(enc) + } + + fn encode_frame(&mut self, pcm: &[f32]) -> Result, FfmpegError> { + let want = FRAME_SAMPLES * self.channels; + if pcm.len() != want { + return Err(FfmpegError::msg(format!( + "Opus frame needs {want} samples, got {}", + pcm.len() + ))); + } + // SAFETY: ctx is an opened encoder; frame owns a buffer of FRAME_SAMPLES interleaved f32 + // samples per channel (allocated in `open`), made writable before the copy; the packet + // data is readable until av_packet_unref. + unsafe { + let r = ffi::av_frame_make_writable(self.frame); + if r < 0 { + return Err(FfmpegError::av("av_frame_make_writable", r)); + } + ptr::copy_nonoverlapping(pcm.as_ptr(), (*self.frame).data[0].cast::(), want); + (*self.frame).pts = self.pts; + let r = ffi::avcodec_send_frame(self.ctx, self.frame); + if r < 0 { + return Err(FfmpegError::av("avcodec_send_frame", r)); + } + self.pts += FRAME_SAMPLES as i64; + let r = ffi::avcodec_receive_packet(self.ctx, self.pkt); + if r == EAGAIN { + // libopus with a 10 ms frame duration yields one packet per frame, so this is not + // expected; an empty packet tells the caller there is nothing to send. + return Ok(Vec::new()); + } + if r < 0 { + return Err(FfmpegError::av("avcodec_receive_packet", r)); + } + let data = + std::slice::from_raw_parts((*self.pkt).data, (*self.pkt).size as usize).to_vec(); + ffi::av_packet_unref(self.pkt); + Ok(data) + } + } +} + +impl AudioEncoder for OpusEncoder { + fn codec_config(&self) -> Vec { + self.head.clone() + } + + fn encode(&mut self, pcm: &[f32]) -> Result, SourceError> { + self.encode_frame(pcm).map_err(SourceError::backend) + } +} + +impl Drop for OpusEncoder { + fn drop(&mut self) { + // SAFETY: each pointer is either null or exclusively owned by us; the FFmpeg free + // functions accept pointers to null and reset them. + unsafe { + ffi::av_frame_free(&mut self.frame); + ffi::av_packet_free(&mut self.pkt); + ffi::avcodec_free_context(&mut self.ctx); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn opus_head_layout() { + let h = opus_head(2, 312, 48_000); + assert_eq!(h.len(), 19); + assert_eq!(&h[..8], b"OpusHead"); + assert_eq!(h[8], 1); + assert_eq!(h[9], 2); + assert_eq!(u16::from_le_bytes([h[10], h[11]]), 312); + assert_eq!(u32::from_le_bytes([h[12], h[13], h[14], h[15]]), 48_000); + assert_eq!(i16::from_le_bytes([h[16], h[17]]), 0); + assert_eq!(h[18], 0); + } + + fn sine(channels: usize, frame: usize) -> Vec { + (0..FRAME_SAMPLES) + .flat_map(|i| { + let t = (frame * FRAME_SAMPLES + i) as f32 / 48_000.0; + let v = 0.5 * (2.0 * std::f32::consts::PI * 440.0 * t).sin(); + std::iter::repeat_n(v, channels) + }) + .collect() + } + + #[test] + fn encodes_one_packet_per_frame() { + if !opus_available() { + eprintln!("libopus not available, skipping"); + return; + } + for channels in [1u8, 2] { + let mut enc = create_opus_encoder(channels).unwrap(); + let head = enc.codec_config(); + assert!(head.starts_with(b"OpusHead")); + assert_eq!(head[9], channels); + let mut total = 0; + for n in 0..100 { + let p = enc.encode(&sine(channels as usize, n)).unwrap(); + assert!(!p.is_empty(), "frame {n} produced no packet"); + total += p.len(); + } + let avg = total / 100; + let kbps = u64::from(channels) * 96_000; + let expect = (kbps / 8 / 100) as usize; + assert!( + avg > 10 && avg < expect * 2, + "average packet {avg} bytes for {kbps} bit/s" + ); + } + assert!(create_opus_encoder(3).is_err()); + } + + #[test] + fn rejects_wrong_frame_length() { + if !opus_available() { + return; + } + let mut enc = create_opus_encoder(2).unwrap(); + assert!(enc.encode(&[0.0; 480]).is_err()); + } + + #[test] + fn decodes_back_to_the_same_level() { + if !opus_available() { + return; + } + // SAFETY: lookup only. + let dec = unsafe { ffi::avcodec_find_decoder(ffi::AVCodecID::AV_CODEC_ID_OPUS) }; + if dec.is_null() { + return; + } + let mut enc = create_opus_encoder(2).unwrap(); + let head = enc.codec_config(); + let mut rms_sum = 0.0f64; + let mut count = 0usize; + // SAFETY: standard decode loop on a context, packet and frame owned by this block and + // freed at its end; extradata is allocated with the required padding. + unsafe { + let mut ctx = ffi::avcodec_alloc_context3(dec); + let extra = ffi::av_mallocz(head.len() + ffi::AV_INPUT_BUFFER_PADDING_SIZE as usize) + .cast::(); + ptr::copy_nonoverlapping(head.as_ptr(), extra, head.len()); + (*ctx).extradata = extra; + (*ctx).extradata_size = head.len() as i32; + (*ctx).sample_rate = SAMPLE_RATE; + ffi::av_channel_layout_default(&mut (*ctx).ch_layout, 2); + assert!(ffi::avcodec_open2(ctx, dec, ptr::null_mut()) >= 0); + let mut pkt = ffi::av_packet_alloc(); + let mut frame = ffi::av_frame_alloc(); + for n in 0..100 { + let data = enc.encode(&sine(2, n)).unwrap(); + assert_eq!(ffi::av_new_packet(pkt, data.len() as i32), 0); + ptr::copy_nonoverlapping(data.as_ptr(), (*pkt).data, data.len()); + assert!(ffi::avcodec_send_packet(ctx, pkt) >= 0); + ffi::av_packet_unref(pkt); + while ffi::avcodec_receive_frame(ctx, frame) >= 0 { + if n >= 20 { + let ns = (*frame).nb_samples as usize; + // The native decoder outputs planar f32; read the left channel. + let left = std::slice::from_raw_parts((*frame).data[0].cast::(), ns); + rms_sum += left.iter().map(|s| f64::from(*s).powi(2)).sum::(); + count += ns; + } + ffi::av_frame_unref(frame); + } + } + ffi::av_frame_free(&mut frame); + ffi::av_packet_free(&mut pkt); + ffi::avcodec_free_context(&mut ctx); + } + assert!(count > 0, "decoder produced no samples"); + let rms = (rms_sum / count as f64).sqrt(); + // A 0.5 amplitude sine has an RMS of about 0.354. + assert!((rms - 0.354).abs() < 0.05, "decoded RMS {rms}"); + } +} diff --git a/crates/ezd-ffmpeg/src/lib.rs b/crates/ezd-ffmpeg/src/lib.rs index 21d0b84..d6cdce3 100644 --- a/crates/ezd-ffmpeg/src/lib.rs +++ b/crates/ezd-ffmpeg/src/lib.rs @@ -16,6 +16,7 @@ // Bindgen constant types (i32 vs u32) differ between platforms; the casts keep one code base portable. #![allow(clippy::unnecessary_cast)] +mod audio; mod color; mod decoder; mod encoder; @@ -27,6 +28,7 @@ mod source; #[cfg(target_os = "linux")] mod vulkan; +pub use audio::{OpusEncoder, create_opus_encoder, opus_available, opus_head}; pub use color::{ColorInfo, HdrMeta}; pub use decoder::{DecodedFrame, Decoder}; pub use encoder::{EncodedPacket, Encoder, EncoderConfig, RawPixelFormat}; diff --git a/packaging/linux/appimage/build.sh b/packaging/linux/appimage/build.sh index 267abbe..17d6d5b 100755 --- a/packaging/linux/appimage/build.sh +++ b/packaging/linux/appimage/build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Builds the ezdisplay AppImage: a minimal GPL FFmpeg (with x264 and the hardware paths the host +# Builds the ezdisplay AppImage: a minimal GPL FFmpeg (with x264, libopus and the hardware paths the host # uses) built from pinned sources, the two binaries linked against it, the bundled adb, and the # AppImage itself. # @@ -79,7 +79,7 @@ fetch() { } log "Downloading pinned sources and tools" -for n in FFMPEG X264 NVCODEC VULKAN APPIMAGETOOL RUNTIME ADB; do fetch $n; done +for n in FFMPEG X264 OPUS NVCODEC VULKAN APPIMAGETOOL RUNTIME ADB; do fetch $n; done # unpack NAME DIR: extracts the archive into an empty DIR (one top-level directory is stripped). unpack() { @@ -112,6 +112,18 @@ unpack X264 "$SRC/x264" make install ) +# --- 3b. libopus (static, PIC; BSD-3-Clause) ---------------------------------------------- + +log "Building libopus" +unpack OPUS "$SRC/opus" +( + cd "$SRC/opus" + ./configure --prefix="$PREFIX" --enable-static --disable-shared --with-pic \ + --disable-doc --disable-extra-programs + make -j"$JOBS" + make install +) + # --- 4. FFmpeg ---------------------------------------------------------------------------- log "Building FFmpeg $FFMPEG_VERSION" @@ -120,11 +132,11 @@ export PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig FFMPEG_CONFIGURE=( --prefix="$PREFIX" --enable-shared --disable-static --enable-pic - --enable-gpl --enable-libx264 + --enable-gpl --enable-libx264 --enable-libopus --disable-autodetect --disable-everything --disable-programs --disable-doc --disable-avdevice --disable-avformat --disable-swresample --disable-network --disable-xlib - --enable-encoder=h264_vaapi,h264_nvenc,libx264 + --enable-encoder=h264_vaapi,h264_nvenc,libx264,libopus --enable-decoder=h264 --enable-parser=h264 --enable-filter=buffer,buffersink,format,null,scale,hwmap,hwupload,scale_vaapi,scale_vulkan @@ -191,6 +203,7 @@ install -m644 "$REPO/LICENSE-MIT" "$REPO/LICENSE-APACHE" "$L/" install -m644 "$REPO/packaging/windows/licenses/FluentSystemIcons-LICENSE.txt" "$L/" install -m644 "$SRC/ffmpeg/COPYING.GPLv2" "$SRC/ffmpeg/COPYING.GPLv3" "$SRC/ffmpeg/LICENSE.md" "$L/" install -m644 "$SRC/x264/COPYING" "$L/x264-COPYING" +install -m644 "$SRC/opus/COPYING" "$L/opus-COPYING" for pkg in glslang-dev glslang-tools spirv-tools; do [ -f "/usr/share/doc/$pkg/copyright" ] && install -m644 "/usr/share/doc/$pkg/copyright" "$L/$pkg-copyright" done @@ -208,6 +221,9 @@ FFmpeg $FFMPEG_VERSION (libavcodec, libavfilter, libavutil, libswscale) x264 $X264_VERSION (statically linked into libavcodec) Licence: GNU GPL v2 or later (see x264-COPYING). Source: $X264_URL +libopus $OPUS_VERSION (statically linked into libavcodec, for the Opus audio encoder) + Licence: BSD 3-Clause (see opus-COPYING). Source: $OPUS_URL + glslang and SPIRV-Tools (statically linked into libavfilter, for scale_vulkan) Licences: see the glslang and spirv-tools copyright files next to this file. @@ -286,7 +302,7 @@ log "Collecting the corresponding sources" SRCOUT=$OUT/ezdisplay-appimage-sources rm -rf "$SRCOUT" mkdir -p "$SRCOUT" -for n in FFMPEG X264 NVCODEC VULKAN; do +for n in FFMPEG X264 OPUS NVCODEC VULKAN; do install -m644 "$CACHE/$(eval "echo \${${n}_FILE}")" "$SRCOUT/" done install -m755 "$HERE/build.sh" "$SRCOUT/" diff --git a/packaging/linux/appimage/pins.env b/packaging/linux/appimage/pins.env index b80275d..3d83df8 100644 --- a/packaging/linux/appimage/pins.env +++ b/packaging/linux/appimage/pins.env @@ -14,6 +14,12 @@ X264_URL=https://snapshot.debian.org/archive/debian/20260714T142842Z/pool/main/x X264_FILE=x264_0.165.3223+git20250910.0480cb0.orig.tar.xz X264_SHA256=de68ca43d0c6e09baeb86a38ed5cde86b0a84a21d6babcc569804201dc9df133 +# libopus source (BSD 3-Clause), built statically into libavcodec for the Opus audio encoder. +OPUS_VERSION=1.5.2 +OPUS_URL=https://downloads.xiph.org/releases/opus/opus-1.5.2.tar.gz +OPUS_FILE=opus-1.5.2.tar.gz +OPUS_SHA256=65c1d2f78b9f2fb20082c38cbe47c951ad5839345876e46941612ee87f9a7ce1 + # nv-codec-headers (MIT): headers FFmpeg uses to load NVENC and CUDA at run time. n12.1.14.1 is # chosen over 13.x so NVENC keeps working with NVIDIA drivers 530 and later. NVCODEC_VERSION=12.1.14.1 From 9d5564fc8234071dacc6fbe2a86f90e3a5a6b953 Mon Sep 17 00:00:00 2001 From: FacuM Date: Sun, 25 Oct 2026 21:22:01 -0300 Subject: [PATCH 03/12] Capture the PC audio on Windows with WASAPI loopback Adds an AudioSource that records the default output device as 48 kHz float in shared loopback mode. The PC keeps playing the sound locally. --- crates/ezd-host-windows/Cargo.toml | 4 + crates/ezd-host-windows/src/audio_math.rs | 142 +++++++++++ crates/ezd-host-windows/src/lib.rs | 17 +- crates/ezd-host-windows/src/win/audio.rs | 297 ++++++++++++++++++++++ crates/ezd-host-windows/src/win/mod.rs | 1 + 5 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 crates/ezd-host-windows/src/audio_math.rs create mode 100644 crates/ezd-host-windows/src/win/audio.rs diff --git a/crates/ezd-host-windows/Cargo.toml b/crates/ezd-host-windows/Cargo.toml index b2c9714..e3d44ab 100644 --- a/crates/ezd-host-windows/Cargo.toml +++ b/crates/ezd-host-windows/Cargo.toml @@ -35,12 +35,16 @@ windows = { version = "0.62.2", features = [ "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Gdi", + "Win32_Media_Audio", "Win32_Devices_Display", "Win32_Storage_FileSystem", + "Win32_System_Com", + "Win32_System_Com_StructuredStorage", "Win32_System_Console", "Win32_System_IO", "Win32_System_Pipes", "Win32_System_Registry", + "Win32_System_Variant", "Win32_System_SystemInformation", "Win32_System_Threading", "Win32_Security", diff --git a/crates/ezd-host-windows/src/audio_math.rs b/crates/ezd-host-windows/src/audio_math.rs new file mode 100644 index 0000000..892ce42 --- /dev/null +++ b/crates/ezd-host-windows/src/audio_math.rs @@ -0,0 +1,142 @@ +//! Pure logic of the WASAPI loopback capture (`win/audio.rs`): the shared-mode format request, +//! frame/sample math, the silent-packet flag and the packet timestamp. Platform independent and +//! unit-tested everywhere. + +/// Sample rate of the audio stream (fixed by the protocol). +pub const SAMPLE_RATE: u32 = 48_000; +/// Requested WASAPI buffer length (about two 10 ms Opus frames). +pub const BUFFER_MS: u32 = 20; +/// `AUDCLNT_BUFFERFLAGS_SILENT`: the packet content must be treated as zeros. +pub const BUFFERFLAG_SILENT: u32 = 0x2; +/// `WAVE_FORMAT_EXTENSIBLE`. +pub const FORMAT_TAG_EXTENSIBLE: u16 = 0xFFFE; +/// `WAVE_FORMAT_IEEE_FLOAT` sub-format tag (the first field of `KSDATAFORMAT_SUBTYPE_IEEE_FLOAT`). +pub const SUBFORMAT_IEEE_FLOAT: u32 = 0x0000_0003; + +/// The fields of the `WAVEFORMATEXTENSIBLE` the capture asks WASAPI for: 32-bit IEEE float, +/// 48 kHz, `channels` interleaved channels. The Windows audio engine converts and resamples the +/// mix to it (`AUTOCONVERTPCM`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WaveFormat { + pub channels: u16, + pub sample_rate: u32, + pub bits_per_sample: u16, + pub block_align: u16, + pub avg_bytes_per_sec: u32, + pub channel_mask: u32, +} + +/// Format for `channels` interleaved float channels at 48 kHz. +pub fn wave_format(channels: u8) -> WaveFormat { + let channels = u16::from(channels.max(1)); + let block_align = channels * 4; + WaveFormat { + channels, + sample_rate: SAMPLE_RATE, + bits_per_sample: 32, + block_align, + avg_bytes_per_sec: SAMPLE_RATE * u32::from(block_align), + channel_mask: channel_mask(channels), + } +} + +/// Speaker positions (`SPEAKER_*` bits) for the usual layouts; 0 (unspecified) for the others. +pub fn channel_mask(channels: u16) -> u32 { + match channels { + 1 => 0x4, // front center + 2 => 0x3, // front left + right + 4 => 0x33, // quad + 6 => 0x3F, // 5.1 + 8 => 0x63F, // 7.1 + _ => 0, + } +} + +/// Requested buffer duration in 100 ns units. +pub fn buffer_duration_hns() -> i64 { + i64::from(BUFFER_MS) * 10_000 +} + +/// Duration of `frames` frames in microseconds. +pub fn frames_to_us(frames: u32) -> u64 { + u64::from(frames) * 1_000_000 / u64::from(SAMPLE_RATE) +} + +/// Timestamp of the first frame of a packet of `frames` frames that was read at `read_us` +/// (`now_us()` units). Packets are read right after the engine produced them, so the last frame +/// is about "now" and the first one is one packet duration earlier. (The packet's QPC position +/// is not used: it is on another clock than `now_us()` and loopback reports it unreliably.) +pub fn packet_pts_us(read_us: u64, frames: u32) -> u64 { + read_us.saturating_sub(frames_to_us(frames)) +} + +/// Interleaved samples of one packet: zeros when it is flagged silent (its buffer content is +/// undefined), else the little-endian `f32` values of `raw` (a trailing partial sample is ignored). +pub fn packet_samples(raw: &[u8], frames: u32, channels: u16, flags: u32) -> Vec { + let count = frames as usize * usize::from(channels); + if flags & BUFFERFLAG_SILENT != 0 { + return vec![0.0; count]; + } + raw.chunks_exact(4) + .take(count) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stereo_format() { + let f = wave_format(2); + assert_eq!(f.channels, 2); + assert_eq!(f.sample_rate, 48_000); + assert_eq!(f.bits_per_sample, 32); + assert_eq!(f.block_align, 8); + assert_eq!(f.avg_bytes_per_sec, 384_000); + assert_eq!(f.channel_mask, 0x3); + } + + #[test] + fn mono_and_zero_channels() { + assert_eq!(wave_format(1).channel_mask, 0x4); + assert_eq!(wave_format(0).channels, 1); + assert_eq!(wave_format(3).channel_mask, 0); + } + + #[test] + fn buffer_is_20_ms() { + assert_eq!(buffer_duration_hns(), 200_000); + } + + #[test] + fn frame_durations() { + assert_eq!(frames_to_us(480), 10_000); + assert_eq!(frames_to_us(0), 0); + } + + #[test] + fn pts_is_one_packet_before_the_read() { + assert_eq!(packet_pts_us(50_000, 480), 40_000); + assert_eq!(packet_pts_us(5_000, 480), 0); + } + + #[test] + fn silent_packets_are_zeros_even_with_garbage() { + let raw = [0xFFu8; 16]; + let s = packet_samples(&raw, 2, 2, BUFFERFLAG_SILENT | 0x1); + assert_eq!(s, vec![0.0; 4]); + } + + #[test] + fn data_packets_decode_floats() { + let mut raw = Vec::new(); + for v in [0.5f32, -0.25, 1.0, 0.0] { + raw.extend_from_slice(&v.to_le_bytes()); + } + assert_eq!(packet_samples(&raw, 2, 2, 0), vec![0.5, -0.25, 1.0, 0.0]); + assert_eq!(packet_samples(&raw, 1, 2, 0), vec![0.5, -0.25]); + assert_eq!(packet_samples(&raw[..6], 2, 2, 0), vec![0.5]); + } +} diff --git a/crates/ezd-host-windows/src/lib.rs b/crates/ezd-host-windows/src/lib.rs index 990a185..53e1721 100644 --- a/crates/ezd-host-windows/src/lib.rs +++ b/crates/ezd-host-windows/src/lib.rs @@ -12,6 +12,7 @@ pub mod accent; pub mod annexb; +pub mod audio_math; pub mod cursor_shape; pub mod display_topology; pub mod hdr_math; @@ -28,7 +29,7 @@ pub mod vdd_xml; mod win; pub use ezd_host_core::PlatformOptions; -use ezd_host_core::{InputBackend, PlatformError, VideoSource}; +use ezd_host_core::{AudioSource, InputBackend, PlatformError, VideoSource}; use std::sync::Arc; /// Creates the virtual display + capture + encode pipeline. @@ -58,6 +59,20 @@ pub fn input_backend() -> Option> { } } +/// System audio capture (WASAPI loopback of the default output device); `None` on other +/// targets. Creating it does no device work: that starts in `AudioSource::start`. The PC keeps +/// playing the audio locally. +pub fn audio_source() -> Option> { + #[cfg(windows)] + { + Some(Box::new(win::audio::WasapiLoopback::new())) + } + #[cfg(not(windows))] + { + None + } +} + /// Platform-specific diagnostics for `ezd-host diag`. pub fn diag_section() -> serde_json::Value { #[cfg(windows)] diff --git a/crates/ezd-host-windows/src/win/audio.rs b/crates/ezd-host-windows/src/win/audio.rs new file mode 100644 index 0000000..acdea07 --- /dev/null +++ b/crates/ezd-host-windows/src/win/audio.rs @@ -0,0 +1,297 @@ +//! System audio capture through WASAPI loopback of the default render device. +//! +//! Known limitation: loopback taps the mix, it does not redirect it, so the PC keeps playing the +//! audio locally while it is streamed. +//! +//! `start` spawns the capture thread `ezd-audio-wasapi`, which owns all COM objects (it joins the +//! MTA), opens the default render endpoint in shared mode with the loopback flag and reads +//! packets into a bounded channel that `next_samples` drains. Nothing touches COM before `start`. +//! If the device goes away (`AUDCLNT_E_DEVICE_INVALIDATED`, default device switched, ...) the +//! thread ends and `next_samples` reports the error; the session then stops the audio only. + +use crate::audio_math::{self, WaveFormat}; +use ezd_host_core::{AudioBlock, AudioSource, SourceError, now_us}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, TrySendError}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::Duration; +use windows::Win32::Foundation::{CloseHandle, HANDLE}; +use windows::Win32::Media::Audio::{ + AUDCLNT_SHAREMODE_SHARED, AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM, + AUDCLNT_STREAMFLAGS_EVENTCALLBACK, AUDCLNT_STREAMFLAGS_LOOPBACK, + AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY, IAudioCaptureClient, IAudioClient, + IMMDeviceEnumerator, MMDeviceEnumerator, WAVEFORMATEX, WAVEFORMATEXTENSIBLE, + WAVEFORMATEXTENSIBLE_0, eConsole, eRender, +}; +use windows::Win32::System::Com::{ + CLSCTX_ALL, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx, CoUninitialize, +}; +use windows::Win32::System::Threading::{CreateEventW, WaitForSingleObject}; +use windows::core::GUID; + +/// Blocks queued between the capture thread and `next_samples` (about 1 s at 20 ms); newer +/// blocks are dropped when the consumer is behind. +const QUEUE_BLOCKS: usize = 50; +/// How long the capture thread waits for the buffer event before it polls anyway (loopback +/// delivers no events while nothing plays on some Windows versions) and checks the stop flag. +const WAIT_MS: u32 = 100; +/// How long `start` waits for the capture thread to open the device. +const INIT_TIMEOUT: Duration = Duration::from_secs(3); +/// `KSDATAFORMAT_SUBTYPE_IEEE_FLOAT`. +const SUBTYPE_IEEE_FLOAT: GUID = GUID::from_u128(0x00000003_0000_0010_8000_00aa00389b71); + +type Failure = Arc>>; + +/// WASAPI loopback [`AudioSource`]. +pub struct WasapiLoopback { + rx: Option>, + stop: Arc, + failure: Failure, + thread: Option>, +} + +impl WasapiLoopback { + pub fn new() -> Self { + Self { + rx: None, + stop: Arc::new(AtomicBool::new(false)), + failure: Arc::new(Mutex::new(None)), + thread: None, + } + } +} + +impl Default for WasapiLoopback { + fn default() -> Self { + Self::new() + } +} + +impl AudioSource for WasapiLoopback { + fn start(&mut self, channels: u8) -> Result<(), SourceError> { + self.stop(); + self.stop = Arc::new(AtomicBool::new(false)); + self.failure = Arc::new(Mutex::new(None)); + let (tx, rx) = mpsc::sync_channel::(QUEUE_BLOCKS); + let (init_tx, init_rx) = mpsc::sync_channel::>(1); + let stop = self.stop.clone(); + let failure = self.failure.clone(); + let thread = std::thread::Builder::new() + .name("ezd-audio-wasapi".into()) + .spawn(move || capture_thread(channels, tx, init_tx, stop, failure)) + .map_err(SourceError::backend)?; + self.thread = Some(thread); + match init_rx.recv_timeout(INIT_TIMEOUT) { + Ok(Ok(())) => { + self.rx = Some(rx); + Ok(()) + } + Ok(Err(e)) => { + self.stop(); + Err(SourceError::msg(e)) + } + Err(_) => { + // Do not join a thread stuck in a driver call; it exits on the stop flag. + self.stop.store(true, Ordering::SeqCst); + self.thread = None; + Err(SourceError::msg("WASAPI loopback did not start in time")) + } + } + } + + fn next_samples(&mut self, timeout: Duration) -> Result, SourceError> { + let Some(rx) = &self.rx else { + return Err(SourceError::msg("audio capture not started")); + }; + match rx.recv_timeout(timeout) { + Ok(b) => Ok(Some(b)), + Err(RecvTimeoutError::Timeout) => Ok(None), + Err(RecvTimeoutError::Disconnected) => { + let e = self.failure.lock().ok().and_then(|mut f| f.take()); + Err(SourceError::msg( + e.unwrap_or_else(|| "WASAPI capture stopped".into()), + )) + } + } + } + + fn stop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + self.rx = None; + if let Some(t) = self.thread.take() { + let _ = t.join(); + } + } +} + +impl Drop for WasapiLoopback { + fn drop(&mut self) { + self.stop(); + } +} + +/// Closes the buffer event on every exit path. +struct EventGuard(HANDLE); +impl Drop for EventGuard { + fn drop(&mut self) { + // SAFETY: the handle was created by `CreateEventW` and is closed once. + unsafe { + let _ = CloseHandle(self.0); + } + } +} + +/// Balances `CoInitializeEx` on every exit path. +struct ComGuard; +impl Drop for ComGuard { + fn drop(&mut self) { + // SAFETY: only constructed after a successful `CoInitializeEx` on this thread. + unsafe { CoUninitialize() } + } +} + +fn capture_thread( + channels: u8, + tx: mpsc::SyncSender, + init_tx: mpsc::SyncSender>, + stop: Arc, + failure: Failure, +) { + // SAFETY: COM is initialised on this thread and released by `ComGuard` after all COM objects + // created inside `run` are dropped. + let hr = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; + if hr.is_err() { + let _ = init_tx.send(Err(format!("CoInitializeEx failed: {hr:?}"))); + return; + } + let _com = ComGuard; + let fmt = audio_math::wave_format(channels); + if let Err(e) = run(&fmt, &tx, &init_tx, &stop) { + tracing::warn!("WASAPI loopback ended: {e}"); + // Reports a failure that happened before `init_tx` was answered, too. + let _ = init_tx.try_send(Err(e.clone())); + if let Ok(mut f) = failure.lock() { + *f = Some(e); + } + } +} + +fn run( + fmt: &WaveFormat, + tx: &mpsc::SyncSender, + init_tx: &mpsc::SyncSender>, + stop: &AtomicBool, +) -> Result<(), String> { + let err = |what: &str, e: windows::core::Error| format!("{what}: {e}"); + // SAFETY: plain COM calls on objects owned by this function; pointers passed to + // `Initialize`/`GetBuffer` outlive the calls, and the packet slice is only used before + // `ReleaseBuffer`. + unsafe { + let enumerator: IMMDeviceEnumerator = + CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL) + .map_err(|e| err("creating the device enumerator", e))?; + let device = enumerator + .GetDefaultAudioEndpoint(eRender, eConsole) + .map_err(|e| err("no default audio output device", e))?; + let client: IAudioClient = device + .Activate(CLSCTX_ALL, None) + .map_err(|e| err("activating the audio client", e))?; + + let wf = WAVEFORMATEXTENSIBLE { + Format: WAVEFORMATEX { + wFormatTag: audio_math::FORMAT_TAG_EXTENSIBLE, + nChannels: fmt.channels, + nSamplesPerSec: fmt.sample_rate, + nAvgBytesPerSec: fmt.avg_bytes_per_sec, + nBlockAlign: fmt.block_align, + wBitsPerSample: fmt.bits_per_sample, + cbSize: (size_of::() - size_of::()) as u16, + }, + Samples: WAVEFORMATEXTENSIBLE_0 { + wValidBitsPerSample: fmt.bits_per_sample, + }, + dwChannelMask: fmt.channel_mask, + SubFormat: SUBTYPE_IEEE_FLOAT, + }; + let flags = AUDCLNT_STREAMFLAGS_LOOPBACK + | AUDCLNT_STREAMFLAGS_EVENTCALLBACK + | AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM + | AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY; + client + .Initialize( + AUDCLNT_SHAREMODE_SHARED, + flags, + audio_math::buffer_duration_hns(), + 0, + (&wf as *const WAVEFORMATEXTENSIBLE).cast::(), + None, + ) + .map_err(|e| err("initialising loopback capture", e))?; + + let event = CreateEventW(None, false, false, None) + .map_err(|e| err("creating the buffer event", e))?; + let _event = EventGuard(event); + client + .SetEventHandle(event) + .map_err(|e| err("setting the buffer event", e))?; + let capture: IAudioCaptureClient = client + .GetService() + .map_err(|e| err("getting the capture client", e))?; + client.Start().map_err(|e| err("starting capture", e))?; + tracing::info!( + channels = fmt.channels, + "WASAPI loopback capture started (default output device)" + ); + let _ = init_tx.send(Ok(())); + + let mut result = Ok(()); + 'outer: while !stop.load(Ordering::SeqCst) { + // Event or timeout: either way drain whatever is queued. + let _ = WaitForSingleObject(event, WAIT_MS); + loop { + if stop.load(Ordering::SeqCst) { + break 'outer; + } + match capture.GetNextPacketSize() { + Ok(0) => break, + Ok(_) => {} + Err(e) => { + result = Err(err("reading the packet size", e)); + break 'outer; + } + } + let mut data: *mut u8 = std::ptr::null_mut(); + let mut frames = 0u32; + let mut buf_flags = 0u32; + if let Err(e) = + capture.GetBuffer(&mut data, &mut frames, &mut buf_flags, None, None) + { + result = Err(err("reading a packet", e)); + break 'outer; + } + let read_us = now_us(); + let raw: &[u8] = if data.is_null() || frames == 0 { + &[] + } else { + std::slice::from_raw_parts(data, frames as usize * usize::from(fmt.block_align)) + }; + let samples = audio_math::packet_samples(raw, frames, fmt.channels, buf_flags); + let _ = capture.ReleaseBuffer(frames); + if frames == 0 { + break; + } + let block = AudioBlock { + pts_us: audio_math::packet_pts_us(read_us, frames), + samples, + }; + match tx.try_send(block) { + Ok(()) | Err(TrySendError::Full(_)) => {} + Err(TrySendError::Disconnected(_)) => break 'outer, + } + } + } + let _ = client.Stop(); + result + } +} diff --git a/crates/ezd-host-windows/src/win/mod.rs b/crates/ezd-host-windows/src/win/mod.rs index f66cb13..bb68f3b 100644 --- a/crates/ezd-host-windows/src/win/mod.rs +++ b/crates/ezd-host-windows/src/win/mod.rs @@ -4,6 +4,7 @@ // rest is compiled out. #![cfg_attr(not(feature = "ffmpeg"), allow(dead_code))] +pub mod audio; pub mod capture; pub mod diag; pub mod display; From 42340acfd7d249780d198401dae7133f59d87787 Mon Sep 17 00:00:00 2001 From: FacuM Date: Sun, 25 Oct 2026 21:23:49 -0300 Subject: [PATCH 04/12] Capture the PC audio on Linux through a PipeWire null sink Each session creates an "ezdisplay" sink, makes it the configured default output and records its monitor at 48 kHz. Stopping puts the previous default back unless it was changed meanwhile; the sink does not linger, so a crashed host leaves nothing behind. --- crates/ezd-host-linux/src/audio.rs | 658 +++++++++++++++++++++++++++++ crates/ezd-host-linux/src/lib.rs | 9 +- 2 files changed, 666 insertions(+), 1 deletion(-) create mode 100644 crates/ezd-host-linux/src/audio.rs diff --git a/crates/ezd-host-linux/src/audio.rs b/crates/ezd-host-linux/src/audio.rs new file mode 100644 index 0000000..d3d581b --- /dev/null +++ b/crates/ezd-host-linux/src/audio.rs @@ -0,0 +1,658 @@ +//! System audio capture through PipeWire. +//! +//! `start` runs a PipeWire main loop on its own thread ("ezd-audio-pw") that +//! +//! 1. creates a null sink named `ezdisplay` (`object.linger=false`, so the daemon removes it as +//! soon as our connection closes, also after a crash), +//! 2. makes it the configured default output through the `default` metadata object, remembering +//! the previous value so `stop` can put it back (after a crash WirePlumber falls back by itself +//! because the node vanishes), and +//! 3. captures the sink's monitor with a stream and hands the samples to `next_samples` through a +//! bounded channel. +//! +//! Nothing touches PipeWire before `start`. + +use ezd_host_core::{AudioBlock, AudioSource, SourceError, now_us}; +use pipewire as pw; +use pw::properties::properties; +use pw::proxy::ProxyT; +use pw::spa::pod::Pod; +use pw::spa::sys as spa_sys; +use std::cell::{Cell, RefCell}; +use std::rc::Rc; +use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender, TryRecvError, sync_channel}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::Duration; + +/// Name of the null sink, also the `target.object` of the capture stream. +const SINK_NAME: &str = "ezdisplay"; +const SAMPLE_RATE: u32 = 48_000; +/// Metadata key that holds the user's chosen default output. +const DEFAULT_SINK_KEY: &str = "default.configured.audio.sink"; +const JSON_TYPE: &str = "Spa:String:JSON"; +/// Blocks queued between the PipeWire thread and `next_samples` before new ones are dropped. +const QUEUE_BLOCKS: usize = 50; +/// How long `start` waits for the sink and the stream to come up. +const START_TIMEOUT: Duration = Duration::from_secs(3); +/// How long `stop` waits for the PipeWire thread to restore the default and exit. +const STOP_TIMEOUT: Duration = Duration::from_secs(2); +/// If the daemon never answers the sync after a failure, the loop quits after this. +const STOP_SYNC_TIMEOUT: Duration = Duration::from_millis(1000); + +/// The capture source returned by [`audio_source`]. +pub struct PipeWireAudio { + running: Option, +} + +struct Running { + tx: pw::channel::Sender, + rx: Receiver, + thread: JoinHandle<()>, + /// Signalled by the thread right before it ends. + exited: Receiver<()>, + error: Arc>>, +} + +enum Cmd { + Stop, +} + +/// Creates the (idle) source; PipeWire is only contacted by `start`. +pub fn audio_source() -> Option> { + Some(Box::new(PipeWireAudio { running: None })) +} + +impl AudioSource for PipeWireAudio { + fn start(&mut self, channels: u8) -> Result<(), SourceError> { + self.stop(); + let channels = channels.clamp(1, 2); + let (cmd_tx, cmd_rx) = pw::channel::channel::(); + let (block_tx, block_rx) = sync_channel::(QUEUE_BLOCKS); + let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); + let (exit_tx, exit_rx) = std::sync::mpsc::channel::<()>(); + let error = Arc::new(Mutex::new(None)); + let thread = { + let error = error.clone(); + std::thread::Builder::new() + .name("ezd-audio-pw".into()) + .spawn(move || { + if let Err(e) = run(channels, block_tx, cmd_rx, &ready_tx) { + tracing::warn!("audio capture failed: {e}"); + let _ = ready_tx.send(Err(e.clone())); + *error.lock().unwrap_or_else(|e| e.into_inner()) = Some(e); + } + let _ = exit_tx.send(()); + }) + .map_err(|e| SourceError::msg(format!("cannot spawn audio thread: {e}")))? + }; + self.running = Some(Running { + tx: cmd_tx, + rx: block_rx, + thread, + exited: exit_rx, + error, + }); + match ready_rx.recv_timeout(START_TIMEOUT) { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => { + self.stop(); + Err(SourceError::msg(e)) + } + Err(_) => { + self.stop(); + Err(SourceError::msg( + "timed out waiting for the PipeWire audio sink and stream", + )) + } + } + } + + fn next_samples(&mut self, timeout: Duration) -> Result, SourceError> { + let Some(r) = &self.running else { + return Err(SourceError::msg("audio capture is not started")); + }; + match r.rx.recv_timeout(timeout) { + Ok(b) => Ok(Some(b)), + Err(RecvTimeoutError::Timeout) => Ok(None), + Err(RecvTimeoutError::Disconnected) => { + let e = r.error.lock().unwrap_or_else(|e| e.into_inner()).clone(); + Err(SourceError::msg(e.unwrap_or_else(|| { + "the PipeWire audio thread ended".into() + }))) + } + } + } + + fn stop(&mut self) { + let Some(r) = self.running.take() else { + return; + }; + // The thread restores the default output before it drops the sink. + let _ = r.tx.send(Cmd::Stop); + match r.exited.recv_timeout(STOP_TIMEOUT) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => { + let _ = r.thread.join(); + } + Err(RecvTimeoutError::Timeout) => { + tracing::warn!("the PipeWire audio thread did not stop in time; detaching it"); + } + } + // Drain so a stale block never reaches a later session. + while !matches!( + r.rx.try_recv(), + Err(TryRecvError::Empty | TryRecvError::Disconnected) + ) {} + } +} + +impl Drop for PipeWireAudio { + fn drop(&mut self) { + self.stop(); + } +} + +/// `audio.position` value for the null sink. +fn position_prop(channels: u8) -> &'static str { + if channels == 1 { "MONO" } else { "FL,FR" } +} + +/// SPA channel ids for the stream format, padded with zeros. +fn position_ids(channels: u8) -> [u32; spa_sys::SPA_AUDIO_MAX_CHANNELS as usize] { + let mut p = [0; spa_sys::SPA_AUDIO_MAX_CHANNELS as usize]; + if channels == 1 { + p[0] = spa_sys::SPA_AUDIO_CHANNEL_MONO; + } else { + p[0] = spa_sys::SPA_AUDIO_CHANNEL_FL; + p[1] = spa_sys::SPA_AUDIO_CHANNEL_FR; + } + p +} + +/// `{"name":"x"}` as stored in the `default.configured.audio.sink` metadata. +fn sink_json(name: &str) -> String { + serde_json::json!({ "name": name }).to_string() +} + +/// The sink name inside a `default.configured.*` metadata value. +fn parse_sink_name(value: &str) -> Option { + let v: serde_json::Value = serde_json::from_str(value).ok()?; + v.get("name")?.as_str().map(str::to_owned) +} + +/// Capture time of the first sample of a block that has just been delivered: the process +/// callback runs when the block is complete, so the block started `samples / channels / rate` +/// earlier. Ignores the (small) graph latency after the capture. +fn block_pts_us(now: u64, samples: usize, channels: usize) -> u64 { + let frames = (samples / channels.max(1)) as u64; + now.saturating_sub(frames * 1_000_000 / SAMPLE_RATE as u64) +} + +/// The `type`/`value` of the configured default sink as the daemon reports it. +type MetaValue = (Option, String); + +/// Tracks `default.configured.audio.sink` of subject 0 from the metadata events. +#[derive(Default)] +struct DefaultSink { + current: Option, + /// Set once we replaced the default: the value to put back. + saved: Option>, +} + +/// Puts the configured default sink back, once. Returns true if it sent anything. Leaves the +/// value alone when something else (the user, another app) changed it after we did. +fn restore_default(meta: Option<&Meta>, ds: &mut DefaultSink) -> bool { + let (Some(meta), Some(previous)) = (meta, ds.saved.take()) else { + return false; + }; + let ours = ds + .current + .as_ref() + .is_some_and(|(_, v)| parse_sink_name(v).as_deref() == Some(SINK_NAME)); + if !ours { + tracing::debug!("the default sink changed meanwhile; not restoring it"); + return false; + } + match previous { + Some((t, v)) => meta.proxy.set_property( + 0, + DEFAULT_SINK_KEY, + t.as_deref().or(Some(JSON_TYPE)), + Some(&v), + ), + None => meta.proxy.set_property(0, DEFAULT_SINK_KEY, None, None), + } + true +} + +struct Meta { + proxy: pw::metadata::Metadata, + _listener: pw::metadata::MetadataListener, +} + +fn run( + channels: u8, + blocks: SyncSender, + cmds: pw::channel::Receiver, + ready: &std::sync::mpsc::Sender>, +) -> Result<(), String> { + let e = |what: &str, err: pw::Error| format!("pipewire {what}: {err}"); + pw::init(); + let mainloop = pw::main_loop::MainLoopRc::new(None).map_err(|x| e("main loop", x))?; + let context = pw::context::ContextRc::new(&mainloop, None).map_err(|x| e("context", x))?; + let core = context.connect_rc(None).map_err(|x| e("connect", x))?; + + let failure: Rc>> = Rc::new(RefCell::new(None)); + // Sync sequence the main loop is waiting for (see `roundtrip` and the stop handler). + let wait_seq: Rc>> = Rc::new(Cell::new(None)); + let _core_listener = { + let (ml, failure, wait_seq) = (mainloop.clone(), failure.clone(), wait_seq.clone()); + core.add_listener_local() + .done(move |id, seq| { + if id == pw::core::PW_ID_CORE && wait_seq.get() == Some(seq.seq()) { + wait_seq.set(None); + ml.quit(); + } + }) + .error({ + let ml = mainloop.clone(); + move |id, _seq, res, msg| { + // Errors on other objects (the node proxy) are reported by their own listeners. + if id == pw::core::PW_ID_CORE { + *failure.borrow_mut() = Some(format!("pipewire core error {res}: {msg}")); + ml.quit(); + } + } + }) + .register() + }; + let roundtrip = || -> Result<(), String> { + let seq = core.sync(0).map_err(|x| e("sync", x))?; + wait_seq.set(Some(seq.seq())); + mainloop.run(); + match failure.borrow().clone() { + Some(f) => Err(f), + None => Ok(()), + } + }; + + // 1. The null sink. + let mut sink_props = properties! { + "factory.name" => "support.null-audio-sink", + "node.name" => SINK_NAME, + "node.description" => SINK_NAME, + "media.class" => "Audio/Sink", + "object.linger" => "false", + "audio.position" => position_prop(channels), + }; + sink_props.insert("audio.rate", SAMPLE_RATE.to_string()); + let sink: pw::node::Node = core + .create_object("adapter", &sink_props) + .map_err(|x| e("create sink", x))?; + let _sink_listener = { + let (ml, failure) = (mainloop.clone(), failure.clone()); + sink.upcast_ref() + .add_listener_local() + .error(move |seq, res, msg| { + *failure.borrow_mut() = Some(format!("pipewire sink error {res} ({seq}): {msg}")); + ml.quit(); + }) + .register() + }; + roundtrip()?; + + // 2. The default output. + let default_sink = Rc::new(RefCell::new(DefaultSink::default())); + let registry = core.get_registry_rc().map_err(|x| e("registry", x))?; + let meta_found: Rc>> = Rc::new(RefCell::new(None)); + let reg_listener = { + let meta_found = meta_found.clone(); + registry + .add_listener_local() + .global(move |g| { + if g.type_ == pw::types::ObjectType::Metadata + && g.props + .is_some_and(|p| p.get("metadata.name") == Some("default")) + { + meta_found.borrow_mut().get_or_insert(g.id); + } + }) + .register() + }; + roundtrip()?; + let meta = match *meta_found.borrow() { + Some(id) => { + let obj = pw::registry::GlobalObject { + id, + permissions: pw::permissions::PermissionFlags::all(), + type_: pw::types::ObjectType::Metadata, + version: 0, + props: None::<&pw::spa::utils::dict::DictRef>, + }; + let proxy: pw::metadata::Metadata = + registry.bind(&obj).map_err(|x| e("bind metadata", x))?; + let listener = { + let ds = default_sink.clone(); + proxy + .add_listener_local() + .property(move |subject, key, type_, value| { + if subject == 0 && key.is_none_or(|k| k == DEFAULT_SINK_KEY) { + ds.borrow_mut().current = + value.map(|v| (type_.map(str::to_owned), v.to_owned())); + } + 0 + }) + .register() + }; + Some(Rc::new(Meta { + proxy, + _listener: listener, + })) + } + None => { + tracing::warn!("no default metadata object; the default output is left unchanged"); + None + } + }; + drop(reg_listener); + if let Some(m) = &meta { + // The properties are replayed on bind; the roundtrip makes sure they arrived. + roundtrip()?; + let mut ds = default_sink.borrow_mut(); + let previous = ds.current.clone(); + tracing::debug!(?previous, "configured default sink before ezdisplay"); + m.proxy.set_property( + 0, + DEFAULT_SINK_KEY, + Some(JSON_TYPE), + Some(&sink_json(SINK_NAME)), + ); + ds.saved = Some(previous); + } + + // 3. The monitor stream. + let props = properties! { + *pw::keys::MEDIA_TYPE => "Audio", + *pw::keys::MEDIA_CATEGORY => "Capture", + *pw::keys::MEDIA_ROLE => "Music", + "target.object" => SINK_NAME, + *pw::keys::STREAM_CAPTURE_SINK => "true", + *pw::keys::NODE_LATENCY => "240/48000", + *pw::keys::NODE_DONT_RECONNECT => "true", + }; + let stream = pw::stream::StreamRc::new(core.clone(), "ezdisplay audio", props) + .map_err(|x| e("stream", x))?; + let ch = channels as usize; + let stream_listener = { + let (ml, failure, ready) = (mainloop.clone(), failure.clone(), ready.clone()); + stream + .add_local_listener_with_user_data(blocks) + .state_changed(move |_, _, old, new| { + tracing::debug!("audio stream state {old:?} -> {new:?}"); + match new { + pw::stream::StreamState::Streaming | pw::stream::StreamState::Paused => { + let _ = ready.send(Ok(())); + } + pw::stream::StreamState::Error(msg) => { + *failure.borrow_mut() = Some(format!("pipewire audio stream error: {msg}")); + ml.quit(); + } + pw::stream::StreamState::Unconnected + if matches!( + old, + pw::stream::StreamState::Streaming | pw::stream::StreamState::Paused + ) => + { + // The sink went away under us. + *failure.borrow_mut() = Some("the audio stream was disconnected".into()); + ml.quit(); + } + _ => {} + } + }) + .process(move |stream, blocks| { + let Some(mut buffer) = stream.dequeue_buffer() else { + return; + }; + let Some(data) = buffer.datas_mut().first_mut() else { + return; + }; + let (offset, size) = (data.chunk().offset() as usize, data.chunk().size() as usize); + let Some(bytes) = data.data() else { + return; + }; + let Some(bytes) = bytes.get(offset..offset.saturating_add(size).min(bytes.len())) + else { + return; + }; + let samples: Vec = bytes + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect(); + if samples.is_empty() { + return; + } + let pts_us = block_pts_us(now_us(), samples.len(), ch); + // Full queue: the consumer is stalled, drop rather than block the graph. + let _ = blocks.try_send(AudioBlock { pts_us, samples }); + }) + .register() + .map_err(|x| e("stream listener", x))? + }; + let mut info = pw::spa::param::audio::AudioInfoRaw::new(); + info.set_format(pw::spa::param::audio::AudioFormat::F32LE); + info.set_rate(SAMPLE_RATE); + info.set_channels(channels as u32); + info.set_position(position_ids(channels)); + let format = pw::spa::pod::serialize::PodSerializer::serialize( + std::io::Cursor::new(Vec::new()), + &pw::spa::pod::Value::Object(pw::spa::pod::Object { + type_: pw::spa::utils::SpaTypes::ObjectParamFormat.as_raw(), + id: pw::spa::param::ParamType::EnumFormat.as_raw(), + properties: info.into(), + }), + ) + .map_err(|x| format!("pipewire audio format: {x:?}"))? + .0 + .into_inner(); + let mut params = [Pod::from_bytes(&format).ok_or("invalid audio format pod")?]; + stream + .connect( + pw::spa::utils::Direction::Input, + None, + pw::stream::StreamFlags::AUTOCONNECT + | pw::stream::StreamFlags::MAP_BUFFERS + | pw::stream::StreamFlags::RT_PROCESS, + &mut params, + ) + .map_err(|x| e("stream connect", x))?; + + // Stop: put the default output back, wait until the daemon has it, then leave. + let _cmds = { + let (ml, ds, wait_seq, core) = ( + mainloop.clone(), + default_sink.clone(), + wait_seq.clone(), + core.clone(), + ); + let meta = meta.clone(); + cmds.attach(mainloop.loop_(), move |Cmd::Stop| { + restore_default(meta.as_deref(), &mut ds.borrow_mut()); + match core.sync(0) { + Ok(seq) => { + wait_seq.set(Some(seq.seq())); + } + Err(_) => ml.quit(), + } + }) + }; + mainloop.run(); + + // Also on the error paths: never leave our sink as the default. + if restore_default(meta.as_deref(), &mut default_sink.borrow_mut()) + && let Ok(seq) = core.sync(0) + { + wait_seq.set(Some(seq.seq())); + let ml = mainloop.clone(); + let timer = mainloop.loop_().add_timer(move |_| ml.quit()); + let _ = timer.update_timer(Some(STOP_SYNC_TIMEOUT), None); + mainloop.run(); + } + drop(stream_listener); + let _ = stream.disconnect(); + drop(sink); + match failure.take() { + Some(f) => Err(f), + None => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::Command; + + #[test] + fn parses_sink_name_from_metadata_json() { + assert_eq!( + parse_sink_name(r#"{"name":"alsa_output.pci-0000_00_1f.3.analog-stereo"}"#).as_deref(), + Some("alsa_output.pci-0000_00_1f.3.analog-stereo") + ); + assert_eq!( + parse_sink_name(r#"{ "name" : "x", "other": 1 }"#).as_deref(), + Some("x") + ); + assert_eq!(parse_sink_name("not json"), None); + assert_eq!(parse_sink_name(r#"{"name":3}"#), None); + assert_eq!(parse_sink_name("{}"), None); + } + + #[test] + fn sink_json_round_trips_through_the_parser() { + let json = sink_json(SINK_NAME); + assert_eq!(json, r#"{"name":"ezdisplay"}"#); + assert_eq!(parse_sink_name(&json).as_deref(), Some(SINK_NAME)); + // Quotes in a name are escaped, not injected. + assert_eq!( + parse_sink_name(&sink_json(r#"a"b"#)).as_deref(), + Some(r#"a"b"#) + ); + } + + #[test] + fn channel_positions() { + assert_eq!(position_prop(1), "MONO"); + assert_eq!(position_prop(2), "FL,FR"); + let p = position_ids(2); + assert_eq!( + &p[..3], + &[ + spa_sys::SPA_AUDIO_CHANNEL_FL, + spa_sys::SPA_AUDIO_CHANNEL_FR, + 0 + ] + ); + assert_eq!(position_ids(1)[0], spa_sys::SPA_AUDIO_CHANNEL_MONO); + } + + #[test] + fn block_pts_subtracts_the_block_duration() { + // 480 stereo frames are 10 ms. + assert_eq!(block_pts_us(1_000_000, 960, 2), 990_000); + assert_eq!(block_pts_us(1_000_000, 480, 1), 990_000); + // Never underflows right after the clock starts. + assert_eq!(block_pts_us(5, 960, 2), 0); + assert_eq!(block_pts_us(100, 0, 0), 100); + } + + #[test] + fn restore_leaves_a_changed_default_alone() { + let mut ds = DefaultSink { + current: Some((None, r#"{"name":"other"}"#.into())), + saved: Some(None), + }; + // No metadata proxy and a foreign value: nothing to send either way. + assert!(!restore_default(None, &mut ds)); + } + + /// The current `default.configured.audio.sink` value as `pw-metadata` prints it. + fn default_sink_value() -> Option { + let out = Command::new("pw-metadata") + .args(["-n", "default", "0", DEFAULT_SINK_KEY]) + .output() + .expect("pw-metadata"); + let text = String::from_utf8_lossy(&out.stdout).into_owned(); + let rest = text.split("value:'").nth(1)?; + Some( + rest.rsplit_once("' type:") + .map_or(rest, |(v, _)| v) + .to_owned(), + ) + } + + /// Puts the user's default output back even if the test panics. + struct RestoreDefault(Option); + impl Drop for RestoreDefault { + fn drop(&mut self) { + let mut c = Command::new("pw-metadata"); + c.args(["-n", "default"]); + match &self.0 { + Some(v) => c.args(["0", DEFAULT_SINK_KEY, v, JSON_TYPE]), + None => c.args(["-d", "0", DEFAULT_SINK_KEY]), + }; + let _ = c.output(); + } + } + + fn sink_node_exists() -> bool { + let out = Command::new("pw-dump").output().expect("pw-dump"); + let dump: serde_json::Value = serde_json::from_slice(&out.stdout).expect("pw-dump json"); + dump.as_array().is_some_and(|objs| { + objs.iter().any(|o| { + o.pointer("/info/props/node.name").and_then(|v| v.as_str()) == Some(SINK_NAME) + }) + }) + } + + /// Needs a running PipeWire + WirePlumber session: + /// `EZD_PIPEWIRE_AUDIO_TEST=1 cargo test -p ezd-host-linux pipewire_audio`. + #[test] + fn pipewire_audio_capture_and_default_restore() { + if std::env::var_os("EZD_PIPEWIRE_AUDIO_TEST").is_none_or(|v| v != "1") { + eprintln!("skipped: set EZD_PIPEWIRE_AUDIO_TEST=1 (needs PipeWire and WirePlumber)"); + return; + } + let before = default_sink_value(); + let _guard = RestoreDefault(before.clone()); + assert!(!sink_node_exists(), "a stale ezdisplay sink exists"); + + let mut src = audio_source().expect("audio source"); + src.start(2).expect("start"); + assert!(sink_node_exists(), "the ezdisplay sink was not created"); + let during = default_sink_value().expect("default sink while running"); + assert_eq!(parse_sink_name(&during).as_deref(), Some(SINK_NAME)); + + let mut got = None; + for _ in 0..20 { + if let Some(b) = src + .next_samples(Duration::from_millis(250)) + .expect("next_samples") + { + got = Some(b); + break; + } + } + let block = got.expect("no audio block arrived from the sink monitor"); + assert!(!block.samples.is_empty() && block.samples.len() % 2 == 0); + assert!(block.pts_us <= now_us()); + + src.stop(); + src.stop(); // idempotent + assert_eq!( + default_sink_value(), + before, + "the previous default was not restored" + ); + assert!(!sink_node_exists(), "the ezdisplay sink outlived stop"); + } +} diff --git a/crates/ezd-host-linux/src/lib.rs b/crates/ezd-host-linux/src/lib.rs index afcefef..75a0275 100644 --- a/crates/ezd-host-linux/src/lib.rs +++ b/crates/ezd-host-linux/src/lib.rs @@ -8,6 +8,7 @@ mod adapter; mod ahead; +mod audio; mod cursor; mod cursor_meta; mod diag; @@ -26,7 +27,7 @@ mod uinput_sink; mod vkdma; pub use ezd_host_core::PlatformOptions; -use ezd_host_core::{InputBackend, PlatformError, UnavailableBackend, VideoSource}; +use ezd_host_core::{AudioSource, InputBackend, PlatformError, UnavailableBackend, VideoSource}; use std::sync::Arc; /// Creates the capture + encode source for one client connection. @@ -64,6 +65,12 @@ pub fn input_backend() -> Option> { } } +/// System audio capture (a PipeWire null sink made the default output while the session runs). +/// Cheap: PipeWire is only contacted by `AudioSource::start`. +pub fn audio_source() -> Option> { + audio::audio_source() +} + /// Platform section of `ezd-host diag`. pub fn diag_section() -> serde_json::Value { diag::section() From 488a65db977c04091272754f1a3a8ba10fbe36a7 Mon Sep 17 00:00:00 2001 From: FacuM Date: Sun, 25 Oct 2026 21:34:39 -0300 Subject: [PATCH 05/12] Play the PC audio on the tablet The Android client advertises audio when the new Audio setting is on (default on), plays PCM through a low-latency AAudio stream behind a 20 ms jitter buffer, and decodes Opus with MediaCodec. --- .../com/librestatic/ezdisplay/Capabilities.kt | 7 + .../com/librestatic/ezdisplay/MainActivity.kt | 5 +- .../ezdisplay/settings/Sections.kt | 33 +- .../ezdisplay/settings/SettingsScreen.kt | 16 +- .../ezdisplay/settings/SettingsStore.kt | 12 + .../src/main/res/drawable/ic_volume_up.xml | 13 + .../app/src/main/res/values-de/strings.xml | 3 + .../app/src/main/res/values-es/strings.xml | 3 + .../app/src/main/res/values-fr/strings.xml | 3 + .../app/src/main/res/values-it/strings.xml | 3 + .../app/src/main/res/values-pt/strings.xml | 3 + android/app/src/main/res/values/strings.xml | 3 + .../settings/SettingsListDetailTest.kt | 5 +- .../ezdisplay/settings/SettingsPhoneTest.kt | 8 + .../settings/SettingsScreenshotTest.kt | 9 + .../ezdisplay/settings/SettingsStoreTest.kt | 10 + crates/ezd-android/Cargo.toml | 2 +- crates/ezd-android/src/audio.rs | 410 ++++++++++++++++++ crates/ezd-android/src/caps.rs | 14 + crates/ezd-android/src/jitter.rs | 279 ++++++++++++ crates/ezd-android/src/lib.rs | 5 +- crates/ezd-android/src/session.rs | 55 ++- 22 files changed, 890 insertions(+), 11 deletions(-) create mode 100644 android/app/src/main/res/drawable/ic_volume_up.xml create mode 100644 crates/ezd-android/src/audio.rs create mode 100644 crates/ezd-android/src/jitter.rs diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/Capabilities.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/Capabilities.kt index ab4ae25..8f08f23 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/Capabilities.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/Capabilities.kt @@ -62,6 +62,13 @@ object Capabilities { return json.toString() } + /** + * [caps] (from [toJson]) with the `audio` switch set. The switch is a user setting that can change between + * sessions, so it is applied on every connect instead of being cached with the rest. + */ + fun withAudio(caps: String, enabled: Boolean): String = + JSONObject(caps).put("audio", enabled).toString() + /** HDR10 support and luminance of the panel; luminance members are 0 when the platform does not report them. */ @Suppress("DEPRECATION") private fun hdr(display: Display): JSONObject { diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt index c678d92..89ee0b4 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt @@ -451,7 +451,10 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall private fun connect() { if (!started || sessionActive) return if (!surfaceReady) return // surfaceCreated reschedules - val caps = capsJson ?: Capabilities.toJson(this, chosenMode).also { capsJson = it } + val caps = Capabilities.withAudio( + capsJson ?: Capabilities.toJson(this, chosenMode).also { capsJson = it }, + InputPrefs.prefs(this).getBoolean(SettingsStore.AUDIO_ENABLED, true), + ) val acc = accessory?.takeIf { usbManager.hasPermission(it) } val wifiChoice = if (tryWifiNext) wifiTarget() else null val route = chooseRoute(acc != null, tryWifiNext, wifiChoice != null) diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/Sections.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/Sections.kt index 056ebfc..481e19c 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/Sections.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/Sections.kt @@ -1,6 +1,7 @@ package com.librestatic.ezdisplay.settings import androidx.annotation.DrawableRes +import androidx.annotation.StringRes import androidx.compose.animation.animateContentSize import androidx.compose.foundation.BorderStroke import androidx.compose.foundation.layout.Arrangement @@ -520,6 +521,30 @@ private fun DoubleClickSlider(ms: Int, actions: SettingsActions, shape: Shape) { /** Palm rejection toggle. */ @Composable fun PalmRow(on: Boolean, onChange: (Boolean) -> Unit, modifier: Modifier = Modifier) { + SwitchRow( + on, onChange, R.drawable.ic_back_hand, R.string.settings_palm_title, R.string.settings_palm_desc, + modifier.testTag("palm_row"), + ) +} + +/** Audio toggle: play the PC's audio on this device. */ +@Composable +fun AudioRow(on: Boolean, onChange: (Boolean) -> Unit, modifier: Modifier = Modifier) { + SwitchRow( + on, onChange, R.drawable.ic_volume_up, R.string.settings_audio_title, R.string.settings_audio_desc, + modifier.testTag("audio_row"), + ) +} + +@Composable +private fun SwitchRow( + on: Boolean, + onChange: (Boolean) -> Unit, + @DrawableRes icon: Int, + @StringRes title: Int, + @StringRes desc: Int, + modifier: Modifier, +) { val colors = MaterialTheme.colorScheme // The whole row is the switch, so TalkBack reads one control. Surface( @@ -527,21 +552,21 @@ fun PalmRow(on: Boolean, onChange: (Boolean) -> Unit, modifier: Modifier = Modif onCheckedChange = onChange, shape = RoundedCornerShape(GroupEnd), color = colors.surfaceBright, - modifier = modifier.fillMaxWidth().semantics { role = Role.Switch }.testTag("palm_row"), + modifier = modifier.fillMaxWidth().semantics { role = Role.Switch }, ) { Row( Modifier.heightIn(min = RowMinHeight).padding(RowPadding), verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(12.dp), ) { - Symbol(R.drawable.ic_back_hand, tint = colors.secondary) + Symbol(icon, tint = colors.secondary) Column(Modifier.weight(1f)) { Text( - stringResource(R.string.settings_palm_title), + stringResource(title), style = MaterialTheme.typography.titleSmall, fontWeight = FontWeight.Bold, ) Text( - stringResource(R.string.settings_palm_desc), + stringResource(desc), style = MaterialTheme.typography.bodySmall, color = colors.onSurfaceVariant, ) } diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsScreen.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsScreen.kt index b66fb22..ec621bb 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsScreen.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsScreen.kt @@ -56,7 +56,7 @@ import com.librestatic.ezdisplay.InputPrefs import com.librestatic.ezdisplay.R /** Settings sections; AirActions only exists while its card is shown (and is always last). */ -enum class SettingsSection { Pen, Zoom, Touch, Connection, AirActions } +enum class SettingsSection { Pen, Zoom, Touch, Audio, Connection, AirActions } private fun airCardShown(state: SettingsState) = state.airActions == AirActions.On || state.airActions == AirActions.Unknown @@ -177,6 +177,8 @@ private fun PhoneMain(state: SettingsState, actions: SettingsActions, onOpenEdit SectionTitle(stringResource(R.string.section_touch)) PalmRow(state.palmRejection, actions::setPalmRejection) AirActionsCard(state.airActions, Modifier.padding(top = profileGap(2))) + SectionTitle(stringResource(R.string.section_audio)) + AudioRow(state.audioEnabled, actions::setAudioEnabled) SectionTitle(stringResource(R.string.section_connection)) ConnectionContent(state, actions) } @@ -227,7 +229,7 @@ private fun ListDetailLayout( ) { val colors = MaterialTheme.colorScheme val sections = buildList { - add(SettingsSection.Pen); add(SettingsSection.Zoom); add(SettingsSection.Touch); add(SettingsSection.Connection) + add(SettingsSection.Pen); add(SettingsSection.Zoom); add(SettingsSection.Touch); add(SettingsSection.Audio); add(SettingsSection.Connection) if (airCardShown(state)) add(SettingsSection.AirActions) } val current = selected.takeIf { it in sections } ?: SettingsSection.Pen @@ -286,6 +288,7 @@ private fun ListDetailLayout( ) SettingsSection.Zoom -> ZoomGroup(state, actions) SettingsSection.Touch -> PalmRow(state.palmRejection, actions::setPalmRejection) + SettingsSection.Audio -> AudioRow(state.audioEnabled, actions::setAudioEnabled) SettingsSection.Connection -> ConnectionContent(state, actions) SettingsSection.AirActions -> AirActionsCard(state.airActions) } @@ -298,6 +301,7 @@ private fun sectionTitle(s: SettingsSection) = when (s) { SettingsSection.Pen -> R.string.settings_profile SettingsSection.Zoom -> R.string.section_zoom SettingsSection.Touch -> R.string.settings_palm_title + SettingsSection.Audio -> R.string.section_audio SettingsSection.Connection -> R.string.section_connection SettingsSection.AirActions -> R.string.section_air_actions } @@ -306,6 +310,7 @@ private fun sectionIcon(s: SettingsSection) = when (s) { SettingsSection.Pen -> R.drawable.ic_stylus SettingsSection.Zoom -> R.drawable.ic_zoom_in SettingsSection.Touch -> R.drawable.ic_back_hand + SettingsSection.Audio -> R.drawable.ic_volume_up SettingsSection.Connection -> R.drawable.ic_wifi SettingsSection.AirActions -> R.drawable.ic_info } @@ -315,6 +320,7 @@ private fun sectionValue(s: SettingsSection, state: SettingsState): String = whe SettingsSection.Pen -> stringResource(Profiles.firstOrNull { it.value == state.profile }?.title ?: R.string.profile_raw) SettingsSection.Zoom -> stringResource(R.string.settings_zoom_summary, formatZoom(state.zoomFactor), state.doubleClickMs) SettingsSection.Touch -> stringResource(if (state.palmRejection) R.string.settings_on else R.string.settings_off) + SettingsSection.Audio -> stringResource(if (state.audioEnabled) R.string.settings_on else R.string.settings_off) SettingsSection.Connection -> connectionSummary(state) SettingsSection.AirActions -> stringResource(R.string.section_air_actions_value) } @@ -420,6 +426,9 @@ private fun WideLayout( WideHeading(stringResource(R.string.section_touch)) PalmRow(state.palmRejection, actions::setPalmRejection) AirActionsCard(state.airActions, Modifier.padding(top = profileGap(3))) + Spacer(Modifier.height(20.dp)) + WideHeading(stringResource(R.string.section_audio)) + AudioRow(state.audioEnabled, actions::setAudioEnabled) } Column(Modifier.weight(1f)) { WideHeading(stringResource(R.string.section_connection)) @@ -434,6 +443,9 @@ private fun WideLayout( PalmRow(state.palmRejection, actions::setPalmRejection) AirActionsCard(state.airActions, Modifier.padding(top = profileGap(2))) Spacer(Modifier.height(20.dp)) + WideHeading(stringResource(R.string.section_audio)) + AudioRow(state.audioEnabled, actions::setAudioEnabled) + Spacer(Modifier.height(20.dp)) WideHeading(stringResource(R.string.section_connection)) ConnectionSummary(state, actions) } diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsStore.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsStore.kt index d32cdf8..6506cad 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsStore.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsStore.kt @@ -97,6 +97,8 @@ data class SettingsState( val doubleClickMs: Int = 300, val palmRejection: Boolean = true, val airActions: AirActions = AirActions.NotSamsung, + /** Play the PC's audio on this device. */ + val audioEnabled: Boolean = true, /** Connect over Wi-Fi when the USB connection is not there. */ val wifiEnabled: Boolean = false, val wifiHosts: List = emptyList(), @@ -177,6 +179,7 @@ interface SettingsActions { fun setDoubleClickMs(ms: Int) fun setPalmRejection(on: Boolean) fun setWifiEnabled(on: Boolean) + fun setAudioEnabled(on: Boolean) /** Opens the PIN dialog for a host that was found or typed. */ fun startPairing(name: String, shortFp: String, addr: String) @@ -212,6 +215,7 @@ class SettingsStore( .coerceIn(InputPrefs.DOUBLE_CLICK_MIN, InputPrefs.DOUBLE_CLICK_MAX), palmRejection = prefs.getBoolean(InputPrefs.PALM_REJECTION, true), airActions = airActions, + audioEnabled = prefs.getBoolean(AUDIO_ENABLED, true), wifiEnabled = prefs.getBoolean(WIFI_ENABLED, false), wifiHosts = wifi.hosts(), deviceFingerprint = wifi.identity(), @@ -271,6 +275,11 @@ class SettingsStore( _state.value = _state.value.copy(palmRejection = on) } + override fun setAudioEnabled(on: Boolean) { + prefs.edit { putBoolean(AUDIO_ENABLED, on) } + _state.value = _state.value.copy(audioEnabled = on) + } + override fun setWifiEnabled(on: Boolean) { prefs.edit { putBoolean(WIFI_ENABLED, on) } _state.value = _state.value.copy(wifiEnabled = on) @@ -341,6 +350,9 @@ class SettingsStore( /** Pref key of the Wi-Fi switch, read by MainActivity. */ const val WIFI_ENABLED = "wifi_enabled" + /** Pref key of the audio switch, read by MainActivity when a session starts. */ + const val AUDIO_ENABLED = "audio_enabled" + fun create(context: Context) = SettingsStore( InputPrefs.prefs(context), readAirActions(context), NativeWifiBackend(context), ) diff --git a/android/app/src/main/res/drawable/ic_volume_up.xml b/android/app/src/main/res/drawable/ic_volume_up.xml new file mode 100644 index 0000000..58b5b5d --- /dev/null +++ b/android/app/src/main/res/drawable/ic_volume_up.xml @@ -0,0 +1,13 @@ + + + + + + + diff --git a/android/app/src/main/res/values-de/strings.xml b/android/app/src/main/res/values-de/strings.xml index d8f4d31..808aff7 100644 --- a/android/app/src/main/res/values-de/strings.xml +++ b/android/app/src/main/res/values-de/strings.xml @@ -76,6 +76,9 @@ %1$s× · %2$d ms Handballenerkennung Ignoriert Finger, solange der Stift in der Nähe ist + Audio + Audio + Gib den Ton des PCs auf dem Tablet wieder Ein Aus Aktuelle Kombination diff --git a/android/app/src/main/res/values-es/strings.xml b/android/app/src/main/res/values-es/strings.xml index 6654a50..cea7168 100644 --- a/android/app/src/main/res/values-es/strings.xml +++ b/android/app/src/main/res/values-es/strings.xml @@ -76,6 +76,9 @@ %1$s× · %2$d ms Rechazo de palma Ignora los dedos mientras el lápiz está cerca + Audio + Audio + Reproducí el audio de la PC en la tablet Activado Desactivado Combinación actual diff --git a/android/app/src/main/res/values-fr/strings.xml b/android/app/src/main/res/values-fr/strings.xml index c970b66..4d6c70e 100644 --- a/android/app/src/main/res/values-fr/strings.xml +++ b/android/app/src/main/res/values-fr/strings.xml @@ -76,6 +76,9 @@ %1$s× · %2$d ms Rejet de la paume Ignore les doigts lorsque le stylet est proche + Audio + Audio + Lisez le son du PC sur la tablette Activé Désactivé Combinaison actuelle diff --git a/android/app/src/main/res/values-it/strings.xml b/android/app/src/main/res/values-it/strings.xml index d7d312a..09d3244 100644 --- a/android/app/src/main/res/values-it/strings.xml +++ b/android/app/src/main/res/values-it/strings.xml @@ -76,6 +76,9 @@ %1$s× · %2$d ms Rifiuto del palmo Ignora le dita mentre la penna è vicina + Audio + Audio + Riproduci l\'audio del PC sul tablet Attivo Disattivato Combinazione attuale diff --git a/android/app/src/main/res/values-pt/strings.xml b/android/app/src/main/res/values-pt/strings.xml index 67e0329..fa3df76 100644 --- a/android/app/src/main/res/values-pt/strings.xml +++ b/android/app/src/main/res/values-pt/strings.xml @@ -76,6 +76,9 @@ %1$s× · %2$d ms Rejeição de palma Ignora os dedos enquanto a caneta está por perto + Áudio + Áudio + Reproduza o áudio do PC no tablet Ativado Desativado Combinação atual diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index d53478a..8330c83 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -76,6 +76,9 @@ %1$s× · %2$d ms Palm rejection Ignores fingers while the pen is near + Audio + Audio + Play the PC\'s audio on the tablet On Off Current combination diff --git a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsListDetailTest.kt b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsListDetailTest.kt index 16cb2a8..42473d7 100644 --- a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsListDetailTest.kt +++ b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsListDetailTest.kt @@ -1,6 +1,8 @@ package com.librestatic.ezdisplay.settings +import androidx.compose.ui.test.assertCountEquals import androidx.compose.ui.test.assertIsSelected +import androidx.compose.ui.test.onAllNodesWithText import androidx.compose.ui.test.onNodeWithText import androidx.compose.ui.test.performClick import androidx.compose.ui.test.performScrollTo @@ -31,7 +33,8 @@ class SettingsListDetailTest : SettingsUiTestBase() { @Test fun sectionValuesSummarizeState() { launch { putString(InputPrefs.PROFILE, "drawing") } rule.onNodeWithText("2× · 300 ms").assertExists() - rule.onNodeWithText(str(R.string.settings_on)).assertExists() + // Touch (palm rejection) and Audio are both on by default. + rule.onAllNodesWithText(str(R.string.settings_on)).assertCountEquals(2) } @Test fun selectingZoomShowsZoomContent() { diff --git a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsPhoneTest.kt b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsPhoneTest.kt index 882be19..4bc4a16 100644 --- a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsPhoneTest.kt +++ b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsPhoneTest.kt @@ -220,6 +220,14 @@ class SettingsPhoneTest : SettingsUiTestBase() { assertEquals(800, prefs.getInt(InputPrefs.DOUBLE_CLICK_MS, 0)) } + @Test fun audioRowTogglesAudio() { + launch() + rule.onNodeWithText(str(R.string.settings_audio_desc)).performScrollTo().performClick() + assertEquals(false, prefs.getBoolean(SettingsStore.AUDIO_ENABLED, true)) + rule.onNodeWithText(str(R.string.settings_audio_desc)).performClick() + assertEquals(true, prefs.getBoolean(SettingsStore.AUDIO_ENABLED, false)) + } + @Test fun palmRowTogglesRejection() { launch() rule.onNodeWithText(str(R.string.settings_palm_title)).performScrollTo().performClick() diff --git a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsScreenshotTest.kt b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsScreenshotTest.kt index 888e0eb..71aa657 100644 --- a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsScreenshotTest.kt +++ b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsScreenshotTest.kt @@ -42,6 +42,7 @@ class SettingsScreenshotTest { override fun setDoubleClickMs(ms: Int) {} override fun setPalmRejection(on: Boolean) {} override fun setWifiEnabled(on: Boolean) {} + override fun setAudioEnabled(on: Boolean) {} override fun startPairing(name: String, shortFp: String, addr: String) {} override fun startPairing(link: PairLink) {} override fun submitPairing(pin: String) {} @@ -141,6 +142,14 @@ class SettingsScreenshotTest { @Test @Config(sdk = [36], qualifiers = "de-w412dp-h915dp-notnight") fun phone_main_german_light() { show(tabletState); capture("phone_main_german_light") } + @Test @Config(sdk = [36], qualifiers = "w412dp-h915dp-night") + fun phone_audio_dark() { + show(tabletState) + rule.onNodeWithTag("audio_row").performScrollTo() + rule.waitForIdle() + capture("phone_audio_dark") + } + @Test @Config(sdk = [36], qualifiers = "w841dp-h701dp-night") fun foldable_dark() { show(tabletState); capture("foldable_dark") } diff --git a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsStoreTest.kt b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsStoreTest.kt index 92ed0fe..d5facea 100644 --- a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsStoreTest.kt +++ b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsStoreTest.kt @@ -125,6 +125,16 @@ class SettingsStoreTest { assertFalse(s.palmRejection) } + @Test fun audioIsOnByDefaultAndPersists() { + assertTrue(s.audioEnabled) + store.setAudioEnabled(false) + assertFalse(prefs.getBoolean(SettingsStore.AUDIO_ENABLED, true)) + assertFalse(s.audioEnabled) + assertFalse(SettingsStore(prefs, AirActions.Off).state.value.audioEnabled) + store.setAudioEnabled(true) + assertTrue(prefs.getBoolean(SettingsStore.AUDIO_ENABLED, false)) + } + @Test fun toJsonContract() { store.setProfile("custom") store.setCustomAction(InputPrefs.CUSTOM_CLICK, "keys") diff --git a/crates/ezd-android/Cargo.toml b/crates/ezd-android/Cargo.toml index e2586ba..5f951fd 100644 --- a/crates/ezd-android/Cargo.toml +++ b/crates/ezd-android/Cargo.toml @@ -27,7 +27,7 @@ android_logger = "0.15" jni = "0.22" libc = "0.2" log = "0.4" -ndk = { version = "0.9", default-features = false, features = ["media", "nativewindow", "api-level-26"] } +ndk = { version = "0.9", default-features = false, features = ["audio", "media", "nativewindow", "api-level-26"] } [lints] workspace = true diff --git a/crates/ezd-android/src/audio.rs b/crates/ezd-android/src/audio.rs new file mode 100644 index 0000000..244e889 --- /dev/null +++ b/crates/ezd-android/src/audio.rs @@ -0,0 +1,410 @@ +//! PC audio playback: an AAudio output stream fed from a jitter buffer. +//! +//! PCM packets are converted and pushed straight into the buffer from the client thread. Opus +//! packets go to a small decode thread that owns a MediaCodec `audio/opus` decoder and pushes +//! its output. AAudio pulls from the buffer on its own real-time thread. If the device goes +//! away (headphones, route change) the stream is reopened once, on a helper thread. + +use crate::jitter::{ + AudioStats, Jitter, SAMPLE_RATE, f32_to_i16, opus_csd1, opus_csd2, opus_pre_skip, pcm_to_f32, +}; +use ezd_proto::{AudioCodec, AudioPacket, AudioStream as StreamInfo, SampleFormat}; +use ndk::audio::{ + AudioCallbackResult, AudioDirection, AudioFormat, AudioPerformanceMode, AudioSharingMode, + AudioStream, AudioStreamBuilder, +}; +use ndk::media::media_codec::{ + DequeuedInputBufferResult, DequeuedOutputBufferInfoResult, MediaCodec, MediaCodecDirection, +}; +use ndk::media::media_format::MediaFormat; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::thread::JoinHandle; +use std::time::Duration; + +const OPUS_MIME: &str = "audio/opus"; +/// `AudioFormat.ENCODING_PCM_FLOAT`. +const ENCODING_PCM_FLOAT: i32 = 4; +const INPUT_WAIT: Duration = Duration::from_millis(5); +const IDLE_POLL: Duration = Duration::from_millis(5); +/// Frames per chunk when the stream wants 16-bit samples. +const CHUNK_SAMPLES: usize = 1024; + +/// True when the device has an Opus decoder. +pub fn opus_available() -> bool { + MediaCodec::from_decoder_type(OPUS_MIME).is_some() +} + +fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// AAudio streams can be used from any thread. +struct SendStream(AudioStream); +// SAFETY: AAudio stream handles are thread-safe; the owned callbacks are `Send`. +unsafe impl Send for SendStream {} + +/// State shared by the player, the AAudio callbacks and the helper threads. +struct Core { + jitter: Mutex, + stats: AudioStats, + channels: usize, + i16_out: bool, + stream: Mutex>, + closing: AtomicBool, + reopened: AtomicBool, +} + +/// Plays one negotiated audio stream until dropped. +pub struct AudioPlayer { + core: Arc, + format: SampleFormat, + opus: Option, + reopen_tx: Sender<()>, + reopen_thread: Option>, + scratch: Vec, +} + +impl AudioPlayer { + /// Opens the output (and the Opus decoder when needed) for `info`. + pub fn new(info: &StreamInfo) -> Result { + if info.sample_rate != SAMPLE_RATE { + return Err(format!("unsupported sample rate {}", info.sample_rate)); + } + if info.channels == 0 { + return Err("audio stream without channels".into()); + } + let channels = info.channels as usize; + let core = Arc::new(Core { + jitter: Mutex::new(Jitter::new(info.sample_rate, channels)), + stats: AudioStats::default(), + channels, + i16_out: info.codec == AudioCodec::Pcm && info.format == SampleFormat::S16, + stream: Mutex::new(None), + closing: AtomicBool::new(false), + reopened: AtomicBool::new(false), + }); + let opus = match info.codec { + AudioCodec::Pcm => None, + AudioCodec::Opus => Some(OpusDecoder::spawn(core.clone(), info)?), + }; + let (reopen_tx, reopen_rx) = channel(); + let stream = open_output(&core, &reopen_tx)?; + tracing::info!( + "audio output {} Hz, {} ch, {}, {} frames buffer (burst {})", + stream.0.sample_rate(), + stream.0.channel_count(), + if core.i16_out { "i16" } else { "f32" }, + stream.0.buffer_size_in_frames(), + stream.0.frames_per_burst(), + ); + *lock(&core.stream) = Some(stream); + let reopen_thread = { + let (core, tx) = (core.clone(), reopen_tx.clone()); + std::thread::Builder::new() + .name("ezd-audio-reopen".into()) + .spawn(move || reopen_loop(&core, &tx, reopen_rx)) + .ok() + }; + Ok(AudioPlayer { + core, + format: info.format, + opus, + reopen_tx, + reopen_thread, + scratch: Vec::new(), + }) + } + + /// Handles one packet from the host without blocking. + pub fn on_packet(&mut self, pkt: AudioPacket) { + use std::sync::atomic::Ordering::Relaxed; + self.core.stats.packets.fetch_add(1, Relaxed); + match &self.opus { + Some(opus) => opus.send(pkt), + None => { + self.scratch.clear(); + pcm_to_f32(&pkt.data, self.format, &mut self.scratch); + push(&self.core, &self.scratch); + } + } + } +} + +impl Drop for AudioPlayer { + fn drop(&mut self) { + self.core.closing.store(true, Ordering::Release); + let _ = self.reopen_tx.send(()); + if let Some(j) = self.reopen_thread.take() { + let _ = j.join(); + } + // The decoder thread joins here, before the stream goes away. + self.opus = None; + // Dropping the stream stops and closes it. + let stream = lock(&self.core.stream).take(); + drop(stream); + let (packets, underruns, dropped) = self.core.stats.snapshot(); + tracing::info!( + "audio stopped: {packets} packets, {underruns} underruns, {dropped} frames dropped" + ); + } +} + +/// Pushes decoded samples into the jitter buffer and counts what it dropped. +fn push(core: &Core, samples: &[f32]) { + let dropped = lock(&core.jitter).push(samples); + if dropped > 0 { + core.stats + .dropped_frames + .fetch_add(dropped as u64, Ordering::Relaxed); + } +} + +fn open_output(core: &Arc, reopen_tx: &Sender<()>) -> Result { + let data_core = core.clone(); + let err_tx = reopen_tx.clone(); + let channels = core.channels; + let i16_out = core.i16_out; + let stream = AudioStreamBuilder::new() + .map_err(|e| format!("audio builder: {e}"))? + .direction(AudioDirection::Output) + .performance_mode(AudioPerformanceMode::LowLatency) + .sharing_mode(AudioSharingMode::Shared) + .sample_rate(SAMPLE_RATE as i32) + .channel_count(channels as i32) + .format(if i16_out { + AudioFormat::PCM_I16 + } else { + AudioFormat::PCM_Float + }) + .data_callback(Box::new(move |_stream, data, frames| { + let samples = frames.max(0) as usize * channels; + let mut jitter = lock(&data_core.jitter); + let mut underrun = false; + if i16_out { + // SAFETY: AAudio hands over `frames * channels` writable samples of the format + // that was requested. + let out = unsafe { std::slice::from_raw_parts_mut(data.cast::(), samples) }; + let mut tmp = [0f32; CHUNK_SAMPLES]; + for chunk in out.chunks_mut(CHUNK_SAMPLES) { + let t = &mut tmp[..chunk.len()]; + underrun |= jitter.pull(t); + for (o, s) in chunk.iter_mut().zip(t.iter()) { + *o = f32_to_i16(*s); + } + } + } else { + // SAFETY: as above, for f32. + let out = unsafe { std::slice::from_raw_parts_mut(data.cast::(), samples) }; + underrun = jitter.pull(out); + } + drop(jitter); + if underrun { + data_core.stats.underruns.fetch_add(1, Ordering::Relaxed); + } + AudioCallbackResult::Continue + })) + .error_callback(Box::new(move |_stream, err| { + tracing::warn!("audio stream error: {err}"); + let _ = err_tx.send(()); + })) + .open_stream() + .map_err(|e| format!("open audio stream: {e}"))?; + // Two bursts keep the latency low while tolerating a late callback. + let burst = stream.frames_per_burst(); + if burst > 0 + && let Err(e) = stream.set_buffer_size_in_frames(burst * 2) + { + tracing::debug!("audio buffer size: {e}"); + } + stream + .request_start() + .map_err(|e| format!("start audio stream: {e}"))?; + Ok(SendStream(stream)) +} + +/// Reopens the output once after an error; never runs inside an AAudio callback. +fn reopen_loop(core: &Arc, tx: &Sender<()>, rx: Receiver<()>) { + while rx.recv().is_ok() { + if core.closing.load(Ordering::Acquire) { + return; + } + if core.reopened.swap(true, Ordering::AcqRel) { + tracing::warn!("audio stream failed again, not reopening"); + continue; + } + let old = lock(&core.stream).take(); + drop(old); + lock(&core.jitter).reset(); + match open_output(core, tx) { + Ok(s) => { + tracing::info!("audio stream reopened"); + *lock(&core.stream) = Some(s); + } + Err(e) => tracing::warn!("cannot reopen the audio stream: {e}"), + } + } +} + +// ---------------------------------------------------------------- Opus + +struct OpusDecoder { + tx: Option>, + join: Option>, +} + +impl OpusDecoder { + /// Starts the decode thread and waits for the codec to be configured. + fn spawn(core: Arc, info: &StreamInfo) -> Result { + let (tx, rx) = channel(); + let (ready_tx, ready_rx) = channel(); + let info = info.clone(); + let join = std::thread::Builder::new() + .name("ezd-audio-opus".into()) + .spawn(move || match create_codec(&info) { + Ok((codec, float)) => { + let _ = ready_tx.send(Ok(())); + decode_loop(&codec, float, &core, rx); + let _ = codec.stop(); + } + Err(e) => { + let _ = ready_tx.send(Err(e)); + } + }) + .map_err(|e| format!("spawn opus thread: {e}"))?; + match ready_rx.recv() { + Ok(Ok(())) => Ok(OpusDecoder { + tx: Some(tx), + join: Some(join), + }), + Ok(Err(e)) => { + let _ = join.join(); + Err(e) + } + Err(_) => Err("opus thread died".into()), + } + } + + fn send(&self, pkt: AudioPacket) { + if let Some(tx) = &self.tx { + let _ = tx.send(pkt); + } + } +} + +impl Drop for OpusDecoder { + fn drop(&mut self) { + self.tx = None; + if let Some(j) = self.join.take() { + let _ = j.join(); + } + } +} + +fn opus_format(info: &StreamInfo, float: bool) -> Result { + let pre_skip = opus_pre_skip(&info.codec_config).ok_or("invalid OpusHead")?; + let mut f = MediaFormat::new(); + f.set_str("mime", OPUS_MIME); + f.set_i32("sample-rate", SAMPLE_RATE as i32); + f.set_i32("channel-count", info.channels as i32); + f.set_buffer("csd-0", &info.codec_config); + f.set_buffer("csd-1", &opus_csd1(pre_skip)); + f.set_buffer("csd-2", &opus_csd2()); + if float { + f.set_i32("pcm-encoding", ENCODING_PCM_FLOAT); + } + Ok(f) +} + +/// Configures and starts the decoder, asking for float output first. Returns the codec and +/// whether float output was requested. +fn create_codec(info: &StreamInfo) -> Result<(MediaCodec, bool), String> { + let mut last = String::new(); + for float in [true, false] { + let codec = MediaCodec::from_decoder_type(OPUS_MIME) + .ok_or_else(|| format!("no decoder for {OPUS_MIME}"))?; + let fmt = opus_format(info, float)?; + match codec.configure(&fmt, None, MediaCodecDirection::Decoder) { + Ok(()) => match codec.start() { + Ok(()) => return Ok((codec, float)), + Err(e) => last = format!("start: {e}"), + }, + Err(e) => last = format!("configure (float={float}): {e}"), + } + tracing::warn!("opus decoder setup failed: {last}"); + } + Err(last) +} + +fn decode_loop(codec: &MediaCodec, requested_float: bool, core: &Core, rx: Receiver) { + // Until the codec reports its output format, assume what was asked for. + let mut float = requested_float; + let mut samples: Vec = Vec::new(); + loop { + match rx.recv_timeout(IDLE_POLL) { + Ok(pkt) => match codec.dequeue_input_buffer(INPUT_WAIT) { + Ok(DequeuedInputBufferResult::Buffer(mut ib)) => { + let buf = ib.buffer_mut(); + let n = pkt.data.len().min(buf.len()); + // SAFETY: `buf` has at least `n` bytes; u8 and MaybeUninit share a layout + // and the regions cannot overlap. + unsafe { + std::ptr::copy_nonoverlapping( + pkt.data.as_ptr(), + buf.as_mut_ptr().cast::(), + n, + ); + } + if let Err(e) = codec.queue_input_buffer(ib, 0, n, pkt.pts_us, 0) { + tracing::warn!("opus queue_input_buffer: {e}"); + return; + } + } + Ok(DequeuedInputBufferResult::TryAgainLater) => { + tracing::warn!("opus decoder input busy, dropped a packet"); + } + Err(e) => { + tracing::warn!("opus dequeue_input_buffer: {e}"); + return; + } + }, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return, + } + loop { + match codec.dequeue_output_buffer(Duration::ZERO) { + Ok(DequeuedOutputBufferInfoResult::Buffer(ob)) => { + let (offset, size) = (ob.info().offset().max(0) as usize, ob.info().size()); + samples.clear(); + let data = ob.buffer(); + let end = (offset + size.max(0) as usize).min(data.len()); + let bytes = data.get(offset..end).unwrap_or(&[]); + if float { + pcm_to_f32(bytes, SampleFormat::F32, &mut samples); + } else { + pcm_to_f32(bytes, SampleFormat::S16, &mut samples); + } + if let Err(e) = codec.release_output_buffer(ob, false) { + tracing::warn!("opus release_output_buffer: {e}"); + return; + } + if !samples.is_empty() { + push(core, &samples); + } + } + Ok(DequeuedOutputBufferInfoResult::OutputFormatChanged) => { + let format = codec.output_format(); + float = format.i32("pcm-encoding") == Some(ENCODING_PCM_FLOAT); + tracing::info!("opus output format changed: {format}"); + } + Ok(DequeuedOutputBufferInfoResult::OutputBuffersChanged) => {} + Ok(DequeuedOutputBufferInfoResult::TryAgainLater) => break, + Err(e) => { + tracing::warn!("opus dequeue_output_buffer: {e}"); + return; + } + } + } + } +} diff --git a/crates/ezd-android/src/caps.rs b/crates/ezd-android/src/caps.rs index 37faa2b..37cbe2c 100644 --- a/crates/ezd-android/src/caps.rs +++ b/crates/ezd-android/src/caps.rs @@ -8,6 +8,7 @@ //! "decoders": [{"codec": "h264", "name": "c2.qti.avc.decoder", "hardware": true, //! "low_latency": true, "max_width": 4096, "max_height": 4096, "max_fps": 240, "hdr10": false}], //! "hdr": {"display_hdr10": true, "max_nits": 1000, "max_avg_nits": 600, "min_millinits": 0}, +//! "audio": true, //! "stylus": true, //! "max_touch_points": 10 //! } @@ -25,6 +26,8 @@ pub struct ParsedCaps { pub caps: ClientCaps, /// HDR10 capabilities for `ClientConfig.hdr`; `None` when the JSON has no `hdr` object. pub hdr: Option, + /// The user wants the PC's audio on this device (`"audio"`, default false). + pub audio: bool, } #[derive(Deserialize)] @@ -37,6 +40,8 @@ struct JsonCaps { #[serde(default)] hdr: Option, #[serde(default)] + audio: bool, + #[serde(default)] stylus: bool, #[serde(default = "default_touch")] max_touch_points: u32, @@ -145,6 +150,7 @@ pub fn parse_caps(json: &str) -> Result { }); Ok(ParsedCaps { hdr, + audio: j.audio, device_name: if j.device_name.is_empty() { "android".into() } else { @@ -250,6 +256,14 @@ mod tests { ); } + #[test] + fn audio_switch_defaults_off() { + assert!(!parse_caps(JSON).unwrap().audio); + let json = r#"{"audio": true, + "display": {"width_px": 1, "height_px": 1, "refresh_rates_mhz": [60000]}}"#; + assert!(parse_caps(json).unwrap().audio); + } + #[test] fn old_json_has_no_hdr() { assert!(parse_caps(JSON).unwrap().hdr.is_none()); diff --git a/crates/ezd-android/src/jitter.rs b/crates/ezd-android/src/jitter.rs new file mode 100644 index 0000000..2c6cfa2 --- /dev/null +++ b/crates/ezd-android/src/jitter.rs @@ -0,0 +1,279 @@ +//! Pure parts of the audio path: the jitter buffer, PCM decoding and the Opus codec-specific data. +//! No Android types, so they build and test on the host. + +use ezd_proto::SampleFormat; +use std::collections::VecDeque; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Output sample rate; the host sends 48 kHz. +pub const SAMPLE_RATE: u32 = 48_000; +/// Playback starts once this much audio is buffered (after the start or an underrun). +pub const TARGET_MS: u32 = 20; +/// Above this much buffered audio the oldest samples are dropped down to [`TARGET_MS`]. +pub const MAX_MS: u32 = 60; +/// Opus seek pre-roll that MediaCodec expects in `csd-2`. +pub const OPUS_SEEK_PREROLL_NS: u64 = 80_000_000; + +/// Counters shared between the packet path, the audio callback and the final log line. +#[derive(Default)] +pub struct AudioStats { + /// Packets received from the host. + pub packets: AtomicU64, + /// Times the output ran dry while playing. + pub underruns: AtomicU64, + /// Sample frames (one sample per channel) dropped because the buffer grew too long. + pub dropped_frames: AtomicU64, +} + +impl AudioStats { + /// `(packets, underruns, dropped frames)`. + pub fn snapshot(&self) -> (u64, u64, u64) { + ( + self.packets.load(Ordering::Relaxed), + self.underruns.load(Ordering::Relaxed), + self.dropped_frames.load(Ordering::Relaxed), + ) + } +} + +/// Interleaved f32 FIFO with a prebuffer and a latency cap. +pub struct Jitter { + channels: usize, + buf: VecDeque, + target: usize, + max: usize, + playing: bool, +} + +impl Jitter { + /// Buffer for `channels` interleaved channels at `rate` Hz. + pub fn new(rate: u32, channels: usize) -> Self { + let channels = channels.max(1); + let samples = |ms: u32| (rate as usize * ms as usize / 1000) * channels; + Jitter { + channels, + buf: VecDeque::with_capacity(samples(MAX_MS) * 2), + target: samples(TARGET_MS), + max: samples(MAX_MS), + playing: false, + } + } + + /// Buffered samples (all channels). + pub fn len(&self) -> usize { + self.buf.len() + } + + /// True when nothing is buffered. + pub fn is_empty(&self) -> bool { + self.buf.is_empty() + } + + /// True while audio is being handed out (not prebuffering). + pub fn is_playing(&self) -> bool { + self.playing + } + + /// Appends `samples` (a whole number of frames). When more than [`MAX_MS`] is buffered the + /// oldest audio is dropped down to [`TARGET_MS`]. Returns the sample frames dropped. + pub fn push(&mut self, samples: &[f32]) -> usize { + self.buf.extend(samples.iter().copied()); + if self.buf.len() <= self.max { + return 0; + } + let drop = (self.buf.len() - self.target) / self.channels * self.channels; + self.buf.drain(..drop); + drop / self.channels + } + + /// Fills `out` with buffered audio. While prebuffering, or for the part that is missing on an + /// underrun, it writes silence. Returns true when an underrun happened (the buffer ran dry + /// while playing); playback then waits for the prebuffer again. + pub fn pull(&mut self, out: &mut [f32]) -> bool { + if !self.playing { + if self.buf.len() < self.target { + out.fill(0.0); + return false; + } + self.playing = true; + } + let n = out.len().min(self.buf.len()); + for (o, s) in out.iter_mut().zip(self.buf.drain(..n)) { + *o = s; + } + if n < out.len() { + out[n..].fill(0.0); + self.playing = false; + return true; + } + false + } + + /// Forgets everything and prebuffers again. + pub fn reset(&mut self) { + self.buf.clear(); + self.playing = false; + } +} + +/// Decodes little-endian interleaved PCM into f32, appending to `out`. A trailing partial +/// sample is ignored. +pub fn pcm_to_f32(data: &[u8], format: SampleFormat, out: &mut Vec) { + match format { + SampleFormat::S16 => out.extend( + data.chunks_exact(2) + .map(|b| i16::from_le_bytes([b[0], b[1]]) as f32 / 32768.0), + ), + SampleFormat::F32 => out.extend( + data.chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])), + ), + } +} + +/// f32 to i16 with clipping. +pub fn f32_to_i16(v: f32) -> i16 { + (v * 32768.0).round().clamp(-32768.0, 32767.0) as i16 +} + +/// Pre-skip (samples at 48 kHz) of an OpusHead header, or `None` when it is not one. +pub fn opus_pre_skip(head: &[u8]) -> Option { + if head.len() < 19 || &head[..8] != b"OpusHead" { + return None; + } + Some(u16::from_le_bytes([head[10], head[11]])) +} + +/// Codec delay for `csd-1`: the pre-skip in nanoseconds, 8 bytes little-endian. +pub fn opus_csd1(pre_skip: u16) -> [u8; 8] { + (pre_skip as u64 * 1_000_000_000 / SAMPLE_RATE as u64).to_le_bytes() +} + +/// Seek pre-roll for `csd-2`: 80 ms in nanoseconds, 8 bytes little-endian. +pub fn opus_csd2() -> [u8; 8] { + OPUS_SEEK_PREROLL_NS.to_le_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// 10 ms of stereo. + fn ms(n: usize, v: f32) -> Vec { + vec![v; 48 * n * 2] + } + + #[test] + fn waits_for_the_prebuffer() { + let mut j = Jitter::new(48_000, 2); + j.push(&ms(10, 0.5)); + let mut out = vec![1.0; 96]; + assert!(!j.pull(&mut out)); + assert!(out.iter().all(|&s| s == 0.0)); + assert!(!j.is_playing()); + // The silence consumed nothing. + assert_eq!(j.len(), 960); + j.push(&ms(10, 0.5)); + assert!(!j.pull(&mut out)); + assert!(out.iter().all(|&s| s == 0.5)); + assert!(j.is_playing()); + assert_eq!(j.len(), 1920 - 96); + } + + #[test] + fn drops_the_oldest_above_the_cap() { + let mut j = Jitter::new(48_000, 2); + j.push(&ms(30, 0.1)); + j.push(&ms(20, 0.2)); + assert_eq!(j.len(), 48 * 50 * 2); + // 70 ms in total: down to 20 ms, keeping the newest. + let dropped = j.push(&ms(20, 0.3)); + assert_eq!(dropped, 48 * 50); + assert_eq!(j.len(), 48 * 20 * 2); + let mut out = vec![0.0; 48 * 20 * 2]; + j.pull(&mut out); + assert!(out.iter().all(|&s| s == 0.3)); + } + + #[test] + fn exactly_the_cap_is_kept() { + let mut j = Jitter::new(48_000, 2); + assert_eq!(j.push(&ms(60, 0.1)), 0); + assert_eq!(j.len(), 48 * 60 * 2); + } + + #[test] + fn underrun_gives_silence_and_prebuffers_again() { + let mut j = Jitter::new(48_000, 2); + j.push(&ms(20, 0.5)); + let mut out = vec![0.0; 48 * 30 * 2]; + assert!(j.pull(&mut out), "ran dry while playing"); + assert!(out[..48 * 20 * 2].iter().all(|&s| s == 0.5)); + assert!(out[48 * 20 * 2..].iter().all(|&s| s == 0.0)); + assert!(!j.is_playing()); + // Less than the target arrives: still silent. + j.push(&ms(10, 0.7)); + let mut out = vec![1.0; 96]; + assert!(!j.pull(&mut out)); + assert!(out.iter().all(|&s| s == 0.0)); + j.push(&ms(10, 0.7)); + assert!(!j.pull(&mut out)); + assert!(out.iter().all(|&s| s == 0.7)); + } + + #[test] + fn drops_whole_frames_for_odd_channel_counts() { + let mut j = Jitter::new(48_000, 3); + j.push(&vec![0.0; 48 * 59 * 3]); + let dropped = j.push(&vec![0.0; 48 * 5 * 3]); + assert_eq!(j.len() % 3, 0); + assert_eq!(dropped * 3 + j.len(), 48 * 64 * 3); + } + + #[test] + fn decodes_s16() { + let mut out = Vec::new(); + let data: Vec = [0i16, 16384, -32768, 32767] + .iter() + .flat_map(|s| s.to_le_bytes()) + .collect(); + pcm_to_f32(&data, SampleFormat::S16, &mut out); + assert_eq!(out, vec![0.0, 0.5, -1.0, 32767.0 / 32768.0]); + } + + #[test] + fn decodes_f32_and_ignores_a_partial_tail() { + let mut out = Vec::new(); + let mut data: Vec = [0.25f32, -0.75] + .iter() + .flat_map(|s| s.to_le_bytes()) + .collect(); + data.push(0xAA); + pcm_to_f32(&data, SampleFormat::F32, &mut out); + assert_eq!(out, vec![0.25, -0.75]); + } + + #[test] + fn converts_f32_to_i16_with_clipping() { + assert_eq!(f32_to_i16(0.0), 0); + assert_eq!(f32_to_i16(0.5), 16384); + assert_eq!(f32_to_i16(-1.0), -32768); + assert_eq!(f32_to_i16(2.0), 32767); + assert_eq!(f32_to_i16(-2.0), -32768); + } + + #[test] + fn builds_the_opus_csd() { + let mut head = b"OpusHead".to_vec(); + head.extend([1, 2]); // version, channels + head.extend(312u16.to_le_bytes()); // pre-skip + head.extend(48_000u32.to_le_bytes()); + head.extend([0, 0, 0]); // gain, mapping family + assert_eq!(opus_pre_skip(&head), Some(312)); + // 312 samples at 48 kHz = 6.5 ms. + assert_eq!(u64::from_le_bytes(opus_csd1(312)), 6_500_000); + assert_eq!(u64::from_le_bytes(opus_csd2()), 80_000_000); + assert_eq!(opus_pre_skip(b"OpusTags"), None); + assert_eq!(opus_pre_skip(&head[..10]), None); + } +} diff --git a/crates/ezd-android/src/lib.rs b/crates/ezd-android/src/lib.rs index 9479bfb..180dc29 100644 --- a/crates/ezd-android/src/lib.rs +++ b/crates/ezd-android/src/lib.rs @@ -1,6 +1,6 @@ //! ezdisplay Android client native library. //! -//! The portable parts ([`caps`], [`input`], [`tracker`], [`status`]) build and test on the host. The JNI +//! The portable parts ([`caps`], [`input`], [`jitter`], [`tracker`], [`status`]) build and test on the host. The JNI //! bridge, session and MediaCodec decoder only exist for `target_os = "android"`. //! //! Kotlin class: `com.librestatic.ezdisplay.NativeBridge` (see `bridge.rs`). @@ -11,10 +11,13 @@ pub mod caps; pub mod color; pub mod input; +pub mod jitter; pub mod status; pub mod tracker; pub mod wifi; +#[cfg(target_os = "android")] +mod audio; #[cfg(target_os = "android")] mod bridge; #[cfg(target_os = "android")] diff --git a/crates/ezd-android/src/session.rs b/crates/ezd-android/src/session.rs index 5fddf1c..66608a4 100644 --- a/crates/ezd-android/src/session.rs +++ b/crates/ezd-android/src/session.rs @@ -1,5 +1,6 @@ //! One connection attempt to the host: client core on a small tokio runtime plus the decoder. +use crate::audio::{AudioPlayer, opus_available}; use crate::caps::ParsedCaps; use crate::decoder::{Cmd, DecoderCtl, SendWindow}; use crate::status::{Phase, Shared}; @@ -11,7 +12,10 @@ use ezd_client_core::{ }; use ezd_link::{BlockingOptions, Link}; use ezd_proto::pair::Fingerprint; -use ezd_proto::{CursorShape, StreamColor, StreamConfig, VideoFrame}; +use ezd_proto::{ + AudioCodec, AudioConfig, AudioPacket, ClientAudio, CursorShape, SampleFormat, StreamColor, + StreamConfig, VideoFrame, +}; use std::fs::File; use std::io::{Read, Write}; use std::net::SocketAddr; @@ -175,9 +179,38 @@ struct Handler { dec: Arc, /// Sent right before the matching `StreamConfig`; SDR BT.709 when the host sends none. color: StreamColor, + /// Output for the PC's audio; present between an `AudioConfig` with a stream and the next one. + audio: Option, } impl ClientHandler for Handler { + fn on_audio_config(&mut self, cfg: &AudioConfig) { + // Drop the previous output first: it holds the audio device. + self.audio = None; + let Some(stream) = &cfg.stream else { + tracing::info!("no audio in this session"); + return; + }; + tracing::info!( + "audio {:?} {} Hz, {} ch, {:?}, {} samples per packet", + stream.codec, + stream.sample_rate, + stream.channels, + stream.format, + stream.frame_samples + ); + match AudioPlayer::new(stream) { + Ok(p) => self.audio = Some(p), + Err(e) => tracing::warn!("audio disabled: {e}"), + } + } + + fn on_audio_packet(&mut self, pkt: AudioPacket) { + if let Some(p) = &mut self.audio { + p.on_packet(pkt); + } + } + fn on_stream_color(&mut self, color: &StreamColor) { tracing::info!( "stream colour {}-bit primaries {} transfer {} matrix {} full_range {}", @@ -374,6 +407,24 @@ async fn connect(target: &Target) -> Result { } } +/// What this device plays: 48 kHz stereo PCM, and Opus when the device has a decoder. +fn client_audio(enabled: bool) -> ClientAudio { + if !enabled { + return ClientAudio::none(); + } + let mut codecs = vec![AudioCodec::Pcm]; + if opus_available() { + codecs.push(AudioCodec::Opus); + } + ClientAudio { + enabled: true, + sample_rates: vec![crate::jitter::SAMPLE_RATE], + max_channels: 2, + formats: vec![SampleFormat::F32, SampleFormat::S16], + codecs, + } +} + fn run( handle: ClientHandle, shared: Arc, @@ -412,10 +463,12 @@ fn run( let mut cfg = ClientConfig::new(caps.device_name, caps.caps); cfg.app_version = env!("CARGO_PKG_VERSION").into(); cfg.hdr = caps.hdr; + cfg.audio = Some(client_audio(caps.audio)); let mut handler = Handler { shared: shared.clone(), dec: dec.clone(), color: StreamColor::default(), + audio: None, }; match run_client(link, cfg, &handle, &mut handler).await { Ok(o) => { From 48f1095665e405a14a48d707c60fa68e1e25adb6 Mon Sep 17 00:00:00 2001 From: FacuM Date: Sun, 25 Oct 2026 21:52:23 -0300 Subject: [PATCH 06/12] Add the audio setting to the host, its CLI and the tray ServiceConfig.audio (auto, pcm, opus, off) feeds the negotiation and decides whether sessions get the platform capture and the FFmpeg Opus encoder. ezd-host takes --audio, diag reports whether capture and Opus are available, and the tray gets an Audio choice plus the negotiated stream in its live status. --- crates/ezd-host-service/src/audio.rs | 49 ++++++++++++ crates/ezd-host-service/src/config.rs | 79 ++++++++++++++++++- crates/ezd-host-service/src/lib.rs | 5 +- crates/ezd-host-service/src/platform.rs | 15 +++- crates/ezd-host-service/src/service.rs | 100 +++++++++++++++++++++--- crates/ezd-host-service/src/status.rs | 4 +- crates/ezd-host/i18n/de/ezd-host.ftl | 1 + crates/ezd-host/i18n/en/ezd-host.ftl | 1 + crates/ezd-host/i18n/es/ezd-host.ftl | 1 + crates/ezd-host/i18n/fr/ezd-host.ftl | 1 + crates/ezd-host/i18n/it/ezd-host.ftl | 1 + crates/ezd-host/i18n/pt/ezd-host.ftl | 1 + crates/ezd-host/src/diag.rs | 11 +++ crates/ezd-host/src/run.rs | 57 +++++++++++++- crates/ezd-tray/i18n/de/ezd-tray.ftl | 14 ++++ crates/ezd-tray/i18n/en/ezd-tray.ftl | 14 ++++ crates/ezd-tray/i18n/es/ezd-tray.ftl | 14 ++++ crates/ezd-tray/i18n/fr/ezd-tray.ftl | 14 ++++ crates/ezd-tray/i18n/it/ezd-tray.ftl | 14 ++++ crates/ezd-tray/i18n/pt/ezd-tray.ftl | 14 ++++ crates/ezd-tray/src/menu_model.rs | 1 + crates/ezd-tray/src/ui/view.rs | 67 +++++++++++++++- crates/ezd-tray/src/ui/view/tests.rs | 51 +++++++++++- 23 files changed, 505 insertions(+), 24 deletions(-) create mode 100644 crates/ezd-host-service/src/audio.rs diff --git a/crates/ezd-host-service/src/audio.rs b/crates/ezd-host-service/src/audio.rs new file mode 100644 index 0000000..6053d12 --- /dev/null +++ b/crates/ezd-host-service/src/audio.rs @@ -0,0 +1,49 @@ +//! The audio parts of a session on this host: platform capture and the optional Opus encoder. + +use ezd_host_core::{AudioBackend, AudioEncoder, AudioSource, SourceError}; + +/// Platform capture plus Opus through FFmpeg when the `ffmpeg` feature is on. +#[derive(Debug, Clone, Copy, Default)] +pub struct HostAudioBackend; + +impl AudioBackend for HostAudioBackend { + fn create_source(&self) -> Option> { + crate::platform::audio_source() + } + + fn opus_available(&self) -> bool { + opus_available() + } + + fn create_opus_encoder(&self, channels: u8) -> Result, SourceError> { + #[cfg(feature = "ffmpeg")] + { + ezd_ffmpeg::create_opus_encoder(channels) + } + #[cfg(not(feature = "ffmpeg"))] + { + let _ = channels; + Err(SourceError::msg("built without ffmpeg")) + } + } +} + +/// True if this build can encode Opus. +pub fn opus_available() -> bool { + #[cfg(feature = "ffmpeg")] + { + ezd_ffmpeg::opus_available() + } + #[cfg(not(feature = "ffmpeg"))] + { + false + } +} + +/// The audio part of the diagnostics report. +pub fn diag_section() -> serde_json::Value { + serde_json::json!({ + "capture": crate::platform::audio_source().is_some(), + "opus": opus_available(), + }) +} diff --git a/crates/ezd-host-service/src/config.rs b/crates/ezd-host-service/src/config.rs index 0369234..e9f4d0c 100644 --- a/crates/ezd-host-service/src/config.rs +++ b/crates/ezd-host-service/src/config.rs @@ -1,10 +1,11 @@ //! Configuration of the service: what the CLI flags and the tray settings both describe. use ezd_host_core::{ - InputBackend, PenMode, PlatformOptions, SessionEvent, SessionOptions, UnavailableBackend, + AudioBackend, InputBackend, PenMode, PlatformOptions, SessionEvent, SessionOptions, + UnavailableBackend, }; use ezd_link::VideoCredit; -use ezd_proto::{HdrMode, LinkKind, Prefs}; +use ezd_proto::{AudioMode, HdrMode, LinkKind, Prefs}; use serde::{Deserialize, Serialize}; use std::net::SocketAddr; use std::sync::Arc; @@ -94,6 +95,32 @@ impl From for HdrMode { } } +/// PC audio streaming policy (serializable mirror of [`AudioMode`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AudioChoice { + /// Uncompressed PCM when the link and the tablet allow it, else Opus. + #[default] + Auto, + /// Prefer PCM even on a slow link. + Pcm, + /// Prefer Opus. + Opus, + /// No audio. + Off, +} + +impl From for AudioMode { + fn from(c: AudioChoice) -> Self { + match c { + AudioChoice::Auto => AudioMode::Auto, + AudioChoice::Pcm => AudioMode::Pcm, + AudioChoice::Opus => AudioMode::Opus, + AudioChoice::Off => AudioMode::Off, + } + } +} + /// Congestion controller of the Wi-Fi link. #[cfg(feature = "wifi")] #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] @@ -187,6 +214,8 @@ pub struct ServiceConfig { pub open_app_on_connect: bool, /// HDR10 streaming policy (`hdr = "auto" | "off"`). pub hdr: HdrChoice, + /// PC audio policy (`audio = "auto" | "pcm" | "opus" | "off"`). + pub audio: AudioChoice, /// Wi-Fi listener; `None` leaves Wi-Fi off. Sessions on it always use the network link kind, /// whatever `link` says. #[cfg(feature = "wifi")] @@ -214,6 +243,7 @@ impl Default for ServiceConfig { pen_mode: PenChoice::default(), open_app_on_connect: true, hdr: HdrChoice::default(), + audio: AudioChoice::default(), #[cfg(feature = "wifi")] wifi: None, } @@ -233,6 +263,7 @@ impl ServiceConfig { encoder: self.encoder.clone(), scale_permille: self.scale_permille, hdr: self.hdr.into(), + audio: self.audio.into(), ..Prefs::default() } } @@ -267,6 +298,9 @@ impl ServiceConfig { )) }) }); + // With audio off no source is ever created, whatever the client asks for. + let audio: Option> = (self.audio != AudioChoice::Off) + .then(|| Arc::new(crate::audio::HostAudioBackend) as Arc); SessionOptions { video_credit: VideoCredit::OnAck { window: self.video_window, @@ -279,6 +313,7 @@ impl ServiceConfig { PenChoice::Mouse => PenMode::Mouse, }, telemetry: self.telemetry, + audio, events, ..SessionOptions::default() } @@ -419,6 +454,46 @@ mod tests { assert_eq!(off.prefs().hdr, HdrMode::Off); } + #[test] + fn audio_defaults_to_auto_and_reads_its_serde_name() { + assert_eq!(ServiceConfig::default().audio, AudioChoice::Auto); + let empty: ServiceConfig = serde_json::from_str("{}").unwrap(); + assert_eq!(empty.audio, AudioChoice::Auto); + assert_eq!(empty.prefs().audio, AudioMode::Auto); + for (name, choice, mode) in [ + ("auto", AudioChoice::Auto, AudioMode::Auto), + ("pcm", AudioChoice::Pcm, AudioMode::Pcm), + ("opus", AudioChoice::Opus, AudioMode::Opus), + ("off", AudioChoice::Off, AudioMode::Off), + ] { + let c: ServiceConfig = + serde_json::from_str(&format!(r#"{{ "audio": "{name}" }}"#)).unwrap(); + assert_eq!(c.audio, choice); + assert_eq!(c.prefs().audio, mode); + assert_eq!(c.wifi_prefs().audio, mode); + let back: ServiceConfig = + serde_json::from_str(&serde_json::to_string(&c).unwrap()).unwrap(); + assert_eq!(back, c); + } + assert!(serde_json::from_str::(r#"{ "audio": "loud" }"#).is_err()); + } + + #[test] + fn audio_off_creates_no_audio_backend() { + let (_tx, rx) = watch::channel(false); + assert!( + ServiceConfig::default() + .session_options(rx.clone(), None) + .audio + .is_some() + ); + let off = ServiceConfig { + audio: AudioChoice::Off, + ..ServiceConfig::default() + }; + assert!(off.session_options(rx, None).audio.is_none()); + } + #[test] fn aoa_defaults_to_auto_and_reads_its_serde_name() { assert_eq!(ServiceConfig::default().aoa, AoaMode::Auto); diff --git a/crates/ezd-host-service/src/lib.rs b/crates/ezd-host-service/src/lib.rs index 65792e8..5f23b4a 100644 --- a/crates/ezd-host-service/src/lib.rs +++ b/crates/ezd-host-service/src/lib.rs @@ -17,6 +17,7 @@ pub mod adb; #[cfg(feature = "aoa")] pub mod aoa; +pub mod audio; pub mod config; pub mod error; #[cfg(feature = "wifi")] @@ -30,7 +31,9 @@ pub mod store; mod wifi; pub use adb::{AdbClient, AdbLocator, is_network_serial}; -pub use config::{AdbMode, AoaMode, HdrChoice, LinkChoice, PenChoice, ServiceConfig, SourceKind}; +pub use config::{ + AdbMode, AoaMode, AudioChoice, HdrChoice, LinkChoice, PenChoice, ServiceConfig, SourceKind, +}; #[cfg(feature = "wifi")] pub use config::{CongestionChoice, WifiConfig}; pub use error::ServiceError; diff --git a/crates/ezd-host-service/src/platform.rs b/crates/ezd-host-service/src/platform.rs index 16b7e60..9bd3694 100644 --- a/crates/ezd-host-service/src/platform.rs +++ b/crates/ezd-host-service/src/platform.rs @@ -1,7 +1,7 @@ //! Thin dispatch to the platform crate of the current target. use crate::error::ServiceError; -use ezd_host_core::{InputBackend, PlatformOptions, VideoSource}; +use ezd_host_core::{AudioSource, InputBackend, PlatformOptions, VideoSource}; use std::sync::Arc; #[cfg(target_os = "linux")] @@ -34,6 +34,19 @@ pub fn input_backend() -> Option> { } } +/// A PC audio capture source of this platform, if it has one. Cheap: capture starts in +/// [`AudioSource::start`]. +pub fn audio_source() -> Option> { + #[cfg(any(target_os = "linux", windows))] + { + imp::audio_source() + } + #[cfg(not(any(target_os = "linux", windows)))] + { + None + } +} + /// Platform part of the diagnostics report. pub fn diag_section() -> serde_json::Value { #[cfg(any(target_os = "linux", windows))] diff --git a/crates/ezd-host-service/src/service.rs b/crates/ezd-host-service/src/service.rs index d80ab82..5c258e1 100644 --- a/crates/ezd-host-service/src/service.rs +++ b/crates/ezd-host-service/src/service.rs @@ -6,7 +6,7 @@ use crate::error::ServiceError; use crate::status::{DeviceInfo, HostEvent, HostStatus, LiveSession, Transport, WifiInfo}; #[cfg(feature = "wifi")] use crate::store::{ForgetError, PairedDevice, StoreError}; -use ezd_host_core::{SessionEvent, VideoSource, run_host_session_with}; +use ezd_host_core::{AudioBackend, SessionEvent, VideoSource, run_host_session_with}; use ezd_link::Link; use ezd_proto::Prefs; use std::net::SocketAddr; @@ -40,6 +40,9 @@ pub struct ServiceHooks { pub create_source: SourceFactory, /// How long a pairing window stays open. pub pairing_ttl: Duration, + /// Replaces the audio backend of the sessions (`None`: the host's own). Not used when the + /// configured audio mode is off. + pub audio_backend: Option>, /// The USB accessory acceptor: backend and timings. #[cfg(feature = "aoa")] pub aoa: crate::aoa::AoaHooks, @@ -52,6 +55,7 @@ impl Default for ServiceHooks { locate_adb: Arc::new(locate_system_adb), create_source: Arc::new(create_source), pairing_ttl: DEFAULT_PAIRING_TTL, + audio_backend: None, #[cfg(feature = "aoa")] aoa: crate::aoa::AoaHooks::default(), } @@ -163,12 +167,14 @@ impl Inner { client_hello, stream, dynamic_range, + audio, .. } => { st.phase = Phase::Streaming(LiveSession { device_name: client_hello.device_name, stream, dynamic_range, + audio, devices: Vec::new(), stats: None, input: None, @@ -536,12 +542,11 @@ async fn run_session( Transport::Tcp | Transport::Aoa => cfg.prefs(), Transport::Wifi => cfg.wifi_prefs(), }; - let session = run_host_session_with( - link, - source, - prefs, - cfg.session_options(end_rx, Some(ev_tx)), - ); + let mut opts = cfg.session_options(end_rx, Some(ev_tx)); + if opts.audio.is_some() && inner.hooks.audio_backend.is_some() { + opts.audio = inner.hooks.audio_backend.clone(); + } + let session = run_host_session_with(link, source, prefs, opts); tokio::pin!(session); let mut stop_forwarded = false; let result = loop { @@ -668,6 +673,7 @@ pub(crate) mod tests { Ok(Box::new(ScriptedSource::new(Duration::from_millis(5), 200))) }), pairing_ttl: Duration::from_secs(60), + audio_backend: None, #[cfg(feature = "aoa")] aoa: crate::aoa::AoaHooks::default(), } @@ -716,27 +722,40 @@ pub(crate) mod tests { /// Completes the handshake on `link` like the tablet. pub(crate) async fn handshake(link: Link) -> (MuxSender, MuxReceiver) { + let (tx, rx, _) = handshake_audio(link, ClientAudio::none()).await; + (tx, rx) + } + + /// Like [`handshake`] with `audio` as the client's audio capabilities; also returns the + /// host's audio decision. + pub(crate) async fn handshake_audio( + link: Link, + audio: ClientAudio, + ) -> (MuxSender, MuxReceiver, Option) { let (tx, mut rx, _t) = link.into_mux(DEFAULT_CHUNK_LEN); tx.send_control(Message::Hello(Hello::new(Role::Client, "t", "tablet"))) .unwrap(); tx.send_control(Message::ClientCaps(caps())).unwrap(); tx.send_control(Message::ClientHdr(ClientHdr::none())) .unwrap(); - tx.send_control(Message::ClientAudio(ClientAudio::none())) - .unwrap(); + tx.send_control(Message::ClientAudio(audio)).unwrap(); let mut seen = 0; + let mut audio_stream = None; while seen < 5 { match timeout(WAIT, rx.recv()).await.unwrap().unwrap().unwrap() { Message::Hello(_) | Message::HostCaps(_) | Message::StreamColor(_) - | Message::AudioConfig(_) | Message::StreamConfig(_) => seen += 1, + Message::AudioConfig(a) => { + audio_stream = a.stream; + seen += 1; + } m => panic!("{m:?}"), } } tx.send_control(Message::ConfigAck).unwrap(); - (tx, rx) + (tx, rx, audio_stream) } pub(crate) async fn wait_status( @@ -802,6 +821,65 @@ pub(crate) mod tests { svc.stop().await; } + fn pcm_client() -> ClientAudio { + ClientAudio { + enabled: true, + sample_rates: vec![48_000], + max_channels: 2, + formats: vec![ezd_proto::SampleFormat::S16], + codecs: vec![ezd_proto::AudioCodec::Pcm, ezd_proto::AudioCodec::Opus], + } + } + + /// One session of a client with audio against a service with the scripted audio backend. + async fn audio_session( + audio: crate::config::AudioChoice, + ) -> ( + Option, + Option, + ) { + let svc = start( + ServiceConfig { audio, ..cfg() }, + ServiceHooks { + audio_backend: Some(Arc::new(ezd_host_core::testing::ScriptedAudioBackend::new())), + ..hooks() + }, + ); + let mut status = svc.status(); + let sock = tokio::net::TcpStream::connect(svc.local_addr().unwrap()) + .await + .unwrap(); + let (_tx, _rx, announced) = + handshake_audio(Link::from_tcp(sock).unwrap(), pcm_client()).await; + let HostStatus::Streaming(live) = + wait_status(&mut status, |s| matches!(s, HostStatus::Streaming(_))).await + else { + unreachable!() + }; + svc.stop().await; + (announced, live.audio) + } + + #[tokio::test] + async fn audio_is_negotiated_and_shown_in_the_status() { + let (announced, live) = audio_session(crate::config::AudioChoice::Auto).await; + let stream = announced.expect("audio negotiated"); + assert_eq!(stream.codec, ezd_proto::AudioCodec::Pcm); + assert_eq!((stream.sample_rate, stream.channels), (48_000, 2)); + assert_eq!(live, Some(stream)); + + let (announced, live) = audio_session(crate::config::AudioChoice::Opus).await; + let stream = announced.expect("audio negotiated"); + assert_eq!(stream.codec, ezd_proto::AudioCodec::Opus); + assert_eq!(live, Some(stream)); + } + + #[tokio::test] + async fn audio_off_streams_no_audio() { + let (announced, live) = audio_session(crate::config::AudioChoice::Off).await; + assert_eq!((announced, live), (None, None)); + } + #[tokio::test] async fn stop_says_bye_and_reports_stopped() { let svc = start(cfg(), hooks()); diff --git a/crates/ezd-host-service/src/status.rs b/crates/ezd-host-service/src/status.rs index 6e27349..68980cf 100644 --- a/crates/ezd-host-service/src/status.rs +++ b/crates/ezd-host-service/src/status.rs @@ -2,7 +2,7 @@ use crate::error::ServiceError; use ezd_host_core::{LiveStats, SessionSummary}; -use ezd_proto::{DynamicRange, InputStatus, StreamConfig}; +use ezd_proto::{AudioStream, DynamicRange, InputStatus, StreamConfig}; use std::net::SocketAddr; use std::time::Instant; @@ -34,6 +34,8 @@ pub struct LiveSession { pub stream: StreamConfig, /// Dynamic range of the stream (HDR10, converted SDR with the reason, or plain SDR). pub dynamic_range: DynamicRange, + /// Negotiated PC audio stream, `None` when the session has no audio. + pub audio: Option, /// Devices seen by the adb watcher. pub devices: Vec, /// Latest live measurements (once per second), `None` until the first window ends. diff --git a/crates/ezd-host/i18n/de/ezd-host.ftl b/crates/ezd-host/i18n/de/ezd-host.ftl index 4cd61bb..16fad28 100644 --- a/crates/ezd-host/i18n/de/ezd-host.ftl +++ b/crates/ezd-host/i18n/de/ezd-host.ftl @@ -23,6 +23,7 @@ run-video-window = Videobilder, die gesendet werden dürfen, ohne dass das Table run-telemetry = Fordert das Tablet auf, sein Telemetrie-Overlay anzuzeigen. run-no-input = Gibt Eingaben vom Tablet nicht weiter. run-pen-mode = Wie der Stift eingespeist wird: "tablet" ist ein Grafiktablett mit Druck und Neigung, "mouse" ist ein absoluter Mauszeiger. +run-audio = PC-Audio, das zum Tablet gestreamt wird: "auto" nimmt unkomprimiertes PCM, wenn die Verbindung es erlaubt, sonst Opus, "pcm" bevorzugt PCM, "opus" bevorzugt Opus, "off" streamt kein Audio. vdd-status = Gibt den Zustand der VDD-Installation als JSON aus (ändert nichts). vdd-setup = Einmalige Einrichtung, als Administrator auszuführen: erlaubt normalen Benutzern, die VDD-Einstellungen zu bearbeiten, weist optional die GPU zu und fügt Modi vorab hinzu, und lädt dann den Treiber neu. diff --git a/crates/ezd-host/i18n/en/ezd-host.ftl b/crates/ezd-host/i18n/en/ezd-host.ftl index 24385f4..a1d12f5 100644 --- a/crates/ezd-host/i18n/en/ezd-host.ftl +++ b/crates/ezd-host/i18n/en/ezd-host.ftl @@ -23,6 +23,7 @@ run-video-window = Video frames that may be sent but not yet acknowledged by the run-telemetry = Ask the tablet to show its telemetry overlay. run-no-input = Do not inject input from the tablet. run-pen-mode = How the pen is injected: "tablet" is a graphics tablet with pressure and tilt, "mouse" is an absolute mouse pointer. +run-audio = PC audio streamed to the tablet: "auto" uses uncompressed PCM when the link allows it and Opus otherwise, "pcm" prefers PCM, "opus" prefers Opus, "off" streams no audio. vdd-status = Print the state of the VDD installation as JSON (changes nothing). vdd-setup = One-time setup, run as administrator: lets normal users edit the VDD settings, optionally assigns the GPU and pre-adds modes, then reloads the driver. diff --git a/crates/ezd-host/i18n/es/ezd-host.ftl b/crates/ezd-host/i18n/es/ezd-host.ftl index 0423112..7a55408 100644 --- a/crates/ezd-host/i18n/es/ezd-host.ftl +++ b/crates/ezd-host/i18n/es/ezd-host.ftl @@ -23,6 +23,7 @@ run-video-window = Fotogramas de video que pueden enviarse sin que la tableta lo run-telemetry = Pide a la tableta que muestre su superposición de telemetría. run-no-input = No inyecta la entrada de la tableta. run-pen-mode = Cómo se inyecta el lápiz: "tablet" es una tableta gráfica con presión e inclinación, "mouse" es un puntero de mouse absoluto. +run-audio = Audio de la PC que se transmite a la tablet: "auto" usa PCM sin comprimir si el enlace lo permite y Opus si no, "pcm" prefiere PCM, "opus" prefiere Opus, "off" no transmite audio. vdd-status = Muestra el estado de la instalación del VDD como JSON (no cambia nada). vdd-setup = Configuración única, ejecutada como administrador: permite a los usuarios normales editar los ajustes del VDD, asigna opcionalmente la GPU y agrega modos de antemano, y recarga el controlador. diff --git a/crates/ezd-host/i18n/fr/ezd-host.ftl b/crates/ezd-host/i18n/fr/ezd-host.ftl index 4c40483..32d741c 100644 --- a/crates/ezd-host/i18n/fr/ezd-host.ftl +++ b/crates/ezd-host/i18n/fr/ezd-host.ftl @@ -23,6 +23,7 @@ run-video-window = Images vidéo pouvant être envoyées sans avoir été acquit run-telemetry = Demande à la tablette d'afficher sa superposition de télémétrie. run-no-input = N'injecte pas les entrées de la tablette. run-pen-mode = Mode d'injection du stylet : "tablet" est une tablette graphique avec pression et inclinaison, "mouse" est un pointeur de souris absolu. +run-audio = Audio du PC diffusé vers la tablette : « auto » utilise le PCM non compressé si la liaison le permet et Opus sinon, « pcm » privilégie le PCM, « opus » privilégie Opus, « off » ne diffuse pas d'audio. vdd-status = Affiche l'état de l'installation du VDD en JSON (ne modifie rien). vdd-setup = Configuration unique, à lancer en administrateur : permet aux utilisateurs normaux de modifier les réglages du VDD, attribue éventuellement le GPU et ajoute des modes à l'avance, puis recharge le pilote. diff --git a/crates/ezd-host/i18n/it/ezd-host.ftl b/crates/ezd-host/i18n/it/ezd-host.ftl index 56ba808..a38845a 100644 --- a/crates/ezd-host/i18n/it/ezd-host.ftl +++ b/crates/ezd-host/i18n/it/ezd-host.ftl @@ -23,6 +23,7 @@ run-video-window = Fotogrammi video che possono essere inviati senza conferma da run-telemetry = Chiede al tablet di mostrare la sovrapposizione di telemetria. run-no-input = Non inietta l'input dal tablet. run-pen-mode = Come viene iniettata la penna: "tablet" è una tavoletta grafica con pressione e inclinazione, "mouse" è un puntatore del mouse assoluto. +run-audio = Audio del PC trasmesso al tablet: "auto" usa il PCM non compresso se il collegamento lo permette e Opus altrimenti, "pcm" preferisce il PCM, "opus" preferisce Opus, "off" non trasmette audio. vdd-status = Mostra lo stato dell'installazione del VDD in JSON (non modifica nulla). vdd-setup = Configurazione una tantum, da eseguire come amministratore: consente agli utenti normali di modificare le impostazioni del VDD, assegna facoltativamente la GPU e aggiunge modalità in anticipo, poi ricarica il driver. diff --git a/crates/ezd-host/i18n/pt/ezd-host.ftl b/crates/ezd-host/i18n/pt/ezd-host.ftl index 192480b..8c6e4b4 100644 --- a/crates/ezd-host/i18n/pt/ezd-host.ftl +++ b/crates/ezd-host/i18n/pt/ezd-host.ftl @@ -23,6 +23,7 @@ run-video-window = Quadros de vídeo que podem ser enviados sem confirmação do run-telemetry = Pede ao tablet que mostre a sobreposição de telemetria. run-no-input = Não injeta a entrada do tablet. run-pen-mode = Como a caneta é injetada: "tablet" é uma mesa digitalizadora com pressão e inclinação, "mouse" é um ponteiro de mouse absoluto. +run-audio = Áudio do PC transmitido para o tablet: "auto" usa PCM sem compressão quando a conexão permite e Opus caso contrário, "pcm" prefere PCM, "opus" prefere Opus, "off" não transmite áudio. vdd-status = Exibe o estado da instalação do VDD em JSON (não altera nada). vdd-setup = Configuração única, executada como administrador: permite que usuários comuns editem as configurações do VDD, atribui opcionalmente a GPU e adiciona modos antecipadamente, e recarrega o driver. diff --git a/crates/ezd-host/src/diag.rs b/crates/ezd-host/src/diag.rs index 5e26ea8..607a5aa 100644 --- a/crates/ezd-host/src/diag.rs +++ b/crates/ezd-host/src/diag.rs @@ -29,6 +29,7 @@ pub fn sections() -> Vec> { } v.push(Box::new(Platform)); v.push(Box::new(Encoders)); + v.push(Box::new(AudioInfo)); v.push(Box::new(AdbInfo)); v.push(Box::new(UsbInfo)); v @@ -47,6 +48,16 @@ pub fn report() -> Value { Value::Object(map) } +struct AudioInfo; +impl Section for AudioInfo { + fn name(&self) -> &'static str { + "audio" + } + fn collect(&self) -> Value { + ezd_host_service::audio::diag_section() + } +} + struct HostInfo; impl Section for HostInfo { fn name(&self) -> &'static str { diff --git a/crates/ezd-host/src/run.rs b/crates/ezd-host/src/run.rs index 59c23b9..c9ef1ac 100644 --- a/crates/ezd-host/src/run.rs +++ b/crates/ezd-host/src/run.rs @@ -5,8 +5,8 @@ use anyhow::{Result, anyhow}; use clap::{Args, Command, ValueEnum}; use ezd_host_core::SessionSummary; use ezd_host_service::{ - AdbMode, AoaMode, HostEvent, HostService, HostStatus, LinkChoice, PenChoice, ServiceConfig, - ServiceError, SourceKind, + AdbMode, AoaMode, AudioChoice, HostEvent, HostService, HostStatus, LinkChoice, PenChoice, + ServiceConfig, ServiceError, SourceKind, }; use i18n_embed::fluent::FluentLanguageLoader; use std::net::ToSocketAddrs; @@ -29,6 +29,14 @@ pub enum AoaArg { Off, } +#[derive(Debug, Clone, Copy, ValueEnum)] +pub enum AudioArg { + Auto, + Pcm, + Opus, + Off, +} + #[derive(Debug, Clone, Copy, ValueEnum)] pub enum PenModeArg { Tablet, @@ -70,6 +78,8 @@ pub struct RunArgs { no_input: bool, #[arg(long, value_enum, default_value = "tablet")] pen_mode: PenModeArg, + #[arg(long, value_enum, default_value = "auto")] + audio: AudioArg, #[cfg(feature = "wifi")] #[arg(long, num_args = 0..=1, value_name = "ADDR", default_missing_value = "0.0.0.0:27183")] wifi: Option, @@ -103,7 +113,8 @@ pub fn localize(cmd: Command, l: &FluentLanguageLoader) -> Command { .mut_arg("video_window", |a| a.help(fl!(l, "run-video-window"))) .mut_arg("telemetry", |a| a.help(fl!(l, "run-telemetry"))) .mut_arg("no_input", |a| a.help(fl!(l, "run-no-input"))) - .mut_arg("pen_mode", |a| a.help(fl!(l, "run-pen-mode"))); + .mut_arg("pen_mode", |a| a.help(fl!(l, "run-pen-mode"))) + .mut_arg("audio", |a| a.help(fl!(l, "run-audio"))); #[cfg(feature = "wifi")] let cmd = cmd .mut_arg("wifi", |a| a.help(fl!(l, "run-wifi"))) @@ -174,6 +185,12 @@ fn config(args: &RunArgs) -> Result { PenModeArg::Tablet => PenChoice::Tablet, PenModeArg::Mouse => PenChoice::Mouse, }, + audio: match args.audio { + AudioArg::Auto => AudioChoice::Auto, + AudioArg::Pcm => AudioChoice::Pcm, + AudioArg::Opus => AudioChoice::Opus, + AudioArg::Off => AudioChoice::Off, + }, // The command line never starts the app: that is the tray's convenience. open_app_on_connect: false, #[cfg(feature = "wifi")] @@ -329,6 +346,7 @@ fn log_session_end(r: Result) { Ok(s) => tracing::info!( end = ?s.end, frames = s.frames_sent, keyframes = s.keyframes_sent, input_events = s.input_events, + audio = ?s.audio, audio_packets = s.audio_packets, audio_dropped = s.audio_dropped, seconds = s.duration.as_secs_f32(), "session finished" ), Err(e) => tracing::warn!("session failed: {e}"), @@ -337,7 +355,38 @@ fn log_session_end(r: Result) { #[cfg(test)] mod tests { - use super::parse_scale; + use super::*; + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + run: RunArgs, + } + + fn audio_of(extra: &[&str]) -> Result { + let mut argv = vec!["run", "--source", "synthetic"]; + argv.extend(extra); + Cli::try_parse_from(argv).and_then(|c| { + config(&c.run) + .map(|c| c.audio) + .map_err(|e| clap::Error::raw(clap::error::ErrorKind::Io, e.to_string())) + }) + } + + #[test] + fn audio_flag_overrides_the_default() { + assert_eq!(audio_of(&[]).unwrap(), AudioChoice::Auto); + for (arg, want) in [ + ("auto", AudioChoice::Auto), + ("pcm", AudioChoice::Pcm), + ("opus", AudioChoice::Opus), + ("off", AudioChoice::Off), + ] { + assert_eq!(audio_of(&["--audio", arg]).unwrap(), want, "{arg}"); + } + assert!(audio_of(&["--audio", "loud"]).is_err()); + } #[test] fn scale_parsing() { diff --git a/crates/ezd-tray/i18n/de/ezd-tray.ftl b/crates/ezd-tray/i18n/de/ezd-tray.ftl index 0a3fb3d..497e34d 100644 --- a/crates/ezd-tray/i18n/de/ezd-tray.ftl +++ b/crates/ezd-tray/i18n/de/ezd-tray.ftl @@ -96,6 +96,20 @@ hdr-reason-old-client = Die Tablet-App ist zu alt für HDR; aktualisiere sie hdr-reason-display-not-hdr = Der Bildschirm des Tablets unterstützt kein HDR10 hdr-reason-no-main10-decoder = Das Tablet kann kein 10-Bit-HDR-Video dekodieren hdr-reason-no-main10-encoder = Diese Grafikkarte kann kein 10-Bit-HEVC kodieren; Umwandlung in SDR +setting-audio = Audio +audio-description = Streamt den Ton dieses Computers zum Tablet. Automatisch sendet unkomprimiertes Audio, wenn die Verbindung es erlaubt. +audio-auto = Automatisch +audio-pcm = PCM (unkomprimiert) +audio-opus = Opus (komprimiert) +audio-off = Aus +stat-audio = Audio +audio-value-none = Keins +audio-value-pcm = PCM { $format }, { $khz } kHz, { $layout } +audio-value-opus = Opus, { $khz } kHz, { $layout } +audio-mono = Mono +audio-stereo = Stereo +audio-format-s16 = 16 Bit +audio-format-f32 = 32 Bit Float ## Fenster: Eingabe input-enabled = Touch- und Stifteingaben an diesen Computer senden diff --git a/crates/ezd-tray/i18n/en/ezd-tray.ftl b/crates/ezd-tray/i18n/en/ezd-tray.ftl index 49dbb13..e1aaea8 100644 --- a/crates/ezd-tray/i18n/en/ezd-tray.ftl +++ b/crates/ezd-tray/i18n/en/ezd-tray.ftl @@ -96,6 +96,20 @@ hdr-reason-old-client = The tablet app is too old for HDR; update it hdr-reason-display-not-hdr = The tablet's screen doesn't support HDR10 hdr-reason-no-main10-decoder = The tablet can't decode 10-bit HDR video hdr-reason-no-main10-encoder = This graphics card can't encode 10-bit HEVC; converting to SDR +setting-audio = Audio +audio-description = Streams this computer's sound to the tablet. Automatic sends uncompressed audio when the link allows it. +audio-auto = Automatic +audio-pcm = PCM (uncompressed) +audio-opus = Opus (compressed) +audio-off = Off +stat-audio = Audio +audio-value-none = None +audio-value-pcm = PCM { $format }, { $khz } kHz, { $layout } +audio-value-opus = Opus, { $khz } kHz, { $layout } +audio-mono = mono +audio-stereo = stereo +audio-format-s16 = 16-bit +audio-format-f32 = 32-bit float ## Window: input input-enabled = Send touch and pen input to this computer diff --git a/crates/ezd-tray/i18n/es/ezd-tray.ftl b/crates/ezd-tray/i18n/es/ezd-tray.ftl index 84b20cc..fb58fbb 100644 --- a/crates/ezd-tray/i18n/es/ezd-tray.ftl +++ b/crates/ezd-tray/i18n/es/ezd-tray.ftl @@ -96,6 +96,20 @@ hdr-reason-old-client = La app de la tablet es muy vieja para HDR; actualizala hdr-reason-display-not-hdr = La pantalla de la tablet no admite HDR10 hdr-reason-no-main10-decoder = La tablet no puede decodificar video HDR de 10 bits hdr-reason-no-main10-encoder = Esta placa de video no puede codificar HEVC de 10 bits; se convierte a SDR +setting-audio = Audio +audio-description = Transmite el sonido de esta computadora a la tablet. Automático envía audio sin comprimir cuando el enlace lo permite. +audio-auto = Automático +audio-pcm = PCM (sin comprimir) +audio-opus = Opus (comprimido) +audio-off = Apagado +stat-audio = Audio +audio-value-none = Ninguno +audio-value-pcm = PCM { $format }, { $khz } kHz, { $layout } +audio-value-opus = Opus, { $khz } kHz, { $layout } +audio-mono = mono +audio-stereo = estéreo +audio-format-s16 = 16 bits +audio-format-f32 = 32 bits flotante ## Ventana: entrada input-enabled = Enviar la entrada táctil y del lápiz a esta computadora diff --git a/crates/ezd-tray/i18n/fr/ezd-tray.ftl b/crates/ezd-tray/i18n/fr/ezd-tray.ftl index 44a5485..07383bb 100644 --- a/crates/ezd-tray/i18n/fr/ezd-tray.ftl +++ b/crates/ezd-tray/i18n/fr/ezd-tray.ftl @@ -96,6 +96,20 @@ hdr-reason-old-client = L'application de la tablette est trop ancienne pour le H hdr-reason-display-not-hdr = L'écran de la tablette ne prend pas en charge le HDR10 hdr-reason-no-main10-decoder = La tablette ne peut pas décoder la vidéo HDR 10 bits hdr-reason-no-main10-encoder = Cette carte graphique ne peut pas encoder du HEVC 10 bits ; conversion en SDR +setting-audio = Audio +audio-description = Diffuse le son de cet ordinateur vers la tablette. Automatique envoie l'audio non compressé quand la liaison le permet. +audio-auto = Automatique +audio-pcm = PCM (non compressé) +audio-opus = Opus (compressé) +audio-off = Désactivé +stat-audio = Audio +audio-value-none = Aucun +audio-value-pcm = PCM { $format }, { $khz } kHz, { $layout } +audio-value-opus = Opus, { $khz } kHz, { $layout } +audio-mono = mono +audio-stereo = stéréo +audio-format-s16 = 16 bits +audio-format-f32 = 32 bits flottant ## Fenêtre : saisie input-enabled = Envoyer la saisie tactile et du stylet à cet ordinateur diff --git a/crates/ezd-tray/i18n/it/ezd-tray.ftl b/crates/ezd-tray/i18n/it/ezd-tray.ftl index fccf95a..085f3b4 100644 --- a/crates/ezd-tray/i18n/it/ezd-tray.ftl +++ b/crates/ezd-tray/i18n/it/ezd-tray.ftl @@ -96,6 +96,20 @@ hdr-reason-old-client = L'app del tablet è troppo vecchia per l'HDR; aggiornala hdr-reason-display-not-hdr = Lo schermo del tablet non supporta l'HDR10 hdr-reason-no-main10-decoder = Il tablet non può decodificare video HDR a 10 bit hdr-reason-no-main10-encoder = Questa scheda grafica non può codificare HEVC a 10 bit; conversione in SDR +setting-audio = Audio +audio-description = Trasmette l'audio di questo computer al tablet. Automatico invia audio non compresso quando il collegamento lo permette. +audio-auto = Automatico +audio-pcm = PCM (non compresso) +audio-opus = Opus (compresso) +audio-off = Disattivato +stat-audio = Audio +audio-value-none = Nessuno +audio-value-pcm = PCM { $format }, { $khz } kHz, { $layout } +audio-value-opus = Opus, { $khz } kHz, { $layout } +audio-mono = mono +audio-stereo = stereo +audio-format-s16 = 16 bit +audio-format-f32 = 32 bit float ## Finestra: input input-enabled = Invia l'input touch e della penna a questo computer diff --git a/crates/ezd-tray/i18n/pt/ezd-tray.ftl b/crates/ezd-tray/i18n/pt/ezd-tray.ftl index aca38a7..7de5936 100644 --- a/crates/ezd-tray/i18n/pt/ezd-tray.ftl +++ b/crates/ezd-tray/i18n/pt/ezd-tray.ftl @@ -96,6 +96,20 @@ hdr-reason-old-client = O app do tablet é antigo demais para HDR; atualize-o hdr-reason-display-not-hdr = A tela do tablet não suporta HDR10 hdr-reason-no-main10-decoder = O tablet não consegue decodificar vídeo HDR de 10 bits hdr-reason-no-main10-encoder = Esta placa de vídeo não consegue codificar HEVC de 10 bits; convertendo para SDR +setting-audio = Áudio +audio-description = Transmite o som deste computador para o tablet. Automático envia áudio sem compressão quando a conexão permite. +audio-auto = Automático +audio-pcm = PCM (sem compressão) +audio-opus = Opus (comprimido) +audio-off = Desligado +stat-audio = Áudio +audio-value-none = Nenhum +audio-value-pcm = PCM { $format }, { $khz } kHz, { $layout } +audio-value-opus = Opus, { $khz } kHz, { $layout } +audio-mono = mono +audio-stereo = estéreo +audio-format-s16 = 16 bits +audio-format-f32 = 32 bits float ## Janela: entrada input-enabled = Enviar a entrada por toque e caneta para este computador diff --git a/crates/ezd-tray/src/menu_model.rs b/crates/ezd-tray/src/menu_model.rs index 393170f..ece8f54 100644 --- a/crates/ezd-tray/src/menu_model.rs +++ b/crates/ezd-tray/src/menu_model.rs @@ -126,6 +126,7 @@ mod tests { encoder_name: "h264_nvenc".into(), }, dynamic_range: ezd_proto::DynamicRange::Sdr, + audio: None, devices: vec![], stats: None, input: None, diff --git a/crates/ezd-tray/src/ui/view.rs b/crates/ezd-tray/src/ui/view.rs index fe457ac..b971633 100644 --- a/crates/ezd-tray/src/ui/view.rs +++ b/crates/ezd-tray/src/ui/view.rs @@ -12,10 +12,10 @@ use crate::state::Lifecycle; use crate::text; use eframe::egui::{self, ComboBox, Margin, RichText, Ui}; use ezd_host_service::{ - AdbMode, AoaMode, DeviceInfo, DeviceState, HdrChoice, HostStatus, LiveSession, PenChoice, - Transport, WifiInfo, + AdbMode, AoaMode, AudioChoice, DeviceInfo, DeviceState, HdrChoice, HostStatus, LiveSession, + PenChoice, Transport, WifiInfo, }; -use ezd_proto::{DynamicRange, SdrReason}; +use ezd_proto::{AudioCodec, AudioStream, DynamicRange, SampleFormat, SdrReason}; use i18n_embed::fluent::FluentLanguageLoader; use i18n_embed_fl::fl; use std::path::PathBuf; @@ -532,6 +532,7 @@ impl SettingsView { if let DynamicRange::SdrToneMapped(reason) = live.dynamic_range { session.push((fl!(l, "stat-hdr"), sdr_reason_text(l, reason))); } + session.push((fl!(l, "stat-audio"), audio_text(l, live.audio.as_ref()))); session.push(( fl!(l, "stat-encoder"), fl!( @@ -744,6 +745,13 @@ impl SettingsView { HdrChoice::Auto => fl!(l, "hdr-auto"), HdrChoice::Off => fl!(l, "hdr-off"), }; + let audio_text = |c: AudioChoice| match c { + AudioChoice::Auto => fl!(l, "audio-auto"), + AudioChoice::Pcm => fl!(l, "audio-pcm"), + AudioChoice::Opus => fl!(l, "audio-opus"), + AudioChoice::Off => fl!(l, "audio-off"), + }; + let audio_now = cfg.audio; let hdr_now = cfg.hdr; let hdr_description = if cfg!(target_os = "linux") { fl!(l, "hdr-hint-linux") @@ -796,6 +804,30 @@ impl SettingsView { }, ); widgets::row_divider(ui, pal); + widgets::labelled_row( + ui, + pal, + &fl!(l, "setting-audio"), + Some(&fl!(l, "audio-description")), + None, + |ui| { + ComboBox::from_id_salt("audio") + .width(COMBO_WIDTH) + .selected_text(audio_text(audio_now)) + .show_ui(ui, |ui| { + for c in [ + AudioChoice::Auto, + AudioChoice::Pcm, + AudioChoice::Opus, + AudioChoice::Off, + ] { + ui.selectable_value(&mut cfg.audio, c, audio_text(c)); + } + }) + .response + }, + ); + widgets::row_divider(ui, pal); widgets::labelled_row( ui, pal, @@ -1590,6 +1622,35 @@ fn tile_grid(ui: &mut Ui, pal: Palette, tiles: &[Tile]) { } } +/// The negotiated PC audio of the session, for the status rows. +fn audio_text(l: &FluentLanguageLoader, audio: Option<&AudioStream>) -> String { + let Some(a) = audio else { + return fl!(l, "audio-value-none"); + }; + let khz = text::decimal(l, format!("{}", f64::from(a.sample_rate) / 1000.0)); + let layout = if a.channels == 1 { + fl!(l, "audio-mono") + } else { + fl!(l, "audio-stereo") + }; + match a.codec { + AudioCodec::Pcm => { + let format = match a.format { + SampleFormat::S16 => fl!(l, "audio-format-s16"), + SampleFormat::F32 => fl!(l, "audio-format-f32"), + }; + fl!( + l, + "audio-value-pcm", + format = format, + khz = khz, + layout = layout + ) + } + AudioCodec::Opus => fl!(l, "audio-value-opus", khz = khz, layout = layout), + } +} + /// Why an HDR source is streamed as converted SDR. fn sdr_reason_text(l: &FluentLanguageLoader, reason: SdrReason) -> String { match reason { diff --git a/crates/ezd-tray/src/ui/view/tests.rs b/crates/ezd-tray/src/ui/view/tests.rs index e575e38..b9bd17e 100644 --- a/crates/ezd-tray/src/ui/view/tests.rs +++ b/crates/ezd-tray/src/ui/view/tests.rs @@ -7,7 +7,7 @@ use eframe::egui::accesskit::Role; use egui_kittest::Harness; use egui_kittest::kittest::{NodeT, Queryable}; use ezd_host_core::LiveStats; -use ezd_host_service::{HdrChoice, LiveSession, Transport}; +use ezd_host_service::{AudioChoice, HdrChoice, LiveSession, Transport}; use ezd_proto::{ClientStats, Codec, DynamicRange, SdrReason, StreamConfig}; use std::time::Duration; @@ -23,6 +23,7 @@ fn live(stats: Option) -> HostStatus { encoder_name: "h264_nvenc".into(), }, dynamic_range: DynamicRange::Sdr, + audio: None, devices: vec![], stats, input: None, @@ -241,6 +242,9 @@ fn changing_hdr_changes_the_config() { h.get_by_role_and_label(Role::Button, "Display").click(); h.run(); assert_eq!(h.state().snap.config.service.hdr, HdrChoice::Auto); + h.get_by_role_and_label(Role::ComboBox, "HDR") + .scroll_to_me(); + h.run(); h.get_by_role_and_label(Role::ComboBox, "HDR").click(); h.run(); h.get_by_label("Off").click(); @@ -253,6 +257,51 @@ fn changing_hdr_changes_the_config() { assert_eq!(c.service.hdr, HdrChoice::Off); } +#[test] +fn changing_audio_changes_the_config() { + let mut h = harness(snapshot(HostStatus::Waiting { devices: vec![] }), "en"); + h.get_by_role_and_label(Role::Button, "Display").click(); + h.run(); + assert_eq!(h.state().snap.config.service.audio, AudioChoice::Auto); + h.get_by_role_and_label(Role::ComboBox, "Audio").click(); + h.run(); + h.get_by_label("Opus (compressed)").click(); + h.run(); + assert_eq!(h.state().snap.config.service.audio, AudioChoice::Opus); + let actions = h.state_mut().take_actions(); + let [Action::SetConfig(c)] = actions.as_slice() else { + panic!("expected one SetConfig, got {actions:?}"); + }; + assert_eq!(c.service.audio, AudioChoice::Opus); +} + +#[test] +fn the_stream_card_shows_the_negotiated_audio() { + let audio = |codec, format| ezd_proto::AudioStream { + codec, + sample_rate: 48_000, + channels: 2, + format, + frame_samples: 480, + codec_config: vec![], + }; + let with = |a| { + let HostStatus::Streaming(mut s) = live(None) else { + unreachable!() + }; + s.audio = a; + HostStatus::Streaming(s) + }; + let h = harness(snapshot(live(None)), "en"); + h.get_by_label("None"); + let pcm = audio(ezd_proto::AudioCodec::Pcm, ezd_proto::SampleFormat::S16); + let h = harness(snapshot(with(Some(pcm))), "en"); + h.get_by_label("PCM 16-bit, 48 kHz, stereo"); + let opus = audio(ezd_proto::AudioCodec::Opus, ezd_proto::SampleFormat::S16); + let h = harness(snapshot(with(Some(opus))), "es"); + h.get_by_label("Opus, 48 kHz, estéreo"); +} + #[cfg(target_os = "linux")] #[test] fn display_page_explains_that_kde_sends_no_hdr() { From 92e4cd7633729ce8f5e4fd4454eedbb3e24ea0bf Mon Sep 17 00:00:00 2001 From: FacuM Date: Sun, 25 Oct 2026 23:01:50 -0300 Subject: [PATCH 07/12] Log the tablet's playback counters every few seconds --- crates/ezd-android/src/audio.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/ezd-android/src/audio.rs b/crates/ezd-android/src/audio.rs index 244e889..0e3e025 100644 --- a/crates/ezd-android/src/audio.rs +++ b/crates/ezd-android/src/audio.rs @@ -57,6 +57,9 @@ struct Core { } /// Plays one negotiated audio stream until dropped. +/// Packets between two debug lines with the playback counters (about 5 s of 10 ms packets). +const STATS_EVERY: u64 = 500; + pub struct AudioPlayer { core: Arc, format: SampleFormat, @@ -120,7 +123,15 @@ impl AudioPlayer { /// Handles one packet from the host without blocking. pub fn on_packet(&mut self, pkt: AudioPacket) { use std::sync::atomic::Ordering::Relaxed; - self.core.stats.packets.fetch_add(1, Relaxed); + let n = self.core.stats.packets.fetch_add(1, Relaxed) + 1; + if n % STATS_EVERY == 0 { + let (_, underruns, dropped) = self.core.stats.snapshot(); + let buffered = lock(&self.core.jitter).len() / self.core.channels; + tracing::debug!( + "audio: {n} packets, {underruns} underruns, {dropped} frames dropped, {} ms buffered", + buffered as u64 * 1000 / u64::from(SAMPLE_RATE) + ); + } match &self.opus { Some(opus) => opus.send(pkt), None => { From 159770f3f1f069530e523097bc82e64d5726b5a5 Mon Sep 17 00:00:00 2001 From: FacuM Date: Tue, 27 Oct 2026 00:17:56 -0300 Subject: [PATCH 08/12] Adapt the tablet's audio buffer to underruns and clock drift The prebuffer starts at 20 ms, grows on every underrun and shrinks after a calm stretch. When the PC clock runs ahead, single frames are dropped until the buffer is back near its target. --- crates/ezd-android/src/audio.rs | 21 ++- crates/ezd-android/src/jitter.rs | 232 +++++++++++++++++++++++++++---- 2 files changed, 221 insertions(+), 32 deletions(-) diff --git a/crates/ezd-android/src/audio.rs b/crates/ezd-android/src/audio.rs index 0e3e025..ba52626 100644 --- a/crates/ezd-android/src/audio.rs +++ b/crates/ezd-android/src/audio.rs @@ -126,9 +126,13 @@ impl AudioPlayer { let n = self.core.stats.packets.fetch_add(1, Relaxed) + 1; if n % STATS_EVERY == 0 { let (_, underruns, dropped) = self.core.stats.snapshot(); - let buffered = lock(&self.core.jitter).len() / self.core.channels; + let (len, target) = { + let j = lock(&self.core.jitter); + (j.len(), j.target_ms()) + }; + let buffered = len / self.core.channels; tracing::debug!( - "audio: {n} packets, {underruns} underruns, {dropped} frames dropped, {} ms buffered", + "audio: {n} packets, {underruns} underruns, {dropped} frames dropped, {} ms buffered, target {target} ms", buffered as u64 * 1000 / u64::from(SAMPLE_RATE) ); } @@ -193,6 +197,7 @@ fn open_output(core: &Arc, reopen_tx: &Sender<()>) -> Result, reopen_tx: &Sender<()>) -> Result, reopen_tx: &Sender<()>) -> Result(), samples) }; - underrun = jitter.pull(out); + (underrun, drift) = jitter.pull_counting(out); } drop(jitter); + if drift > 0 { + data_core + .stats + .dropped_frames + .fetch_add(drift as u64, Ordering::Relaxed); + } if underrun { data_core.stats.underruns.fetch_add(1, Ordering::Relaxed); } diff --git a/crates/ezd-android/src/jitter.rs b/crates/ezd-android/src/jitter.rs index 2c6cfa2..3565c85 100644 --- a/crates/ezd-android/src/jitter.rs +++ b/crates/ezd-android/src/jitter.rs @@ -7,10 +7,22 @@ use std::sync::atomic::{AtomicU64, Ordering}; /// Output sample rate; the host sends 48 kHz. pub const SAMPLE_RATE: u32 = 48_000; -/// Playback starts once this much audio is buffered (after the start or an underrun). -pub const TARGET_MS: u32 = 20; -/// Above this much buffered audio the oldest samples are dropped down to [`TARGET_MS`]. -pub const MAX_MS: u32 = 60; +/// Initial and minimum prebuffer: playback starts once this much audio is buffered. +pub const MIN_TARGET_MS: u32 = 20; +/// The target never grows beyond this. +pub const MAX_TARGET_MS: u32 = 80; +/// The target grows by this much on every underrun. +pub const TARGET_STEP_UP_MS: u32 = 10; +/// The target shrinks by this much after [`CALM_SECS`] without an underrun. +pub const TARGET_STEP_DOWN_MS: u32 = 5; +/// Seconds of playback without an underrun before the target is lowered. +pub const CALM_SECS: u32 = 30; +/// Above `target + HARD_CAP_MS` the oldest samples are dropped down to the target. +pub const HARD_CAP_MS: u32 = 40; +/// Clock drift: above `target + DRIFT_START_MS` single frames are dropped, one per pull. +pub const DRIFT_START_MS: u32 = 20; +/// Clock drift: dropping stops once the buffer is back under `target + DRIFT_STOP_MS`. +pub const DRIFT_STOP_MS: u32 = 10; /// Opus seek pre-roll that MediaCodec expects in `csd-2`. pub const OPUS_SEEK_PREROLL_NS: u64 = 80_000_000; @@ -36,29 +48,48 @@ impl AudioStats { } } -/// Interleaved f32 FIFO with a prebuffer and a latency cap. +/// Interleaved f32 FIFO with an adaptive prebuffer and a latency cap. +/// +/// The target (prebuffer) starts at [`MIN_TARGET_MS`], grows on every underrun and decays after +/// [`CALM_SECS`] of clean playback. Time is counted in samples pulled, not wall clock. pub struct Jitter { + rate: usize, channels: usize, buf: VecDeque, - target: usize, - max: usize, + target_ms: u32, playing: bool, + /// Frames pulled while playing since the last underrun or decay step. + calm_frames: u64, + /// Clock drift: single frames are being dropped, one per pull. + draining: bool, } impl Jitter { /// Buffer for `channels` interleaved channels at `rate` Hz. pub fn new(rate: u32, channels: usize) -> Self { let channels = channels.max(1); - let samples = |ms: u32| (rate as usize * ms as usize / 1000) * channels; + let rate = rate as usize; Jitter { + rate, channels, - buf: VecDeque::with_capacity(samples(MAX_MS) * 2), - target: samples(TARGET_MS), - max: samples(MAX_MS), + buf: VecDeque::with_capacity((rate * 150 / 1000) * channels), + target_ms: MIN_TARGET_MS, playing: false, + calm_frames: 0, + draining: false, } } + /// Samples (all channels) in `ms` milliseconds. + fn samples(&self, ms: u32) -> usize { + (self.rate * ms as usize / 1000) * self.channels + } + + /// Current prebuffer target in milliseconds. + pub fn target_ms(&self) -> u32 { + self.target_ms + } + /// Buffered samples (all channels). pub fn len(&self) -> usize { self.buf.len() @@ -74,45 +105,82 @@ impl Jitter { self.playing } - /// Appends `samples` (a whole number of frames). When more than [`MAX_MS`] is buffered the - /// oldest audio is dropped down to [`TARGET_MS`]. Returns the sample frames dropped. + /// Appends `samples` (a whole number of frames). Above `target + HARD_CAP_MS` the oldest + /// audio is dropped down to the target. Returns the sample frames dropped. pub fn push(&mut self, samples: &[f32]) -> usize { self.buf.extend(samples.iter().copied()); - if self.buf.len() <= self.max { + let target = self.samples(self.target_ms); + if self.buf.len() <= target + self.samples(HARD_CAP_MS) { return 0; } - let drop = (self.buf.len() - self.target) / self.channels * self.channels; + let drop = (self.buf.len() - target) / self.channels * self.channels; self.buf.drain(..drop); + self.draining = false; drop / self.channels } /// Fills `out` with buffered audio. While prebuffering, or for the part that is missing on an /// underrun, it writes silence. Returns true when an underrun happened (the buffer ran dry - /// while playing); playback then waits for the prebuffer again. + /// while playing); playback then waits for the (larger) prebuffer again. + /// + /// When clock drift has built up more than `target + DRIFT_START_MS`, one frame is dropped + /// per call until the buffer is under `target + DRIFT_STOP_MS`. pub fn pull(&mut self, out: &mut [f32]) -> bool { + self.pull_counting(out).0 + } + + /// Like [`Jitter::pull`], also returning the sample frames dropped for clock drift. + pub fn pull_counting(&mut self, out: &mut [f32]) -> (bool, usize) { + let target = self.samples(self.target_ms); if !self.playing { - if self.buf.len() < self.target { + if self.buf.len() < target { out.fill(0.0); - return false; + return (false, 0); } self.playing = true; } + let mut dropped = 0; + if self.draining { + if self.buf.len() < target + self.samples(DRIFT_STOP_MS) { + self.draining = false; + } + } else if self.buf.len() > target + self.samples(DRIFT_START_MS) { + self.draining = true; + } + if self.draining && self.buf.len() > self.channels { + self.buf.drain(..self.channels); + dropped = 1; + } let n = out.len().min(self.buf.len()); for (o, s) in out.iter_mut().zip(self.buf.drain(..n)) { *o = s; } + let frames = (n / self.channels) as u64; if n < out.len() { out[n..].fill(0.0); self.playing = false; - return true; + self.draining = false; + self.calm_frames = 0; + self.target_ms = (self.target_ms + TARGET_STEP_UP_MS).min(MAX_TARGET_MS); + return (true, dropped); } - false + self.calm_frames += frames; + if self.calm_frames >= self.rate as u64 * u64::from(CALM_SECS) { + self.calm_frames = 0; + self.target_ms = self + .target_ms + .saturating_sub(TARGET_STEP_DOWN_MS) + .max(MIN_TARGET_MS); + } + (false, dropped) } - /// Forgets everything and prebuffers again. + /// Forgets everything and prebuffers again (the target is kept). pub fn reset(&mut self) { self.buf.clear(); self.playing = false; + self.draining = false; + self.calm_frames = 0; } } @@ -180,15 +248,34 @@ mod tests { assert_eq!(j.len(), 1920 - 96); } + /// Pulls `n` ms of stereo in 10 ms steps, pushing 10 ms before each pull. + fn run(j: &mut Jitter, n: usize) -> (u64, usize) { + let (mut underruns, mut dropped) = (0, 0); + let mut out = vec![0.0; 48 * 10 * 2]; + for _ in 0..n / 10 { + j.push(&ms(10, 0.1)); + let (u, d) = j.pull_counting(&mut out); + underruns += u64::from(u); + dropped += d; + } + (underruns, dropped) + } + + /// Drains the buffer to cause one underrun. + fn underrun(j: &mut Jitter) { + let mut out = vec![0.0; j.len() + 96]; + assert!(j.pull(&mut out)); + } + #[test] - fn drops_the_oldest_above_the_cap() { + fn hard_cap_drops_the_oldest_down_to_the_target() { let mut j = Jitter::new(48_000, 2); j.push(&ms(30, 0.1)); - j.push(&ms(20, 0.2)); - assert_eq!(j.len(), 48 * 50 * 2); - // 70 ms in total: down to 20 ms, keeping the newest. + j.push(&ms(30, 0.2)); + assert_eq!(j.len(), 48 * 60 * 2); + // Over 20 + 40 ms: down to 20 ms, keeping the newest. let dropped = j.push(&ms(20, 0.3)); - assert_eq!(dropped, 48 * 50); + assert_eq!(dropped, 48 * 60); assert_eq!(j.len(), 48 * 20 * 2); let mut out = vec![0.0; 48 * 20 * 2]; j.pull(&mut out); @@ -211,8 +298,9 @@ mod tests { assert!(out[..48 * 20 * 2].iter().all(|&s| s == 0.5)); assert!(out[48 * 20 * 2..].iter().all(|&s| s == 0.0)); assert!(!j.is_playing()); - // Less than the target arrives: still silent. - j.push(&ms(10, 0.7)); + // The target grew to 30 ms: 20 ms is not enough any more. + assert_eq!(j.target_ms(), 30); + j.push(&ms(20, 0.7)); let mut out = vec![1.0; 96]; assert!(!j.pull(&mut out)); assert!(out.iter().all(|&s| s == 0.0)); @@ -221,6 +309,94 @@ mod tests { assert!(out.iter().all(|&s| s == 0.7)); } + #[test] + fn underruns_raise_the_target_up_to_the_cap() { + let mut j = Jitter::new(48_000, 2); + assert_eq!(j.target_ms(), 20); + let mut seen = Vec::new(); + for _ in 0..9 { + j.push(&ms(MAX_TARGET_MS as usize, 0.1)); + underrun(&mut j); + seen.push(j.target_ms()); + } + assert_eq!(seen, vec![30, 40, 50, 60, 70, 80, 80, 80, 80]); + } + + #[test] + fn the_target_decays_after_clean_playback_down_to_the_floor() { + let mut j = Jitter::new(48_000, 2); + for _ in 0..3 { + j.push(&ms(80, 0.1)); + underrun(&mut j); + } + assert_eq!(j.target_ms(), 50); + // Just under 30 s: unchanged. Steady state buffers exactly the target. + j.push(&ms(50, 0.1)); + run(&mut j, 29_900); + assert_eq!(j.target_ms(), 50); + run(&mut j, 100); + assert_eq!(j.target_ms(), 45); + run(&mut j, 30_000); + assert_eq!(j.target_ms(), 40); + run(&mut j, 30_000 * 10); + assert_eq!(j.target_ms(), 20, "never below the floor"); + } + + #[test] + fn an_underrun_restarts_the_calm_period() { + let mut j = Jitter::new(48_000, 2); + j.push(&ms(30, 0.1)); + underrun(&mut j); + assert_eq!(j.target_ms(), 30); + j.push(&ms(30, 0.1)); + run(&mut j, 20_000); + j.push(&ms(30, 0.1)); + underrun(&mut j); + assert_eq!(j.target_ms(), 40); + j.push(&ms(40, 0.1)); + run(&mut j, 29_000); + assert_eq!(j.target_ms(), 40); + } + + #[test] + fn drift_is_dropped_one_frame_per_pull() { + let mut j = Jitter::new(48_000, 2); + j.push(&ms(40, 0.1)); // exactly target + 20: not yet drifting + let mut out = vec![0.0; 2]; + assert_eq!(j.pull_counting(&mut out), (false, 0)); + j.push(&ms(1, 0.1)); // now above target + 20 ms + let before = j.len(); + let mut dropped = 0; + let mut pulls = 0; + // Pull one frame per call while nothing new arrives: each call removes the frame it + // returns plus at most one dropped frame. + while j.len() > 48 * 30 * 2 - 2 { + let (_, d) = j.pull_counting(&mut out); + assert!(d <= 1); + dropped += d; + pulls += 1; + } + assert!(dropped > 1, "dropped {dropped} frames"); + assert_eq!(dropped, pulls, "one dropped frame per pull"); + assert_eq!(before - j.len(), (dropped + pulls) * 2); + } + + #[test] + fn drift_dropping_stops_under_target_plus_10() { + let mut j = Jitter::new(48_000, 2); + j.push(&ms(55, 0.1)); + let mut out = vec![0.0; 2]; + for _ in 0..10_000 { + j.pull(&mut out); + } + // Stops once under 30 ms and then only the pulled frames leave. + assert!(j.len() < 48 * 30 * 2); + let len = j.len(); + let (_, d) = j.pull_counting(&mut out); + assert_eq!(d, 0); + assert_eq!(j.len(), len.saturating_sub(2)); + } + #[test] fn drops_whole_frames_for_odd_channel_counts() { let mut j = Jitter::new(48_000, 3); From f675a5d840fcdfe12bc1019836bc4bc9c098bec3 Mon Sep 17 00:00:00 2001 From: FacuM Date: Tue, 27 Oct 2026 00:17:56 -0300 Subject: [PATCH 09/12] Restore the NVENC DMA-BUF path on FFmpeg 9 FFmpeg 9 broke three things the path relied on: scale_vulkan can no longer convert RGB to YUV, a CUDA device derived from Vulkan cannot push its context, and filters cannot allocate output from a Vulkan frames context derived from DRM. Frames now stay RGB and NVENC converts them, as on the shared-memory path; Vulkan is derived from CUDA; and captured buffers are imported into a plain Vulkan frames context. HDR needs the YUV conversion, so it takes shared memory. --- crates/ezd-host-linux/src/encode.rs | 95 +++++++++++++++++++---------- 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/crates/ezd-host-linux/src/encode.rs b/crates/ezd-host-linux/src/encode.rs index 24c55a5..3043011 100644 --- a/crates/ezd-host-linux/src/encode.rs +++ b/crates/ezd-host-linux/src/encode.rs @@ -939,11 +939,16 @@ impl Converter { } Kind::Nvenc => { if dma { - // Buffers are mapped into Vulkan by the shared import (see VkImport), then - // scale_vulkan converts them and hwupload copies them to CUDA. + // Buffers are mapped into Vulkan by the shared import (see VkImport), scaled + // there and copied to CUDA by hwupload. They stay RGB and NVENC converts them, + // as on the shared-memory path: since FFmpeg 9, scale_vulkan goes through + // swscale, which cannot convert RGB to YUV on Vulkan. HDR needs that conversion + // (to P010), so it takes shared memory. + if hdr { + return Err("NVENC HDR takes shared-memory frames".into()); + } let import = VkImport::get(&node, in_fmt, frame.width, frame.height)?; - let cu = - derive_device(&import.vk, ff::AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA, None)?; + let cu = import.cu.try_clone()?; let input = GraphInput { pix_fmt: ff::AVPixelFormat::AV_PIX_FMT_VULKAN, width: frame.width, @@ -951,11 +956,7 @@ impl Converter { fps_mhz: self.fps_mhz, hw_frames: Some(import.vk_frames.0), }; - let desc = if hdr { - format!("scale_vulkan=w={w}:h={h}:format=p010:out_range=tv,hwupload") - } else { - format!("scale_vulkan=w={w}:h={h}:format=nv12:out_range=tv,hwupload") - }; + let desc = format!("scale_vulkan=w={w}:h={h},hwupload"); let graph = Graph::build(&input, &desc, &[Some(import.vk.0), Some(cu.0)])?; let (gw, gh) = graph.sink_size(); if (gw, gh) != (w, h) { @@ -971,8 +972,8 @@ impl Converter { pix_fmt: ff::AVPixelFormat::AV_PIX_FMT_CUDA, width: w, height: h, - path: "nvenc-dmabuf", - tag_colorspace: true, + path: "nvenc-dmabuf-rgb", + tag_colorspace: false, }); } if hdr { @@ -1356,6 +1357,25 @@ fn derive_device( } } +/// The default CUDA device. +fn cuda_device() -> Result { + // SAFETY: standard hwdevice construction; the ref is wrapped in BufRef immediately. + unsafe { + let mut dev = ptr::null_mut(); + let r = ff::av_hwdevice_ctx_create( + &mut dev, + ff::AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA, + ptr::null(), + ptr::null_mut(), + 0, + ); + if r < 0 { + return Err(averr("create CUDA device", r)); + } + Ok(BufRef(dev)) + } +} + /// DRM device and a `DRM_PRIME` frames context for imported buffers. fn drm_context( node: &str, @@ -1424,7 +1444,9 @@ fn drm_frames(dev: &BufRef, pix_fmt: ff::AVPixelFormat, w: u32, h: u32) -> Resul /// bounds that to the formats a process sees. Until their command buffers are reused, the last few /// mapped buffers stay referenced. struct VkImport { - /// Vulkan device derived from the node's DRM device, with single-plane images. + /// CUDA device that NVENC encodes on. + cu: BufRef, + /// Vulkan device derived from `cu`, with single-plane images. vk: BufRef, /// Functions of `vk` for the dedicated imports. dma: Arc, @@ -1454,8 +1476,9 @@ impl VkImport { ) -> Result, String> { type Key = (String, i32, u32, u32); struct Cache { - /// Node, its DRM device, the Vulkan device derived from it and its functions. - devices: Vec<(String, BufRef, BufRef, Arc)>, + /// Node, its DRM device, the CUDA device, the Vulkan device derived from CUDA and its + /// functions. + devices: Vec<(String, BufRef, BufRef, BufRef, Arc)>, imports: Vec<(Key, Arc)>, } static CACHE: Mutex = Mutex::new(Cache { @@ -1469,34 +1492,43 @@ impl VkImport { } if !cache.devices.iter().any(|(n, ..)| n == node) { let dev = drm_device(node)?; + // Vulkan is derived from CUDA, not the other way round: since FFmpeg 9 a CUDA device + // derived from Vulkan cannot push its context, and the copy to CUDA fails. + let cu = cuda_device()?; let vk = derive_device( - &dev, + &cu, ff::AVHWDeviceType::AV_HWDEVICE_TYPE_VULKAN, Some(c"disable_multiplane"), )?; // SAFETY: a Vulkan device context, kept alive by the cache for the whole process. let dma = Arc::new(unsafe { VkDma::new(vk.0) }?); - cache.devices.push((node.to_string(), dev, vk, dma)); + cache.devices.push((node.to_string(), dev, cu, vk, dma)); } - let (_, dev, vk, dma) = cache + let (_, dev, cu, vk, dma) = cache .devices .iter() .find(|(n, ..)| n == node) .expect("added above"); let drm = drm_frames(dev, pix_fmt, w, h)?; - let mut vk_frames = ptr::null_mut(); - // SAFETY: valid device and frames refs; the result is wrapped in BufRef. + // A plain context, not one derived from `drm`: filters cannot allocate their output from a + // derived one since FFmpeg 9 (ENOMEM). Mapping into it works the same. + // SAFETY: valid device ref; the context is filled before init and wrapped in BufRef. + let vk_frames = unsafe { ff::av_hwframe_ctx_alloc(vk.0) }; + if vk_frames.is_null() { + return Err("av_hwframe_ctx_alloc failed".into()); + } + let vk_frames = BufRef(vk_frames); + // SAFETY: `vk_frames` is a fresh, uninitialized frames context. let r = unsafe { - ff::av_hwframe_ctx_create_derived( - &mut vk_frames, - ff::AVPixelFormat::AV_PIX_FMT_VULKAN, - vk.0, - drm.0, - ff::AV_HWFRAME_MAP_READ as i32 | ff::AV_HWFRAME_MAP_WRITE as i32, - ) + let fc = &mut *((*vk_frames.0).data as *mut ff::AVHWFramesContext); + fc.format = ff::AVPixelFormat::AV_PIX_FMT_VULKAN; + fc.sw_format = pix_fmt; + fc.width = w as i32; + fc.height = h as i32; + ff::av_hwframe_ctx_init(vk_frames.0) }; if r < 0 { - return Err(averr("derive Vulkan frames", r)); + return Err(averr("create Vulkan frames", r)); } // SAFETY: lookup in a static table; null for formats Vulkan cannot hold. let vk_format = unsafe { ff::av_vkfmt_from_pixfmt(pix_fmt) }; @@ -1504,12 +1536,13 @@ impl VkImport { return Err(format!("no Vulkan format for {pix_fmt:?}")); } let import = Arc::new(VkImport { + cu: cu.try_clone()?, vk: vk.try_clone()?, dma: dma.clone(), // SAFETY: non-null pointer into FFmpeg's static table. vk_format: ash::vk::Format::from_raw(unsafe { *vk_format } as i32), drm_frames: drm, - vk_frames: BufRef(vk_frames), + vk_frames, map_lock: Mutex::new(()), }); cache.imports.push((key, import.clone())); @@ -2501,7 +2534,7 @@ mod tests { .encode(frame, false) .expect("dma-buf import + encode") .expect("packet"); - assert_eq!(p.path(), Some("nvenc-dmabuf")); + assert_eq!(p.path(), Some("nvenc-dmabuf-rgb")); assert!(out.keyframe && out.codec_config.is_some()); check_colors(&decode_first(&out)); } @@ -2532,7 +2565,7 @@ mod tests { .convert(frame, true) .expect("dma-buf import + convert") .expect("converted frame"); - assert_eq!(conv.path(), Some("nvenc-dmabuf")); + assert_eq!(conv.path(), Some("nvenc-dmabuf-rgb")); // CUDA output: the capture buffer stays held until the packet is out. assert!(p.release.is_some()); let out = std::thread::spawn(move || { @@ -2586,7 +2619,7 @@ mod tests { } } } - assert_eq!(p.path(), Some("nvenc-dmabuf")); + assert_eq!(p.path(), Some("nvenc-dmabuf-rgb")); assert!(bad.is_empty(), "{label} frames lost their content: {bad:?}"); } } From 1939a6ce1eaf52c5e6ae012b1bbc77fac0f6f5c1 Mon Sep 17 00:00:00 2001 From: FacuM Date: Tue, 27 Oct 2026 00:17:56 -0300 Subject: [PATCH 10/12] Sync the volume between the PC and the tablet AudioVolume (Control/20, protocol 0.7) carries a perceptual volume and a mute flag both ways: the tablet sends its value when audio starts, then whichever side changes its volume sends it, and echoes are dropped. A client that sends a new Hello mid-session (its app restarted on the same link) now ends the session so the host reopens it. --- .../com/librestatic/ezdisplay/MainActivity.kt | 32 ++ .../com/librestatic/ezdisplay/NativeBridge.kt | 13 + .../com/librestatic/ezdisplay/VolumeSync.kt | 155 ++++++++ .../librestatic/ezdisplay/VolumeSyncTest.kt | 115 ++++++ crates/ezd-android/src/bridge.rs | 57 ++- crates/ezd-android/src/session.rs | 15 +- crates/ezd-client-core/src/handle.rs | 114 +++++- crates/ezd-client-core/src/session.rs | 20 +- crates/ezd-fake-client/src/lib.rs | 7 + crates/ezd-host-core/src/session.rs | 153 +++++++- crates/ezd-host-core/src/source.rs | 16 +- crates/ezd-host-core/src/testing.rs | 42 +- crates/ezd-host-linux/src/audio.rs | 361 +++++++++++++++++- crates/ezd-host-service/src/aoa.rs | 5 +- crates/ezd-host-service/src/service.rs | 11 +- crates/ezd-proto/src/msg.rs | 47 ++- docs/protocol.md | 18 +- 17 files changed, 1144 insertions(+), 37 deletions(-) create mode 100644 android/app/src/main/kotlin/com/librestatic/ezdisplay/VolumeSync.kt create mode 100644 android/app/src/test/kotlin/com/librestatic/ezdisplay/VolumeSyncTest.kt diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt index 1186e00..0ef04a1 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt @@ -14,6 +14,7 @@ import android.content.IntentFilter import android.content.res.Configuration import android.hardware.usb.UsbAccessory import android.hardware.usb.UsbManager +import android.media.AudioManager import android.os.Build import android.os.Bundle import android.os.Handler @@ -96,6 +97,16 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall private lateinit var telemetryView: TextView private val handler = Handler(Looper.getMainLooper()) + private val volumeWatcher by lazy { + VolumeWatcher( + this, handler, + VolumeSync( + AudioManagerVolume(getSystemService(AudioManager::class.java)), + NativeBridge::setLocalVolume, + NativeBridge::pollHostVolume, + ), + ) + } private var chosenMode: android.view.Display.Mode? = null private var capsJson: String? = null @@ -176,6 +187,7 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) launchIntent = intent + volumeControlStream = AudioManager.STREAM_MUSIC // the volume keys always move the PC's audio window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) if (Build.VERSION.SDK_INT >= 28) { window.attributes = window.attributes.also { @@ -479,6 +491,7 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall InputPrefs.prefs(this).getBoolean(SettingsStore.AUDIO_ENABLED, true), ) pushKeyboardState() + volumeWatcher.start() val acc = accessory?.takeIf { usbManager.hasPermission(it) } val wifiChoice = if (tryWifiNext) wifiTarget() else null val route = chooseRoute(acc != null, tryWifiNext, wifiChoice != null) @@ -494,6 +507,7 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall } if (err != null) { Log.w(TAG, "start failed: $err") + volumeWatcher.stop() setStatus( if (route == Route.ACCESSORY) getString(R.string.status_usb_open_failed) else getString(R.string.status_bad_caps, err), @@ -542,6 +556,7 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall private fun stopSession() { releaseAllKeys() + volumeWatcher.stop() if (sessionActive) { NativeBridge.stop() sessionActive = false @@ -580,6 +595,8 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall val now = SystemClock.uptimeMillis() if (now - lastStatsPoll >= STATS_INTERVAL_MS) { lastStatsPoll = now + volumeWatcher.pollHost() + volumeWatcher.checkLocal() updateStats(now) } } @@ -865,6 +882,13 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall } override fun dispatchKeyEvent(event: KeyEvent): Boolean { + if (inputActive()) { + volumeKeyDirection(event.keyCode)?.let { direction -> + // Handled here so the system volume panel stays hidden: its blur stalls the video. + if (event.action == KeyEvent.ACTION_DOWN) volumeWatcher.onKey(direction) + return true + } + } if (inputActive() && isHardwareKey(event)) { val code = event.keyCode val nanos = event.eventTime * 1_000_000L @@ -887,6 +911,14 @@ class MainActivity : Activity(), SurfaceHolder.Callback, Choreographer.FrameCall return super.dispatchKeyEvent(event) } + /** The volume change a volume key asks for, or null for any other key. */ + private fun volumeKeyDirection(code: Int): Int? = when (code) { + KeyEvent.KEYCODE_VOLUME_UP -> AudioManager.ADJUST_RAISE + KeyEvent.KEYCODE_VOLUME_DOWN -> AudioManager.ADJUST_LOWER + KeyEvent.KEYCODE_VOLUME_MUTE -> AudioManager.ADJUST_TOGGLE_MUTE + else -> null + } + /** A key from a physical keyboard that Android does not need for itself. */ private fun isHardwareKey(event: KeyEvent): Boolean { if (event.source and InputDevice.SOURCE_KEYBOARD != InputDevice.SOURCE_KEYBOARD) return false diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/NativeBridge.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/NativeBridge.kt index d10a3af..5bbbed1 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/NativeBridge.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/NativeBridge.kt @@ -93,6 +93,19 @@ object NativeBridge { */ @JvmStatic external fun setKeyboardAttached(attached: Boolean) + /** + * The tablet's media volume as a perceptual 0..1 value plus the mute flag. Kept across + * sessions: the host gets the last value once audio starts, and every later change at once. + * Do not report a value that was just applied from the host ([pollHostVolume]). + */ + @JvmStatic external fun setLocalVolume(volume: Float, muted: Boolean) + + /** + * `[seq, volume, muted (0/1)]`: the latest volume the host sent in this session. `seq` grows + * with every message and is 0 while there is none. + */ + @JvmStatic external fun pollHostVolume(): FloatArray + /** Fires the engine timers that are due; call once per frame. */ @JvmStatic external fun inputTick() diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/VolumeSync.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/VolumeSync.kt new file mode 100644 index 0000000..cb92d89 --- /dev/null +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/VolumeSync.kt @@ -0,0 +1,155 @@ +package com.librestatic.ezdisplay + +import android.content.Context +import android.database.ContentObserver +import android.media.AudioManager +import android.os.Handler +import android.provider.Settings +import android.util.Log +import kotlin.math.roundToInt + +/** The tablet's media volume as the platform exposes it. */ +interface MusicVolume { + val max: Int + var index: Int + var muted: Boolean + + /** + * Moves the volume one step ([AudioManager.ADJUST_RAISE] or `ADJUST_LOWER`) or toggles mute. + * + * Whole steps through [index], not `adjustStreamVolume`: some devices move a finer hidden + * scale on each press, which the step the PC follows only sees every few presses. No flag + * shows the system volume panel either: its blur over the video surface stalls the decoder + * for as long as the panel is up. + */ + fun adjust(direction: Int) { + when (direction) { + AudioManager.ADJUST_TOGGLE_MUTE -> muted = !muted + else -> index = (index + direction).coerceIn(0, max) + } + } +} + +/** [MusicVolume] on top of [AudioManager]'s `STREAM_MUSIC`. */ +class AudioManagerVolume(private val audio: AudioManager) : MusicVolume { + override val max: Int get() = audio.getStreamMaxVolume(AudioManager.STREAM_MUSIC).coerceAtLeast(1) + + override var index: Int + get() = audio.getStreamVolume(AudioManager.STREAM_MUSIC) + set(value) = audio.setStreamVolume(AudioManager.STREAM_MUSIC, value, 0) + + override var muted: Boolean + get() = audio.isStreamMute(AudioManager.STREAM_MUSIC) + set(value) = audio.adjustStreamVolume( + AudioManager.STREAM_MUSIC, + if (value) AudioManager.ADJUST_MUTE else AudioManager.ADJUST_UNMUTE, + 0, + ) +} + +/** Perceptual volume (0..1, what a slider shows) of a volume step. */ +fun volumeOfIndex(index: Int, max: Int): Float = (index.toFloat() / max.coerceAtLeast(1)).coerceIn(0f, 1f) + +/** The volume step closest to the perceptual [volume]. */ +fun indexOfVolume(volume: Float, max: Int): Int = + (volume.coerceIn(0f, 1f) * max).roundToInt().coerceIn(0, max) + +/** + * Keeps the tablet's media volume and the PC's volume in step (protocol 0.7 `AudioVolume`). + * + * The native layer holds the tablet's value and sends it to the host once audio starts, then on + * every change; a volume received from the host is polled with [pollHost] and applied to the + * hardware volume. To avoid sending a host change back, this class remembers the state it last + * saw (applied or reported) as a volume step plus the mute flag and reports only a different + * one: this is the only place that suppresses echoes. + */ +class VolumeSync( + private val music: MusicVolume, + private val report: (volume: Float, muted: Boolean) -> Unit, + private val pollHostVolume: () -> FloatArray, +) { + private var knownIndex = -1 + private var knownMuted = false + private var hostSeq = 0f + + /** Starts a session: reports the current volume (kept natively until audio starts). */ + fun begin() { + hostSeq = 0f + knownIndex = -1 + onLocalChange() + } + + /** The tablet's volume may have changed (volume keys, the system panel, another app). */ + fun onLocalChange() { + val index = music.index + val muted = music.muted + if (index == knownIndex && muted == knownMuted) return + knownIndex = index + knownMuted = muted + report(volumeOfIndex(index, music.max), muted) + } + + /** A volume key pressed on the tablet: changes the volume without the system panel and reports it. */ + fun onKey(direction: Int) { + try { + music.adjust(direction) + } catch (e: SecurityException) { + Log.w(TAG, "cannot set the media volume", e) + } + onLocalChange() + } + + /** Applies a volume the host sent since the last call. */ + fun pollHost() { + val v = pollHostVolume() + if (v.size < 3 || v[0] == 0f || v[0] == hostSeq) return + hostSeq = v[0] + val index = indexOfVolume(v[1], music.max) + val muted = v[2] != 0f + // Remember before applying so the change notification that follows is not reported. + knownIndex = index + knownMuted = muted + try { + if (music.index != index) music.index = index + // Setting a volume can unmute, so the mute state goes last. + if (music.muted != muted) music.muted = muted + } catch (e: SecurityException) { + Log.w(TAG, "cannot set the media volume", e) + } + } + + private companion object { + const val TAG = "VolumeSync" + } +} + +/** Runs [sync]'s local check whenever the system settings (where the volumes live) change. */ +class VolumeWatcher(private val context: Context, handler: Handler, private val sync: VolumeSync) { + private val observer = object : ContentObserver(handler) { + override fun onChange(selfChange: Boolean) = sync.onLocalChange() + } + private var registered = false + + /** Registers the observer (once) and reports the current volume. */ + fun start() { + if (!registered) { + context.contentResolver.registerContentObserver(Settings.System.CONTENT_URI, true, observer) + registered = true + } + sync.begin() + } + + fun stop() { + if (registered) { + context.contentResolver.unregisterContentObserver(observer) + registered = false + } + } + + fun pollHost() = sync.pollHost() + + fun onKey(direction: Int) = sync.onKey(direction) + + /** Safety net for devices that do not notify through the settings provider. */ + fun checkLocal() = sync.onLocalChange() +} diff --git a/android/app/src/test/kotlin/com/librestatic/ezdisplay/VolumeSyncTest.kt b/android/app/src/test/kotlin/com/librestatic/ezdisplay/VolumeSyncTest.kt new file mode 100644 index 0000000..db19e05 --- /dev/null +++ b/android/app/src/test/kotlin/com/librestatic/ezdisplay/VolumeSyncTest.kt @@ -0,0 +1,115 @@ +package com.librestatic.ezdisplay + +import android.media.AudioManager +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class VolumeSyncTest { + private class FakeMusic(override val max: Int = 15, start: Int = 7) : MusicVolume { + override var index = start + override var muted = false + var sets = 0 + var muteCalls = 0 + } + + private class Rig(val music: FakeMusic = FakeMusic()) { + val reports = mutableListOf>() + var host = floatArrayOf(0f, 0f, 0f) + val sync = VolumeSync(music, { v, m -> reports.add(v to m) }, { host }) + } + + @Test fun mapsStepsAndPerceptualVolume() { + assertEquals(0f, volumeOfIndex(0, 15), 0f) + assertEquals(1f, volumeOfIndex(15, 15), 0f) + assertEquals(0.5f, volumeOfIndex(5, 10), 1e-6f) + assertEquals(8, indexOfVolume(0.5f, 15)) + assertEquals(15, indexOfVolume(1.5f, 15)) + assertEquals(0, indexOfVolume(-1f, 15)) + for (i in 0..15) assertEquals(i, indexOfVolume(volumeOfIndex(i, 15), 15)) + } + + @Test fun beginReportsTheCurrentVolume() { + val r = Rig() + r.sync.begin() + assertEquals(listOf(7 / 15f to false), r.reports) + // Nothing changed: no new report. + r.sync.onLocalChange() + assertEquals(1, r.reports.size) + } + + @Test fun reportsLocalChangesOnce() { + val r = Rig() + r.sync.begin() + r.music.index = 9 + r.sync.onLocalChange() + r.sync.onLocalChange() + r.music.muted = true + r.sync.onLocalChange() + assertEquals(listOf(7 / 15f to false, 9 / 15f to false, 9 / 15f to true), r.reports) + } + + @Test fun appliesTheHostVolumeWithoutEchoingIt() { + val r = Rig() + r.sync.begin() + r.host = floatArrayOf(1f, 0.5f, 0f) // 0.5 * 15 = 7.5, rounds to 8 + r.sync.pollHost() + assertEquals(8, r.music.index) + // The settings notification that follows is not sent back, even though 8/15 != 0.5. + r.sync.onLocalChange() + assertEquals(1, r.reports.size) + // The same message again changes nothing. + r.music.index = 3 + r.sync.pollHost() + assertEquals(3, r.music.index) + r.sync.onLocalChange() + assertEquals(listOf(7 / 15f to false, 3 / 15f to false), r.reports) + } + + @Test fun appliesTheHostMuteLast() { + val r = Rig() + r.sync.begin() + r.host = floatArrayOf(1f, 0.2f, 1f) + r.sync.pollHost() + assertEquals(3, r.music.index) + assertTrue(r.music.muted) + r.sync.onLocalChange() + assertEquals(1, r.reports.size) + r.host = floatArrayOf(2f, 0.2f, 0f) + r.sync.pollHost() + assertEquals(false, r.music.muted) + r.sync.onLocalChange() + assertEquals(1, r.reports.size) + } + + @Test fun ignoresMissingOrShortHostData() { + val r = Rig() + r.sync.begin() + r.sync.pollHost() // seq 0: nothing yet + r.host = floatArrayOf(1f) + r.sync.pollHost() + assertEquals(7, r.music.index) + } + + @Test fun aNewSessionStartsFromSequenceZero() { + val r = Rig() + r.sync.begin() + r.host = floatArrayOf(5f, 1f, 0f) + r.sync.pollHost() + assertEquals(15, r.music.index) + r.sync.begin() // the next session numbers its messages from 1 again + r.host = floatArrayOf(1f, 0f, 0f) + r.sync.pollHost() + assertEquals(0, r.music.index) + } + + @Test fun volumeKeysStepAndReport() { + val r = Rig() + r.sync.begin() + r.sync.onKey(AudioManager.ADJUST_RAISE) + r.sync.onKey(AudioManager.ADJUST_TOGGLE_MUTE) + r.music.index = 15 + r.sync.onKey(AudioManager.ADJUST_RAISE) // already at the top: clamped + assertEquals(listOf(7 / 15f to false, 8 / 15f to false, 8 / 15f to true, 1f to true), r.reports) + } +} diff --git a/crates/ezd-android/src/bridge.rs b/crates/ezd-android/src/bridge.rs index dc5a139..d82e996 100644 --- a/crates/ezd-android/src/bridge.rs +++ b/crates/ezd-android/src/bridge.rs @@ -11,7 +11,7 @@ use crate::wifi; use ezd_client_core::now_us; use ezd_input::{InputConfig, InputEngine, ViewTransform}; use ezd_link::identity::parse_fingerprint; -use ezd_proto::{Key, Message}; +use ezd_proto::{AudioVolume, Key, Message}; use jni::EnvUnowned; use jni::errors::ThrowRuntimeExAndDefault; use jni::objects::{JByteArray, JClass, JFloatArray, JIntArray, JLongArray, JObject, JString}; @@ -40,6 +40,8 @@ static INPUT: Mutex = Mutex::new(InputState { }); /// Last hardware keyboard state from Kotlin; a new session starts with it. static KEYBOARD_ATTACHED: AtomicBool = AtomicBool::new(false); +/// The tablet's media volume as last reported by Kotlin; sent to the host with each new session. +static LOCAL_VOLUME: Mutex> = Mutex::new(None); static LAST_INPUT_ERR_S: AtomicU64 = AtomicU64::new(0); #[derive(Default)] @@ -131,6 +133,10 @@ fn start_session(window: Option, target: Target, caps: crate::caps:: log::info!("start {target:?} surface={}", window.is_some()); let s = Session::start(window, target, caps); s.set_keyboard_attached(KEYBOARD_ATTACHED.load(Ordering::Relaxed)); + let volume = *LOCAL_VOLUME.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(v) = volume { + s.set_audio_volume(v); + } *guard = Some(s); } @@ -532,6 +538,55 @@ pub extern "system" fn set_keyboard_attached<'local>( .resolve::() } +/// `external fun setLocalVolume(volume: Float, muted: Boolean)`: the tablet's media volume, +/// perceptual 0..1. Stored also while there is no session; the host gets it once it announces +/// audio, and every later change right away. A no-op for the host without an audio session. +/// Kotlin must not report a volume it just applied from the host (it does the echo suppression). +#[jni::jni_mangle("com.librestatic.ezdisplay.NativeBridge")] +pub extern "system" fn set_local_volume<'local>( + mut env: EnvUnowned<'local>, + _class: JClass<'local>, + volume: jni::sys::jfloat, + muted: jni::sys::jboolean, +) { + env.with_env(|_env| -> jni::errors::Result<()> { + let v = AudioVolume { + volume: if volume.is_nan() { + 0.0 + } else { + volume.clamp(0.0, 1.0) + }, + muted, + }; + *LOCAL_VOLUME.lock().unwrap_or_else(|e| e.into_inner()) = Some(v); + if let Some(s) = session().as_ref() { + s.set_audio_volume(v); + } + Ok(()) + }) + .resolve::() +} + +/// `external fun pollHostVolume(): FloatArray` = `[seq, volume, muted(0/1)]`: the latest volume +/// the host sent in this session. `seq` grows with every message and is 0 while there is none. +/// It is exact in a float up to 2^24 messages, far more than a session sees. +#[jni::jni_mangle("com.librestatic.ezdisplay.NativeBridge")] +pub extern "system" fn poll_host_volume<'local>( + mut env: EnvUnowned<'local>, + _class: JClass<'local>, +) -> JFloatArray<'local> { + env.with_env(|env| -> jni::errors::Result> { + let v: [f32; 3] = match session().as_ref().and_then(|s| s.host_audio_volume()) { + Some((seq, a)) => [seq as f32, a.volume, f32::from(u8::from(a.muted))], + None => [0.0; 3], + }; + let arr = env.new_float_array(v.len())?; + arr.set_region(env, 0, &v)?; + Ok(arr) + }) + .resolve::() +} + /// `external fun inputTick()`: fires due gesture/pen timers. #[jni::jni_mangle("com.librestatic.ezdisplay.NativeBridge")] pub extern "system" fn input_tick<'local>(mut env: EnvUnowned<'local>, _class: JClass<'local>) { diff --git a/crates/ezd-android/src/session.rs b/crates/ezd-android/src/session.rs index 56232da..08d1087 100644 --- a/crates/ezd-android/src/session.rs +++ b/crates/ezd-android/src/session.rs @@ -13,8 +13,8 @@ use ezd_client_core::{ use ezd_link::{BlockingOptions, Link}; use ezd_proto::pair::Fingerprint; use ezd_proto::{ - AudioCodec, AudioConfig, AudioPacket, ClientAudio, CursorShape, SampleFormat, StreamColor, - StreamConfig, VideoFrame, + AudioCodec, AudioConfig, AudioPacket, AudioVolume, ClientAudio, CursorShape, SampleFormat, + StreamColor, StreamConfig, VideoFrame, }; use std::fs::File; use std::io::{Read, Write}; @@ -349,6 +349,17 @@ impl Session { self.handle.set_keyboard_attached(attached); } + /// Reports the local audio volume to the host (kept for the session; sent once the host + /// has announced audio). + pub fn set_audio_volume(&self, volume: AudioVolume) { + self.handle.set_audio_volume(volume); + } + + /// Latest volume of the host with its sequence number. + pub fn host_audio_volume(&self) -> Option<(u64, AudioVolume)> { + self.handle.host_audio_volume() + } + /// Latest cursor state. pub fn cursor(&self) -> Option { self.handle.cursor() diff --git a/crates/ezd-client-core/src/handle.rs b/crates/ezd-client-core/src/handle.rs index ea869c2..852fec4 100644 --- a/crates/ezd-client-core/src/handle.rs +++ b/crates/ezd-client-core/src/handle.rs @@ -3,8 +3,8 @@ use crate::percentile; use ezd_link::MuxSender; use ezd_proto::{ - ClientKeyboard, ClientOptions, ClientStats, ClockSync, CursorShape, CursorState, InputStatus, - Message, StreamColor, + AudioVolume, ClientKeyboard, ClientOptions, ClientStats, ClockSync, CursorShape, CursorState, + InputStatus, Message, StreamColor, }; use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; @@ -81,6 +81,36 @@ pub(crate) struct Inner { /// Set while connected to a host that understands input. input_tx: Mutex>, keyboard: Mutex, + volume: Mutex, +} + +/// Audio volume sync state (`AudioVolume`, protocol 0.7). +#[derive(Default)] +struct VolumeState { + /// Latest local volume set by the application (survives reconnects). + local: Option, + /// Set while connected to a host with an audio stream. + tx: Option, + /// What the host was last told in this connection. + sent: Option, + /// Latest volume received from the host in this connection, with a sequence number that + /// grows with every message (also across connections). + host: Option<(u64, AudioVolume)>, + host_seq: u64, +} + +impl VolumeState { + fn send_if_changed(&mut self) { + let (Some(tx), Some(v)) = (&self.tx, self.local) else { + return; + }; + if self.sent.is_some_and(|s| s.is_same(&v)) { + return; + } + if tx.send_control(Message::AudioVolume(v)).is_ok() { + self.sent = Some(v); + } + } } /// Hardware keyboard state reported by the application. @@ -145,6 +175,7 @@ impl ClientHandle { snapshot: Mutex::new(ClientSnapshot::default()), input_tx: Mutex::new(None), keyboard: Mutex::new(KeyboardState::default()), + volume: Mutex::new(VolumeState::default()), })) } @@ -286,6 +317,41 @@ impl ClientHandle { k.send_if_changed(); } + /// Reports the local audio volume to the host (`AudioVolume`, protocol 0.7). + /// + /// The latest value is kept, also while not connected, and sent as soon as the host has + /// announced an audio stream (so the PC starts at this volume). While connected, a value + /// within [`AudioVolume::EPSILON`] of the last one sent is not sent again. Applying a host + /// volume locally and then reporting the result back is for the caller to avoid. Callable + /// from any thread; never blocks. + pub fn set_audio_volume(&self, volume: AudioVolume) { + let mut v = lock(&self.0.volume); + v.local = Some(volume); + v.send_if_changed(); + } + + /// The latest volume the host sent in this connection and its sequence number (it grows with + /// every message, so a caller can tell a new one from the one it already applied). + pub fn host_audio_volume(&self) -> Option<(u64, AudioVolume)> { + lock(&self.0.volume).host + } + + /// Sets (or clears) the link used for `AudioVolume` and, when set, sends the current local + /// volume. Also forgets the host volume of the previous connection. + pub(crate) fn set_audio_volume_sender(&self, tx: Option) { + let mut v = lock(&self.0.volume); + v.tx = tx; + v.sent = None; + v.host = None; + v.send_if_changed(); + } + + pub(crate) fn store_host_audio_volume(&self, volume: AudioVolume) { + let mut v = lock(&self.0.volume); + v.host_seq += 1; + v.host = Some((v.host_seq, volume)); + } + pub(crate) fn set_input_sender(&self, tx: Option) { *lock(&self.0.input_tx) = tx; } @@ -409,6 +475,50 @@ mod tests { assert_eq!(h.snapshot().inputs_sent, 3); } + #[tokio::test] + async fn volume_is_kept_until_connected_and_sent_only_on_change() { + let vol = |volume| AudioVolume { + volume, + muted: false, + }; + let h = ClientHandle::new(); + h.set_audio_volume(vol(0.3)); // not connected: stored only + let (a, b) = tokio::io::duplex(1 << 16); + let (tx, _task) = ezd_link::mux::spawn_mux(Box::new(a), MuxOptions::default()); + h.set_audio_volume_sender(Some(tx)); + h.set_audio_volume(vol(0.302)); // within the epsilon + h.set_audio_volume(vol(0.6)); + h.set_audio_volume_sender(None); + h.set_audio_volume(vol(0.9)); // disconnected again + let mut rx = ezd_link::mux::MuxReceiver::new(Box::new(b)); + let mut got = Vec::new(); + while let Ok(Ok(Some(m))) = + tokio::time::timeout(std::time::Duration::from_millis(200), rx.recv()).await + { + if let Message::AudioVolume(v) = m { + got.push(v.volume); + } + } + assert_eq!(got, vec![0.3, 0.6]); + } + + #[test] + fn host_volume_has_a_growing_sequence_and_is_cleared_on_connect() { + let h = ClientHandle::new(); + assert_eq!(h.host_audio_volume(), None); + let v = AudioVolume { + volume: 0.5, + muted: true, + }; + h.store_host_audio_volume(v); + h.store_host_audio_volume(v); + assert_eq!(h.host_audio_volume(), Some((2, v))); + h.set_audio_volume_sender(None); + assert_eq!(h.host_audio_volume(), None); + h.store_host_audio_volume(v); + assert_eq!(h.host_audio_volume().map(|(s, _)| s), Some(3)); + } + async fn read_keyboard_messages( h: &ClientHandle, act: impl FnOnce(&ClientHandle), diff --git a/crates/ezd-client-core/src/session.rs b/crates/ezd-client-core/src/session.rs index ed1df16..e0afece 100644 --- a/crates/ezd-client-core/src/session.rs +++ b/crates/ezd-client-core/src/session.rs @@ -3,8 +3,8 @@ use crate::{ClientHandle, now_us}; use ezd_link::{Link, MuxError, MuxReceiver, MuxSender}; use ezd_proto::{ - AudioConfig, AudioPacket, Bye, ClientAudio, ClientCaps, ClientHdr, CursorShape, CursorState, - FrameAck, Hello, Message, Ping, Pong, Role, StreamColor, StreamConfig, VideoFrame, + AudioConfig, AudioPacket, AudioVolume, Bye, ClientAudio, ClientCaps, ClientHdr, CursorShape, + CursorState, FrameAck, Hello, Message, Ping, Pong, Role, StreamColor, StreamConfig, VideoFrame, }; use std::time::Duration; use tokio::time::{MissedTickBehavior, interval, timeout}; @@ -47,6 +47,10 @@ pub trait ClientHandler: Send { /// [`on_stream_config`]: ClientHandler::on_stream_config fn on_audio_config(&mut self, _cfg: &AudioConfig) {} + /// The host's volume changed (protocol 0.7, `AudioVolume`). The latest one is also kept in + /// [`ClientHandle::host_audio_volume`]. Apply it without reporting it back. + fn on_audio_volume(&mut self, _volume: AudioVolume) {} + /// An audio packet. Hand it to the audio thread without blocking; packets only arrive /// when [`on_audio_config`](Self::on_audio_config) announced a stream. fn on_audio_packet(&mut self, _pkt: AudioPacket) {} @@ -305,6 +309,12 @@ pub async fn run_client( if hello.supports_input() { handle.set_input_sender(Some(tx.clone())); } + if hello.supports_audio() + && my_minor_supports_audio + && audio.as_ref().is_some_and(|a| a.stream.is_some()) + { + handle.set_audio_volume_sender(Some(tx.clone())); + } if hello.supports_keyboard() && my_minor_supports_keyboard { handle.set_keyboard_sender(Some(tx.clone())); } @@ -373,6 +383,7 @@ pub async fn run_client( handle.set_input_sender(None); handle.set_keyboard_sender(None); + handle.set_audio_volume_sender(None); let outcome = ClientOutcome { stream, color, @@ -454,6 +465,11 @@ fn on_message( handler.on_audio_packet(p); Ok(None) } + Message::AudioVolume(v) => { + handle.store_host_audio_volume(v); + handler.on_audio_volume(v); + Ok(None) + } Message::InputStatus(st) => { handle.store_input_status(st); Ok(None) diff --git a/crates/ezd-fake-client/src/lib.rs b/crates/ezd-fake-client/src/lib.rs index b98d607..17c8f92 100644 --- a/crates/ezd-fake-client/src/lib.rs +++ b/crates/ezd-fake-client/src/lib.rs @@ -207,6 +207,8 @@ pub struct ClientReport { pub audio_packets: u64, /// Audio payload bytes received. pub audio_bytes: u64, + /// `AudioVolume` messages received from the host. + pub audio_volumes: u64, } impl ClientReport { @@ -386,6 +388,10 @@ impl ClientHandler for Handler { self.report.audio = cfg.stream.clone(); } + fn on_audio_volume(&mut self, _volume: ezd_proto::AudioVolume) { + self.report.audio_volumes += 1; + } + fn on_audio_packet(&mut self, pkt: AudioPacket) { self.report.audio_packets += 1; self.report.audio_bytes += pkt.data.len() as u64; @@ -535,6 +541,7 @@ pub async fn run_fake_client( audio: None, audio_packets: 0, audio_bytes: 0, + audio_volumes: 0, }, all_decode: Vec::new(), all_e2e: Vec::new(), diff --git a/crates/ezd-host-core/src/session.rs b/crates/ezd-host-core/src/session.rs index 929fb14..ab01ae8 100644 --- a/crates/ezd-host-core/src/session.rs +++ b/crates/ezd-host-core/src/session.rs @@ -31,9 +31,9 @@ use crate::source::{ use crate::time::now_us; use ezd_link::{Link, MuxClosed, MuxError, MuxOptions, MuxReceiver, MuxSender, VideoCredit}; use ezd_proto::{ - AudioCodec, AudioConfig, AudioPacket, AudioStream, Bye, ClientAudio, ClientCaps, ClientHdr, - ClientOptions, ClientStats, ClockSync, Codec, DEFAULT_CHUNK_LEN, DynamicRange, Hello, - HostAudio, HostCaps, InputStatus, Message, Ping, Pong, Prefs, ProtoError, Reject, Role, + AudioCodec, AudioConfig, AudioPacket, AudioStream, AudioVolume, Bye, ClientAudio, ClientCaps, + ClientHdr, ClientOptions, ClientStats, ClockSync, Codec, DEFAULT_CHUNK_LEN, DynamicRange, + Hello, HostAudio, HostCaps, InputStatus, Message, Ping, Pong, Prefs, ProtoError, Reject, Role, SampleFormat, StreamColor, StreamConfig, VideoFrame, negotiate_audio, negotiate_hdr, }; use std::sync::Arc; @@ -161,6 +161,9 @@ pub enum EndReason { SourceStopped, /// [`SessionOptions::shutdown`] was set; the client was sent `Bye`. HostShutdown, + /// The client sent a new `Hello`: its app was restarted and opened the link again. An + /// accessory link never reports the app closing it, so this is how its end shows. + ClientRestarted, } /// Result of a finished session. @@ -416,13 +419,17 @@ pub async fn run_host_session_with( // Audio thread: its failure ends the audio only. let mut audio_join: Option> = None; let audio_stream = audio.as_ref().map(|a| a.stream.clone()); + // Volumes of the client, forwarded to the audio thread (which owns the source). + let mut volume_tx: Option> = None; if let (Some(a), Some(src)) = (audio, audio_source) { let (shared, tx) = (shared.clone(), tx.clone()); + let (vtx, vrx) = std::sync::mpsc::channel(); + volume_tx = Some(vtx); let poll = opts.poll_interval; audio_join = Some(std::thread::Builder::new().name("ezd-audio".into()).spawn( move || { let mut src = src; - match audio_loop(src.as_mut(), a.encoder, &a.stream, &tx, &shared, poll) { + match audio_loop(src.as_mut(), a.encoder, &a.stream, &tx, &shared, poll, &vrx) { Ok(()) => {} Err(e) => tracing::warn!("audio stopped, video goes on: {e}"), } @@ -547,8 +554,15 @@ pub async fn run_host_session_with( let _ = t.send(InputCmd::Keyboard(k.attached)); } } + Ok(Some(Message::AudioVolume(v))) => { + tracing::debug!(volume = v.volume, muted = v.muted, "client volume"); + if let Some(t) = &volume_tx { + let _ = t.send(v); + } + } Ok(Some(Message::Bye(b))) => break EndReason::ClientBye(b.reason), Ok(Some(Message::Reject(r))) => break EndReason::ClientBye(r.reason), + Ok(Some(Message::Hello(_))) => break EndReason::ClientRestarted, Ok(Some(other)) => tracing::debug!(?other, "ignoring message"), Ok(None) => break EndReason::ClientEof, Err(e) => break EndReason::LinkError(e.to_string()), @@ -1047,13 +1061,28 @@ fn audio_loop( tx: &MuxSender, sh: &VideoShared, poll: Duration, + volumes: &std::sync::mpsc::Receiver, ) -> Result<(), SourceError> { source.start(stream.channels)?; + // The last volume applied from or sent to the client: a change equal to it is an echo. + let mut last_volume: Option = None; let frame_len = stream.frame_samples as usize * stream.channels as usize; let frame_us = stream.frame_samples as u64 * 1_000_000 / stream.sample_rate.max(1) as u64; let mut pending: Vec = Vec::with_capacity(frame_len * 2); let mut next_pts = 0u64; while !sh.stop.load(Ordering::Acquire) { + while let Ok(v) = volumes.try_recv() { + last_volume = Some(v); + source.set_volume(v); + } + if let Some(v) = source.take_volume_change() + && !last_volume.is_some_and(|l| l.is_same(&v)) + { + last_volume = Some(v); + if tx.send_control(Message::AudioVolume(v)).is_err() { + return Ok(()); + } + } let Some(block) = source.next_samples(poll)? else { continue; }; @@ -1296,6 +1325,44 @@ mod tests { )); } + #[tokio::test] + async fn a_new_hello_ends_the_session_as_a_restart() { + let (host_link, client_link) = Link::duplex_pair(1 << 20); + let src = ScriptedSource::new(Duration::from_millis(5), 3000); + let mut host = tokio::spawn(run_host_session_with( + host_link, + Box::new(src), + Prefs::default(), + fast_opts(), + )); + let (tx, mut rx, _t) = client_link.into_mux(DEFAULT_CHUNK_LEN); + tx.send_control(Message::Hello(Hello::new(Role::Client, "t", "c"))) + .unwrap(); + tx.send_control(Message::ClientCaps(caps())).unwrap(); + tx.send_control(Message::ClientHdr(ClientHdr::none())) + .unwrap(); + tx.send_control(Message::ClientAudio(ClientAudio::none())) + .unwrap(); + while !matches!(recv(&mut rx).await, Message::StreamConfig(_)) {} + tx.send_control(Message::ConfigAck).unwrap(); + // The app restarts and opens the same link again. + tx.send_control(Message::Hello(Hello::new(Role::Client, "t", "c"))) + .unwrap(); + let s = timeout(Duration::from_secs(5), async { + let mut open = true; + loop { + tokio::select! { + r = &mut host => break r.unwrap().unwrap(), + // Keep reading so the host is never blocked on a full link. + m = rx.recv(), if open => open = matches!(m, Ok(Some(_))), + } + } + }) + .await + .unwrap(); + assert_eq!(s.end, EndReason::ClientRestarted); + } + #[tokio::test] async fn no_common_codec_is_rejected() { let (host_link, client_link) = Link::duplex_pair(1 << 16); @@ -1797,6 +1864,84 @@ mod tests { }) } + #[tokio::test] + async fn volume_is_synced_both_ways_without_echoes() { + let backend = ScriptedAudioBackend::new(); + let volume = backend.volume.clone(); + let (host_link, client_link) = Link::duplex_pair(1 << 20); + let host = tokio::spawn(run_host_session_with( + host_link, + Box::new(ScriptedSource::new(Duration::from_millis(5), 3000)), + Prefs::default(), + SessionOptions { + audio: Some(Arc::new(backend)), + ..fast_opts() + }, + )); + let (tx, mut rx, _t) = client_link.into_mux(DEFAULT_CHUNK_LEN); + tx.send_control(Message::Hello(Hello::new(Role::Client, "t", "client"))) + .unwrap(); + tx.send_control(Message::ClientCaps(caps())).unwrap(); + tx.send_control(Message::ClientHdr(ClientHdr::none())) + .unwrap(); + tx.send_control(Message::ClientAudio(pcm_client())).unwrap(); + loop { + if matches!(recv(&mut rx).await, Message::StreamConfig(_)) { + break; + } + } + tx.send_control(Message::ConfigAck).unwrap(); + + // Client to host: applied once to the source, not sent back. + let from_client = AudioVolume { + volume: 0.4, + muted: false, + }; + tx.send_control(Message::AudioVolume(from_client)).unwrap(); + // Host to client: a change at the host's control. + let from_host = AudioVolume { + volume: 0.8, + muted: true, + }; + let mut received = Vec::new(); + let start = Instant::now(); + let mut set_local = false; + while start.elapsed() < Duration::from_secs(5) { + if volume.applied().len() == 1 && !set_local { + set_local = true; + volume.set_local(from_host); + } + match recv(&mut rx).await { + Message::AudioVolume(v) => received.push(v), + Message::VideoFrame(v) => { + tx.send_control(Message::FrameAck(ezd_proto::FrameAck { + frame_id: v.frame_id, + received_us: now_us(), + })) + .unwrap(); + } + _ => {} + } + if !received.is_empty() && start.elapsed() > Duration::from_millis(300) { + break; + } + } + assert_eq!(volume.applied(), vec![from_client]); + // Only the host's own change comes back; the client's value was not echoed. + assert_eq!(received, vec![from_host]); + // The client's echo of the host value is applied but nothing more is sent. + tx.send_control(Message::AudioVolume(from_host)).unwrap(); + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!(volume.applied(), vec![from_client, from_host]); + tx.send_control(Message::Bye(Bye { reason: "t".into() })) + .unwrap(); + timeout(Duration::from_secs(5), host) + .await + .unwrap() + .unwrap() + .unwrap(); + } + #[tokio::test] async fn pcm_audio_is_configured_before_the_stream_and_packets_have_whole_frames() { // USB link, client plays F32: 480 samples * 2 channels * 4 bytes. diff --git a/crates/ezd-host-core/src/source.rs b/crates/ezd-host-core/src/source.rs index d1088fb..0892a04 100644 --- a/crates/ezd-host-core/src/source.rs +++ b/crates/ezd-host-core/src/source.rs @@ -2,8 +2,8 @@ use ezd_link::{MuxClosed, MuxSender}; use ezd_proto::{ - Codec, CursorShape, CursorState, DynamicRange, EncoderCaps, Mastering, StreamColor, - StreamConfig, + AudioVolume, Codec, CursorShape, CursorState, DynamicRange, EncoderCaps, Mastering, + StreamColor, StreamConfig, }; use std::collections::HashSet; use std::sync::{Arc, Mutex}; @@ -225,6 +225,18 @@ pub trait AudioSource: Send { /// Stops capture and releases resources. Also called if `start` failed. fn stop(&mut self); + + /// The output volume if it changed since the last call (the user moved a slider on this + /// host), which the session forwards to the client. Polled by the audio thread on every + /// iteration, so it must not block. Default: no volume control. The session drops values + /// that merely echo the one it applied through [`set_volume`](Self::set_volume). + fn take_volume_change(&mut self) -> Option { + None + } + + /// Applies the volume of the client. The captured samples stay at full level; this only + /// moves the host's own control. Default: ignored. + fn set_volume(&mut self, _volume: AudioVolume) {} } /// An Opus encoder for one audio stream (48 kHz, the negotiated channel count, 10 ms frames). diff --git a/crates/ezd-host-core/src/testing.rs b/crates/ezd-host-core/src/testing.rs index 16370c2..3d728d3 100644 --- a/crates/ezd-host-core/src/testing.rs +++ b/crates/ezd-host-core/src/testing.rs @@ -7,8 +7,8 @@ use crate::source::{ }; use crate::time::now_us; use ezd_proto::{ - Codec, CursorShape, CursorState, EncoderCaps, InputStatus, Key, PenSample, PointerButton, - Scroll, StreamConfig, TouchFrame, + AudioVolume, Codec, CursorShape, CursorState, EncoderCaps, InputStatus, Key, PenSample, + PointerButton, Scroll, StreamConfig, TouchFrame, }; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -175,6 +175,33 @@ pub struct ScriptedAudioBackend { pub fail_opus: bool, /// Samples per channel in each block (default 700). pub block_samples: usize, + /// The volume control shared by every source this backend creates. + pub volume: ScriptedVolume, +} + +/// The volume control of the scripted sources, shared with the test. +/// +/// Like a real mixer it reports a change for every value it is given, including the ones set +/// through `set_volume` (the session must not send those back). +#[derive(Debug, Clone, Default)] +pub struct ScriptedVolume(Arc>); + +#[derive(Debug, Default)] +struct VolumeState { + applied: Vec, + pending: Option, +} + +impl ScriptedVolume { + /// Pretends the user moved the host's volume control. + pub fn set_local(&self, v: AudioVolume) { + self.0.lock().unwrap().pending = Some(v); + } + + /// Every value the session applied with `set_volume`, in order. + pub fn applied(&self) -> Vec { + self.0.lock().unwrap().applied.clone() + } } impl ScriptedAudioBackend { @@ -186,6 +213,7 @@ impl ScriptedAudioBackend { opus: true, fail_opus: false, block_samples: 700, + volume: ScriptedVolume::default(), } } @@ -209,6 +237,7 @@ impl AudioBackend for ScriptedAudioBackend { n: 0, block_samples: self.block_samples, fail_start: self.fail_start, + volume: self.volume.clone(), }) as Box }) } @@ -230,6 +259,7 @@ struct ScriptedAudio { n: u64, block_samples: usize, fail_start: bool, + volume: ScriptedVolume, } impl AudioSource for ScriptedAudio { @@ -254,6 +284,14 @@ impl AudioSource for ScriptedAudio { })) } fn stop(&mut self) {} + fn take_volume_change(&mut self) -> Option { + self.volume.0.lock().unwrap().pending.take() + } + fn set_volume(&mut self, v: AudioVolume) { + let mut st = self.volume.0.lock().unwrap(); + st.applied.push(v); + st.pending = Some(v); + } } struct ScriptedOpus { diff --git a/crates/ezd-host-linux/src/audio.rs b/crates/ezd-host-linux/src/audio.rs index d3d581b..735fa80 100644 --- a/crates/ezd-host-linux/src/audio.rs +++ b/crates/ezd-host-linux/src/audio.rs @@ -10,15 +10,22 @@ //! 3. captures the sink's monitor with a stream and hands the samples to `next_samples` through a //! bounded channel. //! +//! The sink's volume is a control value only: the monitor is captured before the volume is applied +//! (`monitor.channel-volumes` is left off), so the stream always carries full-level audio. The +//! thread watches the sink's `Props` and reports `channelVolumes`/`mute` through +//! `take_volume_change`; `set_volume` writes them back (the PC then starts at the tablet's volume). +//! //! Nothing touches PipeWire before `start`. use ezd_host_core::{AudioBlock, AudioSource, SourceError, now_us}; +use ezd_proto::AudioVolume; use pipewire as pw; use pw::properties::properties; use pw::proxy::ProxyT; use pw::spa::pod::Pod; use pw::spa::sys as spa_sys; use std::cell::{Cell, RefCell}; +use std::collections::HashMap; use std::rc::Rc; use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender, TryRecvError, sync_channel}; use std::sync::{Arc, Mutex}; @@ -31,6 +38,8 @@ const SAMPLE_RATE: u32 = 48_000; /// Metadata key that holds the user's chosen default output. const DEFAULT_SINK_KEY: &str = "default.configured.audio.sink"; const JSON_TYPE: &str = "Spa:String:JSON"; +/// `node.latency` of the sink and the capture stream: 480 samples, 10 ms like our packets. +const NODE_LATENCY: &str = "480/48000"; /// Blocks queued between the PipeWire thread and `next_samples` before new ones are dropped. const QUEUE_BLOCKS: usize = 50; /// How long `start` waits for the sink and the stream to come up. @@ -39,6 +48,10 @@ const START_TIMEOUT: Duration = Duration::from_secs(3); const STOP_TIMEOUT: Duration = Duration::from_secs(2); /// If the daemon never answers the sync after a failure, the loop quits after this. const STOP_SYNC_TIMEOUT: Duration = Duration::from_millis(1000); +/// How long `stop` waits for the playback streams to leave the sink after the default is restored. +const DRAIN_TIMEOUT: Duration = Duration::from_secs(2); +/// How often the drain checks whether the streams have left. +const DRAIN_POLL: Duration = Duration::from_millis(50); /// The capture source returned by [`audio_source`]. pub struct PipeWireAudio { @@ -52,10 +65,13 @@ struct Running { /// Signalled by the thread right before it ends. exited: Receiver<()>, error: Arc>>, + /// The latest volume change seen on the sink, until `take_volume_change` takes it. + volume: Arc>>, } enum Cmd { Stop, + SetVolume(AudioVolume), } /// Creates the (idle) source; PipeWire is only contacted by `start`. @@ -72,12 +88,13 @@ impl AudioSource for PipeWireAudio { let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); let (exit_tx, exit_rx) = std::sync::mpsc::channel::<()>(); let error = Arc::new(Mutex::new(None)); + let volume = Arc::new(Mutex::new(None)); let thread = { - let error = error.clone(); + let (error, volume) = (error.clone(), volume.clone()); std::thread::Builder::new() .name("ezd-audio-pw".into()) .spawn(move || { - if let Err(e) = run(channels, block_tx, cmd_rx, &ready_tx) { + if let Err(e) = run(channels, block_tx, cmd_rx, &ready_tx, &volume) { tracing::warn!("audio capture failed: {e}"); let _ = ready_tx.send(Err(e.clone())); *error.lock().unwrap_or_else(|e| e.into_inner()) = Some(e); @@ -92,6 +109,7 @@ impl AudioSource for PipeWireAudio { thread, exited: exit_rx, error, + volume, }); match ready_rx.recv_timeout(START_TIMEOUT) { Ok(Ok(())) => Ok(()), @@ -124,6 +142,17 @@ impl AudioSource for PipeWireAudio { } } + fn take_volume_change(&mut self) -> Option { + let r = self.running.as_ref()?; + r.volume.lock().unwrap_or_else(|e| e.into_inner()).take() + } + + fn set_volume(&mut self, volume: AudioVolume) { + if let Some(r) = &self.running { + let _ = r.tx.send(Cmd::SetVolume(volume)); + } + } + fn stop(&mut self) { let Some(r) = self.running.take() else { return; @@ -169,6 +198,108 @@ fn position_ids(channels: u8) -> [u32; spa_sys::SPA_AUDIO_MAX_CHANNELS as usize] p } +/// Linear (mixer) gain to the perceptual 0..=1 value a volume slider shows (the cube-root +/// convention of PulseAudio and pavucontrol). Boosts above 100 % clamp to 1. +fn linear_to_perceptual(linear: f32) -> f32 { + if linear.is_nan() { + return 0.0; + } + linear.clamp(0.0, 1.0).cbrt() +} + +/// Inverse of [`linear_to_perceptual`]. +fn perceptual_to_linear(perceptual: f32) -> f32 { + if perceptual.is_nan() { + return 0.0; + } + perceptual.clamp(0.0, 1.0).powi(3) +} + +/// What a `Props` param said about the volume; a field is `None` when the param left it out. +#[derive(Debug, Default, PartialEq)] +struct PropsInfo { + /// Loudest channel, linear. + linear: Option, + mute: Option, +} + +/// Reads `channelVolumes` and `mute` out of a serialized `Props` object. +fn parse_props(bytes: &[u8]) -> Option { + use pw::spa::pod::{Value, ValueArray, deserialize::PodDeserializer}; + let (_, Value::Object(obj)) = PodDeserializer::deserialize_any_from(bytes).ok()? else { + return None; + }; + let mut info = PropsInfo::default(); + for p in obj.properties { + match (p.key, p.value) { + (spa_sys::SPA_PROP_channelVolumes, Value::ValueArray(ValueArray::Float(v))) => { + info.linear = v.into_iter().reduce(f32::max); + } + (spa_sys::SPA_PROP_mute, Value::Bool(m)) => info.mute = Some(m), + _ => {} + } + } + Some(info) +} + +/// A serialized `Props` object that sets the volume of every channel and the mute flag. +fn props_pod(volume: AudioVolume, channels: u8) -> Result, String> { + use pw::spa::pod::{Object, Property, Value, ValueArray, serialize::PodSerializer}; + let linear = perceptual_to_linear(volume.volume); + let obj = Value::Object(Object { + type_: spa_sys::SPA_TYPE_OBJECT_Props, + id: spa_sys::SPA_PARAM_Props, + properties: vec![ + Property::new( + spa_sys::SPA_PROP_channelVolumes, + Value::ValueArray(ValueArray::Float(vec![linear; channels.max(1) as usize])), + ), + Property::new(spa_sys::SPA_PROP_mute, Value::Bool(volume.muted)), + ], + }); + PodSerializer::serialize(std::io::Cursor::new(Vec::new()), &obj) + .map(|(c, _)| c.into_inner()) + .map_err(|e| format!("pipewire volume pod: {e:?}")) +} + +/// Turns the `Props` events of the sink into volume changes. +/// +/// The first event is the state the sink had when we subscribed, not a change by the user, so it +/// is remembered without being reported (the tablet's volume must not be overwritten by it). +#[derive(Default)] +struct VolumeTracker { + last: Option, +} + +impl VolumeTracker { + /// The new volume when `info` changes it. + fn observe(&mut self, info: PropsInfo) -> Option { + let known = self.last.unwrap_or(AudioVolume { + volume: 1.0, + muted: false, + }); + let now = AudioVolume { + volume: info.linear.map_or(known.volume, linear_to_perceptual), + muted: info.mute.unwrap_or(known.muted), + }; + let first = self.last.is_none(); + let changed = !known.is_same(&now); + self.last = Some(now); + (!first && changed).then_some(now) + } +} + +/// The `link.input.node` property of a Link global: the node the link feeds. +fn link_input_node(value: Option<&str>) -> Option { + value?.trim().parse().ok() +} + +/// Number of links that feed `sink` (playback streams into it). Links that start at the sink +/// (our own monitor capture) do not count. +fn links_into(links: &HashMap>, sink: u32) -> usize { + links.values().filter(|i| **i == Some(sink)).count() +} + /// `{"name":"x"}` as stored in the `default.configured.audio.sink` metadata. fn sink_json(name: &str) -> String { serde_json::json!({ "name": name }).to_string() @@ -235,6 +366,7 @@ fn run( blocks: SyncSender, cmds: pw::channel::Receiver, ready: &std::sync::mpsc::Sender>, + volume_out: &Arc>>, ) -> Result<(), String> { let e = |what: &str, err: pw::Error| format!("pipewire {what}: {err}"); pw::init(); @@ -284,11 +416,14 @@ fn run( "media.class" => "Audio/Sink", "object.linger" => "false", "audio.position" => position_prop(channels), + // One quantum of our 10 ms packets; at the default 128 the capture node xruns under load. + *pw::keys::NODE_LATENCY => NODE_LATENCY, }; sink_props.insert("audio.rate", SAMPLE_RATE.to_string()); - let sink: pw::node::Node = core - .create_object("adapter", &sink_props) - .map_err(|x| e("create sink", x))?; + let sink: Rc = Rc::new( + core.create_object("adapter", &sink_props) + .map_err(|x| e("create sink", x))?, + ); let _sink_listener = { let (ml, failure) = (mainloop.clone(), failure.clone()); sink.upcast_ref() @@ -299,17 +434,51 @@ fn run( }) .register() }; + // The global id of our sink, from its own info: matching the registry by name could pick + // the sink of a previous session that the daemon has not removed yet (ids are reused). + let sink_global: Rc>> = Rc::new(Cell::new(None)); + // The volume: report every change of the sink's Props. + let _sink_volume_listener = { + let (out, tracker, global) = ( + volume_out.clone(), + Rc::new(RefCell::new(VolumeTracker::default())), + sink_global.clone(), + ); + let l = sink + .add_listener_local() + .info(move |info| global.set(Some(info.id()))) + .param(move |_seq, id, _index, _next, param| { + if id != pw::spa::param::ParamType::Props { + return; + } + let Some(info) = param.and_then(|p| parse_props(p.as_bytes())) else { + return; + }; + if let Some(v) = tracker.borrow_mut().observe(info) { + *out.lock().unwrap_or_else(|e| e.into_inner()) = Some(v); + } + }) + .register(); + sink.subscribe_params(&[pw::spa::param::ParamType::Props]); + l + }; roundtrip()?; // 2. The default output. let default_sink = Rc::new(RefCell::new(DefaultSink::default())); let registry = core.get_registry_rc().map_err(|x| e("registry", x))?; let meta_found: Rc>> = Rc::new(RefCell::new(None)); - let reg_listener = { - let meta_found = meta_found.clone(); + // Links by id with the node they feed. + let links: Rc>>> = Rc::new(RefCell::new(HashMap::new())); + let _reg_listener = { + let (meta_found, links, links_rm) = (meta_found.clone(), links.clone(), links.clone()); registry .add_listener_local() .global(move |g| { + if g.type_ == pw::types::ObjectType::Link { + let input = link_input_node(g.props.and_then(|p| p.get("link.input.node"))); + links.borrow_mut().insert(g.id, input); + } if g.type_ == pw::types::ObjectType::Metadata && g.props .is_some_and(|p| p.get("metadata.name") == Some("default")) @@ -317,6 +486,9 @@ fn run( meta_found.borrow_mut().get_or_insert(g.id); } }) + .global_remove(move |id| { + links_rm.borrow_mut().remove(&id); + }) .register() }; roundtrip()?; @@ -354,7 +526,6 @@ fn run( None } }; - drop(reg_listener); if let Some(m) = &meta { // The properties are replayed on bind; the roundtrip makes sure they arrived. roundtrip()?; @@ -377,7 +548,7 @@ fn run( *pw::keys::MEDIA_ROLE => "Music", "target.object" => SINK_NAME, *pw::keys::STREAM_CAPTURE_SINK => "true", - *pw::keys::NODE_LATENCY => "240/48000", + *pw::keys::NODE_LATENCY => NODE_LATENCY, *pw::keys::NODE_DONT_RECONNECT => "true", }; let stream = pw::stream::StreamRc::new(core.clone(), "ezdisplay audio", props) @@ -476,13 +647,28 @@ fn run( core.clone(), ); let meta = meta.clone(); - cmds.attach(mainloop.loop_(), move |Cmd::Stop| { - restore_default(meta.as_deref(), &mut ds.borrow_mut()); - match core.sync(0) { - Ok(seq) => { - wait_seq.set(Some(seq.seq())); + let sink = Rc::downgrade(&sink); + cmds.attach(mainloop.loop_(), move |cmd| match cmd { + Cmd::Stop => { + restore_default(meta.as_deref(), &mut ds.borrow_mut()); + match core.sync(0) { + Ok(seq) => { + wait_seq.set(Some(seq.seq())); + } + Err(_) => ml.quit(), + } + } + Cmd::SetVolume(v) => { + let Some(sink) = sink.upgrade() else { + return; + }; + match props_pod(v, channels) { + Ok(bytes) => match Pod::from_bytes(&bytes) { + Some(pod) => sink.set_param(pw::spa::param::ParamType::Props, 0, pod), + None => tracing::warn!("invalid volume pod"), + }, + Err(e) => tracing::warn!("{e}"), } - Err(_) => ml.quit(), } }) }; @@ -498,6 +684,10 @@ fn run( let _ = timer.update_timer(Some(STOP_SYNC_TIMEOUT), None); mainloop.run(); } + // The daemon has the default back, but WirePlumber moves the playback streams off our sink + // asynchronously; removing the sink first yanks them and stalls players. Wait for them. + let sink_id = sink_global.get().unwrap_or_else(|| sink.upcast_ref().id()); + drain_links(&mainloop, &links, sink_id); drop(stream_listener); let _ = stream.disconnect(); drop(sink); @@ -507,6 +697,46 @@ fn run( } } +/// Runs the main loop until no playback link feeds `sink` any more, or [`DRAIN_TIMEOUT`]. +fn drain_links( + mainloop: &pw::main_loop::MainLoopRc, + links: &Rc>>>, + sink: u32, +) { + let before = links_into(&links.borrow(), sink); + if before == 0 { + tracing::debug!("no playback stream on the audio sink ({sink})"); + return; + } + let start = std::time::Instant::now(); + let timed_out = Rc::new(Cell::new(false)); + let timer = { + let (ml, links, timed_out) = (mainloop.clone(), links.clone(), timed_out.clone()); + mainloop.loop_().add_timer(move |_| { + if links_into(&links.borrow(), sink) == 0 { + ml.quit(); + } else if start.elapsed() >= DRAIN_TIMEOUT { + timed_out.set(true); + ml.quit(); + } + }) + }; + let _ = timer.update_timer(Some(DRAIN_POLL), Some(DRAIN_POLL)); + mainloop.run(); + let left = links_into(&links.borrow(), sink); + if timed_out.get() { + tracing::warn!( + "{left} playback stream(s) still linked to the audio sink after {:?}", + start.elapsed() + ); + } else { + tracing::debug!( + "{before} playback link(s) left the audio sink in {:?}", + start.elapsed() + ); + } +} + #[cfg(test)] mod tests { use super::*; @@ -555,6 +785,107 @@ mod tests { assert_eq!(position_ids(1)[0], spa_sys::SPA_AUDIO_CHANNEL_MONO); } + #[test] + fn volume_conversion_is_cubic_and_clamped() { + assert_eq!(linear_to_perceptual(0.0), 0.0); + assert_eq!(linear_to_perceptual(1.0), 1.0); + assert!((linear_to_perceptual(0.125) - 0.5).abs() < 1e-6); + assert_eq!(linear_to_perceptual(1.5), 1.0, "boost clamps"); + assert_eq!(linear_to_perceptual(-1.0), 0.0); + assert_eq!(linear_to_perceptual(f32::NAN), 0.0); + assert!((perceptual_to_linear(0.5) - 0.125).abs() < 1e-6); + assert_eq!(perceptual_to_linear(2.0), 1.0); + for i in 0..=100 { + let p = i as f32 / 100.0; + assert!((linear_to_perceptual(perceptual_to_linear(p)) - p).abs() < 1e-5); + } + } + + #[test] + fn props_pod_round_trips_through_the_parser() { + let v = AudioVolume { + volume: 0.5, + muted: true, + }; + let bytes = props_pod(v, 2).unwrap(); + let info = parse_props(&bytes).unwrap(); + assert_eq!(info.mute, Some(true)); + assert!((info.linear.unwrap() - 0.125).abs() < 1e-6); + assert!(Pod::from_bytes(&bytes).is_some()); + let mono = parse_props(&props_pod(v, 1).unwrap()).unwrap(); + assert_eq!(mono.linear, info.linear); + assert_eq!(parse_props(b"junk"), None); + } + + #[test] + fn props_without_volume_fields_parse_to_none() { + use pw::spa::pod::{Object, Property, Value, serialize::PodSerializer}; + let obj = Value::Object(Object { + type_: spa_sys::SPA_TYPE_OBJECT_Props, + id: spa_sys::SPA_PARAM_Props, + properties: vec![Property::new(spa_sys::SPA_PROP_mute, Value::Bool(false))], + }); + let bytes = PodSerializer::serialize(std::io::Cursor::new(Vec::new()), &obj) + .unwrap() + .0 + .into_inner(); + assert_eq!( + parse_props(&bytes), + Some(PropsInfo { + linear: None, + mute: Some(false) + }) + ); + } + + #[test] + fn tracker_skips_the_initial_state_and_reports_changes() { + let info = |linear, mute| PropsInfo { + linear: Some(linear), + mute: Some(mute), + }; + let mut t = VolumeTracker::default(); + // The state at subscription time is not a user change. + assert_eq!(t.observe(info(1.0, false)), None); + assert_eq!(t.observe(info(1.0, false)), None, "unchanged"); + let v = t.observe(info(0.125, false)).unwrap(); + assert!((v.volume - 0.5).abs() < 1e-6 && !v.muted); + assert_eq!(t.observe(info(0.125, false)), None); + // A mute toggle keeps the volume; a Props event without fields changes nothing. + let m = t.observe(info(0.125, true)).unwrap(); + assert!((m.volume - 0.5).abs() < 1e-6 && m.muted); + assert_eq!(t.observe(PropsInfo::default()), None); + let u = t + .observe(PropsInfo { + linear: None, + mute: Some(false), + }) + .unwrap(); + assert!(!u.muted && (u.volume - 0.5).abs() < 1e-6); + } + + #[test] + fn link_input_node_parses_the_property() { + assert_eq!(link_input_node(Some("42")), Some(42)); + assert_eq!(link_input_node(Some(" 7 ")), Some(7)); + assert_eq!(link_input_node(Some("x")), None); + assert_eq!(link_input_node(None), None); + } + + #[test] + fn only_links_into_the_sink_count() { + let mut links = HashMap::new(); + links.insert(1, Some(10)); // a playback stream into the sink + links.insert(2, Some(10)); + links.insert(3, Some(20)); // our monitor capture feeds another node + links.insert(4, None); + assert_eq!(links_into(&links, 10), 2); + links.remove(&1); + links.remove(&2); + assert_eq!(links_into(&links, 10), 0); + assert_eq!(links_into(&links, 20), 1); + } + #[test] fn block_pts_subtracts_the_block_duration() { // 480 stereo frames are 10 ms. diff --git a/crates/ezd-host-service/src/aoa.rs b/crates/ezd-host-service/src/aoa.rs index 7e5b64e..2ce8c58 100644 --- a/crates/ezd-host-service/src/aoa.rs +++ b/crates/ezd-host-service/src/aoa.rs @@ -435,8 +435,9 @@ async fn start_session( }); if stream_session(inner, cfg, link, Transport::Aoa, stop_rx).await { // The app is still in a session with an earlier host process and cannot tell that it - // ended (an accessory never reports the host closing it). A reset makes it see the cable - // go away; the device then comes back in its normal mode and is switched again. + // ended (an accessory never reports the host closing it), or it restarted mid-session. + // A reset makes it see the cable go away; the device then comes back in its normal mode + // and is switched again. tracing::info!( "{} is still in an old session: resetting it", acc.describe() diff --git a/crates/ezd-host-service/src/service.rs b/crates/ezd-host-service/src/service.rs index b9d33dd..7082323 100644 --- a/crates/ezd-host-service/src/service.rs +++ b/crates/ezd-host-service/src/service.rs @@ -505,8 +505,9 @@ pub(crate) async fn stream_session( finish_session(inner, cfg, link, transport, source, stop_rx).await } -/// Runs a session with an existing source and reports its end. True when the peer opened with -/// something other than its handshake: a client still in a session with an earlier host process. +/// Runs a session with an existing source and reports its end. True when the client is in a +/// stale session: it opened with something other than its handshake (still in a session with an +/// earlier host process), or sent a new `Hello` mid-session (its app restarted on the same link). pub(crate) async fn finish_session( inner: &Inner, cfg: &ServiceConfig, @@ -521,7 +522,11 @@ pub(crate) async fn finish_session( Ok(s) => tracing::debug!(end = ?s.end, "session returned"), Err(e) => tracing::debug!("session failed: {e}"), } - let stale = matches!(result, Err(ezd_host_core::SessionError::Unexpected { .. })); + let stale = match &result { + Err(ezd_host_core::SessionError::Unexpected { .. }) => true, + Ok(s) => s.end == ezd_host_core::EndReason::ClientRestarted, + Err(_) => false, + }; inner.emit(HostEvent::SessionEnded(result.map_err(|e| e.to_string()))); stale } diff --git a/crates/ezd-proto/src/msg.rs b/crates/ezd-proto/src/msg.rs index 057dd8e..1f91f19 100644 --- a/crates/ezd-proto/src/msg.rs +++ b/crates/ezd-proto/src/msg.rs @@ -718,6 +718,31 @@ pub struct AudioConfig { pub stream: Option, } +/// Volume of the audio output (Control/20, both directions, protocol 0.7), sent only when both +/// sides speak minor >= [`MINOR_AUDIO`](crate::MINOR_AUDIO) and the session has audio. +/// +/// The client sends its current value right after it receives a non-off [`AudioConfig`]; after +/// that, whichever side changes its volume sends the new one. A receiver applies it without +/// sending it back. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct AudioVolume { + /// Perceptual volume, 0.0 (silent) to 1.0 (full), what a volume slider shows. + pub volume: f32, + /// Muted, independent of `volume`. + pub muted: bool, +} + +impl AudioVolume { + /// Two volumes closer than this (with the same mute flag) count as equal when suppressing + /// echoes: the round trip through a linear mixer or an integer step loses a little. + pub const EPSILON: f32 = 0.005; + + /// True when `other` is the same volume for echo suppression. + pub fn is_same(&self, other: &AudioVolume) -> bool { + self.muted == other.muted && (self.volume - other.volume).abs() < Self::EPSILON + } +} + /// One audio packet (host to client, Audio/1, protocol 0.7). /// /// `data` is exactly one frame: `frame_samples * channels` interleaved samples for PCM, one Opus @@ -812,7 +837,7 @@ impl Default for StreamColor { /// Any message of protocol 0.x (0.2 added `FrameAck`, 0.3 the input messages, /// `InputStatus` and `ClientOptions`, 0.5 `ClientHdr` and `StreamColor`, 0.6 `ClientKeyboard`, -/// 0.7 audio). +/// 0.7 audio and `AudioVolume`). #[derive(Debug, Clone, PartialEq)] #[allow(clippy::large_enum_variant)] pub enum Message { @@ -854,6 +879,8 @@ pub enum Message { ClientAudio(ClientAudio), /// Control/19 (0.7), host to client. AudioConfig(AudioConfig), + /// Control/20 (0.7), both directions. + AudioVolume(AudioVolume), /// Input/1 (0.3), client to host. Pen(PenSample), /// Input/2 (0.3), client to host. @@ -912,6 +939,7 @@ impl Message { Message::ClientKeyboard(_) => 17, Message::ClientAudio(_) => 18, Message::AudioConfig(_) => 19, + Message::AudioVolume(_) => 20, Message::Pen(_) => 1, Message::Touch(_) => 2, Message::PointerButton(_) => 3, @@ -951,6 +979,7 @@ pub fn encode_message(msg: &Message) -> Result<(Channel, u8, Vec)> { Message::ClientKeyboard(m) => enc(m)?, Message::ClientAudio(m) => enc(m)?, Message::AudioConfig(m) => enc(m)?, + Message::AudioVolume(m) => enc(m)?, Message::Pen(m) => enc(m)?, Message::Touch(m) => enc(m)?, Message::PointerButton(m) => enc(m)?, @@ -1000,6 +1029,7 @@ pub fn decode_message(channel: u8, kind: u8, payload: &[u8]) -> Result Message::ClientKeyboard(dec(c, k, payload)?), (Channel::Control, 18) => Message::ClientAudio(dec(c, k, payload)?), (Channel::Control, 19) => Message::AudioConfig(dec(c, k, payload)?), + (Channel::Control, 20) => Message::AudioVolume(dec(c, k, payload)?), (Channel::Input, 1) => Message::Pen(dec(c, k, payload)?), (Channel::Input, 2) => Message::Touch(dec(c, k, payload)?), (Channel::Input, 3) => Message::PointerButton(dec(c, k, payload)?), @@ -1136,6 +1166,10 @@ mod tests { codec_config: b"OpusHead".to_vec(), }), }), + Message::AudioVolume(AudioVolume { + volume: 0.65, + muted: true, + }), Message::Pen(PenSample { t_us: 1 << 40, x: 65535, @@ -1373,6 +1407,17 @@ mod tests { assert_eq!(decode_message(ch as u8, kind, &p).unwrap(), Some(none)); } + #[test] + fn audio_volume_echo_comparison() { + let v = AudioVolume { + volume: 0.5, + muted: false, + }; + assert!(v.is_same(&AudioVolume { volume: 0.503, ..v })); + assert!(!v.is_same(&AudioVolume { volume: 0.51, ..v })); + assert!(!v.is_same(&AudioVolume { muted: true, ..v })); + } + #[test] fn stream_color_presets() { let sdr = StreamColor::sdr_bt709(); diff --git a/docs/protocol.md b/docs/protocol.md index b78d052..e62d573 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -59,6 +59,7 @@ flowchart LR | Control | 17 | `ClientKeyboard` | client to host | postcard (0.6) | | Control | 18 | `ClientAudio` | client to host | postcard (0.7) | | Control | 19 | `AudioConfig` | host to client | postcard (0.7) | +| Control | 20 | `AudioVolume` | both directions | postcard (0.7) | | Input | 1 | `Pen` (`PenSample`) | client to host | postcard (0.3) | | Input | 2 | `Touch` (`TouchFrame`) | client to host | postcard (0.3) | | Input | 3 | `PointerButton` | client to host | postcard (0.3) | @@ -117,7 +118,7 @@ per push) and yields complete messages. `Hello` carries `proto_major` and `proto_minor` (currently 0 and 7; 0.2 added `FrameAck`, 0.3 the input messages, `InputStatus` and `ClientOptions`, 0.4 the `Key` message, 0.5 `ClientHdr` and -`StreamColor` for HDR10, 0.6 `ClientKeyboard`, 0.7 `ClientAudio`, `AudioConfig` and the Audio channel for +`StreamColor` for HDR10, 0.6 `ClientKeyboard`, 0.7 `ClientAudio`, `AudioConfig`, `AudioVolume` and the Audio channel for PC audio; the Wi-Fi transport is below the framed protocol and needs no minor bump). A different major version is fatal: the receiver answers `Reject` and closes. Minor versions only add new `(channel, kind)` pairs, so peers with different minors interoperate. `Hello::supports_input()` is true @@ -306,6 +307,10 @@ sequenceDiagram opt host minor ≥ 6 C->>H: ClientKeyboard(attached), again whenever it changes end + opt both minor ≥ 7 and AudioConfig carried a stream + C->>H: AudioVolume(volume, muted), first right after AudioConfig, then on every change + H->>C: AudioVolume(volume, muted), on every change of the PC volume + end par alongside video H->>C: CursorShape (once per shape_id), CursorState (latest wins) and @@ -514,6 +519,17 @@ is exactly `frame_samples * channels` interleaved samples for PCM (`frame_sample or `* 4` bytes) and one Opus packet for Opus. Audio is not acknowledged; audio packets carry no credit. If audio capture fails the host stops sending packets and the video session goes on. +Volume: `AudioVolume { volume: f32, muted: bool }` (Control/20) is sent in both directions, only +when both peers speak minor 7 and the session has audio. `volume` is perceptual, 0.0 to 1.0 (what a +volume slider shows; a host whose mixer is linear converts with a cube root), `muted` is separate. +It is a control value, not a gain applied to the stream: the host sends the audio at full level and +the client maps the value onto its own hardware volume. Right after the client receives an +`AudioConfig` with a stream it sends its current value, which the host applies to its output +(so the PC starts at the client's volume). After that, whichever side changes its volume sends the +new value. A receiver applies a received value without sending it back (it remembers the value +and ignores a local change within 0.005 of it with the same `muted`). A host without volume +control never sends it and ignores the client's. + Negotiation (`negotiate_audio`, mode `Auto` / `Pcm` / `Opus` / `Off`): audio is off when the mode is `Off`, the client is disabled or has no channels, or the host has no capture source. PCM needs the client to support `Pcm` and 48 kHz (and, in `Auto`, a link ceiling of at least 8 Mbit/s); its From abf6c8abfb638a084bdad1f16430b5d8d259e34e Mon Sep 17 00:00:00 2001 From: FacuM Date: Tue, 27 Oct 2026 00:27:32 -0300 Subject: [PATCH 11/12] Document PC audio on the tablet --- README.md | 20 +++++++++++++++++--- docs/architecture.md | 2 +- docs/phase0-findings.md | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1c7d82f..214c4c9 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ Working today, over USB with ADB: | Locally drawn cursor | ✅ | ✅ | 🧪 | 🧪 | 🧪 | | Zero-copy hardware encode | ✅ | ✅ | 🧪 | ❌ shared-memory capture | ❌ shared-memory capture | | Touch, pen pressure, tilt and hover | ✅ | ✅ | 🧪 | 🧪 | 🧪 | +| PC audio on the tablet, with volume kept in sync | 🧪 audio only, no volume sync | ✅ | 🧪 | 🧪 | 🧪 | | Tray app with settings and live statistics | ✅ | ✅ | 🧪 | 🧪 | 🧪 | | Installer / packages | ✅ `.exe` | ✅ AppImage, `.deb`, Arch | 🧪 AppImage, `.deb`, Arch | 🧪 AppImage, `.deb`, Arch | 🧪 AppImage, `.deb`, Arch | @@ -68,9 +69,8 @@ Working today, over USB with ADB: headless Mutter (the tray needs the AppIndicator extension there), Sway against a headless sway, Hyprland only through unit tests, and X11 against Xorg with the dummy video driver (i3, xfwm4). X11 works with any window manager or desktop that honours RandR 1.5 monitors; the proprietary -NVIDIA driver is untested. - -**Not yet:** audio. +NVIDIA driver is untested. Audio on Windows has only been tested through unit tests, and on Linux +only on KDE Plasma. ## How it works @@ -96,6 +96,20 @@ All channels share one prioritized, multiplexed link (ADB over USB today), so in updates are never stuck behind a video frame. Details: [docs/architecture.md](docs/architecture.md), wire format: [docs/protocol.md](docs/protocol.md), input path: [docs/input.md](docs/input.md). +## Audio + +The PC's sound plays on the tablet while a session runs. + +- **Linux:** the host adds a PipeWire output named `ezdisplay` and makes it the default for as long + as the session lasts, then puts the previous default back. Its volume and mute follow the tablet's + and the other way round. +- **Windows:** the host captures what the default output plays (WASAPI loopback). The PC keeps + playing it too, and the volume is not synced. +- Over a fast link the audio is uncompressed PCM, otherwise Opus. Choose with `--audio + auto|pcm|opus|off` or `audio = "..."` in `config.toml`, or in the tray settings. +- The tablet buffers 20 to 80 ms of audio, growing the buffer when playback stutters and shrinking + it again once it is steady. + ## USB without developer mode Besides adb, ezdisplay can talk to the tablet over USB with the Android Open Accessory protocol, diff --git a/docs/architecture.md b/docs/architecture.md index 3d56724..44f42a9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -129,6 +129,6 @@ flowchart TB 1. Vertical slice on both hosts: video and local cursor over ADB. 1a core → 1b Android, 1c Linux KDE, 1d Windows in parallel. 2. Input: touch, pen, S Pen button gestures. 3. Transports: Wi-Fi (QUIC with PIN or QR pairing, mDNS discovery) in progress; AOA (USB without developer mode, libusb) done. -4. Audio. +4. Audio: PC audio on the tablet as PCM or Opus (WASAPI loopback on Windows, a PipeWire output on Linux), with volume sync on Linux; done. 5. GNOME, wlroots and X11 backends. 6. Packaging: Windows installer (done, with the tray app), Linux packages (Arch, .deb, AppImage; done) and udev rules, Android `full`/`play` builds. diff --git a/docs/phase0-findings.md b/docs/phase0-findings.md index 3d0edbd..145e113 100644 --- a/docs/phase0-findings.md +++ b/docs/phase0-findings.md @@ -364,7 +364,7 @@ took 1.5-2 ms and the encode 1.7-2 ms per frame; on the UHD 630 they take 2-17 m With `export KWIN_DRM_DEVICES=/dev/dri/card1:/dev/dri/card2` in `~/.config/plasma-workspace/env/kwin-nvidia.sh` (card1 is the GTX 1650), KWin composites on the -NVIDIA GPU and the Intel GPU only drives the panel. The host then encodes with `nvenc-dmabuf`. +NVIDIA GPU and the Intel GPU only drives the panel. The host then encodes with `nvenc-dmabuf-rgb` (since FFmpeg 9 the frames stay RGB and NVENC converts them). Same benchmark, maximize and restore. NVIDIA busy is `utilization.gpu` from `nvidia-smi`, sampled every 100 ms over the whole run (mean / max): From 883ebbd4d9a6318e9c64d2dbd1a55c5806d30e16 Mon Sep 17 00:00:00 2001 From: FacuM Date: Tue, 27 Oct 2026 00:34:58 -0300 Subject: [PATCH 12/12] Link libm for the static libopus in the AppImage FFmpeg FFmpeg's configure links its libopus check with pkg-config's shared flags, which leave out libm, so the check failed on floorf and sqrtf. --- packaging/linux/appimage/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/linux/appimage/build.sh b/packaging/linux/appimage/build.sh index 2be6704..2364ca0 100755 --- a/packaging/linux/appimage/build.sh +++ b/packaging/linux/appimage/build.sh @@ -148,6 +148,8 @@ FFMPEG_CONFIGURE=( --enable-libglslang --extra-cflags="-I$SRC/vulkan-headers/include" --extra-ldflags="-static-libstdc++ -static-libgcc" + # The static libopus needs libm, which opus.pc lists only under Libs.private. + --extra-libs=-lm ) ( cd "$SRC/ffmpeg"