From 5d7192503c30f5b0b08b6b2ff6923125d96c07e5 Mon Sep 17 00:00:00 2001 From: kenny lopez Date: Tue, 4 Aug 2026 10:17:41 +0100 Subject: [PATCH 1/9] Polish Huddle voice controls Signed-off-by: kenny lopez --- desktop/src-tauri/src/huddle/commands.rs | 123 +++++++ desktop/src-tauri/src/huddle/mod.rs | 4 + desktop/src-tauri/src/huddle/pipeline.rs | 14 +- desktop/src-tauri/src/huddle/state.rs | 17 +- desktop/src-tauri/src/huddle/stt.rs | 62 ++-- desktop/src-tauri/src/lib.rs | 10 +- desktop/src/app/AppHuddleShell.tsx | 9 +- desktop/src/app/AppShell.tsx | 4 +- .../app/useAppShellDesktopNotifications.ts | 15 +- desktop/src/app/useHuddlePresentation.ts | 25 +- desktop/src/features/huddle/HuddleContext.tsx | 54 ++- .../features/huddle/HuddleContext.types.ts | 2 + .../huddle/components/AddAgentDialog.tsx | 166 +++++---- .../huddle/components/AgentVoiceMenu.tsx | 125 ++++--- .../features/huddle/components/HuddleBar.tsx | 54 ++- .../components/HuddleProfileControl.tsx | 2 - .../huddle/components/HuddleRoomHeader.tsx | 62 ++-- .../huddle/components/MicControls.tsx | 66 ++-- .../huddle/components/ParticipantList.tsx | 88 +++-- .../src/features/huddle/lib/audioWorklet.ts | 58 ++- .../features/huddle/lib/useHuddlePttState.ts | 6 +- desktop/src/features/notifications/hooks.ts | 2 + .../features/notifications/lib/sound.test.mjs | 18 + .../src/features/notifications/lib/sound.ts | 7 + .../use-feed-desktop-notifications.ts | 7 +- .../src/shared/styles/globals/components.css | 31 +- .../src/shared/ui/chooser-dialog-content.tsx | 12 +- desktop/src/testing/e2eBridge.ts | 42 ++- .../tests/e2e/huddle-transcription.spec.ts | 344 +++++++++++++++++- 29 files changed, 1085 insertions(+), 344 deletions(-) create mode 100644 desktop/src-tauri/src/huddle/commands.rs create mode 100644 desktop/src/features/notifications/lib/sound.test.mjs diff --git a/desktop/src-tauri/src/huddle/commands.rs b/desktop/src-tauri/src/huddle/commands.rs new file mode 100644 index 0000000000..61e71ee78f --- /dev/null +++ b/desktop/src-tauri/src/huddle/commands.rs @@ -0,0 +1,123 @@ +//! Small Huddle controls that mutate an active session. + +use std::sync::atomic::Ordering; + +use tauri::State; +use uuid::Uuid; + +use crate::{app_state::AppState, events, relay::submit_event}; + +use super::{relay_api::validate_pubkey_hex, HuddlePhase}; + +/// Update the clickable microphone control independently from the PTT shortcut. +#[tauri::command] +pub fn set_huddle_manual_mic_unmuted( + enabled: bool, + state: State<'_, AppState>, +) -> Result<(), String> { + let huddle = state.huddle()?; + if !matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active) { + return Err("no active huddle".to_string()); + } + huddle.manual_mic_unmuted.store(enabled, Ordering::Release); + Ok(()) +} + +/// Immediately interrupt the agent's current speech and discard queued TTS. +/// +/// This shares the barge-in cancellation path used when a participant starts +/// talking, so it stops both audio already handed to the player and response +/// text waiting to be synthesized. +#[tauri::command] +pub fn interrupt_huddle_speech(state: State<'_, AppState>) -> Result<(), String> { + let cancel = { + let huddle = state.huddle()?; + if !matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active) { + return Err("no active huddle".to_string()); + } + huddle.tts_cancel.clone() + }; + cancel.store(true, Ordering::Release); + Ok(()) +} + +/// Remove an agent from the active huddle without removing its parent-channel +/// membership. Keeping the parent membership intact means it remains available +/// to rejoin this huddle from the agent picker. +#[tauri::command] +pub async fn remove_agent_from_huddle( + agent_pubkey: String, + state: State<'_, AppState>, +) -> Result<(), String> { + validate_pubkey_hex(&agent_pubkey)?; + + let (ephemeral_channel_id, huddle_generation) = { + let huddle = state.huddle()?; + if !matches!(huddle.phase, HuddlePhase::Connected | HuddlePhase::Active) { + return Err("no active huddle".to_string()); + } + + let is_huddle_agent = huddle + .agent_pubkeys + .lock() + .unwrap_or_else(|error| error.into_inner()) + .iter() + .any(|pubkey| pubkey.eq_ignore_ascii_case(&agent_pubkey)); + if !is_huddle_agent { + return Err("agent is not in this huddle".to_string()); + } + + ( + huddle + .ephemeral_channel_id + .clone() + .ok_or("no ephemeral channel")?, + huddle.huddle_generation, + ) + }; + + let ephemeral_channel_uuid = + Uuid::parse_str(&ephemeral_channel_id).map_err(|error| error.to_string())?; + submit_event( + events::build_remove_member(ephemeral_channel_uuid, &agent_pubkey)?, + &state, + ) + .await?; + + let roster_changed = { + let mut huddle = state.huddle()?; + if !huddle.is_current_huddle(&ephemeral_channel_id, huddle_generation) { + false + } else { + let mut agent_pubkeys = huddle + .agent_pubkeys + .lock() + .unwrap_or_else(|error| error.into_inner()); + let initial_count = agent_pubkeys.len(); + agent_pubkeys.retain(|pubkey| !pubkey.eq_ignore_ascii_case(&agent_pubkey)); + let changed = agent_pubkeys.len() != initial_count; + drop(agent_pubkeys); + + if changed { + huddle + .participants + .retain(|pubkey| !pubkey.eq_ignore_ascii_case(&agent_pubkey)); + if let Some(settings_pubkey) = huddle + .agent_voice_settings + .keys() + .find(|pubkey| pubkey.eq_ignore_ascii_case(&agent_pubkey)) + .cloned() + { + huddle.agent_voice_settings.remove(&settings_pubkey); + } + } + changed + } + }; + + if roster_changed { + state.emit_huddle_state_changed(); + } + + Ok(()) +} diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 99337400c9..94a6179836 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -27,6 +27,7 @@ mod agent_tts_routing; pub mod agent_voice; pub mod agents; pub mod audio_output; +mod commands; pub mod jitter; pub mod models; pub mod pipeline; @@ -67,6 +68,9 @@ pub(super) fn drain_until_shutdown( // ── Re-exports ──────────────────────────────────────────────────────────────── +pub use commands::{ + interrupt_huddle_speech, remove_agent_from_huddle, set_huddle_manual_mic_unmuted, +}; pub use state::{HuddleJoinInfo, HuddlePhase, HuddleState, VoiceInputMode}; pub use transcription::{set_huddle_transcription_enabled, start_stt_pipeline}; pub use tts_settings::set_tts_enabled; diff --git a/desktop/src-tauri/src/huddle/pipeline.rs b/desktop/src-tauri/src/huddle/pipeline.rs index 9572ac25bf..e523ee22bf 100644 --- a/desktop/src-tauri/src/huddle/pipeline.rs +++ b/desktop/src-tauri/src/huddle/pipeline.rs @@ -315,6 +315,7 @@ pub(crate) async fn maybe_start_stt_pipeline( expected_generation, stt_starting, ptt_active_for_stt, + manual_mic_unmuted_for_stt, old_stt, ) = { let mut hs = state.huddle()?; @@ -338,6 +339,11 @@ pub(crate) async fn maybe_start_stt_pipeline( } else { None }; + let manual_mic_unmuted = if hs.voice_input_mode == VoiceInputMode::PushToTalk { + Some(Arc::clone(&hs.manual_mic_unmuted)) + } else { + None + }; ( Arc::clone(&hs.tts_active), Arc::clone(&hs.agent_pubkeys), @@ -345,6 +351,7 @@ pub(crate) async fn maybe_start_stt_pipeline( hs.session_generation.load(Ordering::Acquire), stt_starting, ptt, + manual_mic_unmuted, old, ) }; @@ -352,7 +359,12 @@ pub(crate) async fn maybe_start_stt_pipeline( drop(old_stt); let constructed = tokio::task::spawn_blocking(move || { - stt::SttPipeline::new(model_dir, tts_active, ptt_active_for_stt) + stt::SttPipeline::new( + model_dir, + tts_active, + ptt_active_for_stt, + manual_mic_unmuted_for_stt, + ) }) .await; let (pipeline, text_rx) = match constructed { diff --git a/desktop/src-tauri/src/huddle/state.rs b/desktop/src-tauri/src/huddle/state.rs index 0fe3a46f5a..7acf5fe633 100644 --- a/desktop/src-tauri/src/huddle/state.rs +++ b/desktop/src-tauri/src/huddle/state.rs @@ -15,7 +15,7 @@ use super::{stt, tts}; /// Voice input mode: push-to-talk (PTT) or voice-activity detection (VAD). /// -/// PTT: mic is gated by a global shortcut (Ctrl+Space). Pressing the key sets +/// PTT (the default): mic is gated by a global shortcut (Ctrl+Space). Pressing the key sets /// `ptt_active` and immediately cancels any playing TTS. Releasing the key /// (after a 200 ms delay) stops mic capture and flushes the utterance. /// @@ -26,8 +26,8 @@ use super::{stt, tts}; #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "snake_case")] pub enum VoiceInputMode { - PushToTalk, #[default] + PushToTalk, VoiceActivity, } @@ -135,6 +135,10 @@ pub struct HuddleState { /// Shared with the STT pipeline for mic gating. #[serde(skip)] pub ptt_active: Arc, + /// True while the clickable microphone control is manually unmuted. + /// In PTT mode, either this flag or `ptt_active` opens the STT gate. + #[serde(skip)] + pub manual_mic_unmuted: Arc, } fn serialize_agent_pubkeys(v: &Arc>>, s: S) -> Result @@ -190,6 +194,7 @@ impl Clone for HuddleState { session_generation: Arc::clone(&self.session_generation), voice_input_mode: self.voice_input_mode.clone(), ptt_active: Arc::clone(&self.ptt_active), + manual_mic_unmuted: Arc::clone(&self.manual_mic_unmuted), } } } @@ -221,6 +226,7 @@ impl Default for HuddleState { session_generation: Arc::new(AtomicU64::new(0)), voice_input_mode: VoiceInputMode::default(), ptt_active: Arc::new(AtomicBool::new(false)), + manual_mic_unmuted: Arc::new(AtomicBool::new(true)), } } } @@ -332,6 +338,13 @@ mod tests { assert!(!state.maybe_auto_enable_transcription_for_agents()); } + #[test] + fn defaults_to_push_to_talk_with_an_open_microphone() { + let state = HuddleState::default(); + assert_eq!(state.voice_input_mode, super::VoiceInputMode::PushToTalk); + assert!(state.manual_mic_unmuted.load(Ordering::Acquire)); + } + #[test] fn explicit_user_disable_is_not_undone_by_agent_presence() { let mut state = HuddleState::default(); diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 30a47f449a..70a8088640 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -71,9 +71,10 @@ impl SttPipeline { /// therefore never cancel TTS. Push-to-talk and remote participant speech /// remain explicit, reliable barge-in paths. /// - /// `ptt_active` (optional) is the push-to-talk flag. When `Some`, the STT - /// pipeline only accumulates speech while the flag is true (key held). - /// When `None`, the pipeline runs in continuous VAD mode. + /// `ptt_active` and `manual_mic_unmuted` are present when the PTT shortcut + /// is enabled. The pipeline accepts speech while either input path is open; + /// manual unmute uses normal VAD flushing while a shortcut hold is grouped + /// into one utterance. /// /// Returns `Err` only if the thread cannot be spawned (OS error). /// If model files are missing, the worker logs and exits cleanly — @@ -87,6 +88,7 @@ impl SttPipeline { model_dir: PathBuf, tts_active: Arc, ptt_active: Option>, + manual_mic_unmuted: Option>, ) -> Result<(Self, tokio_mpsc::Receiver), String> { let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); let (text_tx, text_rx) = tokio_mpsc::channel::(64); @@ -94,6 +96,7 @@ impl SttPipeline { let shutdown_worker = Arc::clone(&shutdown); let ptt_active_worker = ptt_active.as_ref().map(Arc::clone); + let manual_mic_unmuted_worker = manual_mic_unmuted.as_ref().map(Arc::clone); let handle = thread::Builder::new() .name("stt-worker".into()) .spawn(move || { @@ -104,6 +107,7 @@ impl SttPipeline { shutdown_worker, tts_active, ptt_active_worker, + manual_mic_unmuted_worker, ) }) .map_err(|e| format!("failed to spawn stt-worker thread: {e}"))?; @@ -203,6 +207,7 @@ fn stt_worker( shutdown: Arc, tts_active: Arc, ptt_active: Option>, + manual_mic_unmuted: Option>, ) { // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── use rubato::{Fft, FixedSync, Resampler}; @@ -275,9 +280,12 @@ fn stt_worker( // ── 5. Main loop ────────────────────────────────────────────────────────── let mut tts_was_active = false; - let mut ptt_was_active = ptt_active + let mut transmit_was_active = ptt_active .as_ref() - .is_some_and(|p| p.load(Ordering::Acquire)); + .is_some_and(|ptt| ptt.load(Ordering::Acquire)) + || manual_mic_unmuted + .as_ref() + .is_some_and(|manual| manual.load(Ordering::Acquire)); loop { // Check shutdown flag before blocking. if shutdown.load(Ordering::Acquire) { @@ -292,20 +300,22 @@ fn stt_worker( } tts_was_active = tts_now; - // Track PTT transitions — flush accumulated speech when key is released. - // The worklet stops sending frames when PTT is inactive, so the normal - // silence-accumulation flush path never runs. We must flush here on the - // active→inactive edge to avoid buffering speech across PTT presses. + // Track the combined manual/PTT transmission edge. When both paths + // close, the worklet stops sending frames, so flush here rather than + // waiting for silence that will never arrive. if let Some(ref ptt) = ptt_active { - let ptt_now = ptt.load(Ordering::Acquire); - if ptt_was_active && !ptt_now && in_speech && !speech_buf.is_empty() { + let transmit_now = ptt.load(Ordering::Acquire) + || manual_mic_unmuted + .as_ref() + .is_some_and(|manual| manual.load(Ordering::Acquire)); + if transmit_was_active && !transmit_now && in_speech && !speech_buf.is_empty() { flush_to_stt(&speech_buf, voiced_frames, &recognizer, &text_tx); speech_buf.clear(); silence_frames = 0; in_speech = false; voiced_frames = 0; } - ptt_was_active = ptt_now; + transmit_was_active = transmit_now; } // Use recv_timeout so we can periodically check the shutdown flag. @@ -343,6 +353,7 @@ fn stt_worker( &tts_active, &mut tts_stopped_at, ptt_active.as_ref(), + manual_mic_unmuted.as_ref(), ); } } @@ -385,11 +396,9 @@ fn resample_chunk(resampler: &mut rubato::Fft, chunk_48k: &[f32]) -> Vec, tts_stopped_at: &mut Option, ptt_active: Option<&Arc>, + manual_mic_unmuted: Option<&Arc>, ) { leftover.extend_from_slice(samples); @@ -413,13 +423,11 @@ fn process_16k_samples( let prob = vad.predict_f32(&clamped); let is_speech = prob > VAD_THRESHOLD; - // PTT gating: when PTT key is not held, treat as silence. - // This causes natural flush when the key is released — silence_frames - // accumulates and the existing flush logic kicks in after - // SILENCE_FLUSH_FRAMES. The 200 ms release delay + ~300 ms silence - // flush gives a natural utterance tail. + let manually_open = manual_mic_unmuted.is_some_and(|manual| manual.load(Ordering::Acquire)); + // Shortcut-enabled mode accepts input from either the held shortcut or + // a manually open microphone. let is_speech = if let Some(ptt) = ptt_active { - is_speech && ptt.load(Ordering::Acquire) + is_speech && (ptt.load(Ordering::Acquire) || manually_open) } else { is_speech }; @@ -478,11 +486,9 @@ fn process_16k_samples( speech_buf.extend_from_slice(&frame); *silence_frames += 1; - // In PTT mode, don't flush on silence — accumulate the entire - // key-hold as one utterance. The PTT release edge in the main - // loop handles the flush. In VAD mode, flush after the silence - // threshold so each natural pause becomes a separate message. - if ptt_active.is_none() && *silence_frames >= SILENCE_FLUSH_FRAMES { + // A manually open microphone behaves like normal VAD. A + // shortcut-only transmission stays grouped until key release. + if (ptt_active.is_none() || manually_open) && *silence_frames >= SILENCE_FLUSH_FRAMES { // End of utterance — transcribe. flush_to_stt(speech_buf, *voiced_frames, recognizer, text_tx); speech_buf.clear(); diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index a672647603..e6c9da24ff 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -56,9 +56,10 @@ use huddle::reconnect::reconnect_huddle_audio; use huddle::{ add_agent_to_huddle, check_pipeline_hotstart, close_huddle_companion, confirm_huddle_active, download_voice_models, end_huddle, get_huddle_agent_pubkeys, get_huddle_state, - get_model_status, get_voice_input_mode, join_huddle, leave_huddle, open_huddle_window, - push_audio_pcm, set_huddle_transcription_enabled, set_tts_enabled, set_voice_input_mode, - speak_agent_message, start_huddle, start_stt_pipeline, HuddlePhase, + get_model_status, get_voice_input_mode, interrupt_huddle_speech, join_huddle, leave_huddle, + open_huddle_window, push_audio_pcm, remove_agent_from_huddle, set_huddle_manual_mic_unmuted, + set_huddle_transcription_enabled, set_tts_enabled, set_voice_input_mode, speak_agent_message, + start_huddle, start_stt_pipeline, HuddlePhase, }; use initial_window::*; use managed_agents::{ @@ -850,7 +851,9 @@ pub fn run() { huddle::agent_voice::set_huddle_agent_tts_enabled, huddle::agent_voice::set_huddle_agent_voice, speak_agent_message, + interrupt_huddle_speech, add_agent_to_huddle, + remove_agent_from_huddle, huddle::agents::sync_agents_to_active_huddle, check_pipeline_hotstart, confirm_huddle_active, @@ -858,6 +861,7 @@ pub fn run() { get_huddle_agent_pubkeys, set_voice_input_mode, get_voice_input_mode, + set_huddle_manual_mic_unmuted, list_audio_output_devices, set_audio_output_device, get_audio_output_device, diff --git a/desktop/src/app/AppHuddleShell.tsx b/desktop/src/app/AppHuddleShell.tsx index 736dad1f6a..2e9a7e57e1 100644 --- a/desktop/src/app/AppHuddleShell.tsx +++ b/desktop/src/app/AppHuddleShell.tsx @@ -56,6 +56,13 @@ export function AppHuddleShell({ data-huddle-window={isRoom} > {isRoom ? null : terminal} +