diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..6f81ecd --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,5 @@ +# Keep the default local ASR build self-contained and deterministic. +# Host-wide BLAS/MKL installations must not silently change BLCVoice's native link graph. +# Developers can still override this environment variable explicitly for controlled experiments. +[env] +TRANSCRIBE_CMAKE_ARGS = { value = "-DTRANSCRIBE_USE_SYSTEM_BLAS=OFF", force = false } diff --git a/Cargo.lock b/Cargo.lock index e83e268..3812f42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -383,6 +383,7 @@ dependencies = [ "reis", "rustix", "tokio", + "wl-clipboard-rs", ] [[package]] @@ -1110,13 +1111,19 @@ checksum = "521e380c0c8afb8d9a1e83a1822ee03556fc3e3e7dbc1fd30be14e37f9cb3f89" dependencies = [ "bit-set", "cssparser", - "foldhash", + "foldhash 0.2.0", "html5ever", "precomputed-hash", "selectors", "tendril", ] +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dpi" version = "0.1.2" @@ -1350,6 +1357,12 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.10" @@ -1367,6 +1380,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -1865,13 +1884,22 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -1880,7 +1908,7 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -3092,6 +3120,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "pango" version = "0.18.3" @@ -3152,6 +3190,17 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap 2.14.1", +] + [[package]] name = "phf" version = "0.13.1" @@ -3763,9 +3812,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.43" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "once_cell", @@ -4767,7 +4816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -5178,6 +5227,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tree_magic_mini" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8765b90061cba6c22b5831f675da109ae5561588290f9fa2317adab2714d5a6" +dependencies = [ + "memchr", + "nom 8.0.0", + "petgraph", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -5466,6 +5526,76 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wayland-backend" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38a91b4eaddff87b1cd1074985e3713da4af2c49742d1b356b2c01670a67a078" +dependencies = [ + "cc", + "downcast-rs", + "rustix", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" +dependencies = [ + "bitflags 2.13.1", + "rustix", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols" +version = "0.32.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d0c813de3daa2ed6520af85a3bd49b0e722a3078506899aa9686fea58dc4b6" +dependencies = [ + "bitflags 2.13.1", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" +dependencies = [ + "bitflags 2.13.1", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0" +dependencies = [ + "proc-macro2", + "quick-xml", + "quote", +] + +[[package]] +name = "wayland-sys" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "pkg-config", +] + [[package]] name = "web-sys" version = "0.3.104" @@ -6058,6 +6188,24 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wl-clipboard-rs" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9651471a32e87d96ef3a127715382b2d11cc7c8bb9822ded8a7cc94072eb0a3" +dependencies = [ + "libc", + "log", + "os_pipe", + "rustix", + "thiserror 2.0.20", + "tree_magic_mini", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-protocols-wlr", +] + [[package]] name = "writeable" version = "0.6.4" diff --git a/apps/desktop/src-tauri/capabilities/overlay.json b/apps/desktop/src-tauri/capabilities/overlay.json index 6f65600..4e52d27 100644 --- a/apps/desktop/src-tauri/capabilities/overlay.json +++ b/apps/desktop/src-tauri/capabilities/overlay.json @@ -1,12 +1,16 @@ { "$schema": "../gen/schemas/desktop-schema.json", "identifier": "overlay-capability", - "description": "Minimal event and visibility permissions for the non-interactive dictation overlay.", + "description": "Window permissions required by the compact dictation overlay.", "windows": ["overlay"], "permissions": [ "core:event:allow-listen", "core:event:allow-unlisten", + "core:window:default", "core:window:allow-show", - "core:window:allow-hide" + "core:window:allow-hide", + "core:window:allow-set-size", + "core:window:allow-set-position", + "core:window:allow-start-dragging" ] } diff --git a/apps/desktop/src-tauri/src/insertion.rs b/apps/desktop/src-tauri/src/insertion.rs index 9dab02c..095e17c 100644 --- a/apps/desktop/src-tauri/src/insertion.rs +++ b/apps/desktop/src-tauri/src/insertion.rs @@ -60,6 +60,12 @@ impl DesktopInsertionService { } pub fn capability(&self) -> Result { + let state = self.lock_state(); + if let Some(inserter) = state.inserter.as_ref() { + return Ok(inserter.capability()); + } + drop(state); + resolve_insertion_capability(self.environment).map_err(|error| { InsertionError::new(InsertionErrorKind::BackendUnavailable, error.to_string()) }) @@ -101,7 +107,8 @@ impl DesktopInsertionService { } InsertionBackend::X11XTest => X11Inserter::connect(X11Options::default()) .map(|inserter| Box::new(inserter) as Box), - InsertionBackend::XdgRemoteDesktopEis => { + InsertionBackend::XdgRemoteDesktopEis + | InsertionBackend::XdgRemoteDesktopEisClipboard => { let inserter = WaylandEisInserter::connect(WaylandEisOptions::new( state.wayland_restore_token.clone(), ))?; diff --git a/apps/desktop/src-tauri/src/main.rs b/apps/desktop/src-tauri/src/main.rs index 52d291c..2460da8 100644 --- a/apps/desktop/src-tauri/src/main.rs +++ b/apps/desktop/src-tauri/src/main.rs @@ -1,5 +1,37 @@ #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] +#[cfg(target_os = "linux")] +fn apply_linux_webkit_compatibility() { + use std::os::unix::process::CommandExt; + use std::process::Command; + + let wayland = std::env::var_os("WAYLAND_DISPLAY").is_some(); + let kde = std::env::var("XDG_CURRENT_DESKTOP") + .ok() + .is_some_and(|desktop| desktop.to_ascii_lowercase().contains("kde")); + let override_present = std::env::var_os("WEBKIT_DISABLE_DMABUF_RENDERER").is_some(); + + if !wayland || !kde || override_present { + return; + } + + let Ok(executable) = std::env::current_exe() else { + return; + }; + + let error = Command::new(executable) + .args(std::env::args_os().skip(1)) + .env("WEBKIT_DISABLE_DMABUF_RENDERER", "1") + .exec(); + + eprintln!( + "failed to restart BLCVoice with the KDE/Wayland WebKit compatibility guard: {error}" + ); +} + fn main() { + #[cfg(target_os = "linux")] + apply_linux_webkit_compatibility(); + blcvoice_desktop_lib::run(); } diff --git a/apps/desktop/src-tauri/src/shortcut.rs b/apps/desktop/src-tauri/src/shortcut.rs index 3946758..1002f55 100644 --- a/apps/desktop/src-tauri/src/shortcut.rs +++ b/apps/desktop/src-tauri/src/shortcut.rs @@ -13,7 +13,9 @@ use crate::coordinator::ShortcutDictationCoordinator; #[cfg(target_os = "linux")] use ashpd::desktop::{ CreateSessionOptions, - global_shortcuts::{BindShortcutsOptions, GlobalShortcuts, NewShortcut}, + global_shortcuts::{ + BindShortcutsOptions, ConfigureShortcutsOptions, GlobalShortcuts, NewShortcut, + }, }; #[cfg(target_os = "linux")] use futures_util::{StreamExt, stream}; @@ -26,6 +28,8 @@ pub const SHORTCUT_DECISION_EVENT: &str = "blcvoice://shortcut-decision"; enum ShortcutRegistrationState { Pending, Registering, + #[cfg(any(target_os = "linux", test))] + ConfigurationRequired, Registered, Failed, Unavailable, @@ -36,6 +40,8 @@ impl ShortcutRegistrationState { match self { Self::Pending => "pending", Self::Registering => "registering", + #[cfg(any(target_os = "linux", test))] + Self::ConfigurationRequired => "configuration required", Self::Registered => "registered", Self::Failed => "failed", Self::Unavailable => "unavailable", @@ -50,6 +56,7 @@ struct ShortcutServiceState { selection_error: Option, registration_state: ShortcutRegistrationState, registration_error: Option, + bound_trigger_description: Option, controller: ShortcutController, } @@ -78,6 +85,7 @@ impl ShortcutService { selection_error, registration_state, registration_error: None, + bound_trigger_description: None, controller: ShortcutController::default(), }), } @@ -92,14 +100,27 @@ impl ShortcutService { if state.backend.is_some() { state.registration_state = ShortcutRegistrationState::Registering; state.registration_error = None; + state.bound_trigger_description = None; } } - fn mark_registered(&self) { + #[cfg(any(target_os = "linux", test))] + fn mark_configuration_required(&self) { + let mut state = self.lock_state(); + if state.backend.is_some() { + state.registration_state = ShortcutRegistrationState::ConfigurationRequired; + state.registration_error = None; + state.bound_trigger_description = None; + state.controller.force_idle(); + } + } + + fn mark_registered(&self, trigger_description: Option) { let mut state = self.lock_state(); if state.backend.is_some() { state.registration_state = ShortcutRegistrationState::Registered; state.registration_error = None; + state.bound_trigger_description = trigger_description; } } @@ -107,6 +128,7 @@ impl ShortcutService { let mut state = self.lock_state(); state.registration_state = ShortcutRegistrationState::Failed; state.registration_error = Some(message.into()); + state.bound_trigger_description = None; state.controller.force_idle(); } @@ -141,7 +163,7 @@ pub struct ShortcutCapabilityDto { linux_display_server: Option, selected_backend: Option, registration_implemented: bool, - registration_state: &'static str, + registration_state: String, registration_error: Option, selection_error: Option, } @@ -203,7 +225,9 @@ fn install_native_shortcut(app: &mut App) { .build(); match app.handle().plugin(plugin) { - Ok(()) => app.state::().mark_registered(), + Ok(()) => app + .state::() + .mark_registered(Some(DEFAULT_DICTATION_TRIGGER.to_owned())), Err(error) => app .state::() .mark_failed(format!("global shortcut registration failed: {error}")), @@ -229,6 +253,26 @@ fn install_portal_shortcut(app: &mut App) { ); } +#[cfg(target_os = "linux")] +#[derive(Debug)] +enum PortalShortcutEvent { + Phase(ShortcutPhase), + BindingChanged(Option), +} + +#[cfg(target_os = "linux")] +fn trigger_for_shortcut( + shortcuts: &[ashpd::desktop::global_shortcuts::Shortcut], +) -> Option { + shortcuts + .iter() + .find(|shortcut| shortcut.id() == DICTATION_SHORTCUT_ID) + .and_then(|shortcut| { + let trigger = shortcut.trigger_description().trim(); + (!trigger.is_empty()).then(|| trigger.to_owned()) + }) +} + #[cfg(target_os = "linux")] async fn run_portal_shortcut(app: AppHandle) -> Result<(), String> { let portal = GlobalShortcuts::new() @@ -257,27 +301,59 @@ async fn run_portal_shortcut(app: AppHandle) -> Result<(), String return Err("XDG portal response did not contain the BLCVoice shortcut".to_owned()); } + let initial_trigger = trigger_for_shortcut(response.shortcuts()); + let activated = portal .receive_activated() .await .map_err(|error| format!("could not subscribe to XDG shortcut activation: {error}"))? .filter_map(|event| async move { - (event.shortcut_id() == DICTATION_SHORTCUT_ID).then_some(ShortcutPhase::Pressed) + (event.shortcut_id() == DICTATION_SHORTCUT_ID) + .then_some(PortalShortcutEvent::Phase(ShortcutPhase::Pressed)) }); let deactivated = portal .receive_deactivated() .await .map_err(|error| format!("could not subscribe to XDG shortcut deactivation: {error}"))? .filter_map(|event| async move { - (event.shortcut_id() == DICTATION_SHORTCUT_ID).then_some(ShortcutPhase::Released) + (event.shortcut_id() == DICTATION_SHORTCUT_ID) + .then_some(PortalShortcutEvent::Phase(ShortcutPhase::Released)) }); + let changed = portal + .receive_shortcuts_changed() + .await + .map_err(|error| format!("could not subscribe to XDG shortcut changes: {error}"))? + .map(|event| PortalShortcutEvent::BindingChanged(trigger_for_shortcut(event.shortcuts()))); + + if let Some(trigger) = initial_trigger { + app.state::() + .mark_registered(Some(trigger)); + } else if portal.version() >= 2 { + app.state::().mark_configuration_required(); + portal + .configure_shortcuts(&session, None, ConfigureShortcutsOptions::default()) + .await + .map_err(|error| format!("could not open XDG shortcut configuration: {error}"))?; + } else { + return Err(format!( + "the desktop portal returned the BLCVoice shortcut without a trigger and GlobalShortcuts v{} cannot open the configuration UI", + portal.version() + )); + } - app.state::().mark_registered(); - - let events = stream::select(activated, deactivated); + let phase_events = stream::select(activated, deactivated); + let events = stream::select(phase_events, changed); futures_util::pin_mut!(events); - while let Some(phase) = events.next().await { - route_shortcut_phase(&app, phase); + while let Some(event) = events.next().await { + match event { + PortalShortcutEvent::Phase(phase) => route_shortcut_phase(&app, phase), + PortalShortcutEvent::BindingChanged(Some(trigger)) => app + .state::() + .mark_registered(Some(trigger)), + PortalShortcutEvent::BindingChanged(None) => { + app.state::().mark_configuration_required(); + } + } } let _ = session.close().await; @@ -306,13 +382,24 @@ fn shortcut_capability_for(state: &ShortcutServiceState) -> ShortcutCapabilityDt DesktopPlatform::Linux => Some(state.environment.linux_display_server().to_string()), DesktopPlatform::Windows | DesktopPlatform::MacOs | DesktopPlatform::Other => None, }; + let registration_state = match state.registration_state { + #[cfg(any(target_os = "linux", test))] + ShortcutRegistrationState::ConfigurationRequired => { + "configuration required · KDE shortcut settings opened".to_owned() + } + ShortcutRegistrationState::Registered => match state.bound_trigger_description.as_deref() { + Some(trigger) => format!("registered · {trigger}"), + None => "registered".to_owned(), + }, + other => other.as_str().to_owned(), + }; ShortcutCapabilityDto { platform: platform.to_string(), linux_display_server, selected_backend: state.backend.map(|backend| backend.to_string()), registration_implemented: state.backend.is_some(), - registration_state: state.registration_state.as_str(), + registration_state, registration_error: state.registration_error.clone(), selection_error: state .selection_error @@ -417,7 +504,7 @@ mod tests { ShortcutDecision::Ignore ); - service.mark_registered(); + service.mark_registered(Some("Ctrl+Shift+Space".to_owned())); assert_eq!( service.handle_phase(ShortcutPhase::Pressed), ShortcutDecision::StartDictation @@ -438,7 +525,7 @@ mod tests { DesktopPlatform::Windows, LinuxDisplayServer::Unknown, )); - service.mark_registered(); + service.mark_registered(Some(DEFAULT_DICTATION_TRIGGER.to_owned())); assert_eq!( service.handle_phase(ShortcutPhase::Pressed), ShortcutDecision::StartDictation @@ -456,4 +543,36 @@ mod tests { ShortcutDecision::Ignore ); } + + #[test] + fn registered_capability_reports_the_actual_trigger_description() { + let service = ShortcutService::for_environment(ShortcutEnvironment::new( + DesktopPlatform::Linux, + LinuxDisplayServer::Wayland, + )); + service.mark_registered(Some("Ctrl+Alt+Space".to_owned())); + + assert_eq!( + service.capability().registration_state, + "registered · Ctrl+Alt+Space" + ); + } + + #[test] + fn missing_portal_trigger_is_not_reported_as_registered() { + let service = ShortcutService::for_environment(ShortcutEnvironment::new( + DesktopPlatform::Linux, + LinuxDisplayServer::Wayland, + )); + service.mark_configuration_required(); + + assert_eq!( + service.capability().registration_state, + "configuration required · KDE shortcut settings opened" + ); + assert_eq!( + service.handle_phase(ShortcutPhase::Pressed), + ShortcutDecision::Ignore + ); + } } diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.json index de2388c..498e6b3 100644 --- a/apps/desktop/src-tauri/tauri.conf.json +++ b/apps/desktop/src-tauri/tauri.conf.json @@ -22,9 +22,9 @@ "label": "overlay", "title": "BLCVoice Dictation", "url": "overlay.html", - "width": 340, - "height": 82, - "center": true, + "width": 268, + "height": 52, + "center": false, "resizable": false, "maximizable": false, "minimizable": false, diff --git a/apps/desktop/ui/index.html b/apps/desktop/ui/index.html index 057662e..cfa6ac7 100644 --- a/apps/desktop/ui/index.html +++ b/apps/desktop/ui/index.html @@ -7,180 +7,190 @@ BLCVoice + -
-
-
-
BLCVOICE · LOCAL DICTATION
-

Speak. It types.

-

Private desktop dictation with local speech recognition and platform-native text delivery.

-
- Audio stays local -
- -
-
+
+
+ -
-
-
+
+
+
-

MODEL

-

Local speech model

+

HOME

+

Ready to dictate

+

Stay in the app you are using. Press the shortcut, speak, then press it again to type.

- Checking -
-

Choose a local model. BLCVoice recommends one from available system memory.

-
-
-
+ Ctrl + Shift + Space + -
-
-
-

SETTINGS

-

Input & language

+
+
+
+

DICTATION

+

Speak where your cursor is

+
+ Preparing
- -
+

Checking your microphone, model and text-insertion capability…

+
+ + +
+ + +
- - +
+
+ ⌁ +
Stay focusedThe floating overlay never takes focus from your target app.
+
+
+ ◎ +
Runs locallyCapture and speech recognition remain on this device.
+
+
+ - - + -
-
-
Backend
-
—
-
-
-
Native format
-
—
-
-
-
+ -
-
-
-
-

MICROPHONE CHECK

-

Verify native capture

-
- Ready -
-

Run a short capture check if dictation input seems wrong. Test audio is discarded after validation.

- -
- - - -
- -
+ + -
BLCVoice does not upload microphone audio. Platform permissions for global shortcuts and text insertion remain controlled by your operating system.
-
+ + + diff --git a/apps/desktop/ui/layout.css b/apps/desktop/ui/layout.css new file mode 100644 index 0000000..9e50ada --- /dev/null +++ b/apps/desktop/ui/layout.css @@ -0,0 +1,329 @@ +html, +body { + height: 100%; + overflow: hidden; +} + +body { + background: + radial-gradient(circle at 82% -10%, color-mix(in srgb, var(--accent) 8%, transparent), transparent 36%), + var(--surface-strong); +} + +.app-frame { + width: 100%; + height: 100vh; + display: grid; + grid-template-columns: 214px minmax(0, 1fr); +} + +.sidebar { + min-width: 0; + display: flex; + flex-direction: column; + padding: 22px 14px 16px; + border-right: 1px solid var(--border); + background: color-mix(in srgb, var(--surface-strong) 92%, transparent); + backdrop-filter: blur(20px); +} + +.brand-row { + display: flex; + align-items: center; + gap: 11px; + padding: 4px 8px 22px; +} + +.brand-row > div:last-child { + min-width: 0; + display: grid; + gap: 2px; +} + +.brand-row strong { + font-size: .92rem; + letter-spacing: -.02em; +} + +.brand-row small, +.sidebar-footer small { + color: var(--muted); + font-size: .66rem; + line-height: 1.35; +} + +.brand-mark { + width: 30px; + height: 30px; + display: flex; + align-items: center; + justify-content: center; + gap: 2px; + flex: 0 0 auto; + border: 1px solid color-mix(in srgb, var(--accent) 24%, var(--border)); + border-radius: 10px; + background: var(--accent-soft); +} + +.brand-mark span { + width: 3px; + border-radius: 999px; + background: var(--accent); +} + +.brand-mark span:nth-child(1) { height: 8px; } +.brand-mark span:nth-child(2) { height: 16px; } +.brand-mark span:nth-child(3) { height: 11px; } + +.side-nav { + display: grid; + gap: 4px; +} + +.nav-item { + width: 100%; + min-height: 39px; + padding: 0 11px; + border: 0; + border-radius: 10px; + background: transparent; + color: var(--muted); + cursor: pointer; + text-align: left; + font: inherit; + font-size: .79rem; + font-weight: 690; + transition: color 120ms ease, background 120ms ease, transform 120ms ease; +} + +.nav-item:hover { + color: inherit; + background: color-mix(in srgb, var(--surface-strong) 84%, transparent); +} + +.nav-item.active { + color: inherit; + background: var(--accent-soft); + box-shadow: inset 2px 0 0 var(--accent); +} + +.nav-item:focus-visible { + outline: 3px solid color-mix(in srgb, var(--accent) 25%, transparent); + outline-offset: 1px; +} + +.sidebar-footer { + display: grid; + grid-template-columns: auto minmax(0, 1fr); + align-items: start; + gap: 9px; + margin-top: auto; + padding: 12px 9px 3px; + border-top: 1px solid var(--border); +} + +.sidebar-footer > div { + min-width: 0; + display: grid; + gap: 2px; +} + +.sidebar-footer strong { + font-size: .69rem; +} + +.privacy-dot { + width: 7px; + height: 7px; + margin-top: 5px; + border-radius: 999px; + background: var(--success); + box-shadow: 0 0 0 4px color-mix(in srgb, var(--success) 12%, transparent); +} + +.workspace { + min-width: 0; + height: 100vh; + overflow-y: auto; + scroll-behavior: smooth; +} + +.view-section { + width: min(830px, calc(100% - 52px)); + margin: 0 auto; + padding: 34px 0 40px; +} + +.view-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 22px; + margin-bottom: 24px; +} + +.view-header > div { + min-width: 0; +} + +.view-header h1 { + margin: 6px 0 8px; + font-size: clamp(2rem, 5vw, 3.25rem); + line-height: 1; + letter-spacing: -.05em; +} + +.compact-header h1 { + font-size: clamp(1.75rem, 4vw, 2.6rem); +} + +.view-header .lead { + max-width: 610px; + font-size: .9rem; +} + +.shortcut-chip { + flex: 0 0 auto; + margin-top: 7px; + padding: 8px 10px; + border: 1px solid var(--border); + border-radius: 10px; + background: var(--surface-strong); + color: var(--muted); + font-size: .65rem; + font-weight: 720; + box-shadow: var(--shadow); +} + +.home-hero { + margin-top: 0; + padding: 27px; + border-radius: 24px; + background: + linear-gradient(135deg, color-mix(in srgb, var(--surface-strong) 94%, transparent), var(--accent-soft)); +} + +.home-hero .hero-copy { + max-width: 590px; +} + +.home-notes { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 10px; + margin-top: 12px; +} + +.home-notes article { + min-width: 0; + display: grid; + grid-template-columns: auto minmax(0, 1fr); + gap: 10px; + padding: 14px; + border: 1px solid var(--border); + border-radius: 14px; + background: var(--surface); +} + +.home-notes article > div { + min-width: 0; + display: grid; + gap: 3px; +} + +.home-notes strong { + font-size: .75rem; +} + +.home-notes small { + color: var(--muted); + font-size: .68rem; + line-height: 1.4; +} + +.home-note-icon { + width: 26px; + height: 26px; + display: grid; + place-items: center; + border-radius: 8px; + background: var(--accent-soft); + color: var(--accent); + font-weight: 800; +} + +.feature-panel { + margin-top: 0; + padding: 22px; + border-radius: 18px; +} + +.settings-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + align-items: start; + gap: 12px; +} + +.settings-card { + min-width: 0; +} + +.model-list { + grid-template-columns: repeat(auto-fit, minmax(230px, 1fr)); +} + +.history-list { + margin-top: 16px; +} + +.diagnostic-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +.diagnostic-item { + padding: 16px; +} + +.footer-note { + margin-top: 14px; +} + +@media (max-width: 720px) { + .app-frame { + grid-template-columns: 172px minmax(0, 1fr); + } + + .sidebar { + padding-inline: 10px; + } + + .brand-row { + padding-inline: 5px; + } + + .view-section { + width: min(100% - 28px, 830px); + padding-top: 24px; + } + + .view-header { + display: grid; + } + + .shortcut-chip { + width: max-content; + margin-top: 0; + } + + .settings-grid, + .home-notes, + .diagnostic-grid { + grid-template-columns: 1fr; + } +} + +@media (prefers-reduced-motion: reduce) { + .nav-item { transition: none; } + .workspace { scroll-behavior: auto; } +} diff --git a/apps/desktop/ui/navigation.js b/apps/desktop/ui/navigation.js new file mode 100644 index 0000000..831b1ab --- /dev/null +++ b/apps/desktop/ui/navigation.js @@ -0,0 +1,48 @@ +"use strict"; + +const layoutStylesheet = document.createElement("link"); +layoutStylesheet.rel = "stylesheet"; +layoutStylesheet.href = "layout.css"; +document.head.append(layoutStylesheet); + +const navItems = Array.from(document.querySelectorAll("[data-view-target]")); +const views = Array.from(document.querySelectorAll("[data-view]")); +const VIEW_STORAGE_KEY = "blcvoice.desktop.view"; + +function activateView(name, { remember = true } = {}) { + const target = views.find((view) => view.dataset.view === name) ?? views[0]; + if (!target) return; + + for (const view of views) { + view.hidden = view !== target; + } + + for (const item of navItems) { + const active = item.dataset.viewTarget === target.dataset.view; + item.classList.toggle("active", active); + if (active) item.setAttribute("aria-current", "page"); + else item.removeAttribute("aria-current"); + } + + document.querySelector(".workspace")?.scrollTo({ top: 0, behavior: "auto" }); + if (remember) { + try { + window.localStorage.setItem(VIEW_STORAGE_KEY, target.dataset.view || "home"); + } catch { + // Navigation remains functional when storage is unavailable. + } + } +} + +for (const item of navItems) { + item.addEventListener("click", () => activateView(item.dataset.viewTarget)); +} + +let initialView = "home"; +try { + initialView = window.localStorage.getItem(VIEW_STORAGE_KEY) || "home"; +} catch { + initialView = "home"; +} + +activateView(initialView, { remember: false }); diff --git a/apps/desktop/ui/overlay.css b/apps/desktop/ui/overlay.css index 64fc706..5ebc5a6 100644 --- a/apps/desktop/ui/overlay.css +++ b/apps/desktop/ui/overlay.css @@ -1,8 +1,8 @@ :root { - color-scheme: light dark; + color-scheme: dark; font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; - background: #111318; - color: #f7f8fa; + background: transparent; + color: #f7f8fb; } * { box-sizing: border-box; } @@ -13,7 +13,7 @@ body { height: 100%; margin: 0; overflow: hidden; - background: #111318; + background: transparent; } body { @@ -23,83 +23,140 @@ body { } .overlay-shell { - width: 100%; - height: 100%; + width: calc(100% - 4px); + height: calc(100% - 4px); display: grid; - grid-template-columns: auto 1fr auto; + grid-template-columns: 34px minmax(0, 1fr) auto auto; align-items: center; - gap: 12px; - padding: 14px 16px; - border: 1px solid rgba(255, 255, 255, 0.12); - background: linear-gradient(180deg, #1c1f26 0%, #14171c 100%); + gap: 7px; + padding: 7px 9px; + border: 1px solid rgba(255, 255, 255, 0.11); + border-radius: 13px; + background: rgba(20, 22, 27, 0.95); + box-shadow: 0 10px 28px rgba(0, 0, 0, 0.3); + backdrop-filter: blur(16px); + -webkit-backdrop-filter: blur(16px); + cursor: grab; + transition: border-color 150ms ease, background 150ms ease; +} + +.overlay-shell:active { cursor: grabbing; } +.overlay-shell > * { pointer-events: none; } + +.overlay-shell.recording { + border-color: rgba(141, 126, 255, 0.42); + background: rgba(23, 22, 31, 0.97); } -.state-indicator { - width: 14px; - height: 14px; +.overlay-shell.working { border-color: rgba(126, 164, 255, 0.3); } +.overlay-shell.success { border-color: rgba(86, 218, 158, 0.34); } +.overlay-shell.failed { border-color: rgba(255, 112, 112, 0.4); } + +.waveform { + height: 25px; + display: flex; + align-items: center; + justify-content: center; + gap: 2px; +} + +.waveform span { + width: 3px; + height: 6px; border-radius: 999px; - background: #8d96a8; - box-shadow: 0 0 0 5px rgba(141, 150, 168, 0.12); + background: #8f98a9; + opacity: 0.62; + transform-origin: center; } -.state-indicator.recording { - background: #ff5263; - box-shadow: 0 0 0 5px rgba(255, 82, 99, 0.16); - animation: pulse 1.15s ease-in-out infinite; +.recording .waveform span { + background: #9b8cff; + opacity: 1; + animation: voice-wave 900ms ease-in-out infinite; } -.state-indicator.working { - background: #8aa7ff; - box-shadow: 0 0 0 5px rgba(138, 167, 255, 0.14); +.recording .waveform span:nth-child(2) { animation-delay: -720ms; } +.recording .waveform span:nth-child(3) { animation-delay: -540ms; } +.recording .waveform span:nth-child(4) { animation-delay: -360ms; } +.recording .waveform span:nth-child(5) { animation-delay: -180ms; } +.recording .waveform span:nth-child(6) { animation-delay: -620ms; } + +.working .waveform span { + background: #88a8ff; + animation: working-wave 1s ease-in-out infinite; } -.state-indicator.success { - background: #57d99a; - box-shadow: 0 0 0 5px rgba(87, 217, 154, 0.14); +.working .waveform span:nth-child(2n) { animation-delay: -500ms; } + +.success .waveform span { + height: 4px; + background: #58dca0; + opacity: 0.95; } -.state-indicator.failed { - background: #ff8a65; - box-shadow: 0 0 0 5px rgba(255, 138, 101, 0.15); +.failed .waveform span { + height: 5px; + background: #ff7777; + opacity: 0.9; } -.copy { +.status-line { min-width: 0; - display: grid; - gap: 3px; + display: flex; + align-items: baseline; + gap: 5px; + overflow: hidden; + white-space: nowrap; } -.copy strong { - font-size: 0.88rem; - line-height: 1.15; +.status-line strong { + flex: none; + font-size: 0.78rem; + line-height: 1; letter-spacing: -0.01em; } -.copy span { +.status-line > span { + min-width: 0; overflow: hidden; - color: #aeb6c6; - font-size: 0.75rem; - line-height: 1.2; + color: #9ba3b2; + font-size: 0.63rem; + line-height: 1; text-overflow: ellipsis; white-space: nowrap; } +.timer { + flex: none; + color: #a99dff; + font-size: 0.62rem; + font-variant-numeric: tabular-nums; + font-weight: 700; +} + .shortcut-hint { - padding: 5px 7px; - border: 1px solid rgba(255, 255, 255, 0.1); + padding: 4px 6px; + border: 1px solid rgba(255, 255, 255, 0.09); border-radius: 7px; - color: #8f98a8; - font-size: 0.62rem; - font-weight: 650; - letter-spacing: 0.02em; + color: #858e9f; + background: rgba(255, 255, 255, 0.035); + font-size: 0.54rem; + font-weight: 700; white-space: nowrap; } -@keyframes pulse { - 0%, 100% { transform: scale(1); opacity: 1; } - 50% { transform: scale(0.78); opacity: 0.7; } +@keyframes voice-wave { + 0%, 100% { height: 5px; transform: scaleY(0.85); } + 50% { height: 21px; transform: scaleY(1); } +} + +@keyframes working-wave { + 0%, 100% { height: 5px; opacity: 0.45; } + 50% { height: 15px; opacity: 1; } } @media (prefers-reduced-motion: reduce) { - .state-indicator { animation: none !important; } + .waveform span { animation: none !important; } + .recording .waveform span:nth-child(odd) { height: 15px; } + .recording .waveform span:nth-child(even) { height: 8px; } } diff --git a/apps/desktop/ui/overlay.html b/apps/desktop/ui/overlay.html index 830c008..5585018 100644 --- a/apps/desktop/ui/overlay.html +++ b/apps/desktop/ui/overlay.html @@ -7,13 +7,23 @@ -
- -
+
+ + +
BLCVoice Ready
- Ctrl Shift Space + + + Ctrl⇧Space
diff --git a/apps/desktop/ui/overlay.js b/apps/desktop/ui/overlay.js index fa13dd4..0b24d36 100644 --- a/apps/desktop/ui/overlay.js +++ b/apps/desktop/ui/overlay.js @@ -1,27 +1,128 @@ "use strict"; const listen = window.__TAURI__?.event?.listen; -const getCurrentWindow = window.__TAURI__?.window?.getCurrentWindow; +const tauriWindow = window.__TAURI__?.window; +const getCurrentWindow = tauriWindow?.getCurrentWindow; const overlayWindow = getCurrentWindow ? getCurrentWindow() : null; +const shell = document.getElementById("overlay-shell"); const title = document.getElementById("overlay-title"); const message = document.getElementById("overlay-message"); -const indicator = document.getElementById("state-indicator"); const shortcutHint = document.getElementById("shortcut-hint"); +const recordingTimer = document.getElementById("recording-timer"); + +const OVERLAY_WIDTH = 268; +const OVERLAY_HEIGHT = 52; +const POSITION_STORAGE_KEY = "blcvoice.overlay.position.v1"; let lifecycleGeneration = 0; let unlisten = null; +let unlistenMoved = null; +let timerId = null; +let recordingStartedAt = null; +let positioned = false; + +function truncate(text, max = 28) { + if (typeof text !== "string") return ""; + const normalized = text.replace(/\s+/g, " ").trim(); + return normalized.length > max ? `${normalized.slice(0, max - 1)}…` : normalized; +} + +function formatElapsed(ms) { + const totalSeconds = Math.max(0, Math.floor(ms / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return `${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}`; +} + +function stopTimer() { + if (timerId !== null) { + window.clearInterval(timerId); + timerId = null; + } + recordingStartedAt = null; + recordingTimer.hidden = true; +} + +function startTimer() { + stopTimer(); + recordingStartedAt = Date.now(); + recordingTimer.textContent = "00:00"; + recordingTimer.hidden = false; + timerId = window.setInterval(() => { + if (recordingStartedAt === null) return; + recordingTimer.textContent = formatElapsed(Date.now() - recordingStartedAt); + }, 250); +} -function render(label, detail, kind, showShortcut = true) { +function render(label, detail, kind, options = {}) { + const { showShortcut = false, showTimer = false } = options; title.textContent = label; - message.textContent = detail; - indicator.className = `state-indicator ${kind}`; + message.textContent = detail || ""; + message.hidden = !detail; + shell.className = `overlay-shell ${kind}`; shortcutHint.hidden = !showShortcut; + if (!showTimer) recordingTimer.hidden = true; +} + +async function ensureCompactSize() { + if (!overlayWindow || !tauriWindow?.LogicalSize) return; + try { + await overlayWindow.setSize(new tauriWindow.LogicalSize(OVERLAY_WIDTH, OVERLAY_HEIGHT)); + } catch { + // The static Tauri config has the same size; this is a runtime safety net. + } +} + +function loadSavedPosition() { + try { + const parsed = JSON.parse(window.localStorage.getItem(POSITION_STORAGE_KEY) || "null"); + if (!parsed || !Number.isFinite(parsed.x) || !Number.isFinite(parsed.y)) return null; + return parsed; + } catch { + return null; + } +} + +async function positionInitially() { + if (positioned || !overlayWindow) return; + + const saved = loadSavedPosition(); + if (saved && tauriWindow?.PhysicalPosition) { + try { + await overlayWindow.setPosition( + new tauriWindow.PhysicalPosition(Math.round(saved.x), Math.round(saved.y)), + ); + positioned = true; + return; + } catch { + // Wayland intentionally does not expose global window coordinates. + } + } + + if (!tauriWindow?.currentMonitor || !tauriWindow?.LogicalPosition) return; + try { + const monitor = await tauriWindow.currentMonitor(); + if (!monitor) return; + + const scale = monitor.scaleFactor || 1; + const logicalX = monitor.position.x / scale + 18; + const logicalHeight = monitor.size.height / scale; + const logicalY = monitor.position.y / scale + Math.max(18, (logicalHeight - OVERLAY_HEIGHT) * 0.45); + await overlayWindow.setPosition( + new tauriWindow.LogicalPosition(Math.round(logicalX), Math.round(logicalY)), + ); + positioned = true; + } catch { + // On Wayland the compositor owns absolute placement; drag remains available. + } } async function showOverlay() { if (!overlayWindow) return; try { + await ensureCompactSize(); + await positionInitially(); await overlayWindow.show(); } catch { // The overlay is advisory UI; a visibility failure must never affect dictation. @@ -45,39 +146,82 @@ function applyLifecycle(payload) { switch (payload.state) { case "starting": - render("Preparing", "Loading the selected local model…", "working"); + stopTimer(); + render("Preparing", "Loading model", "working"); void showOverlay(); break; case "recording": - render("Listening", "Press the shortcut again to stop", "recording"); + startTimer(); + render("Listening", "Speak", "recording", { + showShortcut: true, + showTimer: true, + }); + recordingTimer.hidden = false; void showOverlay(); break; case "finishing": - render("Transcribing", "Processing locally and preparing text submission…", "working", false); + stopTimer(); + render("Transcribing", "Local", "working"); void showOverlay(); break; - case "completed": - render("Submitted", "Complete transcript accepted by the insertion backend", "success", false); + case "completed": { + stopTimer(); + const preview = truncate(payload.text); + render("Sent ✓", preview, "success"); void showOverlay(); - void hideOverlay(generation, 1400); + void hideOverlay(generation, 1250); break; + } case "noSpeech": - render("No speech", "Nothing was inserted", "idle", false); + stopTimer(); + render("No speech", "Nothing sent", "idle"); void showOverlay(); - void hideOverlay(generation, 1500); + void hideOverlay(generation, 1200); break; case "failed": - render("Dictation failed", payload.message || "Open BLCVoice for diagnostics", "failed", false); + stopTimer(); + render( + "Not sent", + payload.recoverableText ? "Text recovered" : "Open BLCVoice", + "failed", + ); void showOverlay(); - void hideOverlay(generation, 3200); + void hideOverlay(generation, 2600); break; default: break; } } +async function watchPosition() { + if (!overlayWindow?.onMoved) return; + try { + unlistenMoved = await overlayWindow.onMoved(({ payload }) => { + if (!payload || !Number.isFinite(payload.x) || !Number.isFinite(payload.y)) return; + window.localStorage.setItem( + POSITION_STORAGE_KEY, + JSON.stringify({ x: payload.x, y: payload.y }), + ); + positioned = true; + }); + } catch { + unlistenMoved = null; + } +} + +function installDragBehavior() { + if (!shell || !overlayWindow?.startDragging) return; + shell.addEventListener("mousedown", (event) => { + if (event.button !== 0) return; + void overlayWindow.startDragging().catch(() => {}); + }); +} + async function bootstrap() { if (!listen || !overlayWindow) return; + await ensureCompactSize(); + await watchPosition(); + installDragBehavior(); try { unlisten = await listen("blcvoice://dictation-lifecycle", (event) => { applyLifecycle(event.payload); @@ -88,10 +232,15 @@ async function bootstrap() { } window.addEventListener("beforeunload", () => { + stopTimer(); if (unlisten) { unlisten(); unlisten = null; } + if (unlistenMoved) { + unlistenMoved(); + unlistenMoved = null; + } }); void bootstrap(); diff --git a/crates/blcvoice-audio-processing/src/lib.rs b/crates/blcvoice-audio-processing/src/lib.rs index bf1fda5..073fa65 100644 --- a/crates/blcvoice-audio-processing/src/lib.rs +++ b/crates/blcvoice-audio-processing/src/lib.rs @@ -450,6 +450,10 @@ impl AudioPreprocessor { let (input_frames, output_frames) = resampler .process_into_buffer(&input_adapter, &mut output_adapter, None) .map_err(|error| ProcessingError::Resampling(error.to_string()))?; + let valid_samples = output_frames + .checked_mul(target_channels) + .ok_or(ProcessingError::BufferSizeOverflow)?; + clamp_resampler_overshoot(&mut output[..valid_samples]); Ok(ProcessReport { input_frames, @@ -657,12 +661,24 @@ fn resample_complete_utterance( output.copy_within(valid_start..valid_end, 0); output.truncate(valid_samples); + clamp_resampler_overshoot(output); Ok(expected_output_frames) })(); resampler.reset(); result } +/// Band-limited resampling can ring slightly beyond the nominal full-scale PCM range even when +/// every source sample is valid. Clamp only finite resampler output; NaN/Inf are deliberately left +/// untouched so the strict ASR input validator still rejects genuinely invalid processing output. +fn clamp_resampler_overshoot(samples: &mut [f32]) { + for sample in samples { + if sample.is_finite() { + *sample = sample.clamp(-1.0, 1.0); + } + } +} + fn validate_channel_conversion(source: u16, target: u16) -> Result<(), ProcessingError> { if source == target || target == 1 { Ok(()) @@ -835,7 +851,7 @@ mod tests { assert!( output[..report.output_frames] .iter() - .all(|sample| sample.is_finite()) + .all(|sample| sample.is_finite() && (-1.0..=1.0).contains(sample)) ); } @@ -978,6 +994,42 @@ mod tests { assert!(processed.samples().iter().all(|sample| sample.is_finite())); } + #[test] + fn complete_utterance_resampling_keeps_bounded_pcm_in_unit_range() { + let source = format(1, 48_000); + let target = format(1, 16_000); + let mut processor = + AudioPreprocessor::with_chunk_frames(source, target, 960).expect("valid processor"); + let mut input = Vec::with_capacity(4_800); + for frame in 0..4_800 { + input.push(if (frame / 120) % 2 == 0 { 1.0 } else { -1.0 }); + } + + let processed = processor + .process_utterance(&input) + .expect("bounded full-scale waveform must resample successfully"); + + assert!( + processed + .samples() + .iter() + .all(|sample| sample.is_finite() && (-1.0..=1.0).contains(sample)) + ); + } + + #[test] + fn resampler_clamp_leaves_non_finite_output_for_strict_validation() { + let mut samples = [1.125, -1.25, 0.5, f32::NAN, f32::INFINITY]; + + clamp_resampler_overshoot(&mut samples); + + assert_eq!(samples[0], 1.0); + assert_eq!(samples[1], -1.0); + assert_eq!(samples[2], 0.5); + assert!(samples[3].is_nan()); + assert!(samples[4].is_infinite()); + } + #[test] fn complete_utterances_reset_resampler_state_between_calls() { let source = format(1, 48_000); diff --git a/crates/blcvoice-insertion-eis/Cargo.toml b/crates/blcvoice-insertion-eis/Cargo.toml index fa1b053..6848eb6 100644 --- a/crates/blcvoice-insertion-eis/Cargo.toml +++ b/crates/blcvoice-insertion-eis/Cargo.toml @@ -17,6 +17,7 @@ futures-util = "0.3" reis = { version = "0.7.1", features = ["tokio"] } rustix = { version = "1", features = ["time"] } tokio = { version = "1", features = ["macros", "net", "rt", "sync", "time"] } +wl-clipboard-rs = "0.9.3" [lints] workspace = true diff --git a/crates/blcvoice-insertion-eis/src/lib.rs b/crates/blcvoice-insertion-eis/src/lib.rs index 46e4606..6ba7397 100644 --- a/crates/blcvoice-insertion-eis/src/lib.rs +++ b/crates/blcvoice-insertion-eis/src/lib.rs @@ -8,6 +8,13 @@ use blcvoice_insertion::{ #[cfg(any(target_os = "linux", test))] const EI_TEXT_MAX_UTF8_BYTES: usize = 254; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SubmissionMode { + Text, + #[cfg(any(target_os = "linux", test))] + ClipboardKeyboard, +} + #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct WaylandEisOptions { restore_token: Option, @@ -25,24 +32,31 @@ impl WaylandEisOptions { } } +const fn capability_for_mode(mode: SubmissionMode) -> InsertionCapability { + let backend = match mode { + SubmissionMode::Text => InsertionBackend::XdgRemoteDesktopEis, + #[cfg(any(target_os = "linux", test))] + SubmissionMode::ClipboardKeyboard => InsertionBackend::XdgRemoteDesktopEisClipboard, + }; + InsertionCapability::new(backend, InsertionAuthorization::XdgRemoteDesktop) +} + +#[cfg(any(not(target_os = "linux"), test))] const fn wayland_eis_capability() -> InsertionCapability { - InsertionCapability::new( - InsertionBackend::XdgRemoteDesktopEis, - InsertionAuthorization::XdgRemoteDesktop, - ) + capability_for_mode(SubmissionMode::Text) } fn validate_text(text: &str) -> Result<(), InsertionError> { if text.is_empty() { return Err(InsertionError::new( InsertionErrorKind::InvalidText, - "cannot submit empty text through ei_text.utf8", + "cannot submit empty text through the Wayland insertion backend", )); } if text.contains('\0') { return Err(InsertionError::new( InsertionErrorKind::InvalidText, - "text contains a NUL byte, which ei_text.utf8 cannot encode", + "text contains a NUL byte, which the Wayland insertion backend cannot encode safely", )); } Ok(()) @@ -87,13 +101,18 @@ mod platform { event::{DeviceCapability, EiEvent}, }; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; + use wl_clipboard_rs::copy::{ + MimeType as ClipboardMimeType, Options as ClipboardOptions, Source, + }; use super::{ - InsertionBackend, InsertionError, InsertionErrorKind, InsertionReceipt, TextInserter, - WaylandEisOptions, utf8_chunks, validate_text, wayland_eis_capability, + InsertionBackend, InsertionError, InsertionErrorKind, InsertionReceipt, SubmissionMode, + TextInserter, WaylandEisOptions, capability_for_mode, utf8_chunks, validate_text, }; const DEVICE_READY_TIMEOUT: Duration = Duration::from_secs(5); + const KEY_LEFTCTRL: u32 = 29; + const KEY_V: u32 = 47; enum WorkerCommand { Insert { @@ -103,6 +122,11 @@ mod platform { Shutdown, } + struct ReadyState { + restore_token: Option, + mode: SubmissionMode, + } + struct DeviceSlot { device: reis::event::Device, resumed: bool, @@ -124,10 +148,14 @@ mod platform { } } - fn has_resumed_text_device(&self) -> bool { - self.devices - .iter() - .any(|slot| slot.resumed && slot.device.has_capability(DeviceCapability::Text)) + fn available_mode(&self) -> Option { + if self.text_device_index().is_some() { + Some(SubmissionMode::Text) + } else if self.keyboard_device_index().is_some() { + Some(SubmissionMode::ClipboardKeyboard) + } else { + None + } } fn handle_event( @@ -147,7 +175,9 @@ mod platform { .seat .bind_capabilities(DeviceCapability::Keyboard | DeviceCapability::Text); context.flush().map_err(|error| { - backend_failure(format!("failed to bind EIS text capabilities: {error}")) + backend_failure(format!( + "failed to bind EIS keyboard/text capabilities: {error}" + )) })?; } EiEvent::DeviceAdded(event) => { @@ -202,6 +232,12 @@ mod platform { .position(|slot| slot.resumed && slot.device.has_capability(DeviceCapability::Text)) } + fn keyboard_device_index(&self) -> Option { + self.devices.iter().position(|slot| { + slot.resumed && slot.device.has_capability(DeviceCapability::Keyboard) + }) + } + fn ensure_emulating(&mut self, index: usize, device: &reis::event::Device) { if self.devices[index].emulating { return; @@ -220,12 +256,25 @@ mod platform { ) -> Result { validate_text(text)?; - let index = self.text_device_index().ok_or_else(|| { - InsertionError::new( - InsertionErrorKind::BackendUnavailable, - "no resumed EIS device exposes the ei_text interface", - ) - })?; + if let Some(index) = self.text_device_index() { + return self.insert_ei_text(context, text, index); + } + if let Some(index) = self.keyboard_device_index() { + return self.insert_clipboard_keyboard(context, text, index); + } + + Err(InsertionError::new( + InsertionErrorKind::BackendUnavailable, + "the compositor no longer exposes a resumed EIS text or keyboard device", + )) + } + + fn insert_ei_text( + &mut self, + context: &ei::Context, + text: &str, + index: usize, + ) -> Result { let device = self.devices[index].device.clone(); self.ensure_emulating(index, &device); @@ -264,11 +313,67 @@ mod platform { submitted, )) } + + fn insert_clipboard_keyboard( + &mut self, + context: &ei::Context, + text: &str, + index: usize, + ) -> Result { + ClipboardOptions::new() + .copy(Source::Bytes(text.as_bytes().into()), ClipboardMimeType::Text) + .map_err(|error| { + InsertionError::new( + InsertionErrorKind::BackendUnavailable, + format!( + "the compositor exposes EIS keyboard input but no ei_text device, and the native Wayland clipboard fallback could not be prepared: {error}" + ), + ) + })?; + + let device = self.devices[index].device.clone(); + self.ensure_emulating(index, &device); + let keyboard = device.interface::().ok_or_else(|| { + InsertionError::new( + InsertionErrorKind::BackendUnavailable, + "the selected EIS device no longer exposes a keyboard interface", + ) + })?; + + keyboard.key(KEY_LEFTCTRL, ei::keyboard::KeyState::Press); + device + .device() + .frame(self.last_serial, monotonic_microseconds()); + keyboard.key(KEY_V, ei::keyboard::KeyState::Press); + device + .device() + .frame(self.last_serial, monotonic_microseconds()); + keyboard.key(KEY_V, ei::keyboard::KeyState::Released); + device + .device() + .frame(self.last_serial, monotonic_microseconds()); + keyboard.key(KEY_LEFTCTRL, ei::keyboard::KeyState::Released); + device + .device() + .frame(self.last_serial, monotonic_microseconds()); + + context.flush().map_err(|error| { + backend_failure(format!( + "failed to submit the EIS clipboard-paste shortcut: {error}" + )) + })?; + + Ok(InsertionReceipt::complete( + InsertionBackend::XdgRemoteDesktopEisClipboard, + text.len(), + )) + } } pub struct WaylandEisInserter { command_tx: UnboundedSender, restore_token: Option, + mode: SubmissionMode, worker: Option>, } @@ -285,9 +390,10 @@ mod platform { })?; match ready_rx.recv() { - Ok(Ok(restore_token)) => Ok(Self { + Ok(Ok(ready)) => Ok(Self { command_tx, - restore_token, + restore_token: ready.restore_token, + mode: ready.mode, worker: Some(worker), }), Ok(Err(error)) => { @@ -311,7 +417,7 @@ mod platform { impl TextInserter for WaylandEisInserter { fn capability(&self) -> super::InsertionCapability { - wayland_eis_capability() + capability_for_mode(self.mode) } fn insert_text(&mut self, text: &str) -> Result { @@ -350,7 +456,7 @@ mod platform { fn worker_main( options: WaylandEisOptions, command_rx: UnboundedReceiver, - ready_tx: mpsc::SyncSender, InsertionError>>, + ready_tx: mpsc::SyncSender>, ) { let runtime = match tokio::runtime::Builder::new_current_thread() .enable_all() @@ -376,7 +482,7 @@ mod platform { async fn run_worker( options: WaylandEisOptions, mut command_rx: UnboundedReceiver, - ready_tx: mpsc::SyncSender, InsertionError>>, + ready_tx: mpsc::SyncSender>, ) -> Result<(), InsertionError> { let remote_desktop = RemoteDesktop::new().await.map_err(|error| { backend_failure(format!( @@ -441,15 +547,18 @@ mod platform { .map_err(|error| backend_failure(format!("EIS sender handshake failed: {error}")))?; let mut state = EiState::new(); - tokio::time::timeout(DEVICE_READY_TIMEOUT, async { - while !state.has_resumed_text_device() { + let mode = tokio::time::timeout(DEVICE_READY_TIMEOUT, async { + loop { + if let Some(mode) = state.available_mode() { + break Ok(mode); + } let event = events .next() .await .ok_or_else(|| { InsertionError::new( InsertionErrorKind::BackendUnavailable, - "the EIS event stream closed before a text device became available", + "the EIS event stream closed before a usable input device became available", ) })? .map_err(|error| { @@ -457,19 +566,23 @@ mod platform { })?; state.handle_event(event, &context)?; } - Ok::<(), InsertionError>(()) }) .await .map_err(|_| { InsertionError::new( InsertionErrorKind::BackendUnavailable, - "the compositor did not expose a resumed ei_text device within five seconds", + "the compositor did not expose a resumed EIS text or keyboard device within five seconds", ) })??; - ready_tx.send(Ok(restore_token)).map_err(|_| { - backend_failure("the EIS owner disappeared before initialization completed") - })?; + ready_tx + .send(Ok(ReadyState { + restore_token, + mode, + })) + .map_err(|_| { + backend_failure("the EIS owner disappeared before initialization completed") + })?; loop { tokio::select! { @@ -483,15 +596,15 @@ mod platform { } event = events.next() => { let event = event - .ok_or_else(|| { - InsertionError::new( - InsertionErrorKind::BackendUnavailable, - "the EIS event stream closed", - ) - })? - .map_err(|error| { - backend_failure(format!("failed to read an EIS event: {error}")) - })?; + .ok_or_else(|| { + InsertionError::new( + InsertionErrorKind::BackendUnavailable, + "the EIS event stream closed", + ) + })? + .map_err(|error| { + backend_failure(format!("failed to read an EIS event: {error}")) + })?; state.handle_event(event, &context)?; } } @@ -593,4 +706,18 @@ mod tests { ); assert!(!capability.semantic_delivery_verifiable()); } + + #[test] + fn clipboard_keyboard_fallback_has_distinct_backend_identity() { + let capability = capability_for_mode(SubmissionMode::ClipboardKeyboard); + assert_eq!( + capability.backend(), + InsertionBackend::XdgRemoteDesktopEisClipboard + ); + assert_eq!( + capability.authorization(), + InsertionAuthorization::XdgRemoteDesktop + ); + assert!(!capability.semantic_delivery_verifiable()); + } } diff --git a/crates/blcvoice-insertion/src/lib.rs b/crates/blcvoice-insertion/src/lib.rs index 8196849..6ff511a 100644 --- a/crates/blcvoice-insertion/src/lib.rs +++ b/crates/blcvoice-insertion/src/lib.rs @@ -12,6 +12,7 @@ pub enum InsertionBackend { MacOsQuartz, X11XTest, XdgRemoteDesktopEis, + XdgRemoteDesktopEisClipboard, } impl fmt::Display for InsertionBackend { @@ -21,6 +22,9 @@ impl fmt::Display for InsertionBackend { Self::MacOsQuartz => formatter.write_str("macOsQuartz"), Self::X11XTest => formatter.write_str("x11XTest"), Self::XdgRemoteDesktopEis => formatter.write_str("xdgRemoteDesktopEis"), + Self::XdgRemoteDesktopEisClipboard => { + formatter.write_str("xdgRemoteDesktopEisClipboard") + } } } } @@ -272,6 +276,14 @@ mod tests { ); } + #[test] + fn clipboard_eis_backend_has_an_explicit_identity() { + assert_eq!( + InsertionBackend::XdgRemoteDesktopEisClipboard.to_string(), + "xdgRemoteDesktopEisClipboard" + ); + } + #[test] fn unknown_linux_session_is_explicitly_unavailable() { assert_eq!( diff --git a/docs/platform-validation.md b/docs/platform-validation.md index c77729f..5861d49 100644 --- a/docs/platform-validation.md +++ b/docs/platform-validation.md @@ -78,12 +78,13 @@ For KDE Plasma 6 and GNOME Wayland, verify: 1. BLCVoice requests keyboard control without unnecessary pointer/touchscreen permission. 2. Portal denial or cancellation returns an actionable permission/capability result without crashing. 3. After `RemoteDesktop.Start()` succeeds, BLCVoice connects through `ConnectToEIS()` and sends input exclusively through EIS for that session. -4. A successful EIS session exposes the required text/input capability or BLCVoice reports the capability as unavailable. +4. When `ei_text` is available, BLCVoice prefers the native EIS text path. When the compositor exposes only a resumed EIS keyboard device, the optional `xdgRemoteDesktopEisClipboard` path may use the native Wayland clipboard plus the authorized EIS keyboard session to issue the paste shortcut; if neither path is available, insertion must report unavailable rather than success. 5. Turkish characters, emoji and text longer than 254 UTF-8 bytes are submitted without splitting a code point. 6. Focus remains on the intended target application while text is submitted. -7. Closing or revoking the portal session transitions subsequent insertion attempts to unavailable/failure. -8. Restore-token behavior, if enabled, remains revocable and does not bypass portal policy. -9. No X11, evdev, uinput or privileged fallback activates on failure. +7. If `xdgRemoteDesktopEisClipboard` is selected, record that exact backend identity, verify the intended target visibly receives the complete transcript, and confirm that BLCVoice does not falsely claim clipboard contents were erased. The transcript may remain in the user's system clipboard or clipboard history and that privacy side effect must be visible in release/user documentation. +8. Closing or revoking the portal session transitions subsequent insertion attempts to unavailable/failure. +9. Restore-token behavior, if enabled, remains revocable and does not bypass portal policy. +10. No X11, evdev, uinput or privileged fallback activates on failure. The XDG RemoteDesktop portal documents EIS as the recommended input mechanism. After an EIS connection is established, RemoteDesktop `Notify*` input methods must not be mixed with that session.