diff --git a/Cargo.lock b/Cargo.lock index 586f2be..a1d4be5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,6 +253,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "anymap3" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5dfbc6d8d2675589ccbe4d0fd61df2419075625f8c1a62325e718e2b0049f9" + [[package]] name = "arbitrary" version = "1.5.0" @@ -583,7 +589,7 @@ dependencies = [ "log", "num-rational", "num-traits", - "pastey", + "pastey 0.1.1", "rayon", "thiserror 2.0.21", "v_frame", @@ -687,6 +693,15 @@ dependencies = [ "bit-vec 0.9.1", ] +[[package]] +name = "bit-set" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" +dependencies = [ + "bit-vec 0.10.2", +] + [[package]] name = "bit-vec" version = "0.8.0" @@ -699,6 +714,16 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +[[package]] +name = "bit-vec" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72503af1e72be411d74ca45cd6b309878fcb35354e252fc2a3a8bda3a97082a0" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "bit_field" version = "0.10.3" @@ -793,6 +818,30 @@ dependencies = [ "piper", ] +[[package]] +name = "borsh" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" +dependencies = [ + "once_cell", + "proc-macro-crate 3.5.0", + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "bstr" version = "1.13.1" @@ -1511,6 +1560,17 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" +[[package]] +name = "derive-new" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1659,6 +1719,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + [[package]] name = "dpi" version = "0.1.2" @@ -1689,6 +1755,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "dyn-hash" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fdab65db9274e0168143841eb8f864a0a21f8b1b8d2ba6812bbe6024346e99e" + [[package]] name = "either" version = "1.19.0" @@ -1839,6 +1911,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e59b668157f32d3459d4bd634861a06ffa2503a461ca5f0916b9f0252605e66" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -1999,6 +2082,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.14" @@ -3969,6 +4062,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -4451,6 +4553,12 @@ dependencies = [ "libc", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "markdown" version = "1.0.0" @@ -4487,6 +4595,16 @@ dependencies = [ "xml5ever", ] +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -4546,6 +4664,16 @@ dependencies = [ "paste", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + [[package]] name = "mime" version = "0.3.17" @@ -4562,6 +4690,15 @@ dependencies = [ "unicase", ] +[[package]] +name = "minijinja" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509eeb2603a84eccabb4d75f6ec8ba23b46bbf441aaeb863d6b83a5887fd51e4" +dependencies = [ + "serde", +] + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -4676,6 +4813,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk" version = "0.9.0" @@ -4735,6 +4887,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "nom-language" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de2bc5b451bfedaef92c90b8939a8fff5770bdcc1fafd6239d086aab8fa6b29" +dependencies = [ + "nom", +] + [[package]] name = "noop_proc_macro" version = "0.3.0" @@ -5474,6 +5635,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pathfinder_geometry" version = "0.5.1" @@ -5763,6 +5930,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -5843,6 +6019,29 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prost" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9f016da56b311cff4659f5e7335c4d5fda5ca86cae4f48b8ce1cf3335246a7d" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a367f131d93b3395fc6207c11e67098493ab040def4ce3085adbf186687f7e" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "ptr_meta" version = "0.3.2" @@ -6079,6 +6278,16 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_distr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d431c2703ccf129de4d45253c03f49ebb22b97d6ad79ee3ecfc7e3f4862c1d8" +dependencies = [ + "num-traits", + "rand 0.10.3", +] + [[package]] name = "rand_pcg" version = "0.10.2" @@ -6100,6 +6309,15 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + [[package]] name = "rav1e" version = "0.8.1" @@ -6177,6 +6395,12 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" @@ -6633,6 +6857,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rustix" version = "1.1.5" @@ -7265,6 +7503,12 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strict-num" version = "0.1.1" @@ -7274,6 +7518,16 @@ dependencies = [ "float-cmp", ] +[[package]] +name = "string-interner" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3df9b59e2eded8d825c7c4363ad339a20fb6bc0b9a4778560f518f59910b15" +dependencies = [ + "hashbrown 0.16.1", + "serde", +] + [[package]] name = "string_cache" version = "0.8.9" @@ -7496,6 +7750,17 @@ dependencies = [ "objc", ] +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -7937,6 +8202,173 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tract-core" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5690191b7344a995507bed7523aeea052f907aa1316c0500540eec801e824a1c" +dependencies = [ + "anyhow", + "anymap3", + "bit-set 0.11.1", + "derive-new", + "downcast-rs 2.0.2", + "dyn-clone", + "erased-serde", + "inventory", + "lazy_static", + "log", + "maplit", + "metrics", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pastey 0.2.3", + "rustfft", + "serde", + "smallvec", + "tract-data", + "tract-linalg", +] + +[[package]] +name = "tract-data" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5feff0e09b8cf92be4932457bda1b2156e348bbaf8311753da7f9b102c4af0b3" +dependencies = [ + "anyhow", + "downcast-rs 2.0.2", + "dyn-clone", + "dyn-hash", + "half", + "inventory", + "itertools 0.15.0", + "lazy_static", + "libm", + "maplit", + "ndarray", + "nom", + "nom-language", + "num-integer", + "num-traits", + "parking_lot", + "smallvec", + "string-interner", +] + +[[package]] +name = "tract-extra" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9b3105cbc51d3a24e6b640e18c7d6cdedebeef5b9435a74518e7658ee93578" +dependencies = [ + "tract-nnef", +] + +[[package]] +name = "tract-hir" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a84e8187c517f1a641dcd1b764f1dca2b6b8d39d4c78b2743e1b69874e65b735" +dependencies = [ + "derive-new", + "log", + "tract-core", +] + +[[package]] +name = "tract-linalg" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d389398bb62713b4dd4fc92c3a36ae1552faf522e17b1ae3f636537893a55c" +dependencies = [ + "byteorder", + "cc", + "derive-new", + "downcast-rs 2.0.2", + "dyn-clone", + "dyn-hash", + "half", + "inventory", + "lazy_static", + "libc", + "log", + "minijinja", + "num-traits", + "pastey 0.2.3", + "tract-data", + "walkdir", +] + +[[package]] +name = "tract-nnef" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "def0bced596beb4318148cac90de95dc9953089bb1df86ce2ef4c733ee450773" +dependencies = [ + "byteorder", + "erased-serde", + "log", + "nom", + "nom-language", + "serde", + "serde_json", + "simd-adler32", + "tar", + "tract-core", + "walkdir", +] + +[[package]] +name = "tract-onnx" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "994cd1cb5ea6747ac82707e39ae9588784f6794f96a4a86d137e71ebc49e2689" +dependencies = [ + "bytes", + "derive-new", + "log", + "memmap2", + "nom", + "nom-language", + "num-integer", + "num-rational", + "num-traits", + "prost", + "smallvec", + "tract-extra", + "tract-hir", + "tract-nnef", + "tract-onnx-opl", +] + +[[package]] +name = "tract-onnx-opl" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee1b86a41af0b03fa2bb6d50c86d79ec363ba12e8c76fb0bff1d899d584bf310" +dependencies = [ + "getrandom 0.4.3", + "log", + "rand 0.10.3", + "rand_distr", + "rustfft", + "tract-extra", + "tract-nnef", +] + +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tray-icon" version = "0.26.1" @@ -7984,6 +8416,12 @@ dependencies = [ "core_maths", ] +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.1" @@ -8375,7 +8813,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38a91b4eaddff87b1cd1074985e3713da4af2c49742d1b356b2c01670a67a078" dependencies = [ "cc", - "downcast-rs", + "downcast-rs 1.2.1", "log", "rustix", "scoped-tls", @@ -8734,6 +9172,7 @@ dependencies = [ "tiny_http", "tokio", "tokio-util", + "tract-onnx", "tray-icon", "ttf-parser", "unicode-normalization", @@ -9460,6 +9899,16 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "xcb" version = "1.7.1" diff --git a/Cargo.toml b/Cargo.toml index 11ac2e8..9e2fa51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,8 @@ url = "2" # Audio cpal = "0.18" +hound = "3.5" +tract-onnx = { version = "=0.23.10", default-features = false } # Text ferrous-opencc = "0.4" @@ -80,7 +82,6 @@ block2 = "0.6" base64 = "0.22" [dev-dependencies] -hound = "3.5" tempfile = "3" ttf-parser = "0.25" tiny_http = "0.12" diff --git a/README.md b/README.md index 0047303..7120bae 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,18 @@ typing in. recording (key combinations toggle with one press), or hold to talk. A floating recording pill shows a live waveform and the elapsed time, with a cancel button beside it; click the pill to finish or press Escape to cancel. + During polishing, Skip inserts the recognized text immediately with local + corrections and snippets still applied. If the server fails, the recording is kept so you can resend it. - **Self-hosted models.** Speech recognition goes to your own `/audio/transcriptions` endpoint; optional cleanup goes to a `/chat/completions` endpoint, with an editable prompt and a Prompt Studio to test it. Test Connection checks each server and lists its models. +- **Skip silence locally.** Embedded Silero VAD trims pauses and non-speech + sound from dictation uploads, with buffers around speech. History and recovery + retain the original recording; Send anyway can bypass a no-speech result once. +- **Fewer hesitation sounds.** Optional local filler-word removal works even + with text cleanup off, while preserving the raw transcript in History. - **Gets better with use.** A dictionary supplies spelling hints, corrections you make right after a paste are learned, and Snippets expand short phrases into longer text. Removing a word or snippet can be undone. @@ -45,6 +52,8 @@ typing in. sets the period beside the one before it; highlights such as active days, busiest day and time spoken; and a Habits section with your streak calendar and weekly rhythm. Everything is computed on device. +- **Test your microphone.** Record up to ten seconds and play it back from + Settings > Dictation. The clip stays in memory and is never saved or sent. - **Fits into macOS.** Menu bar icon, standard app menu, start at login (macOS 13+), optional media pause while recording, English and Simplified Chinese UI, and light, dark, Tokyo Night or automatic themes. diff --git a/README.zh-CN.md b/README.zh-CN.md index cb858df..b1e6dbc 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -24,10 +24,16 @@ - **随处听写。** 可用 Globe/Fn 键、单个修饰键(如右 Command)或任意组合键触发。 单键双击进入免手持录音(组合键按一次即开始或结束),或按住说话。录音时会出现 悬浮条,显示实时波形和已录时长,旁边有取消按钮;点击悬浮条结束录音,按 Esc 取消。 + 润色时可点击“跳过”直接插入识别结果,本机纠错和片段展开仍会执行。 如果服务器出错,录音会被保留,可以重新发送。 - **自托管模型。** 语音识别请求发送到你自己的 `/audio/transcriptions` 接口;可选的 文本整理请求发送到 `/chat/completions` 接口,提示词可编辑,并可在 Prompt Studio 中测试。“测试连接”可检查每个服务器并列出它的模型。 +- **本机过滤静音。** 内置 Silero VAD 在上传听写录音前去掉停顿和非人声,并保留 + 语音前后的缓冲。历史和恢复使用原始录音;未检测到人声时可选择“仍然发送”, + 为这次重发跳过过滤。 +- **减少犹豫音。** 可在本机移除“呃”、um、uh 等语气词,无需开启文本整理; + 历史记录仍保留服务器返回的原始转录。 - **越用越准。** 词典为识别提供拼写提示;粘贴后立即做出的修改会被自动学习; 片段(Snippets)可以把短语展开为更长的文本。删除词条或片段后可以撤销。 - **历史记录。** 用 Cmd+K 搜索、查看原始转录、用保存的音频重新转录、撤销删除, @@ -36,6 +42,8 @@ - **统计。** 最近 7 天、30 天或全部时间的字数、听写次数和每分钟字数,每项都与 上一周期对比;趋势图把当前周期与上一周期并排显示;亮点包括活跃天数、最忙的一天 和说话时长;“习惯”部分展示连续天数日历和每周节奏。全部在本机计算。 +- **测试麦克风。** 在“设置 > 听写”录制最长十秒的声音并试听,音频只保留在内存中, + 不会保存、转写或发送到服务器。 - **融入 macOS。** 菜单栏图标、标准应用菜单、开机启动(macOS 13+)、可选在录音时暂停媒体播放、 英文和简体中文界面,以及浅色、深色、Tokyo Night 或自动主题。 - **键盘与旁白。** 所有控件都能用 Tab 访问并带有旁白(VoiceOver)名称,列表和图表 diff --git a/docs/testing.md b/docs/testing.md index 3aee28a..31c7282 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -126,3 +126,48 @@ Do not reset TCC permissions, change system trust, or delete Keychain items duri - Tab through theme choices, buttons, switches, inputs, and dropdowns. Check focus, hover, selected, disabled, and destructive states. The filled action must retain a clearly contrasting label; switches must remain distinguishable by both thumb position and track color. - With `WHISPER_USER_DATA_DIR` pointing to a temporary directory, use `WHISPER_DEV_SCENE` in a debug build to inspect `pill:recording`, `pill:processing`, `pill:preparing`, `pill:error`, and `pill:fallback`. Tokyo Night uses the same indigo surface family as the workspace, muted blue voice activity, and distinct error/recovery colors. Light and Dark retain their dark floating HUD. - Automated tests cover preference serialization, system-appearance resolution, reading contrast, focus indicators, and selected/action text. They complement visual review; they do not prove native interaction or rendering quality. + +## Filler-word removal regression check + +- In Settings > Speech to text, toggle Remove filler words, restart, and verify the preference persists. Existing settings enable it by default. +- With cleanup off and English selected, dictate “Um, we should go”; delivery should read “We should go” while History's raw transcript retains the hesitation. With Auto selected, “um carro” must remain intact. +- Check Chinese hesitation runs and the exceptions 打呃 and 呃逆. Newlines, ordinary words, and the non-filler Chinese words 嗯, 啊, 额, 那个, 然后, and 就是 must remain. +- Check adjacent fillers leave one space between surviving English words, preserve newlines, and remove filler-owned punctuation including ellipses. `foo uh,uh,bar` should become `foo bar`, `hello. Um... we go` should become `hello. We go`, and `好的,呃。我们走` should become `好的。我们走`. +- Check URLs, emails, backtick-wrapped tokens, member access, attached code punctuation and assignments remain intact (`https://uh.edu/a`, `user@hm.com`, `uh_count`, `uh-oh`, `let uh = 1;`). A spaced hesitation dash (`um - I think`) must still filter in Auto. +- Check filler-only input, including `uh!` and `呃。`, produces no-audio feedback. Punctuation-only server text with no removed filler keeps its existing delivery behavior. Whole snippets must match the raw transcript first (`uh oh` and even the filler-only trigger `hmm`), then the filtered transcript (`Um, my address` for `my address`). Turn removal off and verify the transcript is unchanged. File uploads must continue returning raw server text. +- Tab to the switch and activate it with the keyboard; verify its VoiceOver label in both interface languages and all three themes. + +## Skip cleanup regression check + +- Delay the cleanup server, dictate, and verify Transcribing becomes Polishing only after ASR completes. Skip must appear inside the capsule while Cancel remains beside it. Skip is unavailable during ASR. +- Click Skip and verify immediate insertion of the recognized text with filler removal, Chinese script conversion, known corrections, and snippets. There must be no cleanup failure notice and no second paste when the delayed reply arrives. Cancel must still discard the whole operation. +- Tab to Skip and activate it with Return or Space. Check VoiceOver's “Skip cleanup” label and the “Insert without cleanup” hover hint. Check smooth width changes and emergence, then repeat with Reduce motion for immediate changes. +- Use `WHISPER_DEV_SCENE=pill:cleanup` with isolated data to inspect Light, Dark and Tokyo Night, and both interface languages. +- The pill renames itself "Polishing" after 400 ms of cleanup and offers Skip after 1.5 s. A Dictation the server finishes faster must go from "Transcribing" to the check with neither flashing past. Replay measured timings with `WHISPER_DEV_FLOW_MS=,` (for example `130,170`). +- Use `WHISPER_DEV_SCENE=pill:flow` to watch one whole Dictation on a loop: recording, transcribing, polishing, Skip leaving, and the delivered check. The meter and the label must never overlap, and the pill must not read "Transcribing" again after it has read "Polishing". + +## Microphone loopback regression check + +- In Settings > Dictation, check the microphone control order: Test, Refresh, device picker. Test must use the same resolved device as the next dictation, including Auto, lid changes, unavailable saved devices, and the system-default fallback. +- Test retains at most ten seconds with a live level meter, countdown, and Stop. Even if UI polling is delayed beyond the deadline, manual stop and auto stop must truncate the retained samples to ten seconds at the capture sample rate. Stop early, play, stop playback, replay, and Record again. The progress bar, duration and silence warning should reflect only the retained memory clip. A silent clip remains playable and shows a warning. +- Deny microphone permission, disconnect the selected device, and test a device delivering no frames. Check the named error and Try again. Disconnect output during playback and check its error as well. +- Start dictation with the hotkey during recording and playback. The test must close and yield audio before dictation begins. While dictation records, prepares, or processes, Test is disabled with an explanation. No test may play dictation cues or pause media. +- Close Settings, change section, change device, and drop/reopen the sheet during microphone startup, recording and playback. Each must release the streams and clip, and no old result may reopen the strip. No test audio may appear in History, recovery, storage or requests. +- Check Tab, Return/Space, VoiceOver names, Light/Dark/Tokyo Night, both languages, and Reduce motion. Use `settings:mic-recording`, `settings:mic-ready` and `settings:mic-error` dev scenes with isolated data for visual checks. +- `cargo test -- --ignored live_microphone_loopback` checks real capture and output with microphone authorization; ordinary lifecycle tests use an injected clock and hardware-free audio. + +## Local silence filtering regression check + +- In Settings > Speech to text, verify Skip silence defaults on, persists, and is named for VoiceOver. Check both languages, all themes, and keyboard activation. +- Record speech with pauses and background noise. Only the uploaded copy should be trimmed, preserving a 480 ms lookback and tail. Confirm adopted pre-roll is filtered too. History, recovery, recording duration and Insights words per minute must use the original recording. +- Try silence and broadband noise: VAD should avoid a server request and show the existing no-audio wording with a kept recording. Send anyway must send the original once, while Cancel and Record New still discard it. Turn filtering off and verify upload bytes remain untrimmed. File uploads remain outside this path. +- Retry History audio and verify filtering applies with current settings while the stored audio and duration remain unchanged. When speech is detected, send the trimmed copy; when no speech is detected, send the original stored WAV because retry explicitly requests that recording. Live dictation must still avoid a request and offer Send anyway. Check model/inference failures fail open and a short recording is sent intact. +- Automated tests exercise production frame retention with scripted probabilities, detector failures, partial final frames, a real speech fixture in silence and broadband noise, and independent ONNX Runtime reference probabilities. `cargo test --release sixty_second_recording_timing -- --nocapture` logs the 60-second latency without a brittle CI timing limit. + +### VAD model spike and fixture provenance + +`tract-onnx` 0.23.10 was checked against its current crate API and tested before integration. The general v6.2.2 op15 model fails static typing at a decoder `If` (branch ranks differ), while the official fixed-shape 16 kHz OpenVINO ONNX export loads directly. Both official exports give identical reference probabilities on the fixture. Anonymous in-memory node names are made unique before tract optimization so debug validation and release inference use the same graph; weights and connections remain unchanged. Tract identifies 83 of its 90 frames as speech at >0.3; three seconds of silence and seeded broadband noise identify zero speech frames. A separately evaluated `silero-vad-pure` 0.1.1 produced incorrect low speech probabilities and is not a dependency. + +The small `tests/fixtures/vad-speech.wav` was generated locally with macOS `say -v Samantha` saying “Please send the design review to Thursday afternoon.” and converted with `afconvert -f WAVE -d LEI16@16000 -c 1`. It is 16 kHz mono PCM16, 93 KB, and contains no user recording. `vad-probabilities.json` records the official model's CPU ONNX Runtime probabilities for consecutive 512-sample frames, using zero initial `[2,1,128]` state, 64 samples of left context, and zero padding on the final frame. The production tract test compares every probability within 0.00002. + +On the implementation machine (Apple M2 Pro, macOS 27.2.1), the isolated release timing test processed 60 seconds in **116.397 ms**, including first model initialization. The release executable grew from **44,016,400 bytes** before VAD to **67,235,808 bytes** with VAD (+23,219,408 bytes, about 22.14 MiB). These are executable sizes, not app bundles. `cargo build --release --locked --offline` passed; `otool -L` listed only Apple/system dependencies, with no ONNX Runtime or other new distributed dylib. No release packaging or signing was run. diff --git a/docs/third-party-notices.md b/docs/third-party-notices.md index b7e79cb..579df37 100644 --- a/docs/third-party-notices.md +++ b/docs/third-party-notices.md @@ -79,3 +79,35 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +## Silero VAD v6.2.2 + +The main executable embeds the unmodified 16 kHz fixed-shape ONNX model +[`silero_vad_openvino_16k.onnx`](https://github.com/snakers4/silero-vad/blob/60b7ffa243625ebdc1070275a29f18c87843786a/src/silero_vad/data/silero_vad_openvino_16k.onnx) +from the official v6.2.2 release. The OpenVINO export is a standard ONNX graph; +Whisper executes it with pure Rust `tract-onnx` 0.23.10, without OpenVINO or +ONNX Runtime. It is 1,288,203 bytes, SHA-256 +`7776b81ad1b0350c15d7f1555943b9232eb53e9ca5d989c6d0cea9ebc8664d87`. +No model download or external resource is used at runtime. + +MIT License + +Copyright (c) 2020-present Silero Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/locales/en.json b/locales/en.json index 1204a7a..5d4cf5e 100644 --- a/locales/en.json +++ b/locales/en.json @@ -166,7 +166,12 @@ "serverTitle": "Server", "modelHelp": "The model your server should run. Test Connection lists the ones it offers.", "serverModels": "Models on this server", - "protocolTag": "OpenAI-compatible · no auth" + "protocolTag": "OpenAI-compatible · no auth", + "removeFillerWords": "Remove filler words", + "removeFillerWordsDescription": "Drops hesitation sounds such as “um” and “uh” from transcripts. Works without text cleanup.", + "optionsTitle": "Transcription options", + "skipSilence": "Skip silence", + "skipSilenceDescription": "Removes pauses and non-speech sound before the recording is sent, keeping a short buffer so words are not cut off. Runs on this Mac." }, "llms": { "server": { @@ -330,7 +335,8 @@ "viewTranscript": "View Transcript", "recordNew": "Record New", "resend": "Resend Recording", - "openSettings": "Settings…" + "openSettings": "Settings…", + "sendAnyway": "Send anyway" }, "noAudio": { "title": "We didn’t catch that", @@ -368,7 +374,11 @@ "cancel": "Cancel", "cancelDictation": "Cancel Dictation", "dismiss": "Close", - "closesIn": "Closes in {{seconds}} s; hover to keep open" + "closesIn": "Closes in {{seconds}} s; hover to keep open", + "polishing": "Polishing", + "skip": "Skip", + "skipHint": "Insert without cleanup", + "skipCleanup": "Skip cleanup" }, "keptRecording": "Your {{time}} recording is kept until you close this card." }, @@ -757,5 +767,20 @@ "copy": "Copy Text", "copiedHint": "Copied. Press ⌘V to paste." } + }, + "microphoneTest": { + "test": "Test", + "stop": "Stop", + "play": "Play", + "recordAgain": "Record again", + "close": "Close microphone test", + "tryAgain": "Try again", + "privacy": "Stays on this Mac. Never transcribed or saved.", + "silent": "No sound was picked up. Check the selected microphone.", + "permission": "Microphone permission is missing.", + "device": "The microphone is unavailable or disconnected.", + "noSignal": "The microphone did not deliver an input signal.", + "output": "Audio output is unavailable.", + "busy": "Finish or cancel dictation before testing the microphone." } } diff --git a/locales/zh-CN.json b/locales/zh-CN.json index 2aa5669..34ca9e1 100644 --- a/locales/zh-CN.json +++ b/locales/zh-CN.json @@ -163,7 +163,12 @@ "serverTitle": "服务器", "modelHelp": "服务器要运行的模型。点“测试连接”可列出它提供的模型。", "serverModels": "此服务器上的模型", - "protocolTag": "OpenAI 兼容 · 无鉴权" + "protocolTag": "OpenAI 兼容 · 无鉴权", + "removeFillerWords": "移除语气词", + "removeFillerWordsDescription": "自动去掉转录中的“呃”、um、uh 等犹豫音,无需开启文本整理。", + "optionsTitle": "转写选项", + "skipSilence": "跳过静音", + "skipSilenceDescription": "在这台 Mac 上过滤录音中的停顿和非人声,再发送给服务器;保留语音前后的一小段声音,避免截断词句。" }, "llms": { "server": { @@ -326,7 +331,8 @@ "viewTranscript": "查看转录", "recordNew": "新录音", "resend": "重新发送", - "openSettings": "设置…" + "openSettings": "设置…", + "sendAnyway": "仍然发送" }, "noAudio": { "title": "我们没听清", @@ -364,7 +370,11 @@ "cancel": "取消", "cancelDictation": "取消听写", "dismiss": "关闭", - "closesIn": "{{seconds}} 秒后关闭;悬停可保持打开" + "closesIn": "{{seconds}} 秒后关闭;悬停可保持打开", + "polishing": "润色中", + "skip": "跳过", + "skipHint": "不润色直接插入", + "skipCleanup": "跳过润色" }, "keptRecording": "这段 {{time}} 的录音会一直保留,直到你关闭此卡片。" }, @@ -735,5 +745,20 @@ "copy": "复制文字", "copiedHint": "已复制,按 ⌘V 粘贴。" } + }, + "microphoneTest": { + "test": "测试", + "stop": "停止", + "play": "播放", + "recordAgain": "重新录音", + "close": "关闭麦克风测试", + "tryAgain": "重试", + "privacy": "仅保留在这台 Mac 上,不会被转写或保存。", + "silent": "没有录到声音,请检查所选麦克风。", + "permission": "尚未获得麦克风权限。", + "device": "麦克风不可用或已断开连接。", + "noSignal": "麦克风没有传入音频信号。", + "output": "音频输出设备不可用。", + "busy": "请先结束或取消听写,再测试麦克风。" } } diff --git a/src/app_state.rs b/src/app_state.rs index 115b3d2..9412e04 100644 --- a/src/app_state.rs +++ b/src/app_state.rs @@ -28,6 +28,7 @@ use crate::ui::theme; pub struct DictationSnapshot { pub is_recording: bool, pub is_processing: bool, + pub is_cleaning: bool, pub is_preparing: bool, } diff --git a/src/audio/capture.rs b/src/audio/capture.rs index f3027ba..19c9bd5 100644 --- a/src/audio/capture.rs +++ b/src/audio/capture.rs @@ -37,6 +37,9 @@ pub const PRE_ROLL_MAX_AGE: Duration = Duration::from_millis(2_000); pub const FIRST_AUDIO_TIMEOUT: Duration = Duration::from_millis(10_000); /// No frames for this long during a recording means the device is gone. pub const STALL_TIMEOUT: Duration = Duration::from_millis(1_500); +// The lease includes stream teardown, so a test can yield capture to dictation +// without overlapping inputs or feeding its playback into a new recording. +pub(crate) static AUDIO_INPUT_LEASE: Mutex<()> = Mutex::new(()); const TICK: Duration = Duration::from_millis(50); const LEVEL_WINDOW: usize = ANALYSER_WINDOW; @@ -491,6 +494,10 @@ fn capture_thread( commands: mpsc::Receiver, events: Option, ) { + let _lease = AUDIO_INPUT_LEASE.lock(); + if core.state.lock().phase == Phase::Finished { + return; + } let stream = match open_stream(selector.as_deref(), &core) { Ok(stream) => stream, Err(err) => { diff --git a/src/audio/cues.rs b/src/audio/cues.rs index feb9333..6749840 100644 --- a/src/audio/cues.rs +++ b/src/audio/cues.rs @@ -11,9 +11,6 @@ use std::f64::consts::PI; use std::time::Duration; -use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; -use cpal::{FromSample, SampleFormat, SizedSample}; - use super::AudioError; pub const START_NOTES: [f64; 2] = [523.25, 659.25]; @@ -116,60 +113,12 @@ pub fn play(cue: Cue) { /// Plays a cue and returns once it has finished. pub fn play_blocking(cue: Cue) -> Result<(), AudioError> { - let host = cpal::default_host(); - let device = host - .default_output_device() - .ok_or_else(|| AudioError::Output("no output device".into()))?; - let supported = device - .default_output_config() - .map_err(|e| AudioError::Output(e.to_string()))?; - let config = supported.config(); - let samples = render_cue(cue.notes(), config.sample_rate); - let stream = match supported.sample_format() { - SampleFormat::F32 => build::(&device, config, samples), - SampleFormat::F64 => build::(&device, config, samples), - SampleFormat::I16 => build::(&device, config, samples), - SampleFormat::I32 => build::(&device, config, samples), - SampleFormat::U16 => build::(&device, config, samples), - SampleFormat::U8 => build::(&device, config, samples), - other => Err(AudioError::Output(format!( - "unsupported sample format {other:?}" - ))), - }?; - stream - .play() - .map_err(|e| AudioError::Output(e.to_string()))?; - std::thread::sleep(cue_duration(cue.notes()) + Duration::from_millis(80)); - drop(stream); - Ok(()) -} - -fn build( - device: &cpal::Device, - config: cpal::StreamConfig, - samples: Vec, -) -> Result -where - T: SizedSample + FromSample, -{ - let channels = usize::from(config.channels.max(1)); - let mut position = 0usize; - device - .build_output_stream::( - config, - move |out: &mut [T], _| { - for frame in out.chunks_mut(channels) { - let value = samples.get(position).copied().unwrap_or(0.0); - position += 1; - for slot in frame { - *slot = >::from_sample_(value); - } - } - }, - |err| log::debug!("cue output error: {err}"), - None, - ) - .map_err(|e| AudioError::Output(e.to_string())) + super::playback::Playback::start( + std::sync::Arc::new(render_cue(cue.notes(), 48_000)), + 48_000, + false, + )? + .wait() } #[cfg(test)] diff --git a/src/audio/microphone_test.rs b/src/audio/microphone_test.rs new file mode 100644 index 0000000..45b93cb --- /dev/null +++ b/src/audio/microphone_test.rs @@ -0,0 +1,440 @@ +//! Memory-only loopback lifecycle. The clock is supplied by the caller so the +//! ten-second limit and cancellation can be exercised without audio hardware. + +use std::sync::{Arc, Weak}; +use std::time::{Duration, Instant}; + +use futures::future::BoxFuture; +use parking_lot::Mutex; + +use super::AudioError; +use super::capture::{CaptureEvent, RecordedAudio, Recorder}; +#[cfg(any(debug_assertions, test))] +use super::gate::SpeechGate; +use super::gate::{ANALYSER_WINDOW, window_rms}; +use super::playback::Playback; + +pub const RECORDING_LIMIT: Duration = Duration::from_secs(10); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Problem { + Permission, + Device, + NoSignal, + Output, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum State { + Closed, + Recording, + Ready, + Playing, + Error(Problem), +} + +struct Clip { + samples: Arc>, + rate: u32, + duration: Duration, + silent: bool, +} + +pub struct MicrophoneTest { + state: State, + recorder: Recorder, + events: async_channel::Receiver, + clip: Option, + playback: Option, + started: Instant, + generation: u64, + progress: f32, +} + +impl Default for MicrophoneTest { + fn default() -> Self { + let (tx, events) = async_channel::unbounded(); + Self { + state: State::Closed, + recorder: Recorder::new(Some(Arc::new(move |event| { + let _ = tx.try_send(event); + }))), + events, + clip: None, + playback: None, + started: Instant::now(), + generation: 0, + progress: 0., + } + } +} + +impl MicrophoneTest { + pub fn state(&self) -> State { + self.state + } + pub fn generation(&self) -> u64 { + self.generation + } + pub fn level(&self) -> f32 { + self.recorder.level().unwrap_or(0.) + } + pub fn duration(&self) -> Duration { + self.clip + .as_ref() + .map_or(Duration::ZERO, |clip| clip.duration) + } + pub fn silent(&self) -> bool { + self.clip.as_ref().is_some_and(|clip| clip.silent) + } + pub fn progress(&self) -> f32 { + self.playback + .as_ref() + .map_or(self.progress, Playback::progress) + } + pub fn remaining(&self, now: Instant) -> Duration { + RECORDING_LIMIT.saturating_sub(now.saturating_duration_since(self.started)) + } + + pub fn start( + &mut self, + selector: Option<&str>, + now: Instant, + ) -> (u64, BoxFuture<'static, Result<(), AudioError>>) { + self.begin(now); + (self.generation, self.recorder.start(selector)) + } + + fn begin(&mut self, now: Instant) { + self.close(); + self.started = now; + self.state = State::Recording; + } + + pub fn capture_started( + &mut self, + generation: u64, + result: Result<(), AudioError>, + now: Instant, + ) { + if generation != self.generation || self.state != State::Recording { + return; + } + match result { + Ok(()) => self.started = now, + Err(error) => self.fail(&error), + } + } + + pub fn stop(&mut self) { + if self.state != State::Recording { + return; + } + match self.recorder.stop() { + Some(audio) => self.finish_audio(audio), + None => self.fail(&AudioError::FirstAudioUnavailable), + } + } + + fn finish_audio(&mut self, mut audio: RecordedAudio) { + if !audio.received_audio_data || audio.samples.is_empty() || audio.sample_rate == 0 { + self.fail(&AudioError::FirstAudioUnavailable); + return; + } + // UI polling may stop capture late; every finalization path enforces + // the sample limit before measuring or retaining the clip. + let max_samples = audio.sample_rate as usize * RECORDING_LIMIT.as_secs() as usize; + if audio.samples.len() > max_samples { + audio.samples.truncate(max_samples); + audio.speech_gate = Default::default(); + } + // A short clip can finish before capture has measured its first gate. + if audio.speech_gate.window_count == 0 { + for window in audio.samples.chunks(ANALYSER_WINDOW) { + audio + .speech_gate + .record_window(window_rms(window, ANALYSER_WINDOW)); + } + } + let duration = + Duration::from_secs_f64(audio.samples.len() as f64 / f64::from(audio.sample_rate)); + self.clip = Some(Clip { + samples: Arc::new(audio.samples), + rate: audio.sample_rate, + duration, + silent: audio.speech_gate.is_silence(), + }); + self.state = State::Ready; + self.progress = 0.; + } + + pub fn toggle_playback(&mut self) { + self.toggle_playback_with(|clip| { + Playback::start(clip.samples.clone(), clip.rate, true).map(Some) + }); + } + + fn toggle_playback_with( + &mut self, + start: impl FnOnce(&Clip) -> Result, AudioError>, + ) { + if self.state == State::Playing { + self.playback = None; + self.state = State::Ready; + return; + } + if self.state != State::Ready { + return; + } + let Some(clip) = &self.clip else { + return; + }; + match start(clip) { + Ok(playback) => { + self.playback = playback; + self.state = State::Playing; + self.progress = 0.; + } + Err(error) => self.fail(&error), + } + } + + pub fn tick(&mut self, now: Instant) { + while let Ok(CaptureEvent::DeviceLost { capture_id, .. }) = self.events.try_recv() { + if self.state == State::Recording + && self.recorder.active_capture_id() == Some(capture_id) + { + self.fail(&AudioError::DeviceLost(String::new())); + } + } + if self.state == State::Recording && self.remaining(now).is_zero() { + self.stop(); + } + if self.state == State::Playing && self.playback.as_ref().is_some_and(Playback::is_done) { + let error = self.playback.as_ref().and_then(Playback::error); + self.playback = None; + self.playback_finished(error); + } + } + + fn playback_finished(&mut self, error: Option) { + if let Some(error) = error { + self.fail(&error); + } else { + self.state = State::Ready; + self.progress = 1.; + } + } + + fn fail(&mut self, error: &AudioError) { + self.recorder.cancel(); + self.playback = None; + self.state = State::Error(match error { + AudioError::PermissionDenied => Problem::Permission, + AudioError::FirstAudioUnavailable => Problem::NoSignal, + AudioError::Output(_) => Problem::Output, + _ => Problem::Device, + }); + } + + pub fn close(&mut self) { + self.generation = self.generation.wrapping_add(1); + self.recorder.cancel(); + self.playback = None; + self.clip = None; + self.state = State::Closed; + self.progress = 0.; + } + + #[cfg(debug_assertions)] + pub fn dev_scene(&mut self, scene: &str) { + self.begin(Instant::now() - Duration::from_secs(3)); + if scene == "ready" { + self.finish_audio(RecordedAudio { + samples: vec![0.1; 48_000], + sample_rate: 16_000, + received_audio_data: true, + speech_gate: SpeechGate { + peak_rms: 0.1, + window_count: 30, + }, + }); + } else if scene == "error" { + self.fail(&AudioError::PermissionDenied); + } + } +} + +impl Drop for MicrophoneTest { + fn drop(&mut self) { + self.close(); + } +} + +/// Called before dictation opens or adopts a microphone, including prewarming. +pub fn cancel_for_dictation(test: &Weak>) { + if let Some(test) = test.upgrade() { + test.lock().close(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn recording(now: Instant, amplitude: f32) -> MicrophoneTest { + let mut test = MicrophoneTest::default(); + test.begin(now); + test.recorder = Recorder::with_test_audio(vec![amplitude; 16_000]); + test + } + + #[test] + fn injected_clock_auto_stops_at_ten_seconds_and_manual_stop_is_ready() { + let now = Instant::now(); + let mut test = recording(now, 0.1); + test.tick(now + Duration::from_millis(9999)); + assert_eq!(test.state(), State::Recording); + test.tick(now + RECORDING_LIMIT); + assert_eq!(test.state(), State::Ready); + assert_eq!(test.duration(), Duration::from_secs(1)); + assert!(!test.silent()); + let mut test = recording(now, 0.1); + test.stop(); + assert_eq!(test.state(), State::Ready); + test.begin(now); + assert_eq!(test.state(), State::Recording); + assert!(test.clip.is_none()); + } + + #[test] + fn delayed_ticks_and_manual_stops_retain_at_most_ten_seconds() { + let now = Instant::now(); + for auto_stop in [false, true] { + let mut test = recording(now, 0.1); + test.recorder = Recorder::with_test_audio(vec![0.1; 25 * 16_000]); + if auto_stop { + test.tick(now + Duration::from_secs(90)); + } else { + test.stop(); + } + assert_eq!(test.state(), State::Ready); + let clip = test.clip.as_ref().unwrap(); + assert_eq!(clip.samples.len(), 10 * 16_000); + assert_eq!(test.duration(), RECORDING_LIMIT); + } + } + + #[test] + fn finalization_limits_native_rate_samples_and_measures_only_retained_audio() { + for rate in [16_000, 44_100, 48_000] { + let mut test = recording(Instant::now(), 0.1); + let mut samples = vec![0.; 10 * rate as usize]; + samples.extend(vec![0.5; rate as usize]); + test.finish_audio(RecordedAudio { + samples, + sample_rate: rate, + received_audio_data: true, + speech_gate: SpeechGate { + peak_rms: 0.5, + window_count: 110, + }, + }); + let clip = test.clip.as_ref().unwrap(); + assert_eq!(clip.samples.len(), 10 * rate as usize); + assert_eq!(clip.rate, rate); + assert_eq!(test.duration(), RECORDING_LIMIT); + assert!(test.silent()); + } + } + + #[test] + fn silent_clips_still_allow_playback_and_replay() { + let mut test = recording(Instant::now(), 0.); + test.stop(); + assert!(test.silent()); + test.toggle_playback_with(|_| Ok(None)); + assert_eq!(test.state(), State::Playing); + test.playback_finished(None); + assert_eq!(test.state(), State::Ready); + assert_eq!(test.progress(), 1.); + test.toggle_playback_with(|_| Ok(None)); + test.toggle_playback(); + assert_eq!(test.state(), State::Ready); + assert!(test.clip.is_some()); + test.toggle_playback_with(|_| Err(AudioError::Output("disconnected".into()))); + assert_eq!(test.state(), State::Error(Problem::Output)); + test.close(); + assert_eq!(test.state(), State::Closed); + assert!(test.clip.is_none()); + } + + #[test] + fn errors_are_named_and_late_capture_results_cannot_reopen_a_test() { + for (error, problem) in [ + (AudioError::PermissionDenied, Problem::Permission), + ( + AudioError::DeviceLost("disconnected".into()), + Problem::Device, + ), + (AudioError::FirstAudioUnavailable, Problem::NoSignal), + (AudioError::Output("unavailable".into()), Problem::Output), + ] { + let now = Instant::now(); + let mut test = recording(now, 0.1); + let generation = test.generation(); + test.capture_started(generation, Err(error), now); + assert_eq!(test.state(), State::Error(problem)); + assert!(!test.recorder.is_capture_active()); + test.begin(now); + let generation = test.generation(); + test.close(); + test.capture_started(generation, Ok(()), now); + assert_eq!(test.state(), State::Closed); + } + let mut test = recording(Instant::now(), 0.1); + test.finish_audio(RecordedAudio { + samples: Vec::new(), + sample_rate: 16_000, + received_audio_data: false, + speech_gate: SpeechGate::default(), + }); + assert_eq!(test.state(), State::Error(Problem::NoSignal)); + } + + #[test] + fn dictation_and_drop_cancel_capture_and_free_the_memory_clip() { + let now = Instant::now(); + let test = Arc::new(Mutex::new(recording(now, 0.1))); + let recorder = test.lock().recorder.clone(); + cancel_for_dictation(&Arc::downgrade(&test)); + assert!(!recorder.is_capture_active()); + assert_eq!(test.lock().state(), State::Closed); + let mut test = recording(now, 0.1); + let recorder = test.recorder.clone(); + test.stop(); + let clip = Arc::downgrade(&test.clip.as_ref().unwrap().samples); + drop(test); + assert!(!recorder.is_capture_active()); + assert!(clip.upgrade().is_none()); + } + + #[test] + #[ignore] + fn live_microphone_loopback() { + let mut test = MicrophoneTest::default(); + let (generation, start) = test.start(None, Instant::now()); + let result = futures::executor::block_on(start); + test.capture_started(generation, result, Instant::now()); + assert_eq!(test.state(), State::Recording); + std::thread::sleep(Duration::from_secs(2)); + test.stop(); + assert_eq!(test.state(), State::Ready); + test.toggle_playback(); + while test.state() == State::Playing { + test.tick(Instant::now()); + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!(test.state(), State::Ready); + } +} diff --git a/src/audio/mod.rs b/src/audio/mod.rs index 28dbb25..6c9a9d3 100644 --- a/src/audio/mod.rs +++ b/src/audio/mod.rs @@ -7,6 +7,9 @@ pub mod capture; pub mod cues; pub mod devices; pub mod gate; +pub mod microphone_test; +pub mod playback; +pub mod vad; pub mod validation; pub mod wav; diff --git a/src/audio/models/silero_vad_openvino_16k.onnx b/src/audio/models/silero_vad_openvino_16k.onnx new file mode 100644 index 0000000..6ec529d Binary files /dev/null and b/src/audio/models/silero_vad_openvino_16k.onnx differ diff --git a/src/audio/playback.rs b/src/audio/playback.rs new file mode 100644 index 0000000..a937203 --- /dev/null +++ b/src/audio/playback.rs @@ -0,0 +1,167 @@ +//! Shared in-process output for cues and memory-only microphone tests. + +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::Duration; + +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use cpal::{FromSample, SampleFormat, SizedSample}; +use parking_lot::Mutex; + +use super::{AudioError, wav::resample}; + +#[derive(Default)] +struct Status { + position: AtomicUsize, + length: AtomicUsize, + done: AtomicBool, + error: Mutex>, +} + +pub struct Playback { + cancel: mpsc::Sender<()>, + status: Arc, + thread: Option>, +} + +impl Playback { + pub fn start(samples: Arc>, rate: u32, exclusive: bool) -> Result { + let (tx, rx) = mpsc::channel(); + let status = Arc::new(Status::default()); + let progress = status.clone(); + let thread = std::thread::Builder::new() + .name("whisper-playback".into()) + .spawn(move || { + // Test playback and input share a lease. Cancellation can be + // requested while a previous stream is still closing. + let _lease = exclusive.then(|| super::capture::AUDIO_INPUT_LEASE.lock()); + if rx.try_recv().is_ok() { + progress.done.store(true, Ordering::Release); + return; + } + if let Err(error) = play(&samples, rate, &rx, progress.clone()) { + *progress.error.lock() = Some(error); + } + progress.done.store(true, Ordering::Release); + }) + .map_err(|e| AudioError::Output(e.to_string()))?; + Ok(Self { + cancel: tx, + status, + thread: Some(thread), + }) + } + + pub fn progress(&self) -> f32 { + let length = self.status.length.load(Ordering::Relaxed); + if length == 0 { + return 0.; + } + (self.status.position.load(Ordering::Relaxed) as f32 / length as f32).min(1.) + } + + pub fn is_done(&self) -> bool { + self.status.done.load(Ordering::Acquire) + } + + pub fn error(&self) -> Option { + self.status.error.lock().clone() + } + + pub fn wait(mut self) -> Result<(), AudioError> { + if let Some(thread) = self.thread.take() { + thread + .join() + .map_err(|_| AudioError::Output("output worker failed".into()))?; + } + self.error().map_or(Ok(()), Err) + } +} + +impl Drop for Playback { + fn drop(&mut self) { + let _ = self.cancel.send(()); + } +} + +fn play( + samples: &[f32], + rate: u32, + cancel: &mpsc::Receiver<()>, + status: Arc, +) -> Result<(), AudioError> { + let device = cpal::default_host() + .default_output_device() + .ok_or_else(|| AudioError::Output("no output device".into()))?; + let supported = device + .default_output_config() + .map_err(|e| AudioError::Output(e.to_string()))?; + let config = supported.config(); + let samples = resample(samples, rate, config.sample_rate); + status.length.store(samples.len(), Ordering::Relaxed); + let stream = match supported.sample_format() { + SampleFormat::F32 => build::(&device, config, samples, status.clone()), + SampleFormat::F64 => build::(&device, config, samples, status.clone()), + SampleFormat::I16 => build::(&device, config, samples, status.clone()), + SampleFormat::I32 => build::(&device, config, samples, status.clone()), + SampleFormat::U16 => build::(&device, config, samples, status.clone()), + SampleFormat::U8 => build::(&device, config, samples, status.clone()), + other => Err(AudioError::Output(format!( + "unsupported sample format {other:?}" + ))), + }?; + stream + .play() + .map_err(|e| AudioError::Output(e.to_string()))?; + loop { + match cancel.recv_timeout(Duration::from_millis(10)) { + Ok(()) | Err(mpsc::RecvTimeoutError::Disconnected) => break, + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + if status.error.lock().is_some() { + break; + } + if status.position.load(Ordering::Relaxed) >= status.length.load(Ordering::Relaxed) { + // The callback has filled the last output buffer; let it drain. + let _ = cancel.recv_timeout(Duration::from_millis(80)); + break; + } + } + drop(stream); + Ok(()) +} + +fn build( + device: &cpal::Device, + config: cpal::StreamConfig, + samples: Vec, + status: Arc, +) -> Result +where + T: SizedSample + FromSample, +{ + let channels = usize::from(config.channels.max(1)); + let mut position = 0; + let progress = status.clone(); + device + .build_output_stream::( + config, + move |out, _| { + for frame in out.chunks_mut(channels) { + let value = samples.get(position).copied().unwrap_or(0.); + position += 1; + for slot in frame { + *slot = T::from_sample_(value); + } + } + progress + .position + .store(position.min(samples.len()), Ordering::Relaxed); + }, + move |error| { + *status.error.lock() = Some(AudioError::Output(error.to_string())); + }, + None, + ) + .map_err(|e| AudioError::Output(e.to_string())) +} diff --git a/src/audio/vad.rs b/src/audio/vad.rs new file mode 100644 index 0000000..047f106 --- /dev/null +++ b/src/audio/vad.rs @@ -0,0 +1,420 @@ +//! Local Silero speech probabilities and conservative frame retention. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use anyhow::ensure; +use tract_onnx::prelude::*; + +pub const RATE: u32 = 16_000; +pub const FRAME: usize = 512; +pub const NO_SPEECH_CODE: &str = "LOCAL_NO_SPEECH"; +const BUFFER: usize = 15; +const THRESHOLD: f32 = 0.3; +const MODEL: &[u8] = include_bytes!("models/silero_vad_openvino_16k.onnx"); + +#[derive(Debug, Clone, PartialEq)] +pub struct Summary { + pub speech_duration: Duration, + pub removed_duration: Duration, + pub found_speech: bool, +} + +#[derive(Debug)] +pub struct FilteredAudio { + pub samples: Vec, + /// No summary means inference failed open; the samples are the original. + pub summary: Option, +} + +impl FilteredAudio { + pub fn no_speech(&self) -> bool { + self.summary.as_ref().is_some_and(|s| !s.found_speech) + } +} + +fn model() -> Result, String> { + static MODEL_PLAN: OnceLock, String>> = OnceLock::new(); + MODEL_PLAN + .get_or_init(|| { + (|| -> TractResult> { + let mut model = tract_onnx::onnx() + .model_for_read(&mut std::io::Cursor::new(MODEL))? + .with_input_fact(0, f32::fact([1, 576]).into())? + .with_input_fact(1, f32::fact([2, 1, 128]).into())?; + // The official export contains anonymous constant names that + // collide in tract's debug graph validation. Names do not + // participate in inference; preserve all edges and weights. + for node in model.nodes_mut() { + node.name = format!("silero_{}", node.id); + } + model.into_optimized()?.into_runnable() + })() + .map_err(|error| format!("Silero model: {error:#}")) + }) + .clone() +} + +struct Silero { + session: TypedSimpleState, + recurrent: Tensor, + context: [f32; 64], +} + +impl Silero { + fn new() -> Result { + let plan = model()?; + Ok(Self { + session: plan.spawn().map_err(|e| e.to_string())?, + recurrent: Tensor::zero::(&[2, 1, 128]).map_err(|e| e.to_string())?, + context: [0.; 64], + }) + } + + fn probability(&mut self, frame: &[f32; FRAME]) -> Result { + let result = (|| -> TractResult { + let mut input = [0.; 576]; + input[..64].copy_from_slice(&self.context); + input[64..].copy_from_slice(frame); + self.context.copy_from_slice(&input[512..]); + let outputs = self.session.run(tvec!( + Tensor::from_shape(&[1, 576], &input)?.into(), + self.recurrent.clone().into(), + ))?; + ensure!(outputs.len() == 2, "unexpected Silero output count"); + ensure!( + outputs[1].shape() == [2, 1, 128], + "unexpected Silero state shape" + ); + self.recurrent = outputs[1].clone().into_tensor(); + outputs[0].cast_to_scalar::() + })(); + result.map_err(|e| format!("Silero inference: {e:#}")) + } +} + +/// Runs off the UI thread. A fresh recurrent state belongs to each recording; +/// only the immutable execution plan is shared between retries and dictations. +pub fn filter(samples: &[f32]) -> FilteredAudio { + fail_open(samples, || { + let mut detector = Silero::new()?; + filter_with_detector(samples, |frame| detector.probability(frame)) + }) +} + +/// Creates only the upload copy. The caller keeps the original recording. +pub fn upload_samples(samples: &[f32], rate: u32) -> Option> { + let mono = super::wav::resample(samples, rate, RATE); + let filtered = filter(&mono); + if filtered.summary.is_none() { + return Some(super::wav::encode_wav_16k(samples, rate)); + } + (!filtered.no_speech()).then(|| super::wav::encode_wav_pcm16(&filtered.samples, RATE)) +} + +/// History and recovery contain the untrimmed PCM WAV, so they reapply the +/// same detector with a fresh state when sent again. +pub fn upload_wav(original: bytes::Bytes) -> Option { + match decode_wav(&original) { + Ok((samples, rate)) => { + let mono = super::wav::resample(&samples, rate, RATE); + let filtered = filter(&mono); + if filtered.summary.is_none() { + return Some(original); + } + (!filtered.no_speech()) + .then(|| super::wav::encode_wav_pcm16(&filtered.samples, RATE).into()) + } + Err(error) => { + warn_once(&error); + Some(original) + } + } +} + +fn decode_wav(bytes: &[u8]) -> Result<(Vec, u32), String> { + let mut reader = + hound::WavReader::new(std::io::Cursor::new(bytes)).map_err(|e| e.to_string())?; + let spec = reader.spec(); + if spec.channels == 0 + || spec.sample_rate == 0 + || spec.bits_per_sample != 16 + || spec.sample_format != hound::SampleFormat::Int + { + return Err("unsupported stored WAV format".into()); + } + let samples = reader + .samples::() + .map(|sample| sample.map(|sample| f32::from(sample) / 32768.)) + .collect::, _>>() + .map_err(|e| e.to_string())?; + let channels = usize::from(spec.channels); + if samples.len() % channels != 0 { + return Err("incomplete stored WAV frame".into()); + } + Ok(( + samples + .chunks_exact(channels) + .map(|frame| frame.iter().sum::() / channels as f32) + .collect(), + spec.sample_rate, + )) +} + +fn warn_once(error: &str) { + static WARNED: AtomicBool = AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + log::warn!("silence filtering failed open; sending original audio: {error}"); + } +} + +fn fail_open( + samples: &[f32], + work: impl FnOnce() -> Result, +) -> FilteredAudio { + // A runtime panic is also an inference failure, never a lost recording. + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(work)) { + Ok(Ok(audio)) => audio, + failure => { + let error = match failure { + Ok(Err(error)) => error, + _ => "inference panicked".into(), + }; + warn_once(&error); + FilteredAudio { + samples: samples.to_vec(), + summary: None, + } + } + } +} + +fn duration(samples: usize) -> Duration { + Duration::from_secs_f64(samples as f64 / f64::from(RATE)) +} + +/// The production frame algorithm, with an injectable detector for exact +/// retention and failure tests. Partial final frames are padded for inference +/// but only their original samples can be retained. +fn filter_with_detector( + samples: &[f32], + mut probability: impl FnMut(&[f32; FRAME]) -> Result, +) -> Result { + if samples.len() < FRAME { + return Err("recording is shorter than a VAD frame".into()); + } + if samples.iter().any(|s| !s.is_finite()) { + return Err("non-finite audio sample".into()); + } + let mut speech = Vec::with_capacity(samples.len().div_ceil(FRAME)); + for chunk in samples.chunks(FRAME) { + let mut frame = [0.; FRAME]; + frame[..chunk.len()].copy_from_slice(chunk); + let p = probability(&frame)?; + if !p.is_finite() || !(0.0..=1.0).contains(&p) { + return Err("invalid speech probability".into()); + } + speech.push(p > THRESHOLD); + } + let mut keep = vec![false; speech.len()]; + let mut consecutive = 0; + let mut tail_end: Option = None; + for (index, &spoken) in speech.iter().enumerate() { + if tail_end.is_some_and(|end| index > end) { + tail_end = None; + } + if spoken { + consecutive += 1; + if tail_end.is_some() || consecutive >= 2 { + if tail_end.is_none() { + keep[index.saturating_sub(1 + BUFFER)..=index].fill(true); + } + tail_end = Some(index + BUFFER); + } + } else { + consecutive = 0; + } + if tail_end.is_some() { + keep[index] = true; + } + } + if consecutive == 1 && tail_end.is_none() { + let last = speech.len() - 1; + keep[last.saturating_sub(BUFFER)..=last].fill(true); + } + let mut output = Vec::with_capacity(samples.len()); + let mut speech_samples = 0; + for (index, chunk) in samples.chunks(FRAME).enumerate() { + if keep[index] { + output.extend_from_slice(chunk); + if speech[index] { + speech_samples += chunk.len(); + } + } + } + let found_speech = speech_samples > 0; + if found_speech && output.is_empty() || output.len() > samples.len() { + return Err("implausible retained frame result".into()); + } + let removed_duration = duration(samples.len() - output.len()); + Ok(FilteredAudio { + samples: output, + summary: Some(Summary { + speech_duration: duration(speech_samples), + removed_duration, + found_speech, + }), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scripted(probabilities: &[f32]) -> Vec { + let samples: Vec = (0..probabilities.len()) + .flat_map(|i| std::iter::repeat_n(i as f32, FRAME)) + .collect(); + let mut probabilities = probabilities.iter(); + filter_with_detector(&samples, |_| Ok(*probabilities.next().unwrap())) + .unwrap() + .samples + .chunks(FRAME) + .map(|frame| frame[0] as usize) + .collect() + } + + #[test] + fn onset_keeps_fifteen_preceding_frames_and_fifteen_tail_frames() { + let mut p = vec![0.; 60]; + p[20..22].fill(0.9); + assert_eq!(scripted(&p), (5..37).collect::>()); + } + + #[test] + fn speech_inside_the_tail_refills_it_without_duplication() { + let mut p = vec![0.; 70]; + p[20..22].fill(0.9); + p[30] = 0.9; + assert_eq!(scripted(&p), (5..46).collect::>()); + p[50..52].fill(0.9); + assert_eq!(scripted(&p), (5..67).collect::>()); + } + + #[test] + fn final_unconfirmed_burst_all_silence_and_all_speech() { + let mut p = vec![0.; 40]; + p[39] = 0.9; + assert_eq!(scripted(&p), (24..40).collect::>()); + assert!(scripted(&[0.; 40]).is_empty()); + assert!(scripted(&[THRESHOLD; 40]).is_empty()); + assert_eq!(scripted(&[0.9; 40]), (0..40).collect::>()); + let audio = filter_with_detector(&vec![0.1; FRAME + 13], |_| Ok(0.9)).unwrap(); + assert_eq!(audio.samples.len(), FRAME + 13); + } + + #[test] + fn detector_errors_invalid_probabilities_and_short_audio_fail_open() { + let original = vec![0.25; FRAME * 3]; + for probability in [f32::NAN, -0.1, 1.1] { + let audio = fail_open(&original, || { + filter_with_detector(&original, |_| Ok(probability)) + }); + assert_eq!(audio.samples, original); + assert!(audio.summary.is_none()); + } + let audio = fail_open(&original, || { + filter_with_detector(&original, |_| Err("injected failure".into())) + }); + assert_eq!(audio.samples, original); + assert!(!audio.no_speech()); + let short = vec![0.25; FRAME - 1]; + assert_eq!(filter(&short).samples, short); + let short_wav: bytes::Bytes = super::super::wav::encode_wav_pcm16(&short, RATE).into(); + assert_eq!(upload_wav(short_wav.clone()), Some(short_wav)); + let invalid = bytes::Bytes::from_static(b"invalid WAV"); + assert_eq!(upload_wav(invalid.clone()), Some(invalid)); + } + + fn speech_fixture() -> Vec { + let mut wav = hound::WavReader::new(std::io::Cursor::new(include_bytes!( + "../../tests/fixtures/vad-speech.wav" + ))) + .unwrap(); + wav.samples::() + .map(|s| f32::from(s.unwrap()) / 32768.) + .collect() + } + + #[test] + fn tract_probabilities_match_the_independent_onnx_runtime_reference() { + let reference: Vec = + serde_json::from_str(include_str!("../../tests/fixtures/vad-probabilities.json")) + .unwrap(); + let speech = speech_fixture(); + let mut detector = Silero::new().unwrap(); + for (index, chunk) in speech.chunks(FRAME).enumerate() { + let mut frame = [0.; FRAME]; + frame[..chunk.len()].copy_from_slice(chunk); + let probability = detector.probability(&frame).unwrap(); + assert!( + (probability - reference[index]).abs() < 0.00002, + "frame {index}: {probability} vs {}", + reference[index] + ); + } + } + + fn noise(length: usize) -> Vec { + let mut random = 123456789u32; + (0..length) + .map(|_| { + random ^= random << 13; + random ^= random >> 17; + random ^= random << 5; + (random as f32 / u32::MAX as f32 - 0.5) * 0.1 + }) + .collect() + } + + #[test] + fn embedded_silero_keeps_real_speech_surrounded_by_silence_or_noise() { + let speech = speech_fixture(); + for surrounding in [vec![0.; RATE as usize], noise(RATE as usize)] { + let mut samples = surrounding.clone(); + samples.extend_from_slice(&speech); + samples.extend_from_slice(&surrounding); + let audio = filter(&samples); + let summary = audio.summary.unwrap(); + assert!(summary.found_speech); + assert!(summary.speech_duration > Duration::from_secs(1)); + assert!(summary.removed_duration > Duration::from_millis(500)); + assert!(audio.samples.len() >= speech.len()); + } + for samples in [vec![0.; RATE as usize * 3], noise(RATE as usize * 3)] { + let audio = filter(&samples); + assert!(audio.no_speech()); + assert!(audio.samples.is_empty()); + } + } + + #[test] + fn sixty_second_recording_timing() { + let speech = speech_fixture(); + let samples: Vec = speech + .iter() + .copied() + .cycle() + .take(RATE as usize * 60) + .collect(); + let start = std::time::Instant::now(); + let audio = filter(&samples); + eprintln!( + "Silero 60-second recording: {:.3} ms", + start.elapsed().as_secs_f64() * 1000. + ); + assert!(audio.summary.is_some()); + assert!(!audio.no_speech()); + } +} diff --git a/src/audio/wav.rs b/src/audio/wav.rs index cfdd540..080d447 100644 --- a/src/audio/wav.rs +++ b/src/audio/wav.rs @@ -202,7 +202,6 @@ fn resampled_len(input_len: usize, from_rate: u32, to_rate: u32) -> usize { /// Windowed-sinc (Kaiser) resampler. Downsampling low-passes below the /// target Nyquist so nothing aliases into the speech band. -#[cfg(test)] pub fn resample(input: &[f32], from_rate: u32, to_rate: u32) -> Vec { if from_rate == to_rate || input.is_empty() || from_rate == 0 || to_rate == 0 { return input.to_vec(); diff --git a/src/dictation.rs b/src/dictation.rs index 9372696..dd75f79 100644 --- a/src/dictation.rs +++ b/src/dictation.rs @@ -18,7 +18,7 @@ use tokio_util::sync::CancellationToken; use crate::audio::capture::{CaptureEvent, CaptureId, CaptureStatus, RecordedAudio, Recorder}; use crate::audio::cues; -use crate::history::Recording; +use crate::history::{HistoryStore, Recording}; use crate::i18n::t; use crate::pipeline::{self, PipelineInput, PipelineOutcome, Transcript}; use crate::services::transcription; @@ -44,6 +44,7 @@ enum Phase { pub struct ErrorActions { /// The recording is kept, so it can be sent again without speaking again. pub retry: bool, + pub send_anyway: bool, /// The cause lives in Settings → Speech to text. pub open_settings: bool, } @@ -55,18 +56,26 @@ impl ErrorActions { /// time; an invalid endpoint fails the same way until it is fixed. pub fn for_transcription_failure(code: Option<&str>) -> Self { match code { + Some(crate::audio::vad::NO_SPEECH_CODE) => Self { + retry: true, + send_anyway: true, + open_settings: false, + }, Some(transcription::CONNECTION_FAILED_CODE | transcription::REQUEST_TIMEOUT_CODE) => { Self { retry: true, + send_anyway: false, open_settings: true, } } Some(transcription::CUSTOM_ENDPOINT_INVALID_CODE) => Self { retry: false, + send_anyway: false, open_settings: true, }, _ => Self { retry: true, + send_anyway: false, open_settings: false, }, } @@ -78,6 +87,7 @@ impl ErrorActions { /// resend, or the panel closing drops it. struct FailedRecording { wav: bytes::Bytes, + bypass_vad: bool, duration: Duration, occurred_at: DateTime, } @@ -89,6 +99,7 @@ impl FailedRecording { let wav = audio.to_wav_16k(); audio.is_usable(wav.len()).then(|| Self { wav: wav.into(), + bypass_vad: false, duration, occurred_at, }) @@ -166,6 +177,9 @@ pub struct Dictation { generation: u64, start_lock: bool, cancel: Option, + mic_test: std::sync::Weak>, + skip_cleanup: Option, + is_cleaning: bool, target_pid: Option, started_at: Option<(Instant, DateTime)>, effects_held: bool, @@ -212,6 +226,9 @@ impl Dictation { generation: 0, start_lock: false, cancel: None, + mic_test: Default::default(), + skip_cleanup: None, + is_cleaning: false, target_pid: None, started_at: None, effects_held: false, @@ -230,6 +247,14 @@ impl Dictation { cx.global::().0.clone() } + pub fn attach_microphone_test( + &mut self, + test: &Arc>, + ) { + crate::audio::microphone_test::cancel_for_dictation(&self.mic_test); + self.mic_test = Arc::downgrade(test); + } + pub fn status(&self) -> StatusMirror { self.status.clone() } @@ -248,6 +273,10 @@ impl Dictation { fn set_phase(&mut self, phase: Phase, cx: &mut Context) { let previous = self.phase; self.phase = phase; + if phase != Phase::Processing { + self.is_cleaning = false; + self.skip_cleanup = None; + } self.status.set(phase); if phase != Phase::Preparing { backend::clear_hotkey_start_request(); @@ -262,6 +291,7 @@ impl Dictation { let snapshot = DictationSnapshot { is_recording: self.phase == Phase::Recording, is_processing: self.phase == Phase::Processing, + is_cleaning: self.phase == Phase::Processing && self.is_cleaning, is_preparing: self.phase == Phase::Preparing, }; AppState::global(cx).update(cx, |s, cx| { @@ -309,6 +339,7 @@ impl Dictation { if self.start_lock || self.phase != Phase::Idle { return; } + crate::audio::microphone_test::cancel_for_dictation(&self.mic_test); // A new Dictation replaces the one that failed. self.failed = None; let device = backend::preferred_input_device(&AppState::settings(cx)); @@ -349,6 +380,7 @@ impl Dictation { if self.start_lock || matches!(self.phase, Phase::Recording | Phase::Processing) { return; } + crate::audio::microphone_test::cancel_for_dictation(&self.mic_test); self.start_lock = true; self.push_force_stopped = false; self.failed = None; @@ -446,6 +478,42 @@ impl Dictation { true } + pub fn skip_cleanup(&mut self, cx: &mut Context) { + if self.phase == Phase::Processing && self.is_cleaning { + if let Some(skip) = self.skip_cleanup.take() { + skip.cancel(); + } + self.is_cleaning = false; + self.publish(cx); + } + } + + fn cleanup_control( + &mut self, + cx: &mut Context, + ) -> (CancellationToken, async_channel::Sender) { + let skip = CancellationToken::new(); + self.skip_cleanup = Some(skip.clone()); + let (tx, rx) = async_channel::unbounded(); + let generation = self.generation; + cx.spawn(async move |this, cx| { + while let Ok(running) = rx.recv().await { + let _ = this.update(cx, |this, cx| { + if this.generation == generation && this.phase == Phase::Processing { + this.is_cleaning = running + && this + .skip_cleanup + .as_ref() + .is_some_and(|skip| !skip.is_cancelled()); + this.publish(cx); + } + }); + } + }) + .detach(); + (skip, tx) + } + pub fn cancel_processing(&mut self, cx: &mut Context) { if self.phase != Phase::Processing { return; @@ -515,6 +583,7 @@ impl Dictation { description: t("dictation.errorDescriptions.microphoneUnavailable"), actions: ErrorActions { retry: self.failed.is_some(), + send_anyway: false, open_settings: false, }, kept: self.failed.as_ref().map(|failed| failed.duration), @@ -541,18 +610,19 @@ impl Dictation { let generation = self.generation; let cancel = CancellationToken::new(); self.cancel = Some(cancel.clone()); + let (skip, stage) = self.cleanup_control(cx); let stopped = Instant::now(); + let silence_filtering = AppState::settings(cx).silence_filtering; let handle = crate::services::runtime::spawn(async move { let conversion_cancel = cancel.clone(); let wav = crate::services::runtime::blocking(move || { - if conversion_cancel.is_cancelled() || audio.speech_gate.is_silence() { + if conversion_cancel.is_cancelled() { return None; } - let wav = audio.to_wav_16k(); - audio.is_usable(wav.len()).then(|| bytes::Bytes::from(wav)) + prepare_recording_for_upload(audio, silence_filtering) }) .await; - let wav = match wav { + let (wav, upload) = match wav { Ok(Some(wav)) => wav, Ok(None) => return (PipelineOutcome::NoAudio, bytes::Bytes::new()), Err(_) => return (pipeline_task_failed(), bytes::Bytes::new()), @@ -566,8 +636,12 @@ impl Dictation { return (PipelineOutcome::Cancelled, wav); } let mut input = input.await; - input.audio = wav.clone(); - let outcome = run_prepared_pipeline(input, cancel).await; + let Some(upload) = upload else { + return (no_speech_outcome(), wav); + }; + input.audio = upload; + input.cleanup_stage = Some(stage); + let outcome = run_pipeline_with_key(input, cancel, skip).await; log::info!( "[perf] operation={generation} stage=pipeline_complete elapsed_ms={:.3}", stopped.elapsed().as_secs_f64() * 1000.0 @@ -598,12 +672,19 @@ impl Dictation { let Some(failed) = self.failed.take() else { return; }; + crate::audio::microphone_test::cancel_for_dictation(&self.mic_test); self.push_force_stopped = false; self.generation += 1; self.capture_target(cx); self.set_phase(Phase::Processing, cx); cx.emit(DictationNotice::DismissError); - self.transcribe(failed.wav, failed.duration, failed.occurred_at, cx); + self.transcribe( + failed.wav, + failed.duration, + failed.occurred_at, + failed.bypass_vad, + cx, + ); } /// Forgets the failed recording once its error panel closes. @@ -616,14 +697,23 @@ impl Dictation { wav: bytes::Bytes, duration: Duration, occurred_at: DateTime, + bypass_vad: bool, cx: &mut Context, ) { let generation = self.generation; let input = Self::pipeline_input(wav.clone(), cx); let cancel = CancellationToken::new(); self.cancel = Some(cancel.clone()); + let (skip, stage) = self.cleanup_control(cx); let handle = crate::services::runtime::spawn(async move { - run_prepared_pipeline(input.await, cancel).await + let mut input = input.await; + input.cleanup_stage = Some(stage); + let mode = if bypass_vad { + VadMode::Bypass + } else { + VadMode::Live + }; + run_prepared_pipeline_controlled(input, cancel, skip, mode).await }); cx.spawn(async move |this, cx| { let outcome = handle.await.unwrap_or(PipelineOutcome::Failed { @@ -676,6 +766,7 @@ impl Dictation { } => { self.set_phase(Phase::Idle, cx); let title = match code.as_deref() { + Some(crate::audio::vad::NO_SPEECH_CODE) => t("dictation.noAudio.title"), Some(transcription::CONNECTION_FAILED_CODE) => { t("dictation.errorTitles.transcriptionConnectionFailed") } @@ -699,6 +790,7 @@ impl Dictation { let kept = actions.retry.then_some(duration); self.failed = actions.retry.then_some(FailedRecording { wav, + bypass_vad: actions.send_anyway, duration, occurred_at, }); @@ -725,6 +817,7 @@ impl Dictation { ) -> DictationNotice { self.failed = (!wav.is_empty()).then_some(FailedRecording { wav, + bypass_vad: false, duration, occurred_at, }); @@ -896,6 +989,27 @@ fn pipeline_task_failed() -> PipelineOutcome { } } +// Keep original audio for recovery and History; only the upload copy is filtered. +fn prepare_recording_for_upload( + audio: RecordedAudio, + silence_filtering: bool, +) -> Option<(bytes::Bytes, Option)> { + if audio.speech_gate.is_silence() { + return None; + } + let wav = audio.to_wav_16k(); + if !audio.is_usable(wav.len()) { + return None; + } + let original = bytes::Bytes::from(wav); + let upload = if silence_filtering { + crate::audio::vad::upload_samples(&audio.samples, audio.sample_rate).map(bytes::Bytes::from) + } else { + Some(original.clone()) + }; + Some((original, upload)) +} + /// Enqueue at the UI action, before conversion or audio loading can yield. /// Earlier edits have settled before this read, and later edits cannot change /// this request's rules. Never send optimistic UI projections to the pipeline. @@ -921,12 +1035,65 @@ fn capture_pipeline_input( snippets: db.get_snippets().unwrap_or_default(), settings, cleanup_api_key: None, + cleanup_stage: None, }) } -async fn run_prepared_pipeline( +#[derive(Clone, Copy, PartialEq, Eq)] +enum VadMode { + Live, + HistoryRetry, + Bypass, +} + +#[cfg(test)] +async fn run_prepared_pipeline(input: PipelineInput, cancel: CancellationToken) -> PipelineOutcome { + run_prepared_pipeline_controlled(input, cancel, CancellationToken::new(), VadMode::Live).await +} + +async fn run_prepared_pipeline_controlled( + mut input: PipelineInput, + cancel: CancellationToken, + skip_cleanup: CancellationToken, + mode: VadMode, +) -> PipelineOutcome { + if cancel.is_cancelled() { + return PipelineOutcome::Cancelled; + } + if input.settings.silence_filtering && mode != VadMode::Bypass { + let original = input.audio.clone(); + let filtered = + crate::services::runtime::blocking(move || crate::audio::vad::upload_wav(original)) + .await; + if cancel.is_cancelled() { + return PipelineOutcome::Cancelled; + } + match filtered { + Ok(Some(audio)) => input.audio = audio, + Ok(None) if mode == VadMode::HistoryRetry => { + // History retry explicitly requests this stored recording. + // Send it intact if VAD rejects it; live dictation instead + // keeps the recording and offers Send anyway for confirmation. + } + Ok(None) => return no_speech_outcome(), + Err(error) => log::warn!("VAD worker failed; sending original audio: {error}"), + } + } + run_pipeline_with_key(input, cancel, skip_cleanup).await +} + +fn no_speech_outcome() -> PipelineOutcome { + PipelineOutcome::Failed { + message: "No speech detected locally".into(), + code: Some(crate::audio::vad::NO_SPEECH_CODE.into()), + message_key: Some("dictation.noAudio.description".into()), + } +} + +async fn run_pipeline_with_key( mut input: PipelineInput, cancel: CancellationToken, + skip_cleanup: CancellationToken, ) -> PipelineOutcome { if cancel.is_cancelled() { return PipelineOutcome::Cancelled; @@ -945,7 +1112,7 @@ async fn run_prepared_pipeline( if cancel.is_cancelled() { return PipelineOutcome::Cancelled; } - pipeline::run(input, cancel).await + pipeline::run(input, cancel, skip_cleanup).await } /// Re-transcribes a History row from its saved audio through the same @@ -954,16 +1121,10 @@ pub fn retry_transcription(id: i64, cx: &mut App) -> Task> { let history = AppState::global(cx).read(cx).history_store.clone(); let input = Dictation::pipeline_input(bytes::Bytes::new(), cx); cx.spawn(async move |cx| { - let audio = crate::services::runtime::storage(move || history.retry_audio(id)).await; - let Some(audio) = audio else { - return Err(t("controlPanel.history.audioNotFound")); - }; - let mut input = input.await; - input.audio = audio.into(); - let outcome = - crate::services::runtime::spawn(run_prepared_pipeline(input, CancellationToken::new())) - .await - .map_err(|e| e.to_string())?; + let input = input.await; + let outcome = crate::services::runtime::spawn(transcribe_history_audio(history, id, input)) + .await + .map_err(|e| e.to_string())??; match outcome { PipelineOutcome::Transcribed(transcript) => { let failure = transcript.cleanup_failure.clone(); @@ -1018,6 +1179,25 @@ pub fn retry_transcription(id: i64, cx: &mut App) -> Task> { }) } +async fn transcribe_history_audio( + history: Arc, + id: i64, + mut input: PipelineInput, +) -> Result { + let audio = crate::services::runtime::storage(move || history.retry_audio(id)).await; + let Some(audio) = audio else { + return Err(t("controlPanel.history.audioNotFound")); + }; + input.audio = audio.into(); + Ok(run_prepared_pipeline_controlled( + input, + CancellationToken::new(), + CancellationToken::new(), + VadMode::HistoryRetry, + ) + .await) +} + #[cfg(test)] mod tests { use super::{ErrorActions, capture_pipeline_input, run_prepared_pipeline}; @@ -1042,6 +1222,9 @@ mod tests { generation: 7, start_lock: false, cancel: None, + mic_test: Default::default(), + skip_cleanup: None, + is_cleaning: false, target_pid: None, started_at: None, effects_held: false, @@ -1181,6 +1364,7 @@ mod tests { spoken_text: "words worth keeping".into(), raw_text: "um words worth keeping".into(), cleanup_failure: None, + cleanup_skipped: false, }; for retention_enabled in [false, true] { let saved = history @@ -1248,6 +1432,9 @@ mod tests { generation: 7, start_lock: false, cancel: Some(cancel.clone()), + mic_test: Default::default(), + skip_cleanup: None, + is_cleaning: false, target_pid: None, started_at: None, effects_held: false, @@ -1406,4 +1593,199 @@ mod tests { assert_eq!(disabled.snippets, input.snippets); }); } + + fn vad_input(server: &MockServer, audio: bytes::Bytes) -> crate::pipeline::PipelineInput { + crate::pipeline::PipelineInput { + audio, + settings: Settings { + remote_transcription_url: server.url("/v1"), + use_cleanup_model: false, + ..Settings::default() + }, + dictionary: Vec::new(), + dictionary_priority: Vec::new(), + known_corrections: Vec::new(), + snippets: Vec::new(), + cleanup_api_key: None, + cleanup_stage: None, + } + } + + fn uploaded_wav(server: &MockServer) -> Vec { + let requests = server.requests(); + let body = &requests.iter().find(|r| r.method == "POST").unwrap().body; + let start = body.windows(4).position(|w| w == b"RIFF").unwrap(); + let length = + u32::from_le_bytes(body[start + 4..start + 8].try_into().unwrap()) as usize + 8; + body[start..start + length].to_vec() + } + + fn retry_stored_audio(samples: &[f32], expected_upload: &[u8]) { + let original = crate::audio::wav::encode_wav_pcm16(samples, 16_000); + let dir = tempfile::tempdir().unwrap(); + let db = Arc::new(Database::open_in_memory().unwrap()); + let audio = Arc::new(crate::audio_storage::AudioStorage::new( + dir.path().join("audio"), + )); + let history = Arc::new(crate::history::HistoryStore::new(db.clone(), audio)); + let duration = std::time::Duration::from_secs_f64(samples.len() as f64 / 16_000.); + let transcript = crate::pipeline::Transcript { + text: "previous transcript".into(), + spoken_text: "previous transcript".into(), + raw_text: "previous transcript".into(), + cleanup_failure: None, + cleanup_skipped: false, + }; + let row = history + .record_delivered( + &transcript, + &crate::history::Recording { + wav: &original, + duration, + occurred_at: chrono::Utc::now(), + }, + &Settings::default(), + ) + .result + .unwrap() + .unwrap() + .item; + let server = MockServer::start(|_| Reply::json(200, r#"{"text":"retried words"}"#)); + let input = vad_input(&server, bytes::Bytes::new()); + let outcome = run(super::transcribe_history_audio( + history.clone(), + row.id, + input, + )) + .unwrap(); + let PipelineOutcome::Transcribed(transcript) = outcome else { + panic!("History retry must reach the server: {outcome:?}"); + }; + assert_eq!(uploaded_wav(&server), expected_upload); + let retried = history + .apply_retry(row.id, &transcript) + .result + .unwrap() + .unwrap() + .item; + assert_eq!(retried.text, "retried words"); + assert_eq!(retried.audio_duration_ms, row.audio_duration_ms); + assert_eq!(history.retry_audio(row.id).unwrap(), original); + } + + #[test] + fn history_retry_sends_original_audio_when_vad_finds_no_speech() { + let samples = vec![0.; 32_000]; + let original = crate::audio::wav::encode_wav_pcm16(&samples, 16_000); + assert!(crate::audio::vad::upload_wav(original.clone().into()).is_none()); + retry_stored_audio(&samples, &original); + } + + #[test] + fn history_retry_still_sends_trimmed_audio_when_vad_finds_speech() { + let (_, speech) = + crate::audio::wav::read_wav_mono(include_bytes!("../tests/fixtures/vad-speech.wav")); + let mut samples = vec![0.; 16_000]; + samples.extend(speech); + samples.extend(vec![0.; 16_000]); + let original = crate::audio::wav::encode_wav_pcm16(&samples, 16_000); + let trimmed = crate::audio::vad::upload_wav(original.clone().into()).unwrap(); + assert!(trimmed.len() < original.len()); + retry_stored_audio(&samples, &trimmed); + } + + #[test] + fn vad_rejects_silence_without_a_request_and_send_anyway_bypasses_once() { + let server = MockServer::start(|_| Reply::json(200, r#"{"text":"explicitly sent"}"#)); + let original: bytes::Bytes = + crate::audio::wav::encode_wav_pcm16(&vec![0.; 32_000], 16_000).into(); + let input = vad_input(&server, original.clone()); + let outcome = run(super::run_prepared_pipeline_controlled( + input.clone(), + CancellationToken::new(), + CancellationToken::new(), + super::VadMode::Live, + )); + assert!( + matches!(outcome, PipelineOutcome::Failed { code: Some(ref code), .. } if code == crate::audio::vad::NO_SPEECH_CODE) + ); + assert!(server.requests().is_empty()); + let actions = + ErrorActions::for_transcription_failure(Some(crate::audio::vad::NO_SPEECH_CODE)); + assert!(actions.retry && actions.send_anyway); + assert!(matches!( + run(super::run_prepared_pipeline_controlled( + input.clone(), + CancellationToken::new(), + CancellationToken::new(), + super::VadMode::Bypass + )), + PipelineOutcome::Transcribed(_) + )); + assert_eq!(uploaded_wav(&server), original.as_ref()); + let _ = run(run_prepared_pipeline(input, CancellationToken::new())); + assert_eq!( + server + .requests() + .iter() + .filter(|r| r.method == "POST") + .count(), + 1 + ); + } + + #[test] + fn vad_keeps_original_capture_and_history_retry_filters_only_uploads() { + let (_, speech) = + crate::audio::wav::read_wav_mono(include_bytes!("../tests/fixtures/vad-speech.wav")); + let mut samples = vec![0.; 16_000]; + samples.extend(speech); + samples.extend(vec![0.; 16_000]); + let audio = crate::audio::capture::RecordedAudio { + samples, + sample_rate: 16_000, + received_audio_data: true, + speech_gate: Default::default(), + }; + let original = audio.to_wav_16k(); + let (kept, upload) = super::prepare_recording_for_upload(audio.clone(), true).unwrap(); + assert_eq!(kept.as_ref(), original); + assert!(upload.unwrap().len() < kept.len()); + let server = MockServer::start(|_| Reply::json(200, r#"{"text":"words from history"}"#)); + let input = vad_input(&server, kept.clone()); + assert!(matches!( + run(run_prepared_pipeline(input, CancellationToken::new())), + PipelineOutcome::Transcribed(_) + )); + assert!(uploaded_wav(&server).len() < original.len()); + assert_eq!(kept.as_ref(), original); + let (kept, upload) = super::prepare_recording_for_upload(audio, false).unwrap(); + assert_eq!(upload.unwrap(), kept); + let mut input = vad_input(&server, kept.clone()); + input.settings.silence_filtering = false; + let _ = run(run_prepared_pipeline(input, CancellationToken::new())); + let requests = server.requests(); + assert!( + requests + .last() + .unwrap() + .body + .windows(kept.len()) + .any(|bytes| bytes == kept.as_ref()) + ); + } + + #[test] + fn adopted_preroll_is_filtered_even_without_rms_windows() { + let audio = crate::audio::capture::RecordedAudio { + samples: vec![0.; 16_000], + sample_rate: 16_000, + received_audio_data: true, + speech_gate: Default::default(), + }; + assert!(!audio.speech_gate.is_silence()); + let (original, upload) = super::prepare_recording_for_upload(audio, true).unwrap(); + assert!(!original.is_empty()); + assert!(upload.is_none()); + } } diff --git a/src/history.rs b/src/history.rs index 7475190..a42e7cf 100644 --- a/src/history.rs +++ b/src/history.rs @@ -217,6 +217,7 @@ mod tests { spoken_text: text.into(), raw_text: format!("um {text}"), cleanup_failure: None, + cleanup_skipped: false, } } diff --git a/src/pipeline.rs b/src/pipeline.rs index c2c783c..ba06536 100644 --- a/src/pipeline.rs +++ b/src/pipeline.rs @@ -36,6 +36,7 @@ pub struct PipelineInput { pub known_corrections: Vec, pub snippets: Vec, pub cleanup_api_key: Option, + pub cleanup_stage: Option>, } /// Why cleanup did not run on a transcript that was still delivered raw. @@ -57,6 +58,7 @@ pub struct Transcript { /// The server's transcript before cleanup. pub raw_text: String, pub cleanup_failure: Option, + pub cleanup_skipped: bool, } #[derive(Debug, Clone, PartialEq)] @@ -173,6 +175,14 @@ fn cleanup_failure(err: &CleanupError) -> CleanupFailure { } } +fn filter_fillers(text: &str, settings: &Settings) -> String { + if settings.remove_filler_words { + crate::text::filler_words::remove_filler_words(text, &settings.preferred_language) + } else { + text.to_string() + } +} + /// The deterministic Chinese script pass, run on the cleaned text. pub fn finalize_script(text: &str, settings: &Settings) -> String { let target = resolve_chinese_script_target( @@ -186,7 +196,11 @@ pub fn finalize_script(text: &str, settings: &Settings) -> String { /// Transcribes `input.audio`, filters dictionary echoes, cleans up, converts /// the Chinese script and expands Snippets. Cancelling `cancel` aborts only /// this run. -pub async fn run(input: PipelineInput, cancel: CancellationToken) -> PipelineOutcome { +pub async fn run( + input: PipelineInput, + cancel: CancellationToken, + skip_cleanup: CancellationToken, +) -> PipelineOutcome { let route = match transcription::resolve_transcription_route(&input.settings) { Ok(route) => route, Err(err) => return PipelineOutcome::failed_from(&TranscriptionError::Route(err)), @@ -212,12 +226,30 @@ pub async fn run(input: PipelineInput, cancel: CancellationToken) -> PipelineOut } // A whole trigger is intentional input even when it equals the ASR hint. // Deliver saved text verbatim, without letting cleanup rewrite the trigger. + // Check raw input first so filler tokens in saved triggers keep working. if let Some(text) = whole_snippet_replacement(&raw, &input.snippets) { return PipelineOutcome::Transcribed(Transcript { text, spoken_text: raw.trim().to_string(), raw_text: raw, cleanup_failure: None, + cleanup_skipped: false, + }); + } + let filtered = filter_fillers(&raw, &input.settings); + let fillers_removed = filtered != raw; + if filtered.trim().is_empty() + || (fillers_removed && !filtered.chars().any(char::is_alphanumeric)) + { + return PipelineOutcome::NoAudio; + } + if let Some(text) = whole_snippet_replacement(&filtered, &input.snippets) { + return PipelineOutcome::Transcribed(Transcript { + text, + spoken_text: filtered.trim().to_string(), + raw_text: raw, + cleanup_failure: None, + cleanup_skipped: false, }); } // Check the dictionary alone as well as the full prompt: a script bias @@ -234,13 +266,37 @@ pub async fn run(input: PipelineInput, cancel: CancellationToken) -> PipelineOut if cancel.is_cancelled() { return PipelineOutcome::Cancelled; } - let (cleaned, cleanup_failure) = clean_up(&raw, &input, &cancel).await; + let mut cleanup_skipped = false; + let (cleaned, cleanup_failure) = if cleanup_enabled(&input.settings) { + // A skip issued during ASR cannot arm a future cleanup stage. + let can_skip = !skip_cleanup.is_cancelled(); + let stage = CleanupStage::new(input.cleanup_stage.as_ref()); + let result = tokio::select! { + biased; + _ = cancel.cancelled() => return PipelineOutcome::Cancelled, + _ = skip_cleanup.cancelled(), if can_skip => { + cleanup_skipped = true; + (filtered.clone(), None) + } + result = clean_up(&filtered, &input, &cancel) => result, + }; + drop(stage); + result + } else { + (filtered.clone(), None) + }; if cancel.is_cancelled() { return PipelineOutcome::Cancelled; } // Mishearings were learned from delivered text, so they match it here. + let filtered_cleaned = filter_fillers(&cleaned, &input.settings); + if (fillers_removed || filtered_cleaned != cleaned) + && !filtered_cleaned.chars().any(char::is_alphanumeric) + { + return PipelineOutcome::NoAudio; + } let spoken_text = apply_known_corrections( - &finalize_script(&cleaned, &input.settings), + &finalize_script(&filtered_cleaned, &input.settings), &input.known_corrections, ); if spoken_text.trim().is_empty() { @@ -251,9 +307,29 @@ pub async fn run(input: PipelineInput, cancel: CancellationToken) -> PipelineOut spoken_text, raw_text: raw, cleanup_failure, + cleanup_skipped, }) } +struct CleanupStage<'a>(Option<&'a async_channel::Sender>); + +impl<'a> CleanupStage<'a> { + fn new(sender: Option<&'a async_channel::Sender>) -> Self { + if let Some(sender) = sender { + let _ = sender.try_send(true); + } + Self(sender) + } +} + +impl Drop for CleanupStage<'_> { + fn drop(&mut self) { + if let Some(sender) = self.0 { + let _ = sender.try_send(false); + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -261,6 +337,10 @@ mod tests { use serde_json::json; use std::time::Duration; + async fn run(input: PipelineInput, cancel: CancellationToken) -> PipelineOutcome { + super::run(input, cancel, CancellationToken::new()).await + } + fn input(dictionary: &[&str]) -> PipelineInput { PipelineInput { audio: bytes::Bytes::new(), @@ -270,6 +350,7 @@ mod tests { known_corrections: Vec::new(), snippets: Vec::new(), cleanup_api_key: None, + cleanup_stage: None, } } @@ -492,6 +573,105 @@ mod tests { ); } + #[test] + fn filler_only_transcripts_are_no_audio_without_cleanup() { + let asr = asr("Um, uh, 呃,"); + assert_eq!( + run_test(run(served(&asr, None), CancellationToken::new())), + PipelineOutcome::NoAudio + ); + } + + #[test] + fn punctuation_only_is_no_audio_only_when_fillers_were_removed() { + for raw in ["uh!", "呃。", "Uh... ", "uh ; !"] { + let asr = asr(raw); + assert_eq!( + run_test(run(served(&asr, None), CancellationToken::new())), + PipelineOutcome::NoAudio, + "{raw:?}" + ); + } + let asr = asr("; !"); + let t = transcribed(run_test(run(served(&asr, None), CancellationToken::new()))); + assert_eq!(t.text, "; !"); + let asr = self::asr("uh!"); + let mut input = served(&asr, None); + input.settings.remove_filler_words = false; + let t = transcribed(run_test(run(input, CancellationToken::new()))); + assert_eq!(t.text, "uh!"); + let asr = self::asr("hello"); + let llm = llm(completion("uh ; !")); + assert_eq!( + run_test(run(served(&asr, Some(&llm)), CancellationToken::new())), + PipelineOutcome::NoAudio + ); + } + + #[test] + fn whole_snippets_match_raw_before_filtered_transcripts() { + for (raw, trigger, spoken) in [ + ("uh oh", "uh oh", "uh oh"), + ("Um, my address", "my address", "My address"), + ("hmm", "hmm", "hmm"), + ] { + let asr = asr(raw); + let llm = llm(completion("must not run")); + let mut input = served(&asr, Some(&llm)); + input.snippets = vec![Snippet::new(trigger, "Saved text")]; + if raw == "uh oh" { + input + .snippets + .push(Snippet::new("oh", "Wrong filtered match")); + } + let t = transcribed(run_test(run(input, CancellationToken::new()))); + assert_eq!(t.text, "Saved text"); + assert_eq!(t.raw_text, raw); + assert_eq!(t.spoken_text, spoken); + assert!(llm.requests().is_empty()); + } + } + + #[test] + fn filler_removal_preserves_raw_text_and_can_be_disabled() { + for enabled in [true, false] { + let asr = asr("Um, we should go"); + let mut input = served(&asr, None); + input.settings.remove_filler_words = enabled; + let t = transcribed(run_test(run(input, CancellationToken::new()))); + assert_eq!(t.raw_text, "Um, we should go"); + assert_eq!( + t.text, + if enabled { + "We should go" + } else { + "Um, we should go" + } + ); + } + } + + #[test] + fn filler_removal_wraps_cleanup_and_precedes_whole_snippet_matching() { + let asr = asr("Um, we should go"); + let llm = llm(completion("Uh, we should go.")); + let t = transcribed(run_test(run( + served(&asr, Some(&llm)), + CancellationToken::new(), + ))); + assert_eq!(t.text, "We should go."); + assert!(String::from_utf8_lossy(&llm.requests()[0].body).contains("We should go")); + let asr = self::asr("Um, sig."); + let mut input = served(&asr, None); + input.snippets = vec![Snippet { + trigger: "sig".into(), + replacement: "Best regards".into(), + }]; + let t = transcribed(run_test(run(input, CancellationToken::new()))); + assert_eq!(t.text, "Best regards"); + assert_eq!(t.raw_text, "Um, sig."); + } + #[test] fn dictionary_echo_keeps_candidate_text_without_cleanup_or_delivery() { for (spoken, dictionary) in [ @@ -531,6 +711,86 @@ mod tests { assert!(started.elapsed() < Duration::from_secs(2)); } + #[test] + fn skipping_cleanup_is_prompt_and_finishes_deterministic_steps_once() { + assert_cleanup_skip(Duration::from_millis(30)); + } + + #[test] + fn skipping_cleanup_tolerates_delayed_stage_observation() { + // Simulate a loaded runner observing the cleanup stage late. + assert_cleanup_skip(Duration::from_millis(1100)); + } + + fn assert_cleanup_skip(observation_delay: Duration) { + let asr = asr("Uh, 電腦 克劳德 sig"); + let llm = llm(completion("late reply").delayed(Duration::from_secs(30))); + let mut input = served(&asr, Some(&llm)); + input.settings.preferred_language = "zh-CN".into(); + input.settings.chinese_script_preference = + crate::settings::ChineseScriptPreference::Simplified; + input.known_corrections = corrections(); + input.snippets = vec![Snippet { + trigger: "sig".into(), + replacement: "uh saved text".into(), + }]; + let (tx, rx) = async_channel::unbounded(); + input.cleanup_stage = Some(tx); + let skip = CancellationToken::new(); + let started = std::time::Instant::now(); + let t = transcribed(run_test(async { + let run = super::run(input, CancellationToken::new(), skip.clone()); + let skipping = async { + assert_eq!(rx.recv().await, Ok(true)); + tokio::time::sleep(observation_delay).await; + skip.cancel(); + }; + let (outcome, ()) = tokio::join!(run, skipping); + outcome + })); + assert!(started.elapsed() < Duration::from_secs(5)); + assert_eq!(t.raw_text, "Uh, 電腦 克劳德 sig"); + assert!(t.text.contains("电脑"), "{}", t.text); + assert!(t.text.contains("Claude"), "{}", t.text); + assert!(t.text.ends_with("uh saved text"), "{}", t.text); + assert!(t.cleanup_skipped); + assert_eq!(t.cleanup_failure, None); + assert_eq!(rx.try_recv(), Ok(false)); + assert!(rx.try_recv().is_err()); + assert_eq!(llm.requests().len(), 1); + } + + #[test] + fn early_and_late_cleanup_skips_are_harmless() { + for early in [true, false] { + let asr = asr("hello there"); + let llm = llm(completion("Hello there.")); + let mut input = served(&asr, Some(&llm)); + let (tx, rx) = async_channel::unbounded(); + input.cleanup_stage = Some(tx); + let skip = CancellationToken::new(); + if early { + skip.cancel(); + } + let t = transcribed(run_test(super::run( + input, + CancellationToken::new(), + skip.clone(), + ))); + skip.cancel(); + assert_eq!(t.text, "Hello there."); + assert!(!t.cleanup_skipped); + assert_eq!(rx.try_recv(), Ok(true)); + assert_eq!(rx.try_recv(), Ok(false)); + } + let asr = asr("hello there"); + let mut input = served(&asr, None); + let (tx, rx) = async_channel::unbounded(); + input.cleanup_stage = Some(tx); + let _ = run_test(run(input, CancellationToken::new())); + assert!(rx.try_recv().is_err()); + } + /// Live check against real servers (see [`crate::services::live_tests`]): /// `cargo test -- --ignored pipeline`. #[test] diff --git a/src/services/performance_tests.rs b/src/services/performance_tests.rs index 8334589..6f6f848 100644 --- a/src/services/performance_tests.rs +++ b/src/services/performance_tests.rs @@ -55,9 +55,14 @@ fn performance_live_pipeline() { known_corrections: Vec::new(), snippets: Vec::new(), cleanup_api_key: None, + cleanup_stage: None, }; let pipeline_started = Instant::now(); - let outcome = super::runtime::block_on(pipeline::run(input, CancellationToken::new())); + let outcome = super::runtime::block_on(pipeline::run( + input, + CancellationToken::new(), + CancellationToken::new(), + )); let pipeline_ms = pipeline_started.elapsed().as_secs_f64() * 1000.0; let total_ms = started.elapsed().as_secs_f64() * 1000.0; let PipelineOutcome::Transcribed(transcript) = outcome else { diff --git a/src/settings.rs b/src/settings.rs index 576bc75..2b34307 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -133,6 +133,8 @@ pub struct Settings { pub remote_transcription_url: String, pub remote_transcription_model: String, + pub silence_filtering: bool, + pub remove_filler_words: bool, pub use_cleanup_model: bool, pub cleanup_model: String, pub cleanup_remote_url: String, @@ -170,6 +172,8 @@ impl Default for Settings { chinese_script_preference: ChineseScriptPreference::AsTranscribed, remote_transcription_url: String::new(), remote_transcription_model: String::new(), + silence_filtering: true, + remove_filler_words: true, use_cleanup_model: true, cleanup_model: String::new(), cleanup_remote_url: String::new(), @@ -359,5 +363,24 @@ mod tests { let s: Settings = serde_json::from_str(r#"{"theme":"dark"}"#).unwrap(); assert_eq!(s.theme, ThemePreference::Dark); assert_eq!(s.audio_retention_days, 30); + assert!(s.remove_filler_words); + assert!(s.silence_filtering); + } + + #[test] + fn local_filter_preferences_persist_both_enabled_and_disabled() { + for enabled in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("settings.json"); + let settings = Settings { + remove_filler_words: enabled, + silence_filtering: enabled, + ..Settings::default() + }; + settings.save_to(&path).unwrap(); + let loaded = Settings::load_from(&path); + assert_eq!(loaded.remove_filler_words, enabled); + assert_eq!(loaded.silence_filtering, enabled); + } } } diff --git a/src/text/filler_words.rs b/src/text/filler_words.rs new file mode 100644 index 0000000..9cb8db0 --- /dev/null +++ b/src/text/filler_words.rs @@ -0,0 +1,577 @@ +//! Conservative, deterministic hesitation removal, independent of cleanup. + +const UNIVERSAL: &[&str] = &[ + "uh", "uhm", "umm", "uhh", "uhhh", "ehh", "ehm", "ahm", "hmm", "hm", "mmm", "хм", "ммм", +]; +const ENGLISH: &[&str] = &["um", "ah", "eh"]; + +fn word(c: char) -> bool { + c.is_alphanumeric() || c == '_' +} + +fn horizontal(c: char) -> bool { + matches!(c, ' ' | '\t') +} + +fn sentence_start(text: &str) -> bool { + // A list marker opening the line starts a sentence too. + if let Some(lead) = text + .trim_end_matches(horizontal) + .strip_suffix(['-', '*', '•']) + && lead + .trim_end_matches(horizontal) + .chars() + .next_back() + .is_none_or(|c| c == '\n') + { + return true; + } + text.trim_end_matches([' ', '\t', '"', '\'', '“', '‘', '(']) + .chars() + .next_back() + .is_none_or(|c| matches!(c, '.' | '!' | '?' | '\n' | '。' | '!' | '?')) +} + +fn protected(text: &str, code: &[(usize, usize)], start: usize, end: usize) -> bool { + let before = &text[..start]; + let after = &text[end..]; + let span_start = before + .rfind(char::is_whitespace) + .map_or(0, |i| i + text[i..].chars().next().unwrap().len_utf8()); + let span_end = after + .find(char::is_whitespace) + .map_or(text.len(), |i| end + i); + let span = &text[span_start..span_end]; + span.contains("://") + || span.contains('@') + || inside_code(code, start) + || mentioned(before, after) + || before.ends_with([ + '.', '/', '\\', ':', '_', '-', '=', '#', '$', '%', '&', '+', '~', '@', + ]) + || after.starts_with(['/', '\\', ':', '_', '-', '=', '@', '(', '[']) + || after + .strip_prefix('.') + .is_some_and(|tail| tail.chars().next().is_some_and(char::is_alphanumeric)) + || after.trim_start_matches(horizontal).starts_with('=') +} + +/// A filler wrapped in quotes, brackets or emphasis is being talked about, +/// not hesitated over: `He said "uh" today`, `(hmm)`, `*uh*`. +fn mentioned(before: &str, after: &str) -> bool { + before.ends_with([ + '"', '\'', '“', '‘', '(', '[', '*', '「', '『', '(', '《', '〈', '【', + ]) && after.trim_start_matches(punctuation).starts_with([ + '"', '\'', '”', '’', ')', ']', '*', '」', '』', ')', '》', '〉', '】', + ]) +} + +/// Inline code spans: a run of backticks up to the next run of the same +/// length on that line, as Markdown pairs them. Found once per pass so the +/// scan stays linear on long transcripts. +fn code_spans(text: &str) -> Vec<(usize, usize)> { + let bytes = text.as_bytes(); + let run_end = |mut at: usize| { + while at < bytes.len() && bytes[at] == b'`' { + at += 1; + } + at + }; + let mut spans = Vec::new(); + let mut at = 0; + while at < bytes.len() { + if bytes[at] != b'`' { + at += 1; + continue; + } + let open = at; + at = run_end(at); + let mut scan = at; + while scan < bytes.len() && bytes[scan] != b'\n' { + if bytes[scan] != b'`' { + scan += 1; + continue; + } + let close = run_end(scan); + if close - scan == at - open { + spans.push((open, close)); + at = close; + break; + } + scan = close; + } + } + spans +} + +fn inside_code(code: &[(usize, usize)], at: usize) -> bool { + code.get(code.partition_point(|span| span.1 <= at)) + .is_some_and(|span| span.0 <= at) +} + +fn punctuation(c: char) -> bool { + matches!( + c, + '.' | ',' | ';' | '!' | '?' | '…' | '。' | ',' | ';' | '!' | '?' | '、' + ) +} + +/// Copies surviving text, capitalizing the sentence a removed filler used +/// to open. Only a plain word is recased: code, numbers, and names with their +/// own casing (`iPhone`, `my_var`) are left as written. +fn append_kept(out: &mut String, kept: &str, capitalize: &mut bool) { + for (at, c) in kept.char_indices() { + if !*capitalize { + out.push_str(&kept[at..]); + return; + } + if c.is_alphabetic() { + let rest = &kept[at + c.len_utf8()..]; + // Judge the whole unbroken span, so an address or path is seen + // as one literal (`john@example.com`), not as its first word. + let span = rest.split(char::is_whitespace).next().unwrap_or(""); + let literal = span.contains(['@', '/', '\\', '=', '_']) + || span + .split('.') + .skip(1) + .any(|part| part.starts_with(char::is_alphanumeric)); + let own_casing = literal + || rest + .chars() + .take_while(|c| word(*c)) + .any(|c| c.is_uppercase() || c.is_numeric()); + if own_casing { + out.push(c); + } else { + out.extend(c.to_uppercase()); + } + *capitalize = false; + } else { + *capitalize = + c.is_whitespace() || matches!(c, '"' | '\'' | '“' | '‘' | '’' | '(' | '['); + out.push(c); + } + } +} + +/// A missing language is Auto; short English words then require punctuation +/// to avoid deleting ordinary words in other languages (Portuguese `um`). +pub fn remove_filler_words(text: &str, language: &str) -> String { + let base = language + .split(['-', '_']) + .next() + .unwrap_or("") + .to_lowercase(); + let auto = base.is_empty() || base == "auto"; + let specific: &[&str] = match base.as_str() { + "en" => ENGLISH, + "de" => &["äh", "ähm"], + "fr" => &["euh"], + _ => &[], + }; + let mut current = text.to_string(); + // Removing an adjacent filler can expose the punctuation qualifying an + // Auto English token. Finish that work here so the pass is idempotent. + loop { + let next = remove_pass(¤t, specific, auto); + if next == current { + return next; + } + current = next; + } +} + +fn remove_pass(text: &str, specific: &[&str], auto: bool) -> String { + let code = code_spans(text); + let mut ranges: Vec<(usize, usize, bool)> = Vec::new(); + let mut cursor = 0; + while cursor < text.len() { + let Some(c) = text[cursor..].chars().next() else { + break; + }; + if c == '呃' { + let start = cursor; + while text[cursor..].starts_with('呃') { + cursor += '呃'.len_utf8(); + } + if text[..start].ends_with('打') + || text[cursor..].starts_with('逆') + || protected(text, &code, start, cursor) + { + continue; + } + ranges.push((start, cursor, false)); + } else if word(c) { + let start = cursor; + cursor += c.len_utf8(); + while let Some(c) = text[cursor..] + .chars() + .next() + .filter(|c| word(*c) && *c != '呃') + { + cursor += c.len_utf8(); + } + let token = &text[start..cursor]; + let lower = token.to_lowercase(); + let following = text[cursor..].trim_start_matches(horizontal).chars().next(); + let hesitation = + following.is_none_or(|c| punctuation(c) || matches!(c, '-' | '–' | '—')); + let boundary = text[..start].chars().next_back().is_none_or(|c| !word(c)) + && text[cursor..].chars().next().is_none_or(|c| !word(c)); + if boundary + && (UNIVERSAL.contains(&lower.as_str()) + || specific.contains(&lower.as_str()) + || (auto && hesitation && ENGLISH.contains(&lower.as_str()))) + && !protected(text, &code, start, cursor) + { + let capitalize = c.is_uppercase() && sentence_start(&text[..start]); + ranges.push((start, cursor, capitalize)); + } + } else { + cursor += c.len_utf8(); + } + } + if ranges.is_empty() { + return text.to_string(); + } + let mut groups: Vec = Vec::new(); + for (token_start, token_end, recase) in ranges { + let mut end = token_end; + let mut terminal = None; + // Take every pause mark the filler brought with it, in any order + // (`uh, ... we go`, `uh—, hello`). + loop { + let from = skip_horizontal(text, end); + let run = punctuation_run(text, from); + let next = text[run..].chars().next(); + // Punctuation set off by a space is the filler's only when it + // stands alone (`uh , hello`); `uh .env` keeps its dot. + if run > from && (from == end || next.is_none_or(char::is_whitespace)) { + terminal = terminal_in(text, from, run).or(terminal); + end = run; + continue; + } + let mut rest = text[from..].chars(); + // A hyphen must stand apart to be a pause (`uh-oh` is a word). The + // long dashes pause even when set tight (`Uh—we go`), but before + // a number (`–5`, `–.5`) one may be a minus sign. + if let Some(dash) = rest.next().filter(|c| matches!(c, '-' | '–' | '—')) { + let after = rest.next(); + let pause = if dash == '-' { + after.is_none_or(char::is_whitespace) + } else { + !after.is_some_and(|c| c.is_numeric() || c == '.') + }; + if pause { + end = from + dash.len_utf8(); + continue; + } + } + break; + } + let start = text[..token_start].trim_end_matches(horizontal).len(); + let end = skip_horizontal(text, end); + if let Some(previous) = groups.last_mut().filter(|previous| start <= previous.end) { + previous.end = previous.end.max(end); + previous.recase |= recase; + previous.terminal = terminal.or(previous.terminal); + } else { + groups.push(Group { + start, + end, + recase, + terminal, + }); + } + } + let mut out = String::with_capacity(text.len()); + let mut previous = 0; + let mut capitalize = false; + for group in groups { + append_kept(&mut out, &text[previous..group.start], &mut capitalize); + let right = text[group.end..].chars().next(); + let left = out.chars().next_back(); + // The whole group goes, punctuation included. What stays at the seam + // is the strongest mark the surviving clause still needs: a terminal + // spoken after the filler closes the clause before it, replacing that + // clause's comma; a comma was only ever the filler's own. Punctuation + // already standing before the group is the text's and always stays. + let terminal: String = group + .terminal + .map(|(start, end)| text[start..end].chars().filter(|c| terminal(*c)).collect()) + .unwrap_or_default(); + match left { + Some(c) if comma(c) && !terminal.is_empty() => { + out.pop(); + out.push_str(&terminal); + } + Some(c) if word(c) || matches!(c, '"' | '\'' | '”' | '’' | ')' | ']') => { + out.push_str(&terminal); + } + _ => {} + } + if let (Some(left), Some(right)) = (out.chars().next_back(), right) { + let removed = &text[group.start..group.end]; + // Space that led into the filler still separates what surrounds + // it; space that only trailed it has no place after an opener. + let separated = removed.starts_with(horizontal) + || (removed.ends_with(horizontal) + && !matches!(left, '"' | '\'' | '“' | '‘' | '(' | '[')); + let joined_words = word(left) && word(right) && !(cjk(left) && cjk(right)); + if !left.is_whitespace() && !right.is_whitespace() && (separated || joined_words) { + out.push(' '); + } + } + capitalize |= group.recase; + previous = group.end; + } + append_kept(&mut out, &text[previous..], &mut capitalize); + out +} + +/// One or more fillers removed together, with the punctuation and spacing +/// that belonged to them. +struct Group { + start: usize, + end: usize, + recase: bool, + /// The last sentence-ending punctuation run inside the group. + terminal: Option<(usize, usize)>, +} + +fn comma(c: char) -> bool { + matches!(c, ',' | ';' | ',' | ';' | '、') +} + +fn terminal(c: char) -> bool { + punctuation(c) && !comma(c) +} + +fn cjk(c: char) -> bool { + ('\u{3400}'..='\u{9fff}').contains(&c) +} + +fn skip_horizontal(text: &str, from: usize) -> usize { + text.len() - text[from..].trim_start_matches(horizontal).len() +} + +fn punctuation_run(text: &str, from: usize) -> usize { + text.len() - text[from..].trim_start_matches(punctuation).len() +} + +fn terminal_in(text: &str, start: usize, end: usize) -> Option<(usize, usize)> { + text[start..end].contains(terminal).then_some((start, end)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn filtered(text: &str, language: &str) -> String { + let result = remove_filler_words(text, language); + assert_eq!( + remove_filler_words(&result, language), + result, + "{text:?}/{language}" + ); + result + } + + #[test] + fn adjacent_removals_and_orphan_punctuation() { + for (text, expected) in [ + ("foo uh,uh,bar", "foo bar"), + ("foo uh uh bar", "foo bar"), + ("foo, uh, um, bar", "foo, bar"), + ("uh uh uh", ""), + ("foo uh\nbar", "foo\nbar"), + ("呃。", ""), + ("uh!", ""), + ("Uh... ", ""), + ("hello. Um... we go", "hello. We go"), + ("好的,呃。我们走", "好的。我们走"), + ("Wait, uh, what?", "Wait, what?"), + ] { + assert_eq!(filtered(text, "en"), expected, "{text:?}"); + } + for punctuation in [".", ",", "!", "?", "…", "...", "。", ",", "!", "?", "、"] { + for token in ["uh", "呃", "um"] { + assert_eq!(filtered(&format!("{token}{punctuation}"), "en"), ""); + assert_eq!(filtered(&format!("{token}{punctuation}"), "auto"), ""); + } + } + } + + #[test] + fn seam_keeps_the_punctuation_the_surviving_text_needs() { + for (text, expected) in [ + ("好的呃。我们走", "好的。我们走"), + ("好的,呃。", "好的。"), + ("好的,呃。uh,我们走", "好的。我们走"), + ("Uh ... we go", "We go"), + ("uh , hello", "hello"), + ("so uh... we go", "so... we go"), + ("I was... uh... going", "I was... going"), + ("I think so uh.", "I think so."), + ("Wait, uh", "Wait,"), + ("Dear John, uh\nThank you.", "Dear John,\nThank you."), + ("Dear John, uh\r\nThank you.", "Dear John,\r\nThank you."), + ("Uh, uh, ... we go", "We go"), + ("uh—, hello", "hello"), + ("It is uh –5 degrees.", "It is –5 degrees."), + ("- Uh, `myVariable` is it.", "- `myVariable` is it."), + ("Uh, iPhone works.", "iPhone works."), + ("Uh, my_var works.", "my_var works."), + ("Uh, john@example.com is mine.", "john@example.com is mine."), + ("Uh, example.com is mine.", "example.com is mine."), + ("Uh, [hello there].", "[Hello there]."), + ("Uh, ’tis a fine day.", "’Tis a fine day."), + ("Uh, well. It works.", "Well. It works."), + ("The value is uh –.5.", "The value is –.5."), + ("Really? Uh, yes.", "Really? Yes."), + ("foo uh, bar", "foo bar"), + ("line one\nuh, line two", "line one\nline two"), + ("see uh .env here", "see .env here"), + ("\"Uh, we go\"", "\"We go\""), + ("- Uh, we go\n- Um, stay", "- We go\n- Stay"), + ("Uh, 3.5 is right", "3.5 is right"), + ("uh; bar", "bar"), + ("Uh—we go", "We go"), + ("foo uh!uh?bar", "foo? bar"), + ] { + assert_eq!(filtered(text, "en"), expected, "{text:?}"); + } + } + + #[test] + fn inline_code_is_preserved_whatever_surrounds_the_backticks() { + for text in [ + "`uh`, hello", + "(`uh`) hello", + "`say uh now`", + "run `hm` then `uh` ok", + "``uh``", + "```uh```", + "``a ` uh``", + ] { + assert_eq!(filtered(text, "en"), text); + } + for text in [ + "He said \"uh\" today", + "He said \"uh,\" today", + "(hmm) hello", + "請保留「呃」這個字。", + "请保留(呃)这个字。", + "他说『呃』然后走了", + "*uh* hello", + ] { + assert_eq!(filtered(text, "en"), text); + } + // An unmatched backtick protects nothing. + assert_eq!(filtered("it`s uh, fine", "en"), "it`s fine"); + } + + #[test] + fn urls_emails_and_code_are_preserved() { + for text in [ + "https://uh.edu/a", + "https://example.com/uh?x", + "user@hm.com", + "visit uh.edu today", + "hm.value", + "uh_count", + "uh-oh", + "um-hum", + "let uh = 1;", + "let hm=1;", + "`uh`", + "`uh,`", + ] { + assert_eq!(filtered(text, "en"), text); + } + for prefix in [ + '.', '/', '\\', ':', '_', '-', '=', '#', '$', '%', '&', '+', '~', '@', + ] { + let text = format!("{prefix}uh"); + assert_eq!(filtered(&text, "en"), text); + } + for suffix in [ + "/", "\\", ":", "_", "-", "=", "@", "(", "[", ".a", ".1", " = 1", + ] { + let text = format!("uh{suffix}"); + assert_eq!(filtered(&text, "en"), text); + } + assert_eq!(filtered("um - I think", "auto"), "I think"); + assert_eq!(filtered("Uh, I think so.", "auto"), "I think so."); + } + + #[test] + fn universal_and_language_specific_lists() { + for language in ["auto", "en", "pt", "de", "fr", "zh-CN"] { + for token in UNIVERSAL { + assert_eq!( + filtered(&format!("{token}, hello"), language), + "hello", + "{token}/{language}" + ); + } + } + for (language, tokens) in [ + ("en-US", ENGLISH), + ("de-DE", &["äh", "ähm"][..]), + ("fr", &["euh"][..]), + ] { + for token in tokens { + assert_eq!(filtered(&format!("{token} hello"), language), "hello"); + } + } + } + + #[test] + fn boundaries_and_auto_preserve_ordinary_words() { + for text in [ + "umbrella", + "hmmm-like", + "ahead", + "uh_thing", + "1uh", + "éuh", + "um carro", + "ah bon", + "euh oui", + ] { + assert_eq!(filtered(text, "auto"), text); + } + assert_eq!(filtered("Um, I think", "auto"), "I think"); + assert_eq!(filtered("Um, we should go", "en"), "We should go"); + assert_eq!( + filtered("hello. Um, we should go", "auto"), + "hello. We should go" + ); + assert_eq!(filtered("um carro", "pt-BR"), "um carro"); + } + + #[test] + fn chinese_runs_and_exceptions() { + for language in ["en", "auto", "zh-CN"] { + assert_eq!(filtered("呃呃,我们呃、走吧呃,好", language), "我们走吧好"); + let text = "打呃 呃逆 嗯 啊 额 那个 然后 就是"; + assert_eq!(filtered(text, language), text); + } + } + + #[test] + fn whitespace_punctuation_and_idempotence() { + for (text, expected) in [ + ("a b\tc", "a b\tc"), + ("a uh, \t b\nUm, we go\n\nuh", "a b\nWe go\n\n"), + ("Um... we go", "We go"), + ("um uh,", ""), + ("uh, uhm. 呃,", ""), + ] { + let result = filtered(text, "auto"); + assert_eq!(result, expected, "{text:?}"); + assert_eq!(filtered(&result, "auto"), result); + } + } +} diff --git a/src/text/mod.rs b/src/text/mod.rs index 35d8da8..761ace2 100644 --- a/src/text/mod.rs +++ b/src/text/mod.rs @@ -5,6 +5,7 @@ pub mod chinese_script; pub mod correction_learner; pub mod dictionary_import; pub mod echo_filter; +pub mod filler_words; pub mod format; pub mod hotkeys; pub mod language; diff --git a/src/ui/dev_scene.rs b/src/ui/dev_scene.rs index c9d45f5..be009b3 100644 --- a/src/ui/dev_scene.rs +++ b/src/ui/dev_scene.rs @@ -4,7 +4,7 @@ //! Values: `home`, `insights`, `upload`, `dictionary`, `search`, //! `settings[:general|language|hotkeys|speech|llms|privacy]`, and the pill states //! `pill:recording`, `pill:processing`, `pill:preparing`, `pill:transition`, -//! `pill:error`, `pill:fallback`. +//! `pill:cleanup`, `pill:cleanup-transition`, `pill:flow`, `pill:error`, `pill:fallback`. //! `WHISPER_DEV_SCROLL_FRACTION=0.5` or `1` captures the middle or bottom of Settings. use std::time::Duration; @@ -35,7 +35,9 @@ pub fn scene() -> Option { "dictionary" => Scene::View(PanelView::Dictionary), "search" => Scene::Search, "settings" => Scene::Settings(match arg { - "hotkeys" => SettingsSection::Hotkeys, + "hotkeys" | "mic-recording" | "mic-ready" | "mic-error" | "mic-transition" => { + SettingsSection::Hotkeys + } "speech" => SettingsSection::SpeechToText, "llms" => SettingsSection::Llms, "privacy" => SettingsSection::PrivacyData, @@ -45,7 +47,11 @@ pub fn scene() -> Option { "preparing" => "preparing", "transition" => "transition", "processing" => "processing", + "cleanup" => "cleanup", + "cleanup-transition" => "cleanup-transition", + "flow" => "flow", "error" => "error", + "no-speech" => "no-speech", "fallback" => "fallback", _ => "recording", }), @@ -68,6 +74,69 @@ pub fn apply_pill(cx: &mut App) { return; }; match state { + "flow" => { + // One whole Dictation on a loop: record, transcribe, polish, + // skip, deliver. For recording the pill's motion end to end. + let set = |cx: &mut gpui_kit::AsyncApp, recording, processing, cleaning| { + cx.update(|cx| { + AppState::global(cx).update(cx, |s, cx| { + s.dictation.is_preparing = false; + s.dictation.is_recording = recording; + s.dictation.is_processing = processing; + s.dictation.is_cleaning = cleaning; + cx.notify(); + }) + }); + }; + // `WHISPER_DEV_FLOW_MS=transcribe,polish` replays measured + // server timings instead of the slow defaults. + let stages: Vec = std::env::var("WHISPER_DEV_FLOW_MS") + .ok() + .map(|v| v.split(',').filter_map(|n| n.trim().parse().ok()).collect()) + .unwrap_or_default(); + let transcribe = stages.first().copied().unwrap_or(1600); + let polish = stages.get(1).copied().unwrap_or(2400); + cx.spawn(async move |cx| { + loop { + let executor = cx.background_executor().clone(); + let wait = |ms| executor.timer(Duration::from_millis(ms)); + wait(1200).await; + set(cx, true, false, false); + wait(2200).await; + set(cx, false, true, false); + wait(transcribe).await; + set(cx, false, true, true); + wait(polish).await; + set(cx, false, true, false); + wait(if stages.is_empty() { 250 } else { 20 }).await; + set(cx, false, false, false); + cx.update(|cx| { + Dictation::global(cx) + .update(cx, |_, cx| cx.emit(DictationNotice::Delivered)) + }); + wait(1200).await; + } + }) + .detach(); + } + "cleanup-transition" => { + AppState::global(cx).update(cx, |s, cx| { + s.dictation.is_processing = true; + cx.notify(); + }); + cx.spawn(async |cx| { + loop { + cx.background_executor().timer(Duration::from_secs(2)).await; + cx.update(|cx| { + AppState::global(cx).update(cx, |s, cx| { + s.dictation.is_cleaning = !s.dictation.is_cleaning; + cx.notify(); + }); + }); + } + }) + .detach(); + } "preparing" | "transition" => { AppState::global(cx).update(cx, |s, cx| { s.dictation.is_preparing = true; @@ -105,13 +174,24 @@ pub fn apply_pill(cx: &mut App) { .detach(); } } - "recording" | "processing" => { + "recording" | "processing" | "cleanup" => { AppState::global(cx).update(cx, |s, cx| { s.dictation.is_recording = state == "recording"; - s.dictation.is_processing = state == "processing"; + s.dictation.is_processing = state == "processing" || state == "cleanup"; + s.dictation.is_cleaning = state == "cleanup"; cx.notify(); }); } + "no-speech" => Dictation::global(cx).update(cx, |_, cx| { + cx.emit(DictationNotice::Error { + title: t("dictation.noAudio.title"), + description: t("dictation.noAudio.description"), + actions: ErrorActions::for_transcription_failure(Some( + crate::audio::vad::NO_SPEECH_CODE, + )), + kept: Some(Duration::from_secs(10)), + }); + }), "error" => Dictation::global(cx).update(cx, |_, cx| { cx.emit(DictationNotice::Error { title: t("dictation.errorTitles.transcriptionTimeout"), diff --git a/src/ui/pill.rs b/src/ui/pill.rs index dbe4c84..38c92cb 100644 --- a/src/ui/pill.rs +++ b/src/ui/pill.rs @@ -20,6 +20,8 @@ mod waveform; use state::*; use waveform::*; +pub(crate) use waveform::microphone_level_meter; + use std::cell::RefCell; use std::f32::consts::{FRAC_PI_2, TAU}; use std::rc::Rc; @@ -36,7 +38,7 @@ use crate::backend; use crate::dictation::{Dictation, DictationNotice, ErrorActions}; use crate::macos; -use super::components::{KeyboardFocusable, f32_tx, icon, kbd}; +use super::components::{KeyboardFocusable, color_tx, f32_tx, icon, kbd}; use super::format::format_elapsed; use super::theme::{ DUR_BASE, DUR_FAST, DUR_SLOW, DUR_SNAP, FONT_UI, FONT_VOICE, LINE_READING, LINE_SMALL, Palette, @@ -75,6 +77,20 @@ const FINISH: Duration = Duration::from_millis(DUR_BASE); /// How long the pill stays for the delivered check: pop, hold, then exit. const RESOLVE: Duration = Duration::from_millis(RESOLVE_EXIT_AT + DUR_SNAP); const RESOLVE_EXIT_AT: u64 = 620; +/// Cleanup on a healthy server ends within a second. The pill only renames +/// itself "Polishing" once cleanup outlasts a glance, and only offers Skip +/// once waiting is long enough to act on, so a fast Dictation goes from +/// the meter to the check without a control flashing past. +const POLISH_LABEL_AFTER: Duration = Duration::from_millis(400); +const SKIP_AFTER: Duration = Duration::from_millis(1500); + +/// Whether cleanup has run long enough to be named, and to offer Skip. +fn polish_stage(running: Option) -> (bool, bool) { + ( + running.is_some_and(|d| d >= POLISH_LABEL_AFTER), + running.is_some_and(|d| d >= SKIP_AFTER), + ) +} /// The floating HUD remains dark over any wallpaper, while honoring named themes. fn hud(cx: &App) -> Palette { @@ -193,6 +209,15 @@ pub struct PillView { /// Whether the pointer rests on the cancel control, so the hint above /// the pill speaks for it instead of for the pill. cancel_hovered: bool, + skip_hovered: bool, + /// The processing silhouette and its "Polishing" label, held through + /// the finish so neither changes back as the pill leaves. + think_latched: bool, + polish_latched: bool, + /// When cleanup began, which paces the label and Skip. + polishing_since: Option, + /// What the hint last said, which it keeps saying while it fades out. + hint_last: Option<(DictationSnapshot, bool, bool)>, } impl PillView { @@ -236,6 +261,11 @@ impl PillView { ready_at: None, recorded: Duration::ZERO, cancel_hovered: false, + skip_hovered: false, + think_latched: false, + polish_latched: false, + polishing_since: None, + hint_last: None, } } @@ -300,6 +330,22 @@ impl PillView { self.dipped_at = Some(Instant::now()); } self.was_processing = active && processing; + if processing && snapshot.is_cleaning { + if self.polishing_since.is_none() { + self.polishing_since = Some(Instant::now()); + // The dots stand still under reduced motion, so nothing + // else would redraw the pill when each stage comes due. + for due in [POLISH_LABEL_AFTER, SKIP_AFTER] { + cx.spawn_in(window, async move |this, cx| { + cx.background_executor().timer(due).await; + let _ = this.update(cx, |_, cx| cx.notify()); + }) + .detach(); + } + } + } else { + self.polishing_since = None; + } if !active && self.was_active { // Keep the window for the finish fade, or for the delivered check. let hold = if self.resolved_at.is_some() { @@ -511,6 +557,7 @@ impl PillView { Some((text, false)), ErrorActions { retry: kept.is_some(), + send_anyway: false, open_settings: false, }, kept, @@ -746,18 +793,33 @@ impl Render for PillView { // The hint sits above the pill, aligned with it; toasts take // that spot when present. - let hint = (self.hovered && active && self.toasts.is_empty()) - .then(|| Self::render_hint(&snapshot, self.cancel_hovered, cx)) - .flatten(); + let hint_now = (self.hovered && active && self.toasts.is_empty()) + .then_some((snapshot.clone(), self.cancel_hovered, self.skip_hovered)) + .filter(|(s, cancel, skip)| Self::render_hint(s, *cancel, *skip, cx).is_some()); + let hint_alpha = f32_tx( + "pill-hint", + if hint_now.is_some() { 1. } else { 0. }, + DUR_SNAP, + ease_out, + window, + cx, + ); + if hint_now.is_some() { + self.hint_last = hint_now; + } + // It keeps its last words while it fades, and settles the last + // few points onto its place as it arrives. + let hint = (hint_alpha > 0.001) + .then_some(self.hint_last.as_ref()) + .flatten() + .and_then(|(s, cancel, skip)| Self::render_hint(s, *cancel, *skip, cx)); if let Some(hint) = hint { - let hint = - hint.with_animation("pill-hint", motion(DUR_SNAP, cx), |d, t| d.opacity(t)); root = root.child( div() .absolute() .left_0() .right_0() - .bottom(px(DOCK_INSET + PILL_H + 8.)) + .bottom(px(DOCK_INSET + PILL_H + 8. - 3. * (1. - hint_alpha))) .px(px(DOCK_INSET)) .flex() .map(|d| match dock { @@ -765,7 +827,7 @@ impl Render for PillView { PanelStartPosition::Center => d.justify_center(), PanelStartPosition::BottomRight => d.justify_end(), }) - .child(hint), + .child(hint.opacity(hint_alpha)), ); } } diff --git a/src/ui/pill/cards.rs b/src/ui/pill/cards.rs index a689ee1..3a47858 100644 --- a/src/ui/pill/cards.rs +++ b/src/ui/pill/cards.rs @@ -259,7 +259,11 @@ impl PillView { // costs the user nothing, so it leads, named so it cannot be // mistaken for Record New. let retry = actions.retry.then(|| { - let label = t("dictation.errorActions.resend"); + let label = t(if actions.send_anyway { + "dictation.errorActions.sendAnyway" + } else { + "dictation.errorActions.resend" + }); Self::panel_action("pill-error-retry", "rotate-ccw", label, true, cx).on_click( cx.listener(|this, _, _, cx| { this.error = None; diff --git a/src/ui/pill/state.rs b/src/ui/pill/state.rs index 5db46bf..aac5413 100644 --- a/src/ui/pill/state.rs +++ b/src/ui/pill/state.rs @@ -205,6 +205,7 @@ mod tests { None, ErrorActions { retry: true, + send_anyway: false, open_settings: false, }, None, @@ -298,6 +299,7 @@ mod tests { fn a_card_holding_the_recording_never_times_out() { let retry = ErrorActions { retry: true, + send_anyway: false, open_settings: true, }; // Closing drops the recording, so only the user may close it. diff --git a/src/ui/pill/waveform.rs b/src/ui/pill/waveform.rs index 2a8d787..ca545fa 100644 --- a/src/ui/pill/waveform.rs +++ b/src/ui/pill/waveform.rs @@ -30,6 +30,61 @@ const THINK_GAP: f32 = 5.; const PREPARE_W: f32 = 104.; const PREPARE_DOT: f32 = 6.; const PREPARE_MORPH_MS: u64 = 680; +/// The Skip chip nested in the polishing pill: a capsule concentric with +/// the pill's trailing end, so it reads as a control and not as a label. +const SKIP_H: f32 = 22.; +const SKIP_PAD: f32 = 10.; +const SKIP_INSET: f32 = 4.; +/// Leading padding of the polishing pill, which balances the chip's inset. +const SKIP_LEAD: f32 = 12.; +const THINK_DOTS_W: f32 = THINK_DOTS as f32 * THINK_DOT + (THINK_DOTS - 1) as f32 * THINK_GAP; + +/// Opacities of the outgoing and incoming layer of a swap at progress `t`. +/// The first leaves before the second arrives, so two labels or two +/// silhouettes never sit on top of each other. +pub(super) fn swap(t: f32) -> (f32, f32) { + ( + 1. - (t / 0.5).clamp(0., 1.), + ((t - 0.35) / 0.65).clamp(0., 1.), + ) +} + +/// Where the processing pill's parts sit, measured from its outer edge. +#[derive(Clone, Copy, Debug, PartialEq)] +struct ThinkLayout { + width: f32, + /// Left edge of the dots; the label follows them. + group_x: f32, + /// Left edge of the Skip chip. + skip_x: f32, +} + +/// Lays out the dots, the label and the Skip chip. `polish` moves the label +/// from "Transcribing" to "Polishing" and `skip` brings the chip in; both +/// are eased by the caller, so every part moves on the same curve as the +/// capsule's width. Without the chip the group is centered in a pill no +/// narrower than the timed recording pill. +fn think_layout( + timed_w: f32, + transcribing_w: f32, + polishing_w: f32, + skip_w: f32, + polish: f32, + skip: f32, +) -> ThinkLayout { + let label_w = transcribing_w + (polishing_w - transcribing_w) * polish; + let lead = PILL_PAD + (SKIP_LEAD - PILL_PAD) * skip; + let trail = PILL_PAD + (SKIP_INSET - PILL_PAD) * skip; + let content = THINK_DOTS_W + THINK_LABEL_GAP + label_w + skip * (THINK_LABEL_GAP + skip_w); + let width = (lead + content + trail).max(timed_w); + let slack = width - (lead + content + trail); + ThinkLayout { + width, + group_x: lead + slack / 2., + // The chip rides the trailing edge and settles the last few points. + skip_x: width - trail - skip_w + (1. - skip) * 6., + } +} /// Maps an RMS level to 0..1 linearly in decibels between /// [`LEVEL_FLOOR_DB`] and [`LEVEL_CEIL_DB`]. @@ -57,6 +112,21 @@ fn live_target(rms: f32, index: usize, now_ms: f32, reduce: bool) -> f32 { speech + (1. - level) * BREATH * ENVELOPE[index] * swell } +pub(crate) fn microphone_level_meter(rms: f32, reduce: bool, palette: &Palette) -> Div { + div() + .h(px(24.)) + .flex() + .items_center() + .gap(px(BAR_GAP)) + .children((0..BARS).map(|i| { + div() + .w(px(BAR_W)) + .h(px(BAR_MIN + live_target(rms, i, 0., reduce) * 12.)) + .rounded_full() + .bg(palette.voice) + })) +} + /// A zero-velocity handoff at both ends keeps the dots and bars continuous. fn prepare_ease(t: f32) -> f32 { let t = t.clamp(0., 1.); @@ -226,7 +296,21 @@ impl PillView { let reduce = cx.reduce_motion(); let finishing = !snapshot.is_active(); let recording = snapshot.is_recording; - let processing = snapshot.is_processing; + let polishing = snapshot.is_processing && snapshot.is_cleaning; + // The finish keeps the silhouette it ended on: the meter must not + // flash back under the delivered check, and a pill that has started + // polishing never reads "Transcribing" again. + if !finishing { + self.think_latched = snapshot.is_processing; + self.polish_latched &= snapshot.is_processing; + } + let (named, skippable) = polish_stage( + self.polishing_since + .filter(|_| polishing) + .map(|since| since.elapsed()), + ); + self.polish_latched |= named; + let processing = self.think_latched; if waveform_needs_animation_frame(snapshot, reduce) { window.request_animation_frame(); } @@ -245,7 +329,8 @@ impl PillView { window, cx, ); - let bars_alpha = (1. - thinking) * motion_fx.bars; + let (meter_alpha, think_alpha) = swap(thinking); + let bars_alpha = meter_alpha * motion_fx.bars; let scale = motion_fx.scale; let timed_w = Self::timed_width(window); @@ -339,11 +424,39 @@ impl PillView { // Transcribing names itself beside the dots, so the state reads // without hovering. It keeps the timed recording pill's width, so // the capsule does not jump sideways when recording ends. - let think_label = t("dictation.pill.transcribing"); - let think_w = Self::processing_width(&think_label, timed_w, window); - let think_alpha = thinking * motion_fx.bars; + if !skippable { + self.skip_hovered = false; + } + let labels = [ + t("dictation.pill.transcribing"), + t("dictation.pill.polishing"), + ]; + let label_w = [ + Self::label_width(&labels[0], window), + Self::label_width(&labels[1], window), + ]; + let skip_label = t("dictation.pill.skip"); + let skip_w = Self::label_width(&skip_label, window) + 2. * SKIP_PAD; + let polish = f32_tx( + "pill-polish", + if self.polish_latched { 1. } else { 0. }, + DUR_BASE, + ease_out, + window, + cx, + ); + let skip_t = f32_tx( + "skip-emerge", + if skippable { 1. } else { 0. }, + if skippable { DUR_BASE } else { DUR_SNAP }, + if skippable { ease_out } else { ease_snap }, + window, + cx, + ); + let layout = think_layout(timed_w, label_w[0], label_w[1], skip_w, polish, skip_t); + let think_alpha = think_alpha * motion_fx.bars; let think = (think_alpha > 0.001).then(|| { - let mut dots = div().flex().items_center().gap(px(THINK_GAP)); + let mut dots = div().flex_none().flex().items_center().gap(px(THINK_GAP)); for i in 0..THINK_DOTS { dots = dots.child( div().size(px(THINK_DOT)).rounded_full().bg(p @@ -351,23 +464,106 @@ impl PillView { .opacity(think_opacity(i, now_ms, reduce) * think_alpha)), ); } + let (out, arrive) = swap(polish); + let label = |text: &String, alpha: f32| { + div() + .absolute() + .left_0() + .top_0() + .whitespace_nowrap() + .text_size(px(TEXT_SMALL)) + .line_height(px(LINE_SMALL)) + .font_weight(FontWeight::MEDIUM) + .text_color(p.foreground.opacity(PILL_LABEL * think_alpha * alpha)) + .child(text.clone()) + }; + let group = div() + .absolute() + .left(px(layout.group_x - 1.)) + .top_0() + .bottom_0() + .flex() + .items_center() + .gap(px(THINK_LABEL_GAP)) + .child(dots) + .child( + div() + .relative() + .flex_none() + .w(px(label_w[0].max(label_w[1]))) + .h(px(LINE_SMALL)) + .when(out > 0.001, |d| d.child(label(&labels[0], out))) + .when(arrive > 0.001, |d| d.child(label(&labels[1], arrive))), + ); + // The chip stays for its exit but only acts while it is offered. + let chip = (skip_t > 0.001).then(|| { + let alpha = swap(skip_t).1 * think_alpha; + let wash = color_tx( + "pill-skip-wash", + p.white.opacity(if self.skip_hovered { 0.2 } else { 0.11 }), + DUR_SNAP, + window, + cx, + ); + div() + .id("pill-skip-cleanup") + .absolute() + .left(px(layout.skip_x - 1.)) + .top(px((PILL_H - 2. - SKIP_H) / 2.)) + .w(px(skip_w)) + .h(px(SKIP_H)) + .flex() + .items_center() + .justify_center() + .rounded_full() + .bg(wash.opacity(alpha)) + .whitespace_nowrap() + .text_size(px(TEXT_SMALL)) + .line_height(px(LINE_SMALL)) + .font_weight(FontWeight::MEDIUM) + .text_color(p.foreground.opacity(0.92 * alpha)) + .when(skippable, |d| { + d.keyboard_focusable( + "pill-skip-cleanup", + Role::Button, + t("dictation.pill.skipCleanup"), + cx, + ) + .cursor_pointer() + .active(move |s| s.bg(p.white.opacity(0.26))) + .on_hover(cx.listener(|this, hovered: &bool, _, cx| { + this.skip_hovered = *hovered; + cx.notify(); + })) + .on_mouse_down(MouseButton::Left, |_, _, cx| cx.stop_propagation()) + .on_click(cx.listener(|this, _, _, cx| { + this.drag_origin = None; + cx.stop_propagation(); + Dictation::global(cx).update(cx, |d, cx| d.skip_cleanup(cx)); + })) + }) + .child(skip_label.clone()) + }); + // A fixed-size stage centered in the pill, so the parts keep + // their places while the capsule pops or dips around them. div() .absolute() .inset_0() .flex() .items_center() .justify_center() - .gap(px(THINK_LABEL_GAP)) - .child(dots) .child( div() - .whitespace_nowrap() - .text_size(px(TEXT_SMALL)) - .font_weight(FontWeight::MEDIUM) - .text_color(p.foreground.opacity(PILL_LABEL * think_alpha)) - .child(think_label.clone()), + .relative() + .flex_none() + .w(px(layout.width - 2.)) + .h(px(PILL_H - 2.)) + .child(group) + .children(chip), ) }); + let think_label = labels[usize::from(self.polish_latched)].clone(); + let think_w = layout.width; let check = motion_fx.check.map(|(scale, alpha)| { div() @@ -487,14 +683,6 @@ impl PillView { PILL_W + TIMER_GAP + Self::label_width(TIMER_WIDEST, window) } - /// The processing pill's width: the timed recording pill's, unless its - /// dots and label need more inside the side padding. - fn processing_width(label: &str, timed_w: f32, window: &Window) -> f32 { - let dots = THINK_DOTS as f32 * THINK_DOT + (THINK_DOTS - 1) as f32 * THINK_GAP; - let content = dots + THINK_LABEL_GAP + Self::label_width(label, window); - (content + 2. * PILL_PAD).max(timed_w) - } - /// The cancel control and how far it has emerged: it pops in from .8 /// over 200 ms and leaves in 60 ms. At rest its glyph sits just below /// full strength; pointing at it brings it to full. @@ -582,6 +770,7 @@ impl PillView { pub(super) fn render_hint( snapshot: &DictationSnapshot, on_cancel: bool, + on_skip: bool, cx: &App, ) -> Option
{ let p = hud(cx); @@ -605,6 +794,8 @@ impl PillView { if on_cancel && (snapshot.is_recording || snapshot.is_processing) { row.child(caption(t("dictation.pill.cancelDictation"), p.foreground)) .children(esc) + } else if on_skip && snapshot.is_cleaning { + row.child(caption(t("dictation.pill.skipHint"), p.foreground)) } else if snapshot.is_recording { row.child(caption(t("dictation.pill.finish"), p.foreground)) .child(caption("·".into(), p.subtle_foreground)) @@ -637,6 +828,58 @@ mod tests { 10f32.powf(level / 20.) } + #[test] + fn fast_cleanup_never_shows_its_label_or_skip() { + use super::{POLISH_LABEL_AFTER, SKIP_AFTER, polish_stage}; + assert_eq!(polish_stage(None), (false, false)); + // Measured against a healthy server: 170 ms to 1 s of cleanup. + assert_eq!(polish_stage(at(170)), (false, false)); + assert_eq!(polish_stage(at(350)), (false, false)); + assert_eq!(polish_stage(at(1000)), (true, false)); + assert_eq!(polish_stage(Some(POLISH_LABEL_AFTER)), (true, false)); + assert_eq!(polish_stage(Some(SKIP_AFTER)), (true, true)); + } + + #[test] + fn swapped_layers_never_overlap_at_full_strength() { + assert_eq!(super::swap(0.), (1., 0.)); + assert_eq!(super::swap(1.), (0., 1.)); + for step in 0..=100 { + let (out, arrive) = super::swap(step as f32 / 100.); + assert!(out + arrive <= 1., "{out} + {arrive} at {step}"); + } + } + + #[test] + fn processing_layout_centers_the_label_and_docks_the_skip_chip() { + use super::{PILL_PAD, SKIP_INSET, SKIP_LEAD, THINK_DOTS_W, THINK_LABEL_GAP, think_layout}; + let (timed, transcribing, polishing, skip) = (134., 70., 55., 43.); + // Transcribing: the timed pill's width, with the group centered. + let rest = think_layout(timed, transcribing, polishing, skip, 0., 0.); + let group = THINK_DOTS_W + THINK_LABEL_GAP + transcribing; + assert_eq!(rest.width, timed); + assert!((rest.group_x - (timed - group) / 2.).abs() < 1e-4); + // Polishing: the chip sits at its inset from the trailing edge. + let full = think_layout(timed, transcribing, polishing, skip, 1., 1.); + assert!((full.skip_x + skip + SKIP_INSET - full.width).abs() < 1e-4); + assert!((full.group_x - SKIP_LEAD).abs() < 1e-4); + let label_end = full.group_x + THINK_DOTS_W + THINK_LABEL_GAP + polishing; + assert!(full.skip_x - label_end >= THINK_LABEL_GAP - 1e-4); + // A long label widens the pill instead of crowding its padding. + let long = think_layout(timed, 160., polishing, skip, 0., 0.); + assert!( + (long.width - (2. * PILL_PAD + THINK_DOTS_W + THINK_LABEL_GAP + 160.)).abs() < 1e-4 + ); + // The width changes monotonically while the chip arrives and leaves. + let mut previous = think_layout(timed, transcribing, polishing, skip, 1., 0.).width; + for step in 1..=20 { + let width = + think_layout(timed, transcribing, polishing, skip, 1., step as f32 / 20.).width; + assert!(width >= previous - 1e-4); + previous = width; + } + } + #[test] fn waveform_frames_follow_live_levels_and_motion_preferences() { for snapshot in [ diff --git a/src/ui/settings_modal.rs b/src/ui/settings_modal.rs index c25080f..353f855 100644 --- a/src/ui/settings_modal.rs +++ b/src/ui/settings_modal.rs @@ -186,6 +186,8 @@ pub struct SettingsModal { audio_retention: Entity, microphone: Entity