From 42a33982b128e48647d73a39a9dea2462136c370 Mon Sep 17 00:00:00 2001 From: angel1411337-del <232943451+angel1411337-del@users.noreply.github.com> Date: Wed, 11 Nov 2026 08:27:13 -0500 Subject: [PATCH] feat(voice): add optional local dictation with reviewed insertion Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KowAZudvkUiQS623fiB331 --- .github/workflows/ci.yml | 40 +- Cargo.lock | 309 ++- Cargo.toml | 2 + README.md | 6 + apps/notecraft-cli/src/main.rs | 1 + apps/notecraft/Cargo.toml | 4 + crates/engine/Cargo.toml | 1 + crates/engine/src/commands.rs | 8 + crates/engine/src/json.rs | 6 + crates/engine/src/lib.rs | 2 + crates/engine/src/session.rs | 16 +- crates/engine/src/voice.rs | 132 ++ crates/engine/tests/voice.rs | 45 + crates/ui-egui/Cargo.toml | 6 + crates/ui-egui/src/app.rs | 53 +- crates/ui-egui/src/app/voice.mac.rs | 5 + crates/ui-egui/src/app/voice.rs | 2329 +++++++++++++++++++ crates/ui-egui/src/app/voice.win.rs | 5 + crates/ui-egui/src/backstage.rs | 13 + crates/ui-egui/src/history.rs | 8 + crates/ui-egui/src/pageedit.rs | 27 + crates/ui-egui/src/ribbon.rs | 83 +- crates/voice-core/Cargo.toml | 11 + crates/voice-core/src/lib.rs | 264 +++ crates/voice-core/tests/sessions.rs | 88 + crates/voice-runtime/Cargo.toml | 23 + crates/voice-runtime/examples/transcribe.rs | 32 + crates/voice-runtime/src/audio.rs | 138 ++ crates/voice-runtime/src/capture.rs | 164 ++ crates/voice-runtime/src/lib.rs | 52 + crates/voice-runtime/src/runtime.rs | 383 +++ crates/voice-runtime/src/temporary.rs | 77 + crates/voice-runtime/src/whisper.rs | 243 ++ crates/voice-runtime/src/whisper.win.rs | 7 + crates/voice-runtime/tests/audio.rs | 57 + crates/voice-runtime/tests/process.rs | 53 + crates/voice-runtime/tests/runtime.rs | 24 + docs/architecture.md | 19 + docs/commands.md | 58 + docs/feature-parity.md | 1 + docs/gaps.md | 12 + docs/how-to/README.md | 1 + docs/how-to/dictate.md | 153 ++ i18n/ar-XB.tsv | 115 + i18n/en-XA.tsv | 115 + i18n/en.tsv | 115 + packaging/OSX/Info.plist.in | 1 + packaging/OSX/entitlements.plist | 6 +- packaging/OSX/package.sh | 6 +- xtask/src/main.rs | 2 + 50 files changed, 5288 insertions(+), 33 deletions(-) create mode 100644 crates/engine/src/voice.rs create mode 100644 crates/engine/tests/voice.rs create mode 100644 crates/ui-egui/src/app/voice.mac.rs create mode 100644 crates/ui-egui/src/app/voice.rs create mode 100644 crates/ui-egui/src/app/voice.win.rs create mode 100644 crates/voice-core/Cargo.toml create mode 100644 crates/voice-core/src/lib.rs create mode 100644 crates/voice-core/tests/sessions.rs create mode 100644 crates/voice-runtime/Cargo.toml create mode 100644 crates/voice-runtime/examples/transcribe.rs create mode 100644 crates/voice-runtime/src/audio.rs create mode 100644 crates/voice-runtime/src/capture.rs create mode 100644 crates/voice-runtime/src/lib.rs create mode 100644 crates/voice-runtime/src/runtime.rs create mode 100644 crates/voice-runtime/src/temporary.rs create mode 100644 crates/voice-runtime/src/whisper.rs create mode 100644 crates/voice-runtime/src/whisper.win.rs create mode 100644 crates/voice-runtime/tests/audio.rs create mode 100644 crates/voice-runtime/tests/process.rs create mode 100644 crates/voice-runtime/tests/runtime.rs create mode 100644 docs/how-to/dictate.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 316ffb2a..3ba73309 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,6 +104,42 @@ jobs: if: matrix.platform != 'linux' run: cargo test --workspace --locked + voice: + # The optional dictation feature is off in default builds, so the workspace jobs above never + # compile it. Build, lint and test it here, with the ALSA headers the microphone needs. + name: voice (${{ matrix.platform }}) + strategy: + fail-fast: false + matrix: + include: + - platform: linux + runner: ubuntu-24.04 + - platform: windows + runner: windows-2025 + - platform: macos + runner: macos-15 + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install Linux build dependencies + if: matrix.platform == 'linux' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends pkg-config libasound2-dev libxkbcommon-dev libwayland-dev libx11-dev libxcursor-dev libxrandr-dev libxi-dev libgtk-3-dev libfontconfig1-dev libfreetype-dev libvulkan1 mesa-vulkan-drivers + - name: Install Rust + run: rustup install + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + key: voice + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Check macOS microphone metadata + if: matrix.platform == 'macos' + run: plutil -lint packaging/OSX/Info.plist.in packaging/OSX/entitlements.plist + - run: cargo clippy -p craft-voice-core -p craft-voice-runtime -p notecraft-ui-egui -p notecraft --all-targets --locked --features craft-voice-runtime/microphone,notecraft/voice -- -D warnings + - run: cargo test -p craft-voice-core -p craft-voice-runtime --locked --features craft-voice-runtime/microphone + - run: cargo test -p notecraft-ui-egui --locked --features voice --lib app::voice + static: name: static checks runs-on: ubuntu-24.04 @@ -157,12 +193,12 @@ jobs: ci-status: name: CI runs-on: ubuntu-latest - needs: [fmt, clippy, platform-build-test, static, coverage] + needs: [fmt, clippy, platform-build-test, voice, static, coverage] if: always() steps: - name: Check CI result run: | - if [[ "${{ needs.fmt.result }}" == "failure" || "${{ needs.clippy.result }}" == "failure" || "${{ needs.platform-build-test.result }}" == "failure" || "${{ needs.static.result }}" == "failure" ]]; then + if [[ "${{ needs.fmt.result }}" == "failure" || "${{ needs.clippy.result }}" == "failure" || "${{ needs.platform-build-test.result }}" == "failure" || "${{ needs.voice.result }}" == "failure" || "${{ needs.static.result }}" == "failure" ]]; then echo "Quality checks failed" exit 1 fi diff --git a/Cargo.lock b/Cargo.lock index eb189983..6669ce4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,8 +107,8 @@ dependencies = [ "accesskit_consumer", "hashbrown 0.17.1", "static_assertions", - "windows", - "windows-core", + "windows 0.62.2", + "windows-core 0.62.2", ] [[package]] @@ -171,6 +171,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android-activity" version = "0.6.1" @@ -180,7 +202,7 @@ dependencies = [ "android-properties", "bitflags 2.13.2", "cc", - "jni", + "jni 0.22.4", "libc", "log", "ndk", @@ -666,6 +688,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.5" @@ -825,6 +853,46 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aae284fbaf7d27aa0e292f7677dfbe26503b0d555026f702940805a630eac17" +dependencies = [ + "bitflags 1.3.2", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "cpal" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd307f43cc2a697e2d1f8bc7a1d824b5269e052209e28883e5bc04d095aaa3f" +dependencies = [ + "alsa", + "coreaudio-rs", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -857,6 +925,20 @@ dependencies = [ "writeable", ] +[[package]] +name = "craft-voice-core" +version = "0.0.9" + +[[package]] +name = "craft-voice-runtime" +version = "0.0.9" +dependencies = [ + "cpal", + "craft-voice-core", + "serde_json", + "tempfile", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -913,6 +995,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "data-url" version = "0.3.2" @@ -1527,8 +1615,8 @@ dependencies = [ "read-fonts 0.41.0", "roxmltree 0.21.2", "smallvec", - "windows", - "windows-core", + "windows 0.62.2", + "windows-core 0.62.2", "yeslogic-fontconfig-sys", ] @@ -1811,7 +1899,7 @@ dependencies = [ "log", "presser", "thiserror 2.0.21", - "windows", + "windows 0.62.2", ] [[package]] @@ -2292,6 +2380,22 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -2556,6 +2660,15 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.8.3" @@ -2799,6 +2912,7 @@ dependencies = [ name = "notecraft-engine" version = "0.0.9" dependencies = [ + "craft-voice-core", "notecraft-formats", "notecraft-model", "notecraft-store", @@ -2895,6 +3009,8 @@ version = "0.0.9" dependencies = [ "arboard", "craft-i18n", + "craft-voice-core", + "craft-voice-runtime", "egui", "egui-wgpu", "egui_extras", @@ -2925,6 +3041,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3012,6 +3139,21 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2 0.6.5", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.2.2" @@ -3036,6 +3178,28 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2 0.6.5", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.5", +] + [[package]] name = "objc2-core-data" version = "0.2.2" @@ -5371,7 +5535,7 @@ version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62c35be770821a214dbc362fc26908c853e776c0004294d0b10b8a6bad582f94" dependencies = [ - "jni", + "jni 0.22.4", "log", "ndk-context", "objc2 0.6.5", @@ -5539,9 +5703,9 @@ dependencies = [ "web-sys", "wgpu-naga-bridge", "wgpu-types", - "windows", - "windows-core", - "windows-result", + "windows 0.62.2", + "windows-core 0.62.2", + "windows-result 0.4.1", ] [[package]] @@ -5579,6 +5743,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.62.2" @@ -5586,7 +5760,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ "windows-collections", - "windows-core", + "windows-core 0.62.2", "windows-future", "windows-numerics", ] @@ -5597,7 +5771,17 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" dependencies = [ - "windows-core", + "windows-core 0.62.2", +] + +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", ] [[package]] @@ -5609,7 +5793,7 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link", - "windows-result", + "windows-result 0.4.1", "windows-strings", ] @@ -5619,7 +5803,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" dependencies = [ - "windows-core", + "windows-core 0.62.2", "windows-link", "windows-threading", ] @@ -5658,7 +5842,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" dependencies = [ - "windows-core", + "windows-core 0.62.2", "windows-link", ] @@ -5669,10 +5853,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ "windows-link", - "windows-result", + "windows-result 0.4.1", "windows-strings", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -5691,13 +5884,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -5706,7 +5908,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -5718,20 +5920,35 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -5743,18 +5960,36 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -5767,24 +6002,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index ed4c09c3..43d51b62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,8 @@ repository = "https://github.com/AlienTechSoftware/notecraft" publish = false [workspace.dependencies] +craft-voice-core = { path = "crates/voice-core" } +craft-voice-runtime = { path = "crates/voice-runtime" } notecraft-model = { path = "crates/model" } notecraft-math = { path = "crates/math" } notecraft-diagram = { path = "crates/diagram" } diff --git a/README.md b/README.md index 524dffa5..1ecf70d1 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,12 @@ ## A tour +Optional local dictation is available with `cargo build -p notecraft --features voice`. +Use your own whisper-cli/model, review the transcript, then Insert with normal Undo +and autosave. See [dictation setup and limits](docs/how-to/dictate.md). +The feature is disabled by default. Hardware/model and Windows/macOS voice acceptance +remain unverified. + Every picture below is NoteCraft itself, drawn off screen by its own test harness (`cargo xtask screenshots`), not a mock-up. diff --git a/apps/notecraft-cli/src/main.rs b/apps/notecraft-cli/src/main.rs index b43386f2..f0825957 100644 --- a/apps/notecraft-cli/src/main.rs +++ b/apps/notecraft-cli/src/main.rs @@ -194,6 +194,7 @@ fn import_officeimo(root: &Path, mut args: Vec) -> Result<(String, bool) fn describe(o: &Output) -> String { match o { + Output::Voice(_) => notecraft_engine::output_json(o).to_string(), Output::None => "ok".to_string(), Output::Path(p) => display_path(p), Output::Text(t) => t.clone(), diff --git a/apps/notecraft/Cargo.toml b/apps/notecraft/Cargo.toml index 8a909836..f48d9764 100644 --- a/apps/notecraft/Cargo.toml +++ b/apps/notecraft/Cargo.toml @@ -11,6 +11,10 @@ description = "NoteCraft desktop app" [lints] workspace = true +[features] +default = [] +voice = ["notecraft-ui-egui/voice"] + [dependencies] notecraft-engine = { workspace = true } notecraft-store = { workspace = true } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 83abe059..1a23fbf0 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -11,6 +11,7 @@ description = "NoteCraft engine: the session and the command registry. Every use workspace = true [dependencies] +craft-voice-core = { workspace = true } notecraft-model = { workspace = true } notecraft-formats = { workspace = true } notecraft-store = { workspace = true } diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index ddd3c13d..4c130af1 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -16,6 +16,7 @@ pub type Params = BTreeMap; #[derive(Debug)] pub enum Output { + Voice(crate::VoiceEffect), None, Path(PathBuf), Text(String), @@ -552,6 +553,13 @@ blocks (optional): the justified paragraphs' positions among the page's blocks, run: tree_read, }, CommandSpec { id: "tree.refresh", label: "Reload the tree from disk", params: "", changes_tree: false, run: tree_refresh }, + CommandSpec { id: "voice.start", label: "Start local reviewed dictation", params: "target (required): opaque insertion ticket", changes_tree: false, run: crate::voice::start }, + CommandSpec { id: "voice.stop", label: "Stop capture and finalize dictation", params: "session (required): voice session id", changes_tree: false, run: crate::voice::stop }, + CommandSpec { id: "voice.cancel", label: "Discard the voice session", params: "session (required): voice session id", changes_tree: false, run: crate::voice::cancel }, + CommandSpec { id: "voice.status", label: "Read voice lifecycle status without transcript content", params: "", changes_tree: false, run: crate::voice::status }, + CommandSpec { id: "voice.draft.set", label: "Review and edit the local transcript", params: "session (required): voice session id\nrevision (required): expected transcript revision\ntext (required): reviewed transcript, at most 64 KiB", changes_tree: false, run: crate::voice::draft }, + CommandSpec { id: "voice.retarget", label: "Choose a new insertion position", params: "session (required): voice session id\ntarget (required): opaque insertion ticket", changes_tree: false, run: crate::voice::retarget }, + CommandSpec { id: "voice.commit", label: "Insert the reviewed local transcript", params: "session (required): voice session id\nrevision (required): approved transcript revision", changes_tree: false, run: crate::voice::commit }, ]; pub fn registry() -> &'static [CommandSpec] { diff --git a/crates/engine/src/json.rs b/crates/engine/src/json.rs index 25831ca6..e833e0b5 100644 --- a/crates/engine/src/json.rs +++ b/crates/engine/src/json.rs @@ -15,6 +15,12 @@ pub fn commands_json() -> Value { /// A command's result as JSON. pub fn output_json(o: &Output) -> Value { match o { + Output::Voice(effect) => match effect { + crate::VoiceEffect::Status { session, state, revision } => { + json!({"kind": "status", "session": session, "state": state.name(), "revision": revision}) + } + _ => json!({"kind": effect.kind(), "session": effect.session()}), + }, Output::None => Value::Null, Output::Path(p) => json!(display_path(p)), Output::Text(t) => json!(t), diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 08b126fd..9d29d504 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -15,8 +15,10 @@ mod error; mod json; pub mod sample; mod session; +mod voice; pub use commands::{CommandSpec, Output, Params, registry}; pub use error::EngineError; pub use json::{commands_json, output_json, params_from_json}; pub use session::Session; +pub use voice::{InvocationOrigin, VoiceEffect, VoiceUpdate}; diff --git a/crates/engine/src/session.rs b/crates/engine/src/session.rs index 006dd431..975a4e79 100644 --- a/crates/engine/src/session.rs +++ b/crates/engine/src/session.rs @@ -7,6 +7,8 @@ use std::path::{Path, PathBuf}; /// A notebooks folder, the settings, and a cached copy of the tree. pub struct Session { + pub(crate) voice: craft_voice_core::VoiceSession, + pub(crate) invocation_origin: crate::InvocationOrigin, root: PathBuf, /// Notebooks kept somewhere else and opened anyway. A notebook is a folder, so one can /// live outside the notebooks folder; opening it must not put away the others, which is @@ -33,7 +35,7 @@ impl Session { settings.get(OPEN_KEY).map(|v| v.split(SEP).map(str::trim).filter(|p| !p.is_empty()).map(PathBuf::from).collect()).unwrap_or_default(); let closed = settings.get(CLOSED_KEY).map(|v| v.split(SEP).map(str::trim).filter(|p| !p.is_empty()).map(PathBuf::from).collect()).unwrap_or_default(); - let mut s = Session { root, extra, closed, settings, tree: Vec::new() }; + let mut s = Session { root, extra, closed, settings, tree: Vec::new(), voice: Default::default(), invocation_origin: Default::default() }; s.refresh(); Ok(s) } @@ -158,6 +160,18 @@ impl Session { /// Runs a command by id. A panic inside the command becomes `EngineError::Panicked`. pub fn run(&mut self, id: &str, params: &Params) -> Result { + self.run_with_origin(id, params, crate::InvocationOrigin::Headless) + } + + /// The origin is supplied by trusted native code, never by serialized command parameters. + pub fn run_with_origin(&mut self, id: &str, params: &Params, origin: crate::InvocationOrigin) -> Result { + let previous = std::mem::replace(&mut self.invocation_origin, origin); + let result = self.run_inner(id, params); + self.invocation_origin = previous; + result + } + + fn run_inner(&mut self, id: &str, params: &Params) -> Result { let spec = commands::registry().iter().find(|c| c.id == id).ok_or_else(|| EngineError::UnknownCommand(id.to_string()))?; let result = guarded(|| (spec.run)(self, params))?; if spec.changes_tree && result.is_ok() { diff --git a/crates/engine/src/voice.rs b/crates/engine/src/voice.rs new file mode 100644 index 00000000..de573bfe --- /dev/null +++ b/crates/engine/src/voice.rs @@ -0,0 +1,132 @@ +//! Voice commands are pure state transitions; only the local host executes effects. +use crate::{EngineError, Output, Params, Session}; +use craft_voice_core::{State, Transcript}; +use std::fmt; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum InvocationOrigin { + LocalUi, + Automation, + #[default] + Headless, +} + +#[derive(Clone)] +pub enum VoiceEffect { + Start { session: u64, target: String }, + Stop { session: u64 }, + Cancel { session: u64 }, + Commit { session: u64, revision: u64, target: String, text: String }, + Status { session: u64, state: State, revision: u64 }, +} + +impl VoiceEffect { + pub fn session(&self) -> u64 { + match self { + Self::Start { session, .. } + | Self::Stop { session } + | Self::Cancel { session } + | Self::Commit { session, .. } + | Self::Status { session, .. } => *session, + } + } + pub fn kind(&self) -> &'static str { + match self { + Self::Start { .. } => "start", + Self::Stop { .. } => "stop", + Self::Cancel { .. } => "cancel", + Self::Commit { .. } => "commit", + Self::Status { .. } => "status", + } + } +} + +impl fmt::Debug for VoiceEffect { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("VoiceEffect").field("kind", &self.kind()).field("session", &self.session()).finish() + } +} + +pub enum VoiceUpdate { + Ready(u64), + Stopped(u64), + Transcript(Transcript), + Failed(u64, String), + Applied(u64, Result<(), String>), +} + +fn field<'a>(p: &'a Params, key: &'static str) -> Result<&'a str, EngineError> { + p.get(key).map(String::as_str).ok_or(EngineError::MissingParam(key)) +} +fn number(p: &Params, key: &'static str) -> Result { + field(p, key)?.parse().map_err(|_| EngineError::BadParam(format!("{key} must be a whole number"))) +} +fn error(e: craft_voice_core::VoiceError) -> EngineError { + EngineError::BadParam(e.to_string()) +} + +impl Session { + pub fn voice(&self) -> &craft_voice_core::VoiceSession { + &self.voice + } + + /// Native controller delivery. It is intentionally absent from the command registry. + pub fn voice_update(&mut self, update: VoiceUpdate) -> Result<(), EngineError> { + match update { + VoiceUpdate::Ready(id) => self.voice.ready(id), + VoiceUpdate::Stopped(id) if self.voice.id() == id && self.voice.state() == State::Finalizing => Ok(()), + VoiceUpdate::Stopped(id) => self.voice.stop(id), + VoiceUpdate::Transcript(event) => self.voice.accept(event), + VoiceUpdate::Failed(id, message) => self.voice.fail(id, &message), + VoiceUpdate::Applied(id, result) => self.voice.applied(id, result), + } + .map_err(error) + } + + fn require_local_voice(&self) -> Result<(), EngineError> { + if self.invocation_origin == InvocationOrigin::LocalUi { + Ok(()) + } else { + Err(EngineError::BadParam("voice recording and insertion require the local NoteCraft controls".into())) + } + } +} + +pub(crate) fn start(s: &mut Session, p: &Params) -> Result { + s.require_local_voice()?; + let target = field(p, "target")?.to_string(); + let session = s.voice.start(&target).map_err(error)?; + Ok(Output::Voice(VoiceEffect::Start { session, target })) +} +pub(crate) fn stop(s: &mut Session, p: &Params) -> Result { + s.require_local_voice()?; + let session = number(p, "session")?; + s.voice.stop(session).map_err(error)?; + Ok(Output::Voice(VoiceEffect::Stop { session })) +} +pub(crate) fn cancel(s: &mut Session, p: &Params) -> Result { + s.require_local_voice()?; + let session = number(p, "session")?; + s.voice.cancel(session).map_err(error)?; + Ok(Output::Voice(VoiceEffect::Cancel { session })) +} +pub(crate) fn status(s: &mut Session, _: &Params) -> Result { + Ok(Output::Voice(VoiceEffect::Status { session: s.voice.id(), state: s.voice.state(), revision: s.voice.revision() })) +} +pub(crate) fn draft(s: &mut Session, p: &Params) -> Result { + s.require_local_voice()?; + s.voice.set_draft(number(p, "session")?, number(p, "revision")?, field(p, "text")?).map_err(error)?; + status(s, p) +} +pub(crate) fn retarget(s: &mut Session, p: &Params) -> Result { + s.require_local_voice()?; + s.voice.retarget(number(p, "session")?, field(p, "target")?).map_err(error)?; + status(s, p) +} +pub(crate) fn commit(s: &mut Session, p: &Params) -> Result { + s.require_local_voice()?; + let session = number(p, "session")?; + let revision = number(p, "revision")?; + let text = s.voice.begin_apply(session, revision).map_err(error)?; + Ok(Output::Voice(VoiceEffect::Commit { session, revision, target: s.voice.target().into(), text })) +} diff --git a/crates/engine/tests/voice.rs b/crates/engine/tests/voice.rs new file mode 100644 index 00000000..3384e973 --- /dev/null +++ b/crates/engine/tests/voice.rs @@ -0,0 +1,45 @@ +use craft_voice_core::{State, Transcript}; +use notecraft_engine::{InvocationOrigin, Output, Params, Session, VoiceUpdate, output_json}; +use notecraft_store::{Settings, testing::TempDir}; + +fn p(values: &[(&str, &str)]) -> Params { + values.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect() +} +fn session(t: &TempDir) -> Session { + Session::new(&t.path().join("notebooks"), Settings::at(t.path().join("settings/s.toml"))).unwrap() +} + +#[test] +fn serialized_params_cannot_grant_local_microphone_or_edit_authority() { + let t = TempDir::new("voice-origin"); + let mut s = session(&t); + let args = p(&[("target", "opaque"), ("origin", "LocalUi")]); + assert!(s.run("voice.start", &args).is_err()); + assert!(s.run_with_origin("voice.start", &args, InvocationOrigin::Automation).is_err()); + assert_eq!(s.voice().state(), State::Idle); + assert!(s.run_with_origin("unknown", &args, InvocationOrigin::LocalUi).is_err()); + assert!(s.run("voice.start", &args).is_err()); + s.run_with_origin("voice.start", &args, InvocationOrigin::LocalUi).unwrap(); + assert_eq!(s.voice().state(), State::Preparing); + assert!(s.run("voice.cancel", &p(&[("session", "1")])).is_err()); +} + +#[test] +fn headless_status_and_effect_debug_never_reveal_words() { + let t = TempDir::new("voice-redaction"); + let mut s = session(&t); + s.run_with_origin("voice.start", &p(&[("target", "opaque")]), InvocationOrigin::LocalUi).unwrap(); + s.voice_update(VoiceUpdate::Ready(1)).unwrap(); + s.voice_update(VoiceUpdate::Stopped(1)).unwrap(); + s.voice_update(VoiceUpdate::Transcript(Transcript::new(1, 1, "private spoken words", true))).unwrap(); + let status = s.run("voice.status", &Params::new()).unwrap(); + assert!(!output_json(&status).to_string().contains("private spoken words")); + let effect = s.run_with_origin("voice.commit", &p(&[("session", "1"), ("revision", "1")]), InvocationOrigin::LocalUi).unwrap(); + assert!(matches!(effect, Output::Voice(_))); + assert!(!format!("{effect:?}").contains("private spoken words")); + assert!(!output_json(&effect).to_string().contains("private spoken words")); + assert!(s.run_with_origin("voice.commit", &p(&[("session", "1"), ("revision", "1")]), InvocationOrigin::LocalUi).is_err()); + s.voice_update(VoiceUpdate::Applied(1, Err("stale target".into()))).unwrap(); + assert_eq!(s.voice().state(), State::Review); + assert_eq!(s.voice().draft(), "private spoken words"); +} diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index 164ae816..6151f638 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -10,7 +10,13 @@ description = "NoteCraft egui front end. Swappable: nothing below it depends on [lints] workspace = true +[features] +default = [] +voice = ["dep:craft-voice-core", "dep:craft-voice-runtime"] + [dependencies] +craft-voice-core = { workspace = true, optional = true } +craft-voice-runtime = { workspace = true, optional = true, features = ["microphone"] } craft-i18n = { workspace = true } notecraft-input-language = { path = "../input-language" } notecraft-model = { workspace = true } diff --git a/crates/ui-egui/src/app.rs b/crates/ui-egui/src/app.rs index b344b35a..1588b236 100644 --- a/crates/ui-egui/src/app.rs +++ b/crates/ui-egui/src/app.rs @@ -36,6 +36,8 @@ use web_time::Instant; /// The grid boxes snap to on plain paper, in page points: the smallest squared paper's step. const PLAIN_SNAP_STEP: f32 = 12.0; const AUTOSAVE_AFTER: Duration = Duration::from_millis(900); +#[cfg(feature = "voice")] +mod voice; /// The three-bar button: what OneNote puts at the top left of the navigation pane. const BARS: &str = "\u{2630}"; /// The ArtCraft community chat. Opened in the browser, never fetched by the app. @@ -291,6 +293,7 @@ struct PendingFormat { } struct Editor { + generation: u64, dir: PathBuf, format: PageFormat, title: String, @@ -301,6 +304,11 @@ struct Editor { saved_body: String, /// The text an autosave is writing in the background, until it is written. saving: Option, + /// The body as read back from the page's file after the last load or save. Dictation + /// compares the file with it to see whether another program changed the page. It can + /// differ from `saved_body`, because the store may tidy the body as it writes it. + #[cfg(feature = "voice")] + disk_body: String, dirty_since: Option, blocks: Vec, last_saved: Option, @@ -671,6 +679,8 @@ enum Act { } pub struct NoteCraftApp { + #[cfg(feature = "voice")] + voice: voice::VoiceUi, session: Session, /// Writes autosaves off the window's thread (see [`crate::saver`]). saver: crate::saver::Saver, @@ -801,6 +811,8 @@ impl NoteCraftApp { // The installed fonts are read in the background, so the window opens at once. #[cfg(any(target_os = "linux", target_os = "windows", target_os = "macos"))] crate::fonts::start_catalog(); + #[cfg(feature = "voice")] + let voice = voice::VoiceUi::new(&session.settings); let mode_pref = session.settings.get("theme_mode").and_then(|v| ModePref::parse(&v)).unwrap_or(ModePref::System); let ui_language = session.settings.get("ui_language").map(|value| crate::i18n::normalize_preference(&value)).unwrap_or_else(|| crate::i18n::AUTO.into()); @@ -898,6 +910,8 @@ impl NoteCraftApp { queued_alerts: VecDeque::new(), clipboard: Box::new(crate::clipboard::System), open_attachment_warning_suppressed: false, + #[cfg(feature = "voice")] + voice, welcome_notebook_setup: None, } } @@ -1433,6 +1447,10 @@ impl NoteCraftApp { /// Opens a page without counting it as a visit: what Back and Forward do. fn open_page_inner(&mut self, dir: &Path) { + #[cfg(feature = "voice")] + { + self.voice.editor_epoch = self.voice.editor_epoch.wrapping_add(1); + } self.save_now(); self.code = crate::codeview::State::default(); if let Some(Output::Page(page)) = self.run("page.load", ¶ms(&[("path", &path_str(dir))])) { @@ -1475,6 +1493,9 @@ impl NoteCraftApp { // change" rather than a rename to the fallback word itself. let title_buf = if page.has_title { page.title.clone() } else { String::new() }; self.editor = Some(Editor { + generation: 0, + #[cfg(feature = "voice")] + disk_body: page_body.clone(), dir: dir.to_path_buf(), format: page.format, title_buf, @@ -1606,6 +1627,10 @@ impl NoteCraftApp { if let Some(ed) = self.editor.as_mut() { ed.dirty_since = None; ed.last_saved = Some(ts); + #[cfg(feature = "voice")] + { + ed.disk_body = notecraft_store::page::load_page(&dir).map(|page| page.body).unwrap_or_else(|_| body.clone()); + } ed.saved_body = body; } self.say(crate::tl_id!("ui.saved")); @@ -1647,6 +1672,10 @@ impl NoteCraftApp { if ed.saving.as_ref() == Some(&saved.body) { ed.saving = None; } + #[cfg(feature = "voice")] + { + ed.disk_body = notecraft_store::page::load_page(&ed.dir).map(|page| page.body).unwrap_or_else(|_| saved.body.clone()); + } ed.saved_body = saved.body; } self.say(crate::tl_id!("ui.saved")); @@ -1725,6 +1754,7 @@ impl NoteCraftApp { /// The page text changed: remember to save it, and read the blocks from it again. fn mark_dirty(ed: &mut Editor) { + ed.generation = ed.generation.wrapping_add(1); ed.dirty_since = Some(Instant::now()); let (blocks, in_the_text) = notecraft_formats::boxes::read(ed.format, &ed.body); ed.blocks = blocks; @@ -1737,6 +1767,7 @@ impl NoteCraftApp { /// as they are. Reading them back from the text would throw away an empty new block and /// move the caret out from under the user. fn mark_dirty_from_blocks(ed: &mut Editor) { + ed.generation = ed.generation.wrapping_add(1); // A change that leaves the file as it is, or as it is being written (a click on open // paper, which only places the cursor), is not a change to save. let file_will_hold = ed.saving.as_ref().unwrap_or(&ed.saved_body); @@ -2352,13 +2383,17 @@ impl NoteCraftApp { fn title_band(&mut self, ui: &mut Ui, with_tabs: bool) { let t = self.tokens; let (tab, source, in_table) = (self.ribbon_tab.clone(), self.show_source, self.caret_in_table()); + #[cfg(feature = "voice")] + let mic = self.dictation_mic(); + #[cfg(not(feature = "voice"))] + let mic = ribbon::Mic::Hidden; let title = self.editor.as_ref().map(|e| e.title.trim().to_owned()).filter(|s| !s.is_empty()); let mut acts = Vec::new(); egui::Panel::top("band").frame(egui::Frame::new().fill(t.band)).show_separator_line(false).show(ui, |ui| { ui.spacing_mut().item_spacing = vec2(4.0, 0.0); self.title_row(ui, title.as_deref()); if with_tabs { - ribbon::tab_strip(ui, &tab, source, in_table, &t, &mut acts); + ribbon::tab_strip(ui, &tab, source, in_table, mic, &t, &mut acts); } }); for a in acts { @@ -2614,6 +2649,11 @@ impl NoteCraftApp { } backstage::Act::ClearTelemetry(path) => self.dialog = Some(Dialog::ClearTelemetry { path }), backstage::Act::ResetPreferences => self.dialog = Some(Dialog::ResetPreferences), + #[cfg(feature = "voice")] + backstage::Act::DictationSetup => { + self.backstage = None; + self.open_dictation_setup(); + } backstage::Act::ExportTelemetry(root) => { if let Some(session) = std::fs::read_dir(&root) .ok() @@ -3268,6 +3308,10 @@ impl NoteCraftApp { /// Carries out one thing the ribbon asked for. fn ribbon_act(&mut self, act: ribbon::Act) { match act { + #[cfg(feature = "voice")] + ribbon::Act::Dictate => self.toggle_dictation(), + #[cfg(feature = "voice")] + ribbon::Act::DictationSettings => self.open_dictation_setup(), ribbon::Act::File => self.backstage = Some("Info"), ribbon::Act::PageVersions => { self.save_now(); @@ -3839,6 +3883,7 @@ impl NoteCraftApp { let step = if back { ed.history.undo().cloned() } else { ed.history.redo().cloned() }; match step { Some(step) => { + ed.generation = ed.generation.wrapping_add(1); ed.body = step.body; ed.blocks = step.blocks; ed.boxes = step.boxes; @@ -7333,6 +7378,8 @@ impl NoteCraftApp { self.clipboard_keys(ctx); self.poll_control(ctx); self.poll_import(ctx); + #[cfg(feature = "voice")] + self.poll_voice(ctx); if !self.event_frames.is_empty() { ctx.request_repaint(); } @@ -7365,6 +7412,8 @@ impl NoteCraftApp { self.dialogs(ctx); self.import_progress(ctx); self.frame_edges(ctx); + #[cfg(feature = "voice")] + self.show_voice(ctx); return; } self.top_bar(ui); @@ -7384,6 +7433,8 @@ impl NoteCraftApp { if self.saver.busy() { ctx.request_repaint_after(std::time::Duration::from_millis(50)); } + #[cfg(feature = "voice")] + self.show_voice(ctx); if let Some(since) = self.editor.as_ref().and_then(|e| e.dirty_since) { // Read once: read twice, the second could be past the wait, and the subtraction // below panicked (seen on a slow CI runner). diff --git a/crates/ui-egui/src/app/voice.mac.rs b/crates/ui-egui/src/app/voice.mac.rs new file mode 100644 index 00000000..d6eb33f2 --- /dev/null +++ b/crates/ui-egui/src/app/voice.mac.rs @@ -0,0 +1,5 @@ +//! macOS microphone privacy settings, opened only by the user's explicit action. + +pub(super) fn open_microphone_settings() -> std::io::Result<()> { + std::process::Command::new("open").arg("x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone").spawn().map(|_| ()) +} diff --git a/crates/ui-egui/src/app/voice.rs b/crates/ui-egui/src/app/voice.rs new file mode 100644 index 00000000..c7010df7 --- /dev/null +++ b/crates/ui-egui/src/app/voice.rs @@ -0,0 +1,2329 @@ +//! Native dictation: the mic button and shortcut, the recording bar, the preview at the +//! insertion point, the setup screen, and the reviewed editor transaction. + +use super::*; +use craft_voice_core::State; +use craft_voice_runtime::audio::PcmAudio; +use craft_voice_runtime::{Cancellation, Runtime, RuntimeEvent, Transcriber, WhisperConfig}; +use notecraft_engine::{InvocationOrigin, VoiceEffect, VoiceUpdate}; +use std::hash::{Hash, Hasher}; +use std::io::Read; + +/// Session ids for the setup screen's checks, kept apart from dictation sessions. +const MIC_TEST: u64 = u64::MAX; +const RECOGNIZER_TEST: u64 = u64::MAX - 1; +/// Holding the shortcut longer than this makes it push-to-talk; a shorter press toggles. +const HOLD: Duration = Duration::from_millis(600); +/// How long the microphone check listens. +const MIC_TEST_FOR: Duration = Duration::from_secs(4); +/// Where whisper.cpp's program and models come from, for the setup screen's link. +const WHISPER_HELP_URL: &str = "https://github.com/ggml-org/whisper.cpp#quick-start"; +/// The languages offered by name; any other code can still be typed into settings. +const LANGUAGES: [(&str, &str); 11] = [ + ("auto", "voice.detect_automatically_25a36033"), + ("en", "ui.english"), + ("es", "voice.spanish_8df7f1b3"), + ("fr", "voice.french_44389f6a"), + ("de", "voice.german_da91388c"), + ("pt", "voice.portuguese_23882c57"), + ("it", "voice.italian_485b57ad"), + ("nl", "voice.dutch_334a85f1"), + ("zh", "voice.chinese_15dc1886"), + ("ja", "voice.japanese_9239c22d"), + ("ko", "voice.korean_dec66c5b"), +]; + +pub(super) struct VoiceUi { + pub editor_epoch: u64, + ticket_counter: u64, + target: Option, + runtime: Option, + executable: String, + model: String, + language: String, + device: Option, + devices: Vec, + devices_listed: bool, + shortcut: String, + /// Show the words for review before they go in the page (the default), or put them in at once. + review_first: bool, + review: String, + review_revision: u64, + /// The words in the preview are open for typing. + editing: bool, + /// When the shortcut went down, to tell a tap (toggle) from a hold (push-to-talk). + pressed_at: Option, + pending_stop: bool, + /// An automatic insert was tried for this session, so a failure is not retried every frame. + auto_tried: bool, + seconds: f32, + /// The latest input level, and the loudest since recording began (0 to 1). + level: f32, + loudest: f32, + error: Option, + setup_open: bool, + /// Why the setup screen opened by itself, shown at its top. + setup_reason: Option<&'static str>, + mic_test: Option, + mic_result: Option>, + recognizer_test: Option, + recognizer_result: Option>, +} + +struct MicTest { + runtime: Runtime, + started: Instant, + level: f32, + loudest: f32, + heard_audio: bool, +} + +impl VoiceUi { + pub fn new(settings: ¬ecraft_store::Settings) -> Self { + Self { + editor_epoch: 0, + ticket_counter: 0, + target: None, + runtime: None, + executable: settings.get("voice_executable").unwrap_or_default(), + model: settings.get("voice_model").unwrap_or_default(), + language: settings.get("voice_language").unwrap_or_else(|| "auto".into()), + device: settings.get("voice_device").filter(|v| !v.is_empty()), + devices: Vec::new(), + devices_listed: false, + shortcut: settings.get("voice_shortcut").filter(|k| ["F8", "F9", "F10"].contains(&k.as_str())).unwrap_or_else(|| "F8".into()), + review_first: settings.get("voice_review").is_none_or(|v| v != "false"), + review: String::new(), + review_revision: 0, + editing: false, + pressed_at: None, + pending_stop: false, + auto_tried: false, + seconds: 0.0, + level: 0.0, + loudest: 0.0, + error: None, + setup_open: false, + setup_reason: None, + mic_test: None, + mic_result: None, + recognizer_test: None, + recognizer_result: None, + } + } +} + +/// Where the dictated words go: at the caret, or as new paragraphs at the end of the page when +/// there is no caret or it sits somewhere words cannot go (a table, code, a picture). +#[derive(Clone)] +enum Anchor { + Caret(pageedit::Caret), + End, +} + +#[derive(Clone)] +pub(super) struct InsertionTicket { + id: String, + epoch: u64, + generation: u64, + dir: PathBuf, + body: String, + source: u64, + anchor: Anchor, + /// Input selection at capture time, including unsupported targets that append instead. + caret_snapshot: Option, + across_snapshot: Option, + /// The page's title, to name it if the person moves away before inserting. + title: String, +} + +/// What a person can do about an error, offered as a button beside it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Fix { + None, + MicSettings, + Setup, + PageView, +} + +/// An error from the recorder or recognizer, in words a person can act on. +fn explain(message: &str) -> (String, Fix) { + let m = message.to_lowercase(); + let any = |words: &[&str]| words.iter().any(|w| m.contains(w)); + if any(&[ + "sent no audio", + "no matching input device", + "cannot open microphone", + "cannot list microphones", + "cannot start microphone", + "cannot create microphone stream", + "microphone failed", + "microphone disconnected", + ]) { + let fix = if MIC_SETTINGS { Fix::MicSettings } else { Fix::Setup }; + return ( + crate::i18n::format_message_id("voice.notecraft_can_t_hear_your_microphone_check_that_it_5f82d9f8", &[("system", system_name())]), + fix, + ); + } + if m.contains("buffer overflow") { + return (crate::tl_id!("voice.the_computer_fell_behind_while_recording_try_again_e30d7359").into(), Fix::None); + } + if m.contains("timed out") { + return (crate::tl_id!("voice.recognition_took_too_long_a_smaller_model_such_as_fff73af3").into(), Fix::Setup); + } + if any(&["whisper-cli", "does not advertise", "recognizer exited", "whisper model", "language code"]) { + return (crate::tl_id!("voice.the_speech_recognizer_isn_t_set_up_correctly_open_57ac2283").into(), Fix::Setup); + } + if m.contains("page view") { + return (crate::tl_id!("voice.switch_back_to_the_page_view_to_dictate_e3deb7f8").into(), Fix::PageView); + } + (message.into(), Fix::None) +} + +fn system_name() -> &'static str { + #[cfg(target_os = "windows")] + { + "Windows" + } + #[cfg(target_os = "macos")] + { + "macOS" + } + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + { + crate::tl_id!("voice.your_system_3a1356ff") + } +} + +/// Whether this system has a microphone privacy page NoteCraft can open. +const MIC_SETTINGS: bool = cfg!(any(target_os = "windows", target_os = "macos")); + +#[cfg(target_os = "windows")] +#[path = "voice.win.rs"] +mod platform; +#[cfg(target_os = "macos")] +#[path = "voice.mac.rs"] +mod platform; + +/// Opens the system's microphone privacy settings. +fn open_microphone_settings() -> Result<(), String> { + #[cfg(any(target_os = "windows", target_os = "macos"))] + let opened = platform::open_microphone_settings(); + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + let opened: std::io::Result<()> = Err(std::io::Error::other("no microphone settings page on this system")); + opened.map_err(|e| crate::i18n::format_message_id("voice.cannot_open_the_microphone_settings_error_b3a9cb13", &[("error", &e.to_string())])) +} + +/// Listens without recognizing anything: the microphone check only wants the levels. +struct Listen; + +impl Transcriber for Listen { + fn capabilities(&self) -> craft_voice_core::Capabilities { + craft_voice_core::Capabilities::default() + } + fn prepare(&mut self, _: &Cancellation) -> Result<(), String> { + Ok(()) + } + fn transcribe(&mut self, _: &PcmAudio, _: &Cancellation) -> Result { + Ok(String::new()) + } +} + +/// What the shortcut's release means: after a hold it ends a push-to-talk recording; after a +/// tap it does nothing, and the next press ends the recording instead. +fn release_finishes(held_for: Duration) -> bool { + held_for >= HOLD +} + +/// Every non-repeat press of `key` with no modifiers in this frame's input, taken out of it so +/// the page underneath does not also see it. Auto-repeat presses are dropped too. +fn take_press(ctx: &Context, key: Key) -> bool { + ctx.input_mut(|i| { + let mut hit = false; + i.events.retain(|event| match event { + egui::Event::Key { key: k, pressed: true, repeat, modifiers, .. } if *k == key && modifiers.is_none() => { + hit |= !*repeat; + false + } + _ => true, + }); + hit + }) +} + +/// Seconds as minutes and seconds, for the recording bar. +fn clock(seconds: f32) -> String { + let whole = seconds.max(0.0) as u32; + format!("{}:{:02}", whole / 60, whole % 60) +} + +/// Adds literal transcript paragraphs and keeps container block counts in step. +fn add_paragraphs(blocks: &mut Vec, boxes: &mut [notecraft_model::Container], at: usize, lines: &[&str]) { + let at = at.min(blocks.len()); + let before = blocks.len(); + for (k, line) in lines.iter().enumerate() { + blocks.insert(at + k, notecraft_formats::Block::Paragraph(vec![notecraft_formats::Span { text: (*line).into(), ..Default::default() }])); + } + if !boxes.is_empty() && boxes.iter().map(|c| c.blocks).sum::() == before { + let owner = at.saturating_sub(1); + let mut start = 0; + for c in boxes.iter_mut() { + if owner < start + c.blocks { + c.blocks += lines.len(); + return; + } + start += c.blocks; + } + if let Some(last) = boxes.last_mut() { + last.blocks += lines.len(); + } + } +} + +/// Adjacent spans with the same marks are one semantic run, regardless of parser chunking. +fn normalized(blocks: &[notecraft_formats::Block]) -> Vec { + use notecraft_formats::{Block, Span}; + fn spans(spans: &mut Vec) { + let mut joined: Vec = Vec::new(); + for span in std::mem::take(spans) { + if span.text.is_empty() && span.image.is_none() { + continue; + } + if let Some(last) = joined.last_mut() { + let mut a = last.clone(); + a.text.clear(); + let mut b = span.clone(); + b.text.clear(); + if a == b { + last.text.push_str(&span.text); + continue; + } + } + joined.push(span); + } + *spans = joined; + } + let mut blocks = blocks.to_vec(); + for block in &mut blocks { + match block { + Block::Paragraph(s) | Block::Heading { spans: s, .. } | Block::Quote(s) | Block::Item { spans: s, .. } => spans(s), + Block::Table { head, rows } => { + for cell in head.iter_mut().chain(rows.iter_mut().flatten()) { + spans(cell); + } + } + _ => {} + } + } + blocks +} + +/// Parse an inert position token, then replace it inside the Span model so transcript +/// punctuation cannot become markup. Serialization is checked before the edit is applied. +fn insert_words(block: &mut notecraft_formats::Block, caret: &pageedit::Caret, words: &str) -> Result { + let mut stable = block.clone(); + pageedit::set_caret_text(&mut stable, &caret.text); + if normalized(&[stable]) != normalized(std::slice::from_ref(block)) { + return Err(crate::tl_id!("voice.this_cursor_position_cannot_preserve_the_existing_ccbd81ed").into()); + } + let words = spaced(&caret.text, caret.selection, words); + let mut marker = "NoteCraftVoiceInsertionPoint".to_string(); + while caret.text.contains(&marker) || words.contains(&marker) { + marker.push('X'); + } + let (source, _) = pageedit::insert_at_caret(&caret.text, caret.selection, &marker); + pageedit::set_caret_text(block, &source); + let marked = pageedit::caret_text(block); + let position = marked.find(&marker).ok_or(crate::tl_id!("voice.choose_a_cursor_position_in_the_paragraph_s_words_6538d9ab"))?; + let at = marked[..position].chars().count() + words.chars().count(); + let spans = match block { + notecraft_formats::Block::Paragraph(s) + | notecraft_formats::Block::Heading { spans: s, .. } + | notecraft_formats::Block::Quote(s) + | notecraft_formats::Block::Item { spans: s, .. } => s, + _ => return Err(crate::tl_id!("voice.choose_a_paragraph_before_inserting_3f802b0c").into()), + }; + let span = spans + .iter_mut() + .find(|s| s.text.contains(&marker)) + .ok_or(crate::tl_id!("voice.choose_a_cursor_position_outside_formatting_marks_54a40d44"))?; + span.text = span.text.replacen(&marker, &words, 1); + let text = pageedit::caret_text(block); + let at = at.min(text.chars().count()); + Ok(pageedit::Caret { text, selection: (at, at), place: Some(at), focus: true, ..caret.clone() }) +} + +/// The caret at the end of block `at`. +fn caret_at_end(ed: &Editor, at: usize) -> Option { + let text = pageedit::caret_text(ed.blocks.get(at)?); + let n = text.chars().count(); + Some(pageedit::Caret { block: at, text, selection: (n, n), place: Some(n), focus: true, ..Default::default() }) +} + +fn fingerprint(file: &Path) -> Result { + let input = std::fs::File::open(file) + .map_err(|e| crate::i18n::format_message_id("voice.cannot_read_the_page_source_error_3c53fd4d", &[("error", &e.to_string())]))?; + let mut bytes = Vec::new(); + input.take(16 * 1024 * 1024 + 1).read_to_end(&mut bytes).map_err(|e| e.to_string())?; + if bytes.len() > 16 * 1024 * 1024 { + return Err("this page is too large for a dictation target".into()); + } + let mut hash = std::collections::hash_map::DefaultHasher::new(); + bytes.hash(&mut hash); + Ok(hash.finish()) +} + +fn page_file(ed: &Editor) -> PathBuf { + ed.dir.join(if ed.format == PageFormat::Html { "index.html" } else { "index.md" }) +} + +fn validate_voice_block(block: ¬ecraft_formats::Block) -> Result<(), String> { + use notecraft_formats::Block; + let spans = match block { + Block::Paragraph(spans) | Block::Heading { spans, .. } | Block::Quote(spans) | Block::Item { spans, .. } => spans, + _ => return Err(crate::tl_id!("voice.dictation_currently_supports_paragraphs_headings_q_4db15ad2").into()), + }; + if spans.iter().any(|span| span.code || span.math) { + return Err(crate::tl_id!("voice.choose_a_paragraph_without_inline_code_or_equation_bf91894f").into()); + } + Ok(()) +} + +/// Below this peak level (about -40 dBFS) the microphone is taken to be hearing nothing. +const SILENT: f32 = 0.01; + +/// A level from 0 to 1 as the fill of a meter that reads in decibels, from -60 dBFS to 0. +fn meter(level: f32) -> f32 { + if level <= 0.0 { 0.0 } else { ((20.0 * level.log10() + 60.0) / 60.0).clamp(0.0, 1.0) } +} + +/// What to tell the person when audio arrives but none of it is sound. +fn input_hint(seconds: f32, loudest: f32) -> Option<&'static str> { + (seconds >= 2.0 && loudest < SILENT).then_some(crate::tl_id!("voice.no_sound_from_the_microphone_yet_check_that_the_ri_84320e7b")) +} + +/// Dictation arrives as a sentence of its own. Next to words already on the line it needs the +/// space typing would have put there; next to punctuation or white space it needs none. +fn spaced(line: &str, selection: (usize, usize), text: &str) -> String { + let chars: Vec = line.chars().collect(); + let (from, to) = (selection.0.min(selection.1), selection.0.max(selection.1)); + // Look past the marks of a styled neighbour to the word it holds. + let word = |c: &&char| !matches!(**c, '*' | '~'); + let before = chars.get(..from).unwrap_or_default().iter().rev().find(word); + let after = chars.get(to..).unwrap_or_default().iter().find(word); + let mut out = String::with_capacity(text.len() + 2); + if let (Some(b), Some(f)) = (before, text.chars().next()) + && !b.is_whitespace() + && !"([{\"'\u{201c}\u{2018}\u{bf}\u{a1}/-".contains(*b) + && f.is_alphanumeric() + { + out.push(' '); + } + out.push_str(text); + if let (Some(a), Some(l)) = (after, text.chars().last()) + && a.is_alphanumeric() + && !l.is_whitespace() + { + out.push(' '); + } + out +} + +fn voice_cache_root(session: &Session) -> Result { + let parent = session.settings.path().parent().unwrap_or_else(|| Path::new(".")); + let parent = std::fs::canonicalize(parent).map_err(|_| crate::tl_id!("voice.cannot_locate_the_local_settings_folder_6346b636"))?; + let in_notebook = |path: &Path| { + path.starts_with(session.root()) + || session.opened_elsewhere().iter().any(|root| std::fs::canonicalize(root).is_ok_and(|root| path.starts_with(root))) + || path.ancestors().any(|dir| notecraft_store::tree::read_node(dir).is_some_and(|node| node.kind == notecraft_model::NodeKind::Notebook)) + }; + if in_notebook(&parent) { + return Err(crate::tl_id!("voice.keep_the_settings_folder_outside_notebooks_before_a55ba6bc").into()); + } + let root = parent.join("voice-cache"); + std::fs::create_dir_all(&root) + .map_err(|e| crate::i18n::format_message_id("voice.cannot_create_the_voice_cache_error_05777284", &[("error", &e.to_string())]))?; + let root = std::fs::canonicalize(root) + .map_err(|e| crate::i18n::format_message_id("voice.cannot_resolve_the_voice_cache_error_902cca25", &[("error", &e.to_string())]))?; + if in_notebook(&root) { + return Err(crate::tl_id!("voice.the_voice_cache_resolves_into_a_notebook_choose_a_1f9a56e8").into()); + } + Ok(root) +} + +impl NoteCraftApp { + /// Where dictated words would go now: the caret when it is somewhere words can go, the end + /// of the page otherwise. Fails only when the page cannot take words at all. + fn capture_voice_target(&mut self) -> Result { + if self.show_source { + return Err(crate::tl_id!("voice.return_to_the_page_view_before_dictating_2c363339").into()); + } + let ed = self.editor.as_ref().ok_or(crate::tl_id!("voice.open_a_page_to_dictate_into_0300e9a6"))?; + if !ed.writable { + return Err(crate::tl_id!("voice.this_page_is_read_only_here_so_dictation_can_t_wri_47a8e613").into()); + } + // An autosave still being written changes the file; take it in before reading the file. + self.take_saves(true); + let ed = self.editor.as_ref().ok_or(crate::tl_id!("voice.the_page_closed_fc708395"))?; + let disk = notecraft_store::page::load_page(&ed.dir) + .map_err(|_| crate::tl_id!("voice.cannot_read_the_page_from_disk_reload_it_then_dict_b17aa77a"))?; + if disk.body != ed.disk_body { + return Err(crate::tl_id!("voice.another_program_changed_this_page_reload_it_then_d_cc71cb4e").into()); + } + self.save_now(); + let ed = self.editor.as_ref().ok_or(crate::tl_id!("voice.the_page_closed_fc708395"))?; + if ed.dirty_since.is_some() { + return Err(crate::tl_id!("voice.notecraft_couldn_t_save_the_page_so_it_won_t_add_d_5c189933").into()); + } + let anchor = match &ed.caret { + Some(caret) + if ed.across.is_none_or(|a| a.inside_one_block()) + && ed.blocks.get(caret.block).is_some_and(|b| validate_voice_block(b).is_ok()) + && caret.selection.0.max(caret.selection.1) <= caret.text.chars().count() => + { + Anchor::Caret(caret.clone()) + } + _ => Anchor::End, + }; + self.voice.ticket_counter = + self.voice.ticket_counter.checked_add(1).ok_or(crate::tl_id!("voice.reopen_the_app_to_start_a_new_target_sequence_907696fc"))?; + Ok(InsertionTicket { + id: self.voice.ticket_counter.to_string(), + epoch: self.voice.editor_epoch, + generation: ed.generation, + dir: ed.dir.clone(), + body: ed.body.clone(), + source: fingerprint(&page_file(ed))?, + title: ed.title.clone(), + caret_snapshot: ed.caret.clone(), + across_snapshot: ed.across, + anchor, + }) + } + + /// Focus and pending placement are rendering details. Logical selection and page identity + /// decide whether a recorded insertion position still describes the user's current target. + fn cursor_matches_ticket(&self, ticket: &InsertionTicket) -> bool { + self.editor.as_ref().is_some_and(|ed| { + ed.across == ticket.across_snapshot + && match (&ed.caret, &ticket.caret_snapshot) { + (None, None) => true, + (Some(current), Some(saved)) => { + current.block == saved.block + && current.text == saved.text + && current.selection == saved.selection + && current.cell == saved.cell + && current.span == saved.span + } + _ => false, + } + }) + } + + /// The page is as it was when `ticket` was taken, so its position still means the same place. + fn ticket_is_current(&self, ticket: &InsertionTicket) -> bool { + let Some(ed) = self.editor.as_ref() else { return false }; + let same_page = !self.show_source + && ed.writable + && ticket.epoch == self.voice.editor_epoch + && ticket.dir == ed.dir + && ticket.generation == ed.generation + && ticket.body == ed.body + && fingerprint(&page_file(ed)).is_ok_and(|f| f == ticket.source); + same_page + && self.cursor_matches_ticket(ticket) + && match &ticket.anchor { + Anchor::End => true, + Anchor::Caret(caret) => { + ed.blocks.get(caret.block).is_some_and(|b| pageedit::caret_text(b) == caret.text && validate_voice_block(b).is_ok()) + } + } + } + + fn apply_voice_text(&mut self, ticket: &InsertionTicket, text: &str) -> Result<(), String> { + if text.len() > craft_voice_core::MAX_TRANSCRIPT_BYTES { + return Err(crate::tl_id!("voice.the_words_are_longer_than_64_kib_shorten_them_befo_74a29916").into()); + } + // Each line becomes a paragraph of its own. + let lines: Vec<&str> = text.split(['\r', '\n']).map(str::trim).filter(|l| !l.is_empty()).collect(); + let Some((first, rest)) = lines.split_first() else { + return Err(crate::tl_id!("voice.there_are_no_words_to_insert_d397d6a5").into()); + }; + if !self.ticket_is_current(ticket) { + return Err(crate::tl_id!("voice.the_page_changed_while_you_were_talking_click_wher_2d22a184").into()); + } + let ed = self.editor.as_mut().ok_or(crate::tl_id!("voice.the_page_closed_fc708395"))?; + // History stores blocks/containers, but not indexed paragraph/table layout. Refuse + // the narrow case that would shift those indexes rather than corrupt unrelated layout. + if let Anchor::Caret(caret) = &ticket.anchor + && !rest.is_empty() + { + let end = caret.selection.0.max(caret.selection.1); + if caret.text.chars().count() > end { + return Err(crate::tl_id!("voice.multiline_needs_line_end").into()); + } + + let at = caret.block + 1; + let mut shifted = ed.columns.clone(); + if shifted.insert_block(at) || ed.justified.blocks().iter().any(|(block, _)| *block >= at) { + return Err(crate::tl_id!("voice.layout_would_shift").into()); + } + } + let mut blocks = ed.blocks.clone(); + let mut boxes = ed.boxes.clone(); + let before_caret = ed.caret.clone(); + let mut next_caret = before_caret.clone(); + let last = match &ticket.anchor { + Anchor::Caret(caret) => { + let block = blocks.get_mut(caret.block).ok_or(crate::tl_id!("voice.the_paragraph_closed_868b6d17"))?; + let tag = pageedit::tag_of(block).0; + next_caret = Some(insert_words(block, caret, first)?); + if pageedit::tag_of(block).0 != tag { + return Err(crate::tl_id!("voice.these_words_would_become_a_page_tag_edit_the_tag_m_bfed5fa8").into()); + } + add_paragraphs(&mut blocks, &mut boxes, caret.block + 1, rest); + (!rest.is_empty()).then_some(caret.block + rest.len()) + } + Anchor::End => { + let at = blocks.len(); + add_paragraphs(&mut blocks, &mut boxes, at, &lines); + Some(at + lines.len() - 1) + } + }; + let first_new = match &ticket.anchor { + Anchor::Caret(c) => c.block + 1, + Anchor::End => ed.blocks.len(), + }; + if blocks + .iter() + .skip(first_new) + .take(if matches!(ticket.anchor, Anchor::End) { lines.len() } else { rest.len() }) + .any(|block| pageedit::tag_of(block).0.is_some()) + { + return Err(crate::tl_id!("voice.these_words_would_become_a_page_tag_edit_the_tag_m_bfed5fa8").into()); + } + let arrangement = (ed.arranged && !boxes.is_empty()).then_some(boxes.as_slice()); + let body = notecraft_formats::boxes::write(ed.format, &blocks, arrangement); + let (read_back, read_boxes) = notecraft_formats::boxes::read(ed.format, &body); + if normalized(&read_back) != normalized(&blocks) || read_boxes.as_deref() != arrangement { + return Err(crate::tl_id!("voice.this_page_format_cannot_preserve_these_literal_wor_fc42673c").into()); + } + ed.history.remember_caret(match &ticket.anchor { + Anchor::Caret(c) => Some(c.clone()), + Anchor::End => before_caret, + }); + ed.blocks = blocks; + ed.boxes = boxes; + ed.body = body; + ed.caret = next_caret; + if let Some(last) = last { + ed.caret = caret_at_end(ed, last); + } + Self::mark_dirty_from_blocks(ed); + ed.history.record(crate::history::Step::new( + ed.body.clone(), + ed.blocks.clone(), + ed.boxes.clone(), + ed.caret.clone(), + crate::history::Kind::Edit, + )); + Ok(()) + } + + fn voice_command(&mut self, command: &str, p: &Params) -> Result<(), String> { + let output = self.session.run_with_origin(command, p, InvocationOrigin::LocalUi).map_err(|e| e.to_string())?; + match output { + Output::Voice(VoiceEffect::Start { session, .. }) => { + self.voice.review.clear(); + self.voice.review_revision = 0; + self.voice.seconds = 0.0; + self.voice.level = 0.0; + self.voice.loudest = 0.0; + self.voice.pending_stop = false; + self.voice.editing = false; + self.voice.auto_tried = false; + let setup = (|| -> Result { + let root = voice_cache_root(&self.session)?; + let mut config = WhisperConfig::new(self.voice.executable.clone().into(), self.voice.model.clone().into(), root); + config.language = self.voice.language.clone(); + Runtime::start(session, Box::new(config), self.voice.device.clone()) + })(); + match setup { + Ok(runtime) => self.voice.runtime = Some(runtime), + Err(message) => { + let _ = self.session.voice_update(VoiceUpdate::Failed(session, message.clone())); + return Err(message); + } + } + } + Output::Voice(VoiceEffect::Stop { .. }) => { + if let Some(runtime) = &self.voice.runtime { + runtime.stop(); + } + } + Output::Voice(VoiceEffect::Cancel { .. }) => { + self.voice.runtime.take(); + self.voice.target = None; + self.voice.review.clear(); + self.voice.editing = false; + self.voice.pressed_at = None; + self.voice.pending_stop = false; + } + Output::Voice(VoiceEffect::Commit { session, target, text, .. }) => { + let result = match self.voice.target.clone().filter(|t| t.id == target) { + Some(ticket) => self.apply_voice_text(&ticket, &text), + None => Err(crate::tl_id!("voice.click_where_the_words_should_go_and_choose_insert_473a713a").into()), + }; + let _ = self.session.voice_update(VoiceUpdate::Applied(session, result.clone())); + result?; + self.voice.error = None; + self.voice.runtime.take(); + self.voice.target = None; + self.voice.review.clear(); + self.voice.editing = false; + } + _ => {} + } + Ok(()) + } + + fn start_voice(&mut self) -> Result<(), String> { + let ticket = self.capture_voice_target()?; + let id = ticket.id.clone(); + self.voice.target = Some(ticket); + self.voice.error = None; + self.voice_command("voice.start", ¶ms(&[("target", &id)])) + } + + fn stop_voice(&mut self) -> Result<(), String> { + if self.session.voice().state() == State::Preparing { + self.voice.pending_stop = true; + return Ok(()); + } + if self.session.voice().state() != State::Recording { + return Ok(()); + } + let session = self.session.voice().id().to_string(); + self.voice_command("voice.stop", ¶ms(&[("session", &session)])) + } + + /// The title of the page the words were for, when a different page (or none) is open now. + pub(super) fn target_elsewhere(&self) -> Option { + let ticket = self.voice.target.as_ref()?; + (self.editor.as_ref().map(|ed| &ed.dir) != Some(&ticket.dir)).then(|| ticket.title.clone()) + } + + fn commit_voice(&mut self) -> Result<(), String> { + self.commit_with(false) + } + + /// Insert on the open page although the words were for another one: the person chose this. + fn commit_on_this_page(&mut self) -> Result<(), String> { + self.commit_with(true) + } + + fn commit_with(&mut self, other_page: bool) -> Result<(), String> { + if !other_page && let Some(title) = self.target_elsewhere() { + return Err(crate::i18n::format_message_id("voice.these_words_were_for_the_page_title_choose_insert_27760264", &[("title", &title)])); + } + let current = self.session.voice(); + if current.state() != State::Review || self.voice.review_revision != current.revision() || self.voice.review != current.draft() { + return Err(crate::tl_id!("voice.the_displayed_draft_is_not_the_accepted_revision_r_3002e242").into()); + } + // The page changed while the words were being spoken or reviewed: rather than refuse, put + // them where the person is now, which is where they are looking. + let mut moved = false; + if let Some(ticket) = self.voice.target.clone() + && !self.ticket_is_current(&ticket) + { + let fresh = self.capture_voice_target()?; + let session = self.session.voice().id().to_string(); + self.voice_command("voice.retarget", ¶ms(&[("session", &session), ("target", &fresh.id)]))?; + self.voice.target = Some(fresh); + self.voice.review_revision = self.session.voice().revision(); + moved = true; + } + let current = self.session.voice(); + let session = current.id().to_string(); + let revision = current.revision().to_string(); + self.voice_command("voice.commit", ¶ms(&[("session", &session), ("revision", &revision)]))?; + self.say(if moved { + crate::tl_id!("voice.the_page_changed_while_you_were_talking_so_the_wor_41829e1a") + } else { + crate::tl_id!("voice.dictation_inserted_ctrl_z_takes_it_back_a6eb393d") + }); + Ok(()) + } + + fn cancel_voice(&mut self) { + let session = self.session.voice().id().to_string(); + let _ = self.voice_command("voice.cancel", ¶ms(&[("session", &session)])); + self.voice.runtime.take(); + self.voice.review.clear(); + self.voice.target = None; + self.voice.editing = false; + self.voice.pressed_at = None; + self.voice.pending_stop = false; + } + + /// The program and model are chosen and exist, so recording can start. + fn dictation_ready(&self) -> bool { + Path::new(&self.voice.executable).is_file() && Path::new(&self.voice.model).metadata().is_ok_and(|m| m.is_file() && m.len() > 0) + } + + /// Starts dictating, or opens setup first when the recognizer is not chosen yet. + fn begin_dictation(&mut self) { + self.voice.error = None; + if !self.dictation_ready() { + self.voice.setup_open = true; + self.voice.setup_reason = Some(crate::tl_id!("voice.choose_the_speech_recognizer_and_a_model_to_start_4a44889b")); + return; + } + self.voice.mic_test = None; + self.voice.recognizer_test = None; + self.voice.setup_open = false; + if let Err(e) = self.start_voice() { + self.voice.error = Some(e); + } + } + + fn finish_dictation(&mut self) { + if let Err(e) = self.stop_voice() { + self.voice.error = Some(e); + } + } + + /// What the mic button and the shortcut do: start when idle, finish while listening. + pub(super) fn toggle_dictation(&mut self) { + match self.session.voice().state() { + State::Preparing | State::Recording => self.finish_dictation(), + State::Idle | State::Complete | State::Cancelled | State::Failed => self.begin_dictation(), + // The preview holds the choices while words are finishing or waiting for review. + State::Finalizing | State::Review | State::Applying => {} + } + } + + pub(super) fn dictation_mic(&self) -> crate::ribbon::Mic { + match self.session.voice().state() { + State::Preparing | State::Recording => crate::ribbon::Mic::Live, + State::Finalizing | State::Review | State::Applying => crate::ribbon::Mic::Busy, + State::Idle | State::Complete | State::Cancelled | State::Failed => crate::ribbon::Mic::Ready, + } + } + + pub(super) fn open_dictation_setup(&mut self) { + self.voice.setup_open = true; + self.voice.setup_reason = None; + } + + /// The shortcut, Enter and Esc, taken before the page sees them: while words are being + /// spoken, Enter must not split the paragraph the words are going into. + fn dictation_keys(&mut self, ctx: &Context) { + let state = self.session.voice().state(); + if let Some(key) = Key::from_name(&self.voice.shortcut) { + if take_press(ctx, key) && (state.active() || !(self.voice.setup_open || self.dialog.is_some() || self.backstage.is_some())) { + let live = matches!(state, State::Preparing | State::Recording); + self.toggle_dictation(); + self.voice.pressed_at = if live { None } else { Some(Instant::now()) }; + } + // A hold ends when the key comes up, when its release was missed (the key is simply not + // down any more), or when the window loses focus with the key held: another window + // would get the release, and the microphone would stay open. + if let Some(at) = self.voice.pressed_at { + let (released, focused) = ctx.input(|i| (i.key_released(key) || !i.key_down(key), i.focused)); + if released || !focused { + self.voice.pressed_at = None; + if !focused || release_finishes(at.elapsed()) { + self.finish_dictation(); + } + } + } + } + if matches!(state, State::Preparing | State::Recording | State::Finalizing) { + if take_press(ctx, Key::Escape) { + self.cancel_voice(); + } else if state != State::Finalizing && take_press(ctx, Key::Enter) { + self.finish_dictation(); + } + } + } + + pub(super) fn poll_voice(&mut self, ctx: &Context) { + self.dictation_keys(ctx); + let events = self.voice.runtime.as_mut().map(Runtime::poll).unwrap_or_default(); + let mut finished = false; + for event in events { + let update = match event { + RuntimeEvent::Ready { session } => Some(VoiceUpdate::Ready(session)), + RuntimeEvent::Stopped { session } => Some(VoiceUpdate::Stopped(session)), + RuntimeEvent::Progress { session, seconds, level } => { + if session == self.session.voice().id() { + self.voice.seconds = seconds; + self.voice.level = level; + self.voice.loudest = self.voice.loudest.max(level); + } + None + } + RuntimeEvent::Transcript(t) => { + finished |= t.is_final; + Some(VoiceUpdate::Transcript(t)) + } + RuntimeEvent::Error { session, message } => { + finished = true; + self.voice.error = Some(message.clone()); + Some(VoiceUpdate::Failed(session, message)) + } + }; + if let Some(update) = update { + let _ = self.session.voice_update(update); + } + } + if finished { + self.voice.runtime.take(); + } + if self.voice.pending_stop && self.session.voice().state() == State::Recording { + self.voice.pending_stop = false; + self.finish_dictation(); + } + if self.voice.review_revision != self.session.voice().revision() && !self.voice.editing { + self.voice.review = self.session.voice().draft().into(); + self.voice.review_revision = self.session.voice().revision(); + } + self.auto_insert(); + self.poll_setup_checks(); + if self.voice.runtime.is_some() || self.voice.mic_test.is_some() || self.voice.recognizer_test.is_some() { + ctx.request_repaint_after(Duration::from_millis(30)); + } + } + + /// With review turned off, finished words go straight into the page; Undo takes them back. + fn auto_insert(&mut self) { + let voice = self.session.voice(); + // Only words from a finished final pass go in by themselves. Words left over from a recording + // that failed (a disconnected microphone, say) are an unfinished preview: they wait for review. + if self.voice.review_first + || self.voice.auto_tried + || voice.state() != State::Review + || voice.error().is_some() + || voice.draft().trim().is_empty() + { + return; + } + self.voice.auto_tried = true; + if self.dialog.is_some() + || self.backstage.is_some() + || self.voice.setup_open + || !self.voice.target.as_ref().is_some_and(|ticket| self.ticket_is_current(ticket)) + { + self.voice.error = Some(crate::tl_id!("voice.target_changed_review_required").into()); + return; + } + if let Err(e) = self.commit_voice() { + self.voice.error = Some(e); + } + } + + fn start_mic_test(&mut self) { + self.voice.mic_result = None; + self.voice.mic_test = None; + match Runtime::start(MIC_TEST, Box::new(Listen), self.voice.device.clone()) { + Ok(runtime) => self.voice.mic_test = Some(MicTest { runtime, started: Instant::now(), level: 0.0, loudest: 0.0, heard_audio: false }), + Err(e) => self.voice.mic_result = Some(Err(explain(&e).0)), + } + } + + fn start_recognizer_test(&mut self) { + self.voice.recognizer_result = None; + self.voice.recognizer_test = None; + let started = (|| -> Result { + let root = voice_cache_root(&self.session)?; + let mut config = WhisperConfig::new(self.voice.executable.clone().into(), self.voice.model.clone().into(), root); + config.language = self.voice.language.clone(); + // A second of silence: enough for the recognizer to load the model and answer. + let silence = PcmAudio::new(vec![0; craft_voice_runtime::audio::SAMPLE_RATE])?; + Runtime::from_audio(RECOGNIZER_TEST, Box::new(config), silence) + })(); + match started { + Ok(runtime) => self.voice.recognizer_test = Some(runtime), + Err(e) => self.voice.recognizer_result = Some(Err(explain(&e).0)), + } + } + + fn poll_setup_checks(&mut self) { + if let Some(test) = self.voice.mic_test.as_mut() { + let mut failed = None; + for event in test.runtime.poll() { + match event { + RuntimeEvent::Ready { .. } => test.heard_audio = true, + RuntimeEvent::Progress { level, .. } => { + test.heard_audio = true; + test.level = level; + test.loudest = test.loudest.max(level); + } + RuntimeEvent::Error { message, .. } => failed = Some(explain(&message).0), + _ => {} + } + } + if failed.is_some() || test.started.elapsed() >= MIC_TEST_FOR { + let result = match failed { + Some(why) => Err(why), + None if !test.heard_audio => Err(crate::tl_id!("voice.no_audio_arrived_from_the_microphone_check_the_sel_68498934").into()), + None if test.loudest < SILENT => Err(crate::tl_id!("voice.the_microphone_sends_audio_but_it_is_silent_check_d7cf0faa").into()), + None => Ok(crate::i18n::format_message_id( + "voice.heard_you_loudest_sound_decibels_db_ce75d644", + &[("decibels", &format!("{:.0}", 20.0 * test.loudest.log10()))], + )), + }; + self.voice.mic_result = Some(result); + self.voice.mic_test = None; + } + } + if let Some(runtime) = self.voice.recognizer_test.as_mut() { + let mut result = None; + for event in runtime.poll() { + match event { + RuntimeEvent::Transcript(t) if t.is_final => { + result = Some(Ok(crate::tl_id!("voice.the_recognizer_and_the_model_work_11e481ee").to_string())) + } + RuntimeEvent::Error { message, .. } => result = Some(Err(explain(&message).0)), + _ => {} + } + } + if let Some(result) = result { + self.voice.recognizer_result = Some(result); + self.voice.recognizer_test = None; + } + } + } + + fn save_voice_settings(&mut self) { + let review = if self.voice.review_first { "true" } else { "false" }; + for (key, value) in [ + ("voice_executable", self.voice.executable.as_str()), + ("voice_model", self.voice.model.as_str()), + ("voice_language", self.voice.language.as_str()), + ("voice_device", self.voice.device.as_deref().unwrap_or("")), + ("voice_shortcut", self.voice.shortcut.as_str()), + ("voice_review", review), + ] { + if let Err(error) = self.session.settings.set(key, value) { + self.voice.error = + Some(crate::i18n::format_message_id("voice.cannot_save_dictation_settings_error_fa8703c7", &[("error", &error.to_string())])); + break; + } + } + } + + fn submit_voice_edit(&mut self, action: &mut Option<&str>) { + let id = self.session.voice().id().to_string(); + let revision = self.session.voice().revision().to_string(); + let text = self.voice.review.clone(); + match self.voice_command("voice.draft.set", ¶ms(&[("session", &id), ("revision", &revision), ("text", &text)])) { + Ok(()) => self.voice.review_revision = self.session.voice().revision(), + Err(e) => { + self.voice.error = Some(e); + // Keep the rejected words visible. Approval of an older revision needs a + // later rendered review and a new explicit Insert action. + if *action == Some("commit") { + *action = None; + } + } + } + } + + fn act_on_fix(&mut self, fix: Fix) { + match fix { + Fix::MicSettings => { + if let Err(e) = open_microphone_settings() { + self.voice.error = Some(e); + } + } + Fix::Setup => self.open_dictation_setup(), + Fix::PageView => { + self.show_source = false; + self.voice.error = None; + } + Fix::None => {} + } + } +} + +// i18n-ui-text: label +fn primary(label: &str, t: &Tokens) -> egui::Button<'static> { + egui::Button::new(egui::RichText::new(label.to_string()).strong().color(t.band_text)).fill(t.band) +} + +/// A level meter that reads in decibels: a track with the level filled in. +fn level_bar(ui: &mut Ui, level: f32, t: &Tokens) { + let (rect, resp) = ui.allocate_exact_size(vec2(110.0, 8.0), Sense::hover()); + let painter = ui.painter_at(rect); + painter.rect_filled(rect, 4.0, t.faint); + let fill = meter(level); + if fill > 0.0 { + let lit = egui::Rect::from_min_size(rect.min, vec2(rect.width() * fill, rect.height())); + painter.rect_filled(lit, 4.0, if fill > 0.92 { t.warn } else { t.ok }); + } + resp.on_hover_text(crate::tl_id!("voice.microphone_level_7b9bf79a")); +} + +impl NoteCraftApp { + pub(super) fn show_voice(&mut self, ctx: &Context) { + self.dictation_bar(ctx); + self.dictation_preview(ctx); + if self.voice.setup_open { + self.dictation_setup(ctx); + } + } + + /// The bar along the bottom while dictation is listening or finishing, and for errors. + fn dictation_bar(&mut self, ctx: &Context) { + let state = self.session.voice().state(); + let busy = matches!(state, State::Preparing | State::Recording | State::Finalizing); + // An error during review is shown with the words, in the preview. + let error = if state == State::Review { None } else { self.voice.error.clone() }; + if !busy && error.is_none() { + return; + } + let t = self.tokens; + let mut action: Option<&str> = None; + let mut fix = Fix::None; + egui::Area::new(egui::Id::new("dictation-bar")).order(egui::Order::Foreground).anchor(Align2::CENTER_BOTTOM, vec2(0.0, -40.0)).show( + ctx, + |ui| { + egui::Frame::NONE + .fill(t.panel) + .stroke(egui::Stroke::new(1.0, if state == State::Recording { t.warn } else { t.accent })) + .corner_radius(20.0) + .inner_margin(egui::Margin::symmetric(16, 10)) + .shadow(egui::epaint::Shadow { offset: [0, 4], blur: 16, spread: 0, color: egui::Color32::from_black_alpha(60) }) + .show(ui, |ui| { + ui.horizontal(|ui| { + ui.spacing_mut().item_spacing.x = 10.0; + match state { + State::Preparing => { + ui.add(egui::Spinner::new().size(14.0)); + ui.label(egui::RichText::new(crate::tl_id!("voice.starting_the_microphone_26a22bd2")).color(t.text)); + } + State::Recording => { + let (dot, _) = ui.allocate_exact_size(vec2(12.0, 12.0), Sense::hover()); + let pulse = 0.55 + 0.45 * (ui.input(|i| i.time) as f32 * 4.0).sin().abs(); + ui.painter().circle_filled(dot.center(), 6.0, t.warn.gamma_multiply(pulse)); + ui.label(egui::RichText::new(crate::tl_id!("voice.listening_dc353487")).strong().color(t.strong)); + level_bar(ui, self.voice.level, &t); + ui.label(egui::RichText::new(clock(self.voice.seconds)).monospace().color(t.muted)); + } + State::Finalizing => { + ui.add(egui::Spinner::new().size(14.0)); + ui.label(egui::RichText::new(crate::tl_id!("voice.finishing_90218af7")).color(t.text)); + } + _ => { + if let Some(message) = &error { + let (words, f) = explain(message); + fix = f; + ui.add(egui::Label::new(egui::RichText::new(words).color(t.warn)).wrap()).on_hover_text(message); + } + } + } + if matches!(state, State::Preparing | State::Recording) + && ui + .add(primary(crate::tl_id!("voice.done_e9b450d1"), &t)) + .on_hover_text(crate::i18n::format_message_id( + "voice.enter_or_shortcut_25ef81f2", + &[("shortcut", &self.voice.shortcut)], + )) + .clicked() + { + action = Some("done"); + } + if busy && ui.button(crate::tl_id!("ui.cancel")).on_hover_text(crate::tl_id!("voice.esc_1f7a4f9e")).clicked() { + action = Some("cancel"); + } + if !busy { + let label = match fix { + Fix::MicSettings => Some(crate::tl_id!("voice.microphone_settings_2a04af24")), + Fix::Setup => Some(crate::tl_id!("voice.dictation_setup_b70a0aa8")), + Fix::PageView => Some(crate::tl_id!("voice.page_view_6a49efac")), + Fix::None => None, + }; + if let Some(label) = label + && ui.add(primary(label, &t)).clicked() + { + action = Some("fix"); + } + if ui.button(crate::tl_id!("voice.dismiss_70afe9ef")).clicked() { + action = Some("dismiss"); + } + } + }); + if state == State::Recording + && let Some(hint) = input_hint(self.voice.seconds, self.voice.loudest) + { + ui.label(egui::RichText::new(hint).size(12.0).color(t.warn)); + } + }); + }, + ); + if busy { + ctx.request_repaint(); + } + match action { + Some("done") => self.finish_dictation(), + Some("cancel") => self.cancel_voice(), + Some("fix") => self.act_on_fix(fix), + Some("dismiss") => self.voice.error = None, + _ => {} + } + } + + /// Where the preview sits: under the line the words go into, or under the end of the page. + fn preview_anchor(&self, ctx: &Context) -> Option<(egui::Pos2, f32)> { + let ticket = self.voice.target.as_ref()?; + let ed = self.editor.as_ref()?; + if ticket.epoch != self.voice.editor_epoch + || ticket.dir != ed.dir + || ticket.generation != ed.generation + || !self.cursor_matches_ticket(ticket) + { + return None; + } + let pass = ctx.cumulative_pass_nr(); + match self.voice.target.as_ref().map(|t| &t.anchor)? { + Anchor::Caret(caret) => { + let (at, column, drawn, block) = ctx.data(|d| d.get_temp::<(egui::Rect, egui::Rect, u64, usize)>(pageedit::caret_anchor_id()))?; + (drawn == pass && block == caret.block).then(|| (egui::pos2(column.left(), at.bottom() + 6.0), column.width())) + } + Anchor::End => { + let (below, drawn) = ctx.data(|d| d.get_temp::<(egui::Rect, u64)>(pageedit::page_end_anchor_id()))?; + (drawn == pass).then(|| (egui::pos2(below.left(), below.top() + 6.0), below.width())) + } + } + } + + /// The words as they come in, at the place they will go; then, for review, the same words + /// with Insert, Edit and Discard. + fn dictation_preview(&mut self, ctx: &Context) { + let state = self.session.voice().state(); + let draft = self.session.voice().draft().to_string(); + let showing = match state { + State::Recording | State::Finalizing => !draft.trim().is_empty(), + State::Review => true, + _ => false, + }; + if !showing { + return; + } + let t = self.tokens; + let at_end = matches!(self.voice.target.as_ref().map(|t| &t.anchor), Some(Anchor::End)); + let area = egui::Area::new(egui::Id::new("dictation-preview")).order(egui::Order::Foreground); + let (area, width) = match self.preview_anchor(ctx) { + Some((pos, width)) => (area.fixed_pos(pos), width.clamp(280.0, 620.0)), + None => (area.anchor(Align2::CENTER_BOTTOM, vec2(0.0, -100.0)), 520.0), + }; + let mut action: Option<&str> = None; + let mut edited = false; + let mut fix = Fix::None; + let elsewhere = self.target_elsewhere(); + area.show(ctx, |ui| { + let review = state == State::Review; + let fill = if review { t.panel.lerp_to_gamma(t.accent, 0.10) } else { t.panel }; + egui::Frame::NONE + .fill(fill) + .stroke(egui::Stroke::new(1.0, t.accent.gamma_multiply(if review { 1.0 } else { 0.5 }))) + .corner_radius(8.0) + .inner_margin(egui::Margin::same(12)) + .shadow(egui::epaint::Shadow { offset: [0, 3], blur: 12, spread: 0, color: egui::Color32::from_black_alpha(45) }) + .show(ui, |ui| { + ui.set_width(width); + let place = + if at_end { crate::tl_id!("voice.at_the_end_of_the_page_24302b0e") } else { crate::tl_id!("voice.at_your_cursor_e3b613bb") }; + if !review { + ui.label( + egui::RichText::new(crate::i18n::format_message_id("voice.preview_going_place_2eeaa180", &[("place", place)])) + .size(11.5) + .color(t.muted), + ); + ui.add(egui::Label::new(egui::RichText::new(&draft).italics().color(t.muted)).wrap()); + return; + } + if self.voice.review.trim().is_empty() && !self.voice.editing { + ui.label(egui::RichText::new(crate::tl_id!("voice.no_speech_was_recognized_7ff9a913")).strong().color(t.strong)); + ui.label(egui::RichText::new(crate::tl_id!("voice.try_again_a_little_closer_to_the_microphone_30f96ada")).color(t.muted)); + ui.horizontal(|ui| { + if ui.add(primary(crate::tl_id!("voice.try_again_042c862e"), &t)).clicked() { + action = Some("again"); + } + if ui.button(crate::tl_id!("voice.discard_36fff63c")).clicked() { + action = Some("discard"); + } + }); + return; + } + let caption = match &elsewhere { + Some(title) => crate::i18n::format_message_id("voice.these_words_were_for_the_page_title_c7a50f95", &[("title", title)]), + None => crate::i18n::format_message_id("voice.ready_to_insert_place_7f3f4d09", &[("place", place)]), + }; + ui.label(egui::RichText::new(caption).size(11.5).color(t.muted)); + if self.voice.editing { + let label = ui.label(egui::RichText::new(crate::tl_id!("voice.dictated_words_8c96e44a")).size(11.5).color(t.muted)).id; + edited = ui + .add( + egui::TextEdit::multiline(&mut self.voice.review) + .desired_rows(3) + .desired_width(f32::INFINITY) + .char_limit(craft_voice_core::MAX_TRANSCRIPT_BYTES), + ) + .labelled_by(label) + .changed(); + } else { + ui.add(egui::Label::new(egui::RichText::new(&self.voice.review).size(15.0).color(t.strong)).wrap()); + } + ui.add_space(4.0); + ui.horizontal(|ui| { + let (label, choice) = if elsewhere.is_some() { + (crate::tl_id!("voice.insert_on_this_page_f7f93d5b"), "move") + } else { + (crate::tl_id!("voice.insert_here_1fe34d33"), "commit") + }; + if ui.add_enabled(!self.voice.review.trim().is_empty(), primary(label, &t)).clicked() { + action = Some(choice); + } + let edit = if self.voice.editing { crate::tl_id!("voice.done_editing_b4938456") } else { crate::tl_id!("ui.edit") }; + if ui.button(edit).clicked() { + action = Some("edit"); + } + if ui.button(crate::tl_id!("voice.discard_36fff63c")).clicked() { + action = Some("discard"); + } + }); + if let Some(message) = self + .voice + .error + .clone() + .or_else(|| self.session.voice().error().map(str::to_string)) + .filter(|m| elsewhere.is_none() || !m.contains(crate::tl_id!("voice.insert_on_this_page_f7f93d5b"))) + { + let (words, f) = explain(&message); + fix = f; + ui.add(egui::Label::new(egui::RichText::new(words).size(12.0).color(t.warn)).wrap()); + if f == Fix::MicSettings && ui.button(crate::tl_id!("voice.microphone_settings_2a04af24")).clicked() { + action = Some("fix"); + } + } + }); + }); + if edited { + self.submit_voice_edit(&mut action); + } + match action { + Some("commit") => { + if let Err(e) = self.commit_voice() { + self.voice.error = Some(e); + } + } + Some("move") => { + if let Err(e) = self.commit_on_this_page() { + self.voice.error = Some(e); + } + } + Some("edit") => self.voice.editing = !self.voice.editing, + Some("discard") => { + self.cancel_voice(); + self.say(crate::tl_id!("voice.dictation_discarded_2159761f")); + } + Some("again") => { + self.cancel_voice(); + self.begin_dictation(); + } + Some("fix") => self.act_on_fix(fix), + _ => {} + } + } + + /// Choosing the recognizer, model, language and microphone, with a check for each half. + fn dictation_setup(&mut self, ctx: &Context) { + if !self.voice.devices_listed { + self.voice.devices_listed = true; + match craft_voice_runtime::microphone_devices() { + Ok(names) => self.voice.devices = names, + Err(error) => self.voice.mic_result = Some(Err(explain(&error).0)), + } + } + let t = self.tokens; + let mut open = true; + let mut changed = false; + let mut action: Option<&str> = None; + let busy = self.session.voice().state().active(); + egui::Window::new(crate::tl_id!("voice.dictation_setup_b70a0aa8")) + .open(&mut open) + .collapsible(false) + .resizable(false) + .default_width(480.0) + .frame(modal_frame(&t)) + .show(ctx, |ui| { + if let Some(reason) = self.voice.setup_reason { + ui.label(egui::RichText::new(reason).strong().color(t.accent)); + ui.add_space(4.0); + } + ui.label(crate::tl_id!("voice.dictation_runs_on_this_computer_with_whisper_cpp_c_fe751c46")); + ui.hyperlink_to(crate::tl_id!("voice.where_to_get_whisper_cli_and_a_model_3b6c0fcb"), WHISPER_HELP_URL); + ui.add_space(8.0); + ui.add_enabled_ui(!busy, |ui| { + let found = |ok: bool| { + if ok { + egui::RichText::new(crate::tl_id!("voice.found_bbba8413")).color(t.ok) + } else { + egui::RichText::new(crate::tl_id!("voice.not_found_475c8486")).color(t.warn) + } + }; + let program = ui.label(egui::RichText::new(crate::tl_id!("voice.recognizer_program_whisper_cli_eb53d9da")).strong()).id; + ui.horizontal(|ui| { + changed |= ui.add(egui::TextEdit::singleline(&mut self.voice.executable).desired_width(300.0)).labelled_by(program).changed(); + if ui.button(crate::tl_id!("voice.browse_c1b20f52")).clicked() + && let Some(path) = rfd::FileDialog::new().set_title(crate::tl_id!("voice.choose_whisper_cli_db09c1c2")).pick_file() + { + self.voice.executable = path.to_string_lossy().into_owned(); + changed = true; + } + ui.label(found(Path::new(&self.voice.executable).is_file())); + }); + let model = ui.label(egui::RichText::new(crate::tl_id!("voice.speech_model_3c5f0bfe")).strong()).id; + ui.horizontal(|ui| { + changed |= ui.add(egui::TextEdit::singleline(&mut self.voice.model).desired_width(300.0)).labelled_by(model).changed(); + if ui.button(crate::tl_id!("voice.browse_c1b20f52")).clicked() + && let Some(path) = rfd::FileDialog::new().set_title(crate::tl_id!("voice.choose_a_speech_model_f18c2ce5")).pick_file() + { + self.voice.model = path.to_string_lossy().into_owned(); + changed = true; + } + ui.label(found(Path::new(&self.voice.model).is_file())); + }); + ui.label( + egui::RichText::new(crate::tl_id!("voice.models_whose_name_ends_in_en_understand_english_on_41b266d0")) + .size(11.5) + .color(t.muted), + ); + ui.add_space(6.0); + egui::Grid::new("dictation-setup-choices").num_columns(2).spacing(vec2(12.0, 8.0)).show(ui, |ui| { + ui.label(egui::RichText::new(crate::tl_id!("voice.language_89b86ab0")).strong()); + let current = LANGUAGES + .iter() + .find(|(code, _)| *code == self.voice.language) + .map_or_else(|| self.voice.language.as_str(), |(_, name)| crate::i18n::t_id(name)); + egui::ComboBox::from_id_salt("dictation-language").selected_text(current).show_ui(ui, |ui| { + for (code, name) in LANGUAGES { + changed |= ui.selectable_value(&mut self.voice.language, code.to_string(), crate::i18n::t_id(name)).changed(); + } + }); + ui.end_row(); + ui.label(egui::RichText::new(crate::tl_id!("voice.microphone_24280537")).strong()); + ui.horizontal(|ui| { + egui::ComboBox::from_id_salt("dictation-device") + .selected_text(self.voice.device.as_deref().unwrap_or(crate::tl_id!("voice.system_default_9d8d3808"))) + .width(220.0) + .show_ui(ui, |ui| { + changed |= + ui.selectable_value(&mut self.voice.device, None, crate::tl_id!("voice.system_default_9d8d3808")).changed(); + for name in &self.voice.devices { + changed |= ui.selectable_value(&mut self.voice.device, Some(name.clone()), name).changed(); + } + }); + if ui.button(crate::tl_id!("voice.refresh_56e3badc")).clicked() { + match craft_voice_runtime::microphone_devices() { + Ok(names) => self.voice.devices = names, + Err(e) => self.voice.mic_result = Some(Err(explain(&e).0)), + } + } + }); + ui.end_row(); + ui.label(egui::RichText::new(crate::tl_id!("voice.shortcut_e2012dea")).strong()); + egui::ComboBox::from_id_salt("dictation-shortcut").selected_text(self.voice.shortcut.clone()).show_ui(ui, |ui| { + for key in ["F8", "F9", "F10"] { + changed |= ui.selectable_value(&mut self.voice.shortcut, key.into(), key).changed(); + } + }); + ui.end_row(); + }); + ui.label( + egui::RichText::new(crate::i18n::format_message_id( + "voice.tap_shortcut_to_start_and_again_to_finish_or_hold_6f592a2d", + &[("shortcut", &self.voice.shortcut)], + )) + .size(11.5) + .color(t.muted), + ); + changed |= ui + .checkbox(&mut self.voice.review_first, crate::tl_id!("voice.show_the_words_for_review_before_inserting_them_d313939a")) + .changed(); + }); + ui.add_space(6.0); + ui.separator(); + ui.label(egui::RichText::new(crate::tl_id!("voice.check_your_setup_b61b421a")).strong()); + ui.horizontal(|ui| { + let testing = self.voice.mic_test.is_some(); + if ui.add_enabled(!busy && !testing, egui::Button::new(crate::tl_id!("voice.test_microphone_49e76fed"))).clicked() { + action = Some("mic"); + } + if let Some(test) = &self.voice.mic_test { + ui.label(crate::tl_id!("voice.say_something_9aa1505d")); + level_bar(ui, test.level, &t); + } else if let Some(result) = &self.voice.mic_result { + match result { + Ok(words) => ui.label(egui::RichText::new(words).color(t.ok)), + Err(words) => ui.add(egui::Label::new(egui::RichText::new(words).color(t.warn)).wrap()), + }; + } + }); + ui.horizontal(|ui| { + let testing = self.voice.recognizer_test.is_some(); + if ui + .add_enabled(!busy && !testing && self.dictation_ready(), egui::Button::new(crate::tl_id!("voice.test_recognizer_f042c45f"))) + .clicked() + { + action = Some("recognizer"); + } + if testing { + ui.add(egui::Spinner::new().size(14.0)); + ui.label(crate::tl_id!("voice.loading_the_model_45a4f448")); + } else if let Some(result) = &self.voice.recognizer_result { + match result { + Ok(words) => ui.label(egui::RichText::new(words).color(t.ok)), + Err(words) => ui.add(egui::Label::new(egui::RichText::new(words).color(t.warn)).wrap()), + }; + } + }); + if MIC_SETTINGS + && ui + .link(crate::i18n::format_message_id("voice.open_system_microphone_settings_11e3ebb5", &[("system", system_name())])) + .clicked() + { + action = Some("settings"); + } + ui.add_space(8.0); + ui.horizontal(|ui| { + if ui.add_enabled(self.dictation_ready(), primary(crate::tl_id!("voice.done_e9b450d1"), &t)).clicked() { + action = Some("close"); + } + if !self.dictation_ready() { + ui.label(egui::RichText::new(crate::tl_id!("voice.choose_a_program_and_a_model_to_finish_82dc257d")).color(t.muted)); + } + }); + }); + if changed { + self.save_voice_settings(); + self.voice.recognizer_result = None; + } + match action { + Some("mic") => self.start_mic_test(), + Some("recognizer") => self.start_recognizer_test(), + Some("settings") => self.act_on_fix(Fix::MicSettings), + Some("close") => open = false, + _ => {} + } + if !open { + self.voice.setup_open = false; + self.voice.setup_reason = None; + self.voice.mic_test = None; + self.voice.recognizer_test = None; + } + } +} + +#[cfg(test)] +mod tests { + use super::super::*; + use super::{InvocationOrigin, State, VoiceUpdate}; + use notecraft_store::{Settings, testing::TempDir}; + + fn app(format: PageFormat) -> (TempDir, NoteCraftApp) { + let t = TempDir::new("voice-insertion"); + let settings = Settings::at(t.path().join("settings.toml")); + settings.set("privacy_welcome_seen", "true").expect("privacy preference"); + settings.set("pressure_unavailable_never", "true").expect("test preference"); + let mut session = Session::new(&t.path().join("notebooks"), settings).expect("session"); + session.run("notebook.create", ¶ms(&[("name", "Voice"), ("format", format.id())])).expect("notebook"); + session + .run( + "page.save", + ¶ms(&[ + ("path", "voice/new-section-1/untitled-page"), + ( + "body", + if format == PageFormat::Html { "

Keep bold words

\n

Next

\n" } else { "Keep **bold** words\n\nNext\n" }, + ), + ]), + ) + .expect("page"); + let mut app = NoteCraftApp::new(session); + let path = t.path().join("notebooks/voice/new-section-1/untitled-page"); + app.open_page(&path); + let ed = app.editor.as_mut().expect("editor"); + ed.caret = Some(pageedit::Caret { block: 0, text: pageedit::caret_text(&ed.blocks[0]), selection: (0, 0), ..Default::default() }); + (t, app) + } + + #[test] + fn both_formats_insert_literal_unicode_and_support_one_undo() { + for format in PageFormat::ALL { + let (_t, mut app) = app(format); + let before = app.editor.as_ref().expect("editor").blocks.clone(); + let target = app.capture_voice_target().expect("target"); + app.apply_voice_text(&target, "Hola, mundo. مرحبا ").expect("insert"); + let after = app.editor.as_ref().expect("editor").blocks.clone(); + assert_eq!(after[1], before[1]); + let notecraft_formats::Block::Paragraph(spans) = &after[0] else { panic!("paragraph") }; + assert!(spans.iter().any(|s| s.text.contains("Hola, mundo. مرحبا"))); + assert!(spans.iter().any(|s| s.text == "bold" && s.bold)); + app.save_now(); + let dir = app.editor.as_ref().expect("editor").dir.clone(); + let page = notecraft_store::page::load_page(&dir).expect("saved"); + let (persisted, _) = notecraft_formats::boxes::read(format, &page.body); + assert_eq!(persisted, after); + app.step_back(true); + assert_eq!(app.editor.as_ref().expect("editor").blocks, before); + app.step_back(false); + assert_eq!(app.editor.as_ref().expect("editor").blocks, after); + app.save_now(); + app.capture_voice_target().expect("a second recording can target the saved page"); + } + } + + #[test] + fn dictated_words_get_a_space_next_to_existing_words() { + assert_eq!(super::spaced("Keep **bold** words", (4, 4), "hello there"), " hello there"); + assert_eq!(super::spaced("Keep **bold** words", (0, 0), "Hello."), "Hello. "); + assert_eq!(super::spaced("Keep **bold** words", (11, 11), "er"), " er"); + assert_eq!(super::spaced("Keep **bold** words", (11, 11), "*"), "*"); + assert_eq!(super::spaced("end (", (5, 5), "aside"), "aside"); + assert_eq!(super::spaced("", (0, 0), "first words"), "first words"); + for format in PageFormat::ALL { + let (_t, mut app) = app(format); + app.editor.as_mut().expect("editor").caret.as_mut().expect("caret").selection = (4, 4); + let target = app.capture_voice_target().expect("target"); + app.apply_voice_text(&target, "hello there").expect("insert"); + let line = pageedit::caret_text(&app.editor.as_ref().expect("editor").blocks[0]); + assert_eq!(line, "Keep hello there **bold** words"); + } + } + + #[test] + fn the_caret_lands_after_inserted_punctuation_not_inside_the_next_word() { + for format in PageFormat::ALL { + for (words, before_caret) in [("$5", "prefix $5 "), ("*", "prefix * "), ("(a)", "prefix (a) ")] { + let (_t, mut app) = app(format); + let ed = app.editor.as_mut().expect("editor"); + ed.blocks = + vec![notecraft_formats::Block::Paragraph(vec![notecraft_formats::Span { text: "prefix suffix".into(), ..Default::default() }])]; + ed.rebuild_body(); + ed.caret = Some(pageedit::Caret { block: 0, text: pageedit::caret_text(&ed.blocks[0]), selection: (7, 7), ..Default::default() }); + NoteCraftApp::mark_dirty_from_blocks(ed); + app.save_now(); + let target = app.capture_voice_target().expect("target"); + app.apply_voice_text(&target, words).expect("insert"); + let ed = app.editor.as_ref().expect("editor"); + let caret = ed.caret.as_ref().expect("caret"); + let drawn: String = match &ed.blocks[0] { + notecraft_formats::Block::Paragraph(spans) => spans.iter().map(|s| s.text.as_str()).collect(), + other => panic!("paragraph expected, got {other:?}"), + }; + assert_eq!(drawn, format!("{before_caret}suffix"), "{format:?}: the words read as typed"); + let written: String = caret.text.chars().take(caret.selection.0).collect(); + let rest: String = caret.text.chars().skip(caret.selection.0).collect(); + assert!(rest == "suffix", "{format:?} {words:?}: caret sits before {rest:?} in {:?} (after {written:?})", caret.text); + } + } + } + + #[test] + fn the_panel_warns_when_the_microphone_hears_nothing() { + assert_eq!(super::input_hint(1.0, 0.0), None, "too early to tell"); + assert!(super::input_hint(2.5, 0.0).is_some(), "two seconds of digital silence"); + assert_eq!(super::input_hint(2.5, 0.2), None, "speech was heard"); + assert_eq!(super::meter(0.0), 0.0); + assert_eq!(super::meter(1.0), 1.0); + assert!((super::meter(0.01) - 1.0 / 3.0).abs() < 1e-4, "-40 dBFS fills a third"); + } + + #[test] + fn changed_or_reopened_pages_reject_the_original_target() { + let (_t, mut app) = app(PageFormat::Cmgfm); + let target = app.capture_voice_target().expect("target"); + let dir = app.editor.as_ref().expect("editor").dir.clone(); + app.open_page(&dir); + assert!(app.apply_voice_text(&target, "wrong place").is_err()); + } + + #[test] + fn external_changes_reject_insertion_without_overwriting_the_file() { + let (_t, mut app) = app(PageFormat::Cmgfm); + let target = app.capture_voice_target().expect("target"); + let file = app.editor.as_ref().expect("editor").dir.join("index.md"); + std::fs::write(&file, "external words").expect("external edit"); + assert!(app.apply_voice_text(&target, "do not write").is_err()); + assert_eq!(std::fs::read_to_string(file).expect("read"), "external words"); + assert!(app.capture_voice_target().is_err(), "retargeting must not authorize writing a stale editor over an external change"); + } + + #[test] + fn insertion_cannot_approve_words_different_from_the_visible_draft() { + let (_t, mut app) = app(PageFormat::Cmgfm); + let ticket = app.capture_voice_target().expect("target"); + app.session.run_with_origin("voice.start", ¶ms(&[("target", &ticket.id)]), InvocationOrigin::LocalUi).expect("session"); + app.session.voice_update(VoiceUpdate::Ready(1)).expect("ready"); + app.session.voice_update(VoiceUpdate::Stopped(1)).expect("stop"); + app.session.voice_update(VoiceUpdate::Transcript(craft_voice_core::Transcript::new(1, 1, "accepted draft", true))).expect("draft"); + app.voice.target = Some(ticket); + app.voice.review = "different unaccepted draft".into(); + app.voice.review_revision = 1; + assert!(app.commit_voice().is_err()); + assert_eq!(app.session.voice().state(), State::Review); + app.voice.review = "accepted draft".into(); + assert!(app.stop_voice().is_ok(), "releasing a hold after finalization is harmless"); + app.commit_voice().expect("insert approved words"); + let once = app.editor.as_ref().expect("editor").body.clone(); + assert!(app.commit_voice().is_err()); + assert_eq!(app.editor.as_ref().expect("editor").body, once); + } + + #[cfg(unix)] + #[test] + fn linked_cache_cannot_redirect_audio_into_a_notebook() { + let (t, mut app) = app(PageFormat::Cmgfm); + let inside = t.path().join("notebooks/voice/audio-cache"); + std::fs::create_dir(&inside).expect("cache target"); + std::os::unix::fs::symlink(&inside, t.path().join("voice-cache")).expect("cache link"); + assert!(app.start_voice().is_err(), "linked cache must fail before starting a runtime"); + assert_eq!(app.session.voice().state(), State::Failed); + assert_eq!(std::fs::read_dir(&inside).expect("cache").count(), 0); + } + + /// Words that look like markup either go in exactly as said, or are refused with the + /// draft, the page and the file untouched. Which one depends on what the page format's + /// serializer can keep literal, so both outcomes are checked rather than assumed. + #[test] + fn literal_punctuation_is_kept_exactly_or_refused_with_the_draft_and_page_unchanged() { + for format in PageFormat::ALL { + for words in ["*literal*", "[a](b)", r"C:\notes\page", "#important buy milk", "----"] { + let (_t, mut app) = app(format); + app.editor.as_mut().expect("editor").caret = None; + let ticket = app.capture_voice_target().expect("target"); + let before = app.editor.as_ref().expect("editor").body.clone(); + let disk_before = std::fs::read(super::page_file(app.editor.as_ref().expect("editor"))).expect("original page"); + app.session.run_with_origin("voice.start", ¶ms(&[("target", &ticket.id)]), InvocationOrigin::LocalUi).expect("start"); + app.session.voice_update(VoiceUpdate::Ready(1)).expect("ready"); + app.session.voice_update(VoiceUpdate::Stopped(1)).expect("stop"); + app.session.voice_update(VoiceUpdate::Transcript(craft_voice_core::Transcript::new(1, 1, words, true))).expect("draft"); + app.voice.target = Some(ticket); + app.voice.review = words.into(); + app.voice.review_revision = 1; + if app.commit_voice().is_err() { + assert_eq!(app.session.voice().state(), State::Review); + assert_eq!(app.session.voice().draft(), words); + assert_eq!(app.voice.review, words); + assert_eq!(app.editor.as_ref().expect("editor").body, before); + assert_eq!(std::fs::read(super::page_file(app.editor.as_ref().expect("editor"))).expect("page"), disk_before); + } else { + app.save_now(); + let ed = app.editor.as_ref().expect("editor"); + let last = ed.blocks.last().expect("inserted paragraph"); + assert_eq!(pageedit::block_text(last), words, "{format:?} {words:?} must go in exactly as said"); + let page = notecraft_store::page::load_page(&ed.dir).expect("saved"); + let (persisted, _) = notecraft_formats::boxes::read(format, &page.body); + assert_eq!(persisted, ed.blocks, "{format:?} {words:?} must reopen as inserted"); + } + } + } + } + + #[test] + fn rejected_edit_suppresses_pending_insert_and_keeps_rejected_words() { + let (_t, mut app) = app(PageFormat::Cmgfm); + let ticket = app.capture_voice_target().expect("target"); + app.session.run_with_origin("voice.start", ¶ms(&[("target", &ticket.id)]), InvocationOrigin::LocalUi).expect("start"); + app.session.voice_update(VoiceUpdate::Ready(1)).expect("ready"); + app.session.voice_update(VoiceUpdate::Stopped(1)).expect("stop"); + app.session.voice_update(VoiceUpdate::Transcript(craft_voice_core::Transcript::new(1, 1, "old accepted words", true))).expect("draft"); + app.voice.target = Some(ticket); + app.voice.review = "é".repeat(craft_voice_core::MAX_TRANSCRIPT_BYTES / 2 + 1); + let rejected = app.voice.review.clone(); + app.voice.review_revision = 1; + let before = app.editor.as_ref().expect("editor").body.clone(); + let mut action = Some("commit"); + app.submit_voice_edit(&mut action); + assert_eq!(action, None, "a rejected edit cannot approve the prior draft in this frame"); + assert_eq!(app.voice.review, rejected); + assert_eq!(app.session.voice().state(), State::Review); + assert_eq!(app.editor.as_ref().expect("editor").body, before); + assert!(app.commit_voice().is_err()); + } + + #[test] + fn a_caret_in_code_or_maths_sends_the_words_to_the_end_of_the_page() { + for format in PageFormat::ALL { + for code in [true, false] { + let (_t, mut app) = app(format); + let ed = app.editor.as_mut().expect("editor"); + ed.blocks = vec![notecraft_formats::Block::Paragraph(vec![ + notecraft_formats::Span { text: "before ".into(), ..Default::default() }, + notecraft_formats::Span { text: "abc".into(), code, math: !code, ..Default::default() }, + notecraft_formats::Span { text: " after".into(), ..Default::default() }, + ])]; + ed.rebuild_body(); + ed.caret = Some(pageedit::Caret { block: 0, text: pageedit::caret_text(&ed.blocks[0]), selection: (10, 10), ..Default::default() }); + NoteCraftApp::mark_dirty_from_blocks(ed); + app.save_now(); + let line = app.editor.as_ref().expect("editor").blocks[0].clone(); + let target = app.capture_voice_target().expect("the end of the page takes the words"); + assert!(matches!(target.anchor, super::Anchor::End)); + app.apply_voice_text(&target, "literal words.").expect("insert"); + let ed = app.editor.as_ref().expect("editor"); + assert_eq!(ed.blocks[0], line, "{format:?}: the code or maths line is untouched"); + assert_eq!(ed.blocks.len(), 2); + assert_eq!(pageedit::block_text(&ed.blocks[1]), "literal words."); + } + } + } + + #[test] + fn rejected_insertion_does_not_add_a_history_step() { + // " " is always refused. The others are refused only where the page format cannot + // keep them literal; when one goes in instead, it must be exactly one Undo step. + for words in [" ", "*literal*", "[a](b)", r"C:\notes\page", "----"] { + let (_t, mut app) = app(PageFormat::Cmgfm); + let target = app.capture_voice_target().expect("target"); + let before = app.editor.as_ref().expect("editor").blocks.clone(); + let refused = app.apply_voice_text(&target, words).is_err(); + assert!(refused || !words.trim().is_empty(), "blank words are always refused"); + app.step_back(true); + assert_eq!(app.editor.as_ref().expect("editor").blocks, before, "{words:?}, refused: {refused}"); + } + } + + #[cfg(unix)] + #[test] + fn linked_cache_in_an_externally_opened_notebook_is_rejected() { + let (t, mut app) = app(PageFormat::Cmgfm); + let mut other = Session::new(&t.path().join("external-root"), app.session.settings.clone()).expect("other session"); + other.run("notebook.create", ¶ms(&[("name", "Elsewhere")])).expect("notebook"); + let notebook = t.path().join("external-root/elsewhere"); + app.session.open_notebook(¬ebook).expect("open elsewhere"); + let inside = notebook.join("voice-cache"); + std::fs::create_dir(&inside).expect("cache target"); + std::os::unix::fs::symlink(&inside, t.path().join("voice-cache")).expect("link"); + assert!(app.start_voice().is_err()); + assert!(std::fs::read_dir(inside).expect("cache").next().is_none()); + } + + #[test] + fn setup_errors_do_not_leave_a_session_permanently_preparing() { + let (t, mut app) = app(PageFormat::Cmgfm); + app.session.settings = Settings::at(t.path().join("absent/settings.toml")); + assert!(app.start_voice().is_err()); + assert_eq!(app.session.voice().state(), State::Failed); + } + + #[test] + fn single_block_selection_preserves_boxes_and_rejects_later_edits() { + for format in PageFormat::ALL { + let (_t, mut app) = app(format); + let ed = app.editor.as_mut().expect("editor"); + ed.caret.as_mut().expect("caret").selection = (0, 4); + ed.arranged = true; + ed.boxes = vec![notecraft_model::Container::new(14.0, 22.0, 420.0, 2)]; + ed.rebuild_body(); + NoteCraftApp::mark_dirty_from_blocks(ed); + let target = app.capture_voice_target().expect("target"); + let before_boxes = app.editor.as_ref().expect("editor").boxes.clone(); + app.apply_voice_text(&target, "Replace").expect("replace selection"); + let ed = app.editor.as_mut().expect("editor"); + assert!(pageedit::caret_text(&ed.blocks[0]).starts_with("Replace **bold** words")); + assert_eq!(ed.boxes, before_boxes); + let after = ed.body.clone(); + assert!(app.apply_voice_text(&target, "again").is_err()); + assert_eq!(app.editor.as_ref().expect("editor").body, after); + } + } + + fn harness(app: NoteCraftApp, mode: Mode) -> egui_kittest::Harness<'static, NoteCraftApp> { + let mut app = app; + app.set_theme(Theme { mode, accent: Accent::Default }); + // The page's own editor has no accessible name yet, which is why the app's UI tests turn + // the harness check off; the dictation fields are checked by name below instead. + egui_kittest::Harness::builder() + .with_size(vec2(1200.0, 800.0)) + .with_accessibility_check(false) + .wgpu() + .build_ui_state(|ui, app: &mut NoteCraftApp| app.show(ui), app) + } + + fn review_picture(h: &mut egui_kittest::Harness<'static, NoteCraftApp>, name: &str) { + let picture = h.render().expect("render"); + if let Some(dir) = std::env::var_os("NOTECRAFT_UI_VISUAL_DIR") { + let dir = PathBuf::from(dir); + std::fs::create_dir_all(&dir).expect("visual folder"); + picture.save(dir.join(format!("{name}.png"))).expect("review image"); + } + } + + #[test] + fn the_mic_button_opens_setup_until_a_recognizer_is_chosen() { + use egui_kittest::kittest::Queryable; + for (name, mode) in [("light", Mode::Light), ("dark", Mode::Dark)] { + let (_t, app) = app(PageFormat::Cmgfm); + let mut h = harness(app, mode); + for _ in 0..4 { + h.step(); + } + h.get_by_label("Dictate").click(); + for _ in 0..4 { + h.step(); + } + assert!(h.state().voice.setup_open, "no recognizer yet: the button opens setup"); + assert_eq!(h.state().session.voice().state(), State::Idle, "and does not start recording"); + for field in ["Recognizer program (whisper-cli)", "Speech model"] { + let named = h.query_all_by_role_and_label(egui::accesskit::Role::TextInput, field).count(); + assert_eq!(named, 1, "the {field:?} field is named for screen readers"); + } + let _ = h.get_by_label("Test microphone"); + review_picture(&mut h, &format!("dictation-setup-{name}")); + } + } + + /// Replays transcript data through the real engine to inspect UI states. + /// This does not exercise microphone capture or speech recognition. + fn listening(app: &mut NoteCraftApp, words: &str) { + let target = app.capture_voice_target().expect("target"); + app.session.run_with_origin("voice.start", ¶ms(&[("target", &target.id)]), InvocationOrigin::LocalUi).expect("start"); + app.voice.target = Some(target); + app.session.voice_update(VoiceUpdate::Ready(1)).expect("ready"); + app.voice.seconds = 7.0; + app.voice.level = 0.3; + app.voice.loudest = 0.3; + if !words.is_empty() { + app.session.voice_update(VoiceUpdate::Transcript(craft_voice_core::Transcript::new(1, 1, words, false))).expect("preview"); + } + } + + #[test] + fn the_recording_bar_and_preview_show_at_the_insertion_point() { + use egui_kittest::kittest::Queryable; + for (name, mode) in [("light", Mode::Light), ("dark", Mode::Dark)] { + let (_t, app) = app(PageFormat::Cmgfm); + + let mut h = harness(app, mode); + for _ in 0..4 { + h.step(); + } + // Click into the first line, as a person would, once the page is up. + let ed = h.state_mut().editor.as_mut().expect("editor"); + ed.caret = + Some(pageedit::Caret { block: 0, text: pageedit::caret_text(&ed.blocks[0]), selection: (0, 0), focus: true, ..Default::default() }); + for _ in 0..2 { + h.step(); + } + listening(h.state_mut(), "Remember to call the venue about the date"); + for _ in 0..4 { + h.step(); + } + review_picture(&mut h, &format!("dictation-listening-{name}")); + let _ = h.get_by_label("Listening"); + let _ = h.get_by_label("Done"); + let _ = h.get_by_label("Finish dictating"); + let _ = h.get_by_label("Preview, going at your cursor"); + + h.state_mut().session.voice_update(VoiceUpdate::Stopped(1)).expect("stop"); + h.state_mut() + .session + .voice_update(VoiceUpdate::Transcript(craft_voice_core::Transcript::new(1, 2, "Remember to call the venue about the date.", true))) + .expect("final"); + for _ in 0..4 { + h.step(); + } + let _ = h.get_by_label("Ready to insert at your cursor"); + review_picture(&mut h, &format!("dictation-review-{name}")); + h.get_by_label("Insert here").click(); + for _ in 0..4 { + h.step(); + } + assert_eq!(h.state().session.voice().state(), State::Complete); + let body = h.state().editor.as_ref().expect("editor").body.clone(); + // The line took the keyboard with the caret at its end, so that is where the words go, + // with the space typing would have put before them. + assert!(body.contains("Keep **bold** words Remember to call the venue about the date.\n"), "inserted at the caret:\n{body}"); + } + } + + #[test] + fn enter_finishes_and_escape_cancels_without_touching_the_page() { + let (_t, app) = app(PageFormat::Cmgfm); + + let mut h = harness(app, Mode::Light); + h.step(); + listening(h.state_mut(), ""); + let before = h.state().editor.as_ref().expect("editor").body.clone(); + h.key_press(egui::Key::Enter); + h.step(); + assert_eq!(h.state().session.voice().state(), State::Finalizing, "Enter finishes"); + assert_eq!(h.state().editor.as_ref().expect("editor").body, before, "and does not split the paragraph"); + h.key_press(egui::Key::Escape); + h.step(); + assert_eq!(h.state().session.voice().state(), State::Cancelled, "Esc cancels"); + assert_eq!(h.state().editor.as_ref().expect("editor").body, before); + } + + #[test] + fn with_no_cursor_the_words_become_a_paragraph_at_the_end() { + for format in PageFormat::ALL { + for boxed in [false, true] { + let (_t, mut app) = app(format); + let ed = app.editor.as_mut().expect("editor"); + ed.caret = None; + if boxed { + ed.arranged = true; + ed.boxes = vec![notecraft_model::Container::new(10.0, 10.0, 300.0, 1), notecraft_model::Container::new(10.0, 200.0, 300.0, 1)]; + ed.rebuild_body(); + NoteCraftApp::mark_dirty_from_blocks(ed); + } + let before = app.editor.as_ref().expect("editor").blocks.clone(); + let target = app.capture_voice_target().expect("target"); + assert!(matches!(target.anchor, super::Anchor::End)); + app.apply_voice_text(&target, "A new thought.").expect("insert"); + let ed = app.editor.as_ref().expect("editor"); + assert_eq!(ed.blocks[..before.len()], before[..], "{format:?}: nothing before it changes"); + assert_eq!(pageedit::block_text(&ed.blocks[before.len()]), "A new thought."); + assert_eq!(ed.caret.as_ref().map(|c| c.block), Some(before.len()), "the caret follows the words"); + if boxed { + assert_eq!(ed.boxes.iter().map(|c| c.blocks).collect::>(), vec![1, 2], "{format:?}: the last box grows"); + } + app.save_now(); + let page = notecraft_store::page::load_page(&app.editor.as_ref().expect("editor").dir).expect("saved"); + let (blocks, boxes) = notecraft_formats::boxes::read(format, &page.body); + assert_eq!(pageedit::block_text(&blocks[before.len()]), "A new thought.", "{format:?} boxed={boxed}"); + assert_eq!(boxes.is_some(), boxed); + } + } + } + + #[test] + fn several_lines_become_several_paragraphs_and_one_undo_takes_them_back() { + for format in PageFormat::ALL { + let (_t, mut app) = app(format); + let before = app.editor.as_ref().expect("editor").blocks.clone(); + let caret = app.editor.as_mut().expect("editor").caret.as_mut().expect("caret"); + let end = caret.text.chars().count(); + caret.selection = (end, end); + let target = app.capture_voice_target().expect("target"); + app.apply_voice_text(&target, "First line\n\nSecond line\nThird line").expect("insert"); + let ed = app.editor.as_ref().expect("editor"); + assert!(pageedit::block_text(&ed.blocks[0]).ends_with(" First line"), "{format:?}: the first line follows the existing words"); + assert_eq!(pageedit::block_text(&ed.blocks[1]), "Second line"); + assert_eq!(pageedit::block_text(&ed.blocks[2]), "Third line"); + assert_eq!(ed.blocks[3], before[1], "the next paragraph moves down"); + assert_eq!(ed.caret.as_ref().map(|c| (c.block, c.selection.0)), Some((2, "Third line".len()))); + app.step_back(true); + assert_eq!(app.editor.as_ref().expect("editor").blocks, before); + } + } + + #[test] + fn a_page_changed_during_review_takes_the_words_at_the_cursor_now() { + let (_t, mut app) = app(PageFormat::Cmgfm); + + listening(&mut app, ""); + app.session.voice_update(VoiceUpdate::Stopped(1)).expect("stop"); + app.session.voice_update(VoiceUpdate::Transcript(craft_voice_core::Transcript::new(1, 1, "late words", true))).expect("final"); + app.poll_voice(&egui::Context::default()); + // Typed into the page while the words waited, then moved the cursor to the next line. + let ed = app.editor.as_mut().expect("editor"); + ed.caret = Some(pageedit::Caret { block: 0, text: "Typed meanwhile".into(), selection: (15, 15), ..Default::default() }); + pageedit::set_caret_text(&mut ed.blocks[0], "Typed meanwhile"); + ed.rebuild_body(); + NoteCraftApp::mark_dirty_from_blocks(ed); + app.save_now(); + let ed = app.editor.as_mut().expect("editor"); + ed.caret = Some(pageedit::Caret { block: 1, text: pageedit::caret_text(&ed.blocks[1]), selection: (4, 4), ..Default::default() }); + app.commit_voice().expect("inserted where the cursor is now"); + let ed = app.editor.as_ref().expect("editor"); + assert_eq!(pageedit::block_text(&ed.blocks[0]), "Typed meanwhile", "what was typed is kept"); + assert_eq!(pageedit::block_text(&ed.blocks[1]), "Next late words"); + assert_eq!(app.session.voice().state(), State::Complete); + } + + #[test] + fn with_review_off_finished_words_go_straight_in() { + let (_t, mut app) = app(PageFormat::Cmgfm); + + app.voice.review_first = false; + listening(&mut app, ""); + app.session.voice_update(VoiceUpdate::Stopped(1)).expect("stop"); + app.session.voice_update(VoiceUpdate::Transcript(craft_voice_core::Transcript::new(1, 1, "Straight in", true))).expect("final"); + app.poll_voice(&egui::Context::default()); + assert_eq!(app.session.voice().state(), State::Complete); + assert!(pageedit::block_text(&app.editor.as_ref().expect("editor").blocks[0]).starts_with("Straight in Keep")); + } + + /// Opens a second page in the same section and returns its folder. + fn open_second_page(t: &TempDir, app: &mut NoteCraftApp) -> PathBuf { + let created = app + .session + .run("node.create", ¶ms(&[("parent", "voice/new-section-1"), ("kind", "page"), ("title", "Second page"), ("format", "cmgfm")])) + .expect("second page"); + let Output::Path(path) = created else { panic!("a page path") }; + let path = if path.is_absolute() { path } else { t.path().join("notebooks").join(path) }; + let rel = path.strip_prefix(t.path().join("notebooks")).expect("in notebooks").to_string_lossy().replace('\\', "/"); + app.session.run("page.save", ¶ms(&[("path", &rel), ("body", "Other page words\n")])).expect("second page words"); + app.open_page(&path); + path + } + + #[test] + fn finished_words_never_land_on_another_page_without_asking() { + for review_first in [false, true] { + let (t, mut app) = app(PageFormat::Cmgfm); + + app.voice.review_first = review_first; + listening(&mut app, ""); + let first = app.editor.as_ref().expect("editor").dir.clone(); + let first_body = notecraft_store::page::load_page(&first).expect("first").body; + app.session.voice_update(VoiceUpdate::Stopped(1)).expect("stop"); + // The person opens another page before the final words arrive. + let second = open_second_page(&t, &mut app); + app.session + .voice_update(VoiceUpdate::Transcript(craft_voice_core::Transcript::new(1, 1, "meant for the first page", true))) + .expect("final"); + app.poll_voice(&egui::Context::default()); + if review_first { + assert!(app.commit_voice().is_err(), "Insert on another page asks first"); + } + assert_eq!(app.session.voice().state(), State::Review, "the words wait (review_first={review_first})"); + assert_eq!(notecraft_store::page::load_page(&second).expect("second").body.trim(), "Other page words"); + assert_eq!(notecraft_store::page::load_page(&first).expect("first").body, first_body); + assert!(app.target_elsewhere().is_some_and(|title| !title.is_empty()), "the preview names the page the words were for"); + app.commit_on_this_page().expect("an explicit choice moves them"); + app.save_now(); + assert!(notecraft_store::page::load_page(&second).expect("second").body.contains("meant for the first page")); + } + } + + #[test] + fn losing_focus_ends_a_held_recording() { + let (_t, app) = app(PageFormat::Cmgfm); + + let mut h = harness(app, Mode::Light); + h.step(); + listening(h.state_mut(), ""); + h.state_mut().voice.pressed_at = Some(Instant::now() - std::time::Duration::from_millis(100)); + h.input_mut().focused = false; + h.input_mut().events.push(egui::Event::WindowFocused(false)); + h.step(); + assert_eq!(h.state().session.voice().state(), State::Finalizing, "the window lost focus with the key held"); + } + + #[test] + fn a_released_key_ends_a_hold_even_when_the_release_event_was_missed() { + let (_t, app) = app(PageFormat::Cmgfm); + + let mut h = harness(app, Mode::Light); + h.step(); + listening(h.state_mut(), ""); + // Held for a second, and the key is up now, but no release event ever arrived. + h.state_mut().voice.pressed_at = Some(Instant::now() - std::time::Duration::from_secs(1)); + h.step(); + assert_eq!(h.state().session.voice().state(), State::Finalizing); + // A tap leaves recording on, as before. + let (_t, app) = self::app(PageFormat::Cmgfm); + + let mut h = harness(app, Mode::Light); + h.step(); + listening(h.state_mut(), ""); + h.state_mut().voice.pressed_at = Some(Instant::now() - std::time::Duration::from_millis(100)); + h.step(); + assert_eq!(h.state().session.voice().state(), State::Recording, "a tap toggles: recording goes on"); + } + + #[test] + fn a_failed_recording_is_never_inserted_on_its_own() { + let (_t, mut app) = app(PageFormat::Cmgfm); + + app.voice.review_first = false; + listening(&mut app, "half a sentence"); + let before = app.editor.as_ref().expect("editor").body.clone(); + app.voice.error = Some("microphone disconnected".into()); + app.session.voice_update(VoiceUpdate::Failed(1, "microphone disconnected".into())).expect("failed"); + app.poll_voice(&egui::Context::default()); + assert_eq!(app.session.voice().state(), State::Review, "the partial words stay for review"); + assert_eq!(app.session.voice().draft(), "half a sentence"); + assert_eq!(app.editor.as_ref().expect("editor").body, before, "nothing was inserted"); + } + + #[test] + fn caret_insertion_rejects_new_tags_without_invisible_characters() { + for format in PageFormat::ALL { + let (_t, mut app) = app(format); + let ticket = app.capture_voice_target().expect("target"); + let before = app.editor.as_ref().expect("editor").blocks.clone(); + assert!(app.apply_voice_text(&ticket, "#idea").is_err()); + assert_eq!(app.editor.as_ref().expect("editor").blocks, before); + } + } + + #[test] + fn errors_are_explained_with_something_to_do() { + use super::{Fix, explain}; + let (words, fix) = explain("The microphone opened but sent no audio for 5 seconds. Check …"); + assert!(words.contains("can't hear your microphone")); + assert_eq!(fix, if super::MIC_SETTINGS { Fix::MicSettings } else { Fix::Setup }); + assert_eq!(explain("choose an existing absolute path to whisper-cli").1, Fix::Setup); + assert_eq!(explain("choose an existing local Whisper model file").1, Fix::Setup); + assert_eq!(explain("recognizer timed out").1, Fix::Setup); + assert_eq!(explain("Return to the page view before dictating.").1, Fix::PageView); + assert_eq!(explain("something unforeseen"), ("something unforeseen".to_string(), Fix::None)); + } + + #[test] + fn successful_insertion_clears_an_earlier_ui_error() { + let (_t, mut app) = app(PageFormat::Cmgfm); + listening(&mut app, ""); + app.session.voice_update(VoiceUpdate::Stopped(1)).expect("stop"); + app.session.voice_update(VoiceUpdate::Transcript(craft_voice_core::Transcript::new(1, 1, "Reviewed words", true))).expect("final"); + app.poll_voice(&egui::Context::default()); + app.voice.error = Some("earlier insertion failure".into()); + app.commit_voice().expect("insert"); + assert!(app.voice.error.is_none(), "completed dictation must not show a stale error bar"); + } + + #[test] + fn setup_and_review_messages_use_catalogs_with_named_placeholders() { + for text in ["Dictation setup", "Dictated words", "Test microphone", "No speech was recognized."] { + assert_eq!(crate::i18n::translate_for("en-XA", text), format!("$$${text}@@@")); + assert_eq!(crate::i18n::pseudo_unmarked(&crate::i18n::translate_for("ar-XB", text)), text); + } + let message = crate::i18n::format_message_for("en-XA", "Ready to insert {place}", &[("place", "PLACE")]); + assert_eq!(message, "$$$Ready to insert PLACE@@@"); + for (_, id) in super::LANGUAGES { + assert!(crate::i18n::english_text(id).is_some(), "speech language label {id} has an English source"); + } + } + + #[test] + fn idle_shortcut_does_not_open_setup_behind_another_dialog() { + let (_t, mut app) = app(PageFormat::Cmgfm); + app.voice.setup_open = true; + let ctx = egui::Context::default(); + ctx.begin_pass(egui::RawInput { + events: vec![egui::Event::Key { key: Key::F8, physical_key: None, pressed: true, repeat: false, modifiers: egui::Modifiers::NONE }], + ..Default::default() + }); + app.dictation_keys(&ctx); + let mut output = ctx.end_pass(); + output.textures_delta.clear(); + assert_eq!(app.session.voice().state(), State::Idle); + assert!(app.voice.pressed_at.is_none()); + assert!(app.voice.runtime.is_none()); + } + + #[test] + fn multiline_insertion_with_trailing_content_requires_explicit_append() { + for format in PageFormat::ALL { + let (_t, mut app) = app(format); + let ed = app.editor.as_mut().expect("editor"); + pageedit::set_caret_text(&mut ed.blocks[0], "before **AFTER**"); + ed.caret = Some(pageedit::Caret { block: 0, text: pageedit::caret_text(&ed.blocks[0]), selection: (7, 7), ..Default::default() }); + ed.rebuild_body(); + NoteCraftApp::mark_dirty_from_blocks(ed); + let target = app.capture_voice_target().expect("target"); + let before = app.editor.as_ref().expect("editor").blocks.clone(); + assert!(app.apply_voice_text(&target, "one\ntwo").is_err(), "existing suffix must not appear between transcript lines"); + assert_eq!(app.editor.as_ref().expect("editor").blocks, before); + } + } + + #[test] + fn automatic_insertion_keeps_review_when_the_page_or_selection_changed() { + let (_t, mut app) = app(PageFormat::Cmgfm); + app.voice.review_first = false; + listening(&mut app, ""); + let ed = app.editor.as_mut().expect("editor"); + pageedit::set_caret_text(&mut ed.blocks[0], "Changed existing words"); + ed.caret = Some(pageedit::Caret { block: 0, text: pageedit::caret_text(&ed.blocks[0]), selection: (0, 7), ..Default::default() }); + ed.rebuild_body(); + NoteCraftApp::mark_dirty_from_blocks(ed); + let before = ed.body.clone(); + app.session.voice_update(VoiceUpdate::Stopped(1)).expect("stop"); + app.session.voice_update(VoiceUpdate::Transcript(craft_voice_core::Transcript::new(1, 1, "final transcript", true))).expect("final"); + app.poll_voice(&egui::Context::default()); + assert_eq!(app.session.voice().state(), State::Review, "changed targets need explicit Insert"); + assert_eq!(app.editor.as_ref().expect("editor").body, before, "new selections must not be silently replaced"); + assert_eq!(app.voice.review, "final transcript"); + } + + #[test] + fn moving_the_caret_cannot_move_the_preview_away_from_its_insertion_ticket() { + let (_t, mut app) = app(PageFormat::Cmgfm); + let ticket = app.capture_voice_target().expect("target"); + app.voice.target = Some(ticket); + app.editor.as_mut().expect("editor").caret.as_mut().expect("caret").selection = (4, 4); + let ctx = egui::Context::default(); + let spot = egui::Rect::from_min_size(egui::pos2(50.0, 50.0), vec2(1.0, 20.0)); + let line = egui::Rect::from_min_size(egui::pos2(40.0, 40.0), vec2(300.0, 40.0)); + let pass = ctx.cumulative_pass_nr(); + ctx.data_mut(|data| data.insert_temp(pageedit::caret_anchor_id(), (spot, line, pass, 0_usize))); + assert!(app.preview_anchor(&ctx).is_none(), "a moved caret is not the saved insertion point"); + } + + #[test] + fn multiline_insertion_cannot_reassign_indexed_layout_to_different_blocks() { + for format in PageFormat::ALL { + let (_t, mut app) = app(format); + let ed = app.editor.as_mut().expect("editor"); + ed.blocks.push(notecraft_formats::Block::Table { + head: vec![vec![notecraft_formats::Span { text: "Column".into(), ..Default::default() }]], + rows: vec![], + }); + ed.columns.set(2, 0, 244.0, &[160.0]); + ed.justified.set(1, Some(notecraft_model::justify::Strength::High)); + ed.rebuild_body(); + NoteCraftApp::mark_dirty_from_blocks(ed); + NoteCraftApp::remember_step(ed); + let dir = ed.dir.clone(); + notecraft_store::page::set_table_columns(&dir, &ed.columns).expect("save widths"); + notecraft_store::page::set_justified(&dir, &ed.justified).expect("save justification"); + app.save_now(); + let ed = app.editor.as_ref().expect("editor"); + let before = ed.body.clone(); + let columns = ed.columns.clone(); + let justified = ed.justified.clone(); + let caret = app.editor.as_mut().expect("editor").caret.as_mut().expect("caret"); + let end = caret.text.chars().count(); + caret.selection = (end, end); + let ticket = app.capture_voice_target().expect("target"); + assert!(app.apply_voice_text(&ticket, "First line\nSecond line").is_err(), "indexed layout must not migrate to different content"); + let ed = app.editor.as_ref().expect("editor"); + assert_eq!(ed.body, before); + assert_eq!(ed.columns, columns); + assert_eq!(ed.justified, justified); + assert_eq!(notecraft_store::page::table_columns(&dir).expect("widths"), columns); + assert_eq!(notecraft_store::page::justified(&dir).expect("justification"), justified); + app.editor.as_mut().expect("editor").caret = None; + let end = app.capture_voice_target().expect("append target"); + app.apply_voice_text(&end, "First appended line\nSecond appended line").expect("end append preserves all indexes"); + let after = app.editor.as_ref().expect("editor").body.clone(); + app.save_now(); + app.step_back(true); + assert_eq!(app.editor.as_ref().expect("editor").body, before); + app.step_back(false); + assert_eq!(app.editor.as_ref().expect("editor").body, after); + app.save_now(); + assert_eq!(notecraft_store::page::table_columns(&dir).expect("widths"), columns); + assert_eq!(notecraft_store::page::justified(&dir).expect("justification"), justified); + } + } + + #[test] + fn a_tap_toggles_and_a_hold_talks() { + assert!(!super::release_finishes(std::time::Duration::from_millis(150)), "a tap leaves recording on"); + assert!(super::release_finishes(std::time::Duration::from_millis(900)), "letting go after a hold finishes"); + assert_eq!(super::clock(0.4), "0:00"); + assert_eq!(super::clock(75.2), "1:15"); + } +} diff --git a/crates/ui-egui/src/app/voice.win.rs b/crates/ui-egui/src/app/voice.win.rs new file mode 100644 index 00000000..cc8f7d3f --- /dev/null +++ b/crates/ui-egui/src/app/voice.win.rs @@ -0,0 +1,5 @@ +//! Windows microphone privacy settings, opened only by the user's explicit action. + +pub(super) fn open_microphone_settings() -> std::io::Result<()> { + std::process::Command::new("explorer.exe").arg("ms-settings:privacy-microphone").spawn().map(|_| ()) +} diff --git a/crates/ui-egui/src/backstage.rs b/crates/ui-egui/src/backstage.rs index ad9e3ba8..c61f45b0 100644 --- a/crates/ui-egui/src/backstage.rs +++ b/crates/ui-egui/src/backstage.rs @@ -41,6 +41,9 @@ pub enum Act { ClearTelemetry(std::path::PathBuf), ExportTelemetry(std::path::PathBuf), ResetPreferences, + /// Open the dictation setup screen. + #[cfg(feature = "voice")] + DictationSetup, /// Another page of the File screen: Info, New or Open. Page(&'static str), /// A side entry that is listed but not built yet. @@ -476,6 +479,16 @@ fn options_page( }); ui.add_space(12.0); ui.label(egui::RichText::new(crate::tl_id!("ui.export_creates_a_user_selected_zip_then_removes_the_66f2bf3400")).size(11.5).color(t.muted)); + #[cfg(feature = "voice")] + { + ui.add_space(18.0); + ui.separator(); + ui.label(egui::RichText::new(crate::tl_id!("voice.dictation_2a7fe421")).strong().size(15.0).color(t.strong)); + ui.label(egui::RichText::new(crate::tl_id!("voice.the_speech_recognizer_model_language_microphone_an_3fece2c0")).size(11.5).color(t.muted)); + if open_action(ui, crate::tl_id!("voice.dictation_setup_6be1e51d"), 190.0, 34.0, t, true).clicked() { + acts.push(Act::DictationSetup); + } + } ui.add_space(18.0); ui.separator(); ui.label(egui::RichText::new(crate::tl_id!("ui.application_preferences")).strong().size(15.0).color(t.strong)); diff --git a/crates/ui-egui/src/history.rs b/crates/ui-egui/src/history.rs index d250e1c8..087c361a 100644 --- a/crates/ui-egui/src/history.rs +++ b/crates/ui-egui/src/history.rs @@ -160,6 +160,14 @@ impl History { pub fn here(&self) -> Option<&Step> { self.steps.get(self.at) } + + /// Keep the selection before an explicit transaction, without inventing an edit. + #[cfg(feature = "voice")] + pub fn remember_caret(&mut self, caret: Option) { + if let Some(step) = self.steps.get_mut(self.at) { + step.caret = caret; + } + } } #[cfg(test)] diff --git a/crates/ui-egui/src/pageedit.rs b/crates/ui-egui/src/pageedit.rs index 0f815c23..1afded9d 100644 --- a/crates/ui-egui/src/pageedit.rs +++ b/crates/ui-egui/src/pageedit.rs @@ -30,6 +30,26 @@ fn spans_of(block: &Block) -> Option<&[Span]> { use notecraft_model::Tag; +/// Where the caret was drawn this frame: its rectangle, line rectangle, pass and block. +#[cfg(feature = "voice")] +pub fn caret_anchor_id() -> egui::Id { + egui::Id::new("notecraft-caret-anchor") +} + +/// The room under the page and pass: where appended dictation will appear. +#[cfg(feature = "voice")] +pub fn page_end_anchor_id() -> egui::Id { + egui::Id::new("notecraft-page-end-anchor") +} + +#[cfg(feature = "voice")] +fn remember_caret_anchor(ui: &Ui, out: &egui::text_edit::TextEditOutput, caret: &Caret) { + let at = caret.selection.0.max(caret.selection.1).min(caret.text.chars().count()); + let spot = out.galley.pos_from_cursor(egui::text::CCursor::new(at)).translate(out.galley_pos.to_vec2()); + let pass = ui.ctx().cumulative_pass_nr(); + ui.ctx().data_mut(|d| d.insert_temp(caret_anchor_id(), (spot, out.response.rect, pass, caret.block))); +} + /// Where a selection that a right click collapsed is to be put back: `(block, primary, secondary)`. /// Set by the container, taken by the block's text box on the next frame. A text box moves its /// selection to the pointer the moment any button goes down, so a menu that acts on the words @@ -1532,6 +1552,11 @@ pub fn show( let mut changed = show_blocks(ui, blocks, caret, page_dir, t, page, all, borders, cols, justified, across, shown_title, 1.0); // Room under the page: a click there writes at the end, the way an empty page does. let (_rect, below) = ui.allocate_exact_size(vec2(ui.available_width(), 140.0_f32.min(ui.available_height().max(40.0))), Sense::click()); + #[cfg(feature = "voice")] + { + let pass = ui.ctx().cumulative_pass_nr(); + ui.ctx().data_mut(|d| d.insert_temp(page_end_anchor_id(), (_rect, pass))); + } if below.clicked() { apply(Action::ClickBelow, blocks, caret, &mut changed); } @@ -1818,6 +1843,8 @@ pub fn show_blocks( .show(ui) }) }; + #[cfg(feature = "voice")] + remember_caret_anchor(ui, &out, c); let mut cursor_range = out.cursor_range; let (left, right, shift) = ui.input(|input| { let mut left = false; diff --git a/crates/ui-egui/src/ribbon.rs b/crates/ui-egui/src/ribbon.rs index 5aae0e6e..6add0441 100644 --- a/crates/ui-egui/src/ribbon.rs +++ b/crates/ui-egui/src/ribbon.rs @@ -23,9 +23,45 @@ pub const FONT_SIZES: [u8; 16] = [8, 9, 10, 11, 12, 14, 16, 18, 20, 22, 24, 26, /// hand. It goes when the caret leaves, and the ribbon falls back to Home. pub const TABLE_TAB: &str = "Table"; +/// The dictation button in the tab strip, and how it should look. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Mic { + /// Built without dictation. + #[default] + Hidden, + /// Ready to start dictating. + Ready, + /// Words are finishing or waiting for review; the button waits too. + Busy, + /// Listening now: the button finishes. + Live, +} + +/// A microphone: a capsule on a stand. +fn paint_mic(painter: &egui::Painter, center: egui::Pos2, colour: egui::Color32) { + let capsule = Rect::from_center_size(center + vec2(0.0, -3.0), vec2(6.0, 10.0)); + painter.rect_filled(capsule, 3.0, colour); + let stroke = egui::Stroke::new(1.5, colour); + let cup: Vec = (0..=12) + .map(|k| { + let a = std::f32::consts::PI * k as f32 / 12.0; + center + vec2(5.5 * a.cos(), -1.0 + 5.5 * a.sin()) + }) + .collect(); + painter.add(egui::Shape::line(cup, stroke)); + painter.line_segment([center + vec2(0.0, 4.5), center + vec2(0.0, 7.5)], stroke); + painter.line_segment([center + vec2(-3.0, 7.5), center + vec2(3.0, 7.5)], stroke); +} + /// What a ribbon control asks the window to do. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Act { + /// Start dictating at the caret, or finish. + #[cfg(feature = "voice")] + Dictate, + /// Open the dictation setup screen. + #[cfg(feature = "voice")] + DictationSettings, /// The File tab: open the File screen. File, /// Show the saved versions of the open page. @@ -526,7 +562,7 @@ pub fn tabs(source: bool, in_table: bool) -> Vec<&'static str> { /// Office. The chosen tab is a raised tab in the ribbon's own colour that runs down into the /// ribbon below it; the others are white writing on the band, lit when the pointer is over /// one. -pub fn tab_strip(ui: &mut Ui, tab: &str, source: bool, in_table: bool, t: &Tokens, acts: &mut Vec) { +pub fn tab_strip(ui: &mut Ui, tab: &str, source: bool, in_table: bool, mic: Mic, t: &Tokens, acts: &mut Vec) { let chosen = resolve_tab(tab, source, in_table); ui.horizontal(|ui| { ui.spacing_mut().item_spacing.x = 0.0; @@ -555,6 +591,51 @@ pub fn tab_strip(ui: &mut Ui, tab: &str, source: bool, in_table: bool, t: &Token acts.push(if name == FILE_TAB { Act::File } else { Act::Tab(name.to_string()) }); } } + if mic != Mic::Hidden { + ui.add_space(12.0); + let live = mic == Mic::Live; + let label = if live { crate::tl_id!("voice.finish_dictating_0f7d9f4b") } else { crate::tl_id!("voice.dictate_3eaff69c") }; + let font = egui::FontId::proportional(12.5); + let galley = ui.painter().layout_no_wrap(label.to_string(), font.clone(), t.band_text); + let (rect, resp) = ui.allocate_exact_size(vec2(galley.size().x + 40.0, TAB_STRIP_H), Sense::click()); + let painter = ui.painter_at(rect); + let pill = rect.shrink2(vec2(2.0, 4.0)); + if live || (resp.hovered() && mic == Mic::Ready) { + painter.rect_filled(pill, 10.0, t.band_hover); + } + let ink = if mic == Mic::Busy { t.band_text.gamma_multiply(0.5) } else { t.band_text }; + paint_mic(&painter, egui::pos2(rect.min.x + 16.0, rect.center().y), ink); + painter.text(egui::pos2(rect.min.x + 28.0, rect.center().y + 1.0), Align2::LEFT_CENTER, label, font, ink); + let enabled = mic != Mic::Busy; + resp.widget_info(move || egui::WidgetInfo::labeled(egui::WidgetType::Button, enabled, label)); + let tip = match mic { + Mic::Live => crate::tl_id!("voice.finish_dictating_enter_esc_cancels_2151a686"), + Mic::Busy => crate::tl_id!("voice.insert_or_discard_the_dictated_words_first_df8983ce"), + _ => crate::tl_id!("voice.dictate_at_the_cursor_or_at_the_end_of_the_page_12635b46"), + }; + let resp = resp.on_hover_text(tip); + #[cfg(feature = "voice")] + { + if resp.clicked() && enabled { + acts.push(Act::Dictate); + } + let (gear, settings) = ui.allocate_exact_size(vec2(22.0, TAB_STRIP_H), Sense::click()); + let hover = settings.hovered(); + ui.painter().text( + gear.center(), + Align2::CENTER_CENTER, + "\u{22ef}", + egui::FontId::proportional(14.0), + if hover { t.band_text } else { t.band_text.gamma_multiply(0.7) }, + ); + settings.widget_info(|| egui::WidgetInfo::labeled(egui::WidgetType::Button, true, crate::tl_id!("voice.dictation_setup_b70a0aa8"))); + if settings.on_hover_text(crate::tl_id!("voice.dictation_setup_b70a0aa8")).clicked() { + acts.push(Act::DictationSettings); + } + } + #[cfg(not(feature = "voice"))] + let _ = (resp, acts.len()); + } }); } diff --git a/crates/voice-core/Cargo.toml b/crates/voice-core/Cargo.toml new file mode 100644 index 00000000..d2edde80 --- /dev/null +++ b/crates/voice-core/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "craft-voice-core" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true +description = "Provider-independent voice session and transcript contracts" + +[lints] +workspace = true diff --git a/crates/voice-core/src/lib.rs b/crates/voice-core/src/lib.rs new file mode 100644 index 00000000..6bb44ff5 --- /dev/null +++ b/crates/voice-core/src/lib.rs @@ -0,0 +1,264 @@ +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +//! Host-independent voice sessions. No UI, I/O, provider, or notebook dependency. + +use std::fmt; + +pub const MAX_TRANSCRIPT_BYTES: usize = 64 * 1024; +pub const MAX_SECONDS: usize = 60; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum State { + #[default] + Idle, + Preparing, + Recording, + Finalizing, + Review, + Applying, + Complete, + Cancelled, + Failed, +} + +impl State { + pub fn name(self) -> &'static str { + match self { + Self::Idle => "idle", + Self::Preparing => "preparing", + Self::Recording => "recording", + Self::Finalizing => "finalizing", + Self::Review => "review", + Self::Applying => "applying", + Self::Complete => "complete", + Self::Cancelled => "cancelled", + Self::Failed => "failed", + } + } + + pub fn active(self) -> bool { + matches!(self, Self::Preparing | Self::Recording | Self::Finalizing | Self::Applying) + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Capabilities { + pub batch_transcription: bool, + pub partial_transcripts: bool, + pub language_selection: bool, + pub speech_output: bool, + pub conversation: bool, +} + +#[derive(Clone, PartialEq, Eq)] +pub struct Transcript { + pub session: u64, + pub revision: u64, + pub text: String, + pub is_final: bool, +} + +impl Transcript { + pub fn new(session: u64, revision: u64, text: impl Into, is_final: bool) -> Self { + Self { session, revision, text: text.into(), is_final } + } +} + +impl fmt::Debug for Transcript { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Transcript") + .field("session", &self.session) + .field("revision", &self.revision) + .field("bytes", &self.text.len()) + .field("is_final", &self.is_final) + .finish() + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum VoiceError { + InvalidState, + StaleSession, + StaleRevision, + TooLong, + Empty, + InvalidTarget, + Exhausted, +} + +impl fmt::Display for VoiceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::InvalidState => "that action is not available in this voice state", + Self::StaleSession => "that voice session has ended", + Self::StaleRevision => "the transcript changed; review it again", + Self::TooLong => "the transcript exceeds 64 KiB", + Self::Empty => "there are no words to insert", + Self::InvalidTarget => "the host must supply a nonempty target ticket of at most 128 bytes", + Self::Exhausted => "voice sequence exhausted; reopen the app", + }) + } +} + +impl std::error::Error for VoiceError {} + +#[derive(Clone, Default)] +pub struct VoiceSession { + id: u64, + state: State, + revision: u64, + draft: String, + target: String, + error: Option, +} + +impl fmt::Debug for VoiceSession { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("VoiceSession") + .field("id", &self.id) + .field("state", &self.state) + .field("revision", &self.revision) + .field("draft_bytes", &self.draft.len()) + .finish() + } +} + +impl VoiceSession { + pub fn id(&self) -> u64 { + self.id + } + pub fn state(&self) -> State { + self.state + } + pub fn revision(&self) -> u64 { + self.revision + } + pub fn draft(&self) -> &str { + &self.draft + } + pub fn target(&self) -> &str { + &self.target + } + pub fn error(&self) -> Option<&str> { + self.error.as_deref() + } + + pub fn start(&mut self, target: &str) -> Result { + if self.state.active() || self.state == State::Review { + return Err(VoiceError::InvalidState); + } + validate_target(target)?; + self.id = self.id.checked_add(1).ok_or(VoiceError::Exhausted)?; + self.state = State::Preparing; + self.revision = 0; + self.draft.clear(); + self.target = target.into(); + self.error = None; + Ok(self.id) + } + + fn check(&self, id: u64, states: &[State]) -> Result<(), VoiceError> { + if id != self.id { + return Err(VoiceError::StaleSession); + } + if !states.contains(&self.state) { + return Err(VoiceError::InvalidState); + } + Ok(()) + } + + pub fn ready(&mut self, id: u64) -> Result<(), VoiceError> { + self.check(id, &[State::Preparing])?; + self.state = State::Recording; + Ok(()) + } + + pub fn stop(&mut self, id: u64) -> Result<(), VoiceError> { + self.check(id, &[State::Recording])?; + self.state = State::Finalizing; + Ok(()) + } + + pub fn accept(&mut self, event: Transcript) -> Result<(), VoiceError> { + let states: &[State] = if event.is_final { &[State::Finalizing] } else { &[State::Recording, State::Finalizing] }; + self.check(event.session, states)?; + if event.revision <= self.revision { + return Err(VoiceError::StaleRevision); + } + validate_text(&event.text)?; + self.revision = event.revision; + self.draft = event.text; + if event.is_final { + self.state = State::Review; + } + Ok(()) + } + + pub fn set_draft(&mut self, id: u64, expected: u64, text: &str) -> Result<(), VoiceError> { + self.check(id, &[State::Review])?; + if expected != self.revision { + return Err(VoiceError::StaleRevision); + } + validate_text(text)?; + self.revision = self.revision.checked_add(1).ok_or(VoiceError::Exhausted)?; + self.draft = text.into(); + self.error = None; + Ok(()) + } + + pub fn retarget(&mut self, id: u64, target: &str) -> Result<(), VoiceError> { + self.check(id, &[State::Review])?; + validate_target(target)?; + self.revision = self.revision.checked_add(1).ok_or(VoiceError::Exhausted)?; + self.target = target.into(); + self.error = None; + Ok(()) + } + + pub fn begin_apply(&mut self, id: u64, expected: u64) -> Result { + self.check(id, &[State::Review])?; + if expected != self.revision { + return Err(VoiceError::StaleRevision); + } + if self.draft.trim().is_empty() { + return Err(VoiceError::Empty); + } + self.state = State::Applying; + Ok(self.draft.clone()) + } + + pub fn applied(&mut self, id: u64, result: Result<(), String>) -> Result<(), VoiceError> { + self.check(id, &[State::Applying])?; + self.state = if result.is_ok() { State::Complete } else { State::Review }; + self.error = result.err(); + if self.state == State::Complete { + self.draft.clear(); + self.target.clear(); + } + Ok(()) + } + + pub fn cancel(&mut self, id: u64) -> Result<(), VoiceError> { + self.check(id, &[State::Preparing, State::Recording, State::Finalizing, State::Review, State::Failed])?; + self.state = State::Cancelled; + self.draft.clear(); + self.target.clear(); + self.error = None; + Ok(()) + } + + pub fn fail(&mut self, id: u64, message: &str) -> Result<(), VoiceError> { + self.check(id, &[State::Preparing, State::Recording, State::Finalizing])?; + self.state = if self.draft.trim().is_empty() { State::Failed } else { State::Review }; + self.error = Some(message.chars().take(1024).collect()); + Ok(()) + } +} + +fn validate_text(text: &str) -> Result<(), VoiceError> { + if text.len() > MAX_TRANSCRIPT_BYTES { Err(VoiceError::TooLong) } else { Ok(()) } +} + +fn validate_target(target: &str) -> Result<(), VoiceError> { + if target.is_empty() || target.len() > 128 { Err(VoiceError::InvalidTarget) } else { Ok(()) } +} diff --git a/crates/voice-core/tests/sessions.rs b/crates/voice-core/tests/sessions.rs new file mode 100644 index 00000000..551a6a62 --- /dev/null +++ b/crates/voice-core/tests/sessions.rs @@ -0,0 +1,88 @@ +use craft_voice_core::{MAX_TRANSCRIPT_BYTES, State, Transcript, VoiceError, VoiceSession}; + +fn recording() -> (VoiceSession, u64) { + let mut s = VoiceSession::default(); + let id = s.start("opaque-target").unwrap(); + s.ready(id).unwrap(); + (s, id) +} +fn review() -> (VoiceSession, u64) { + let (mut s, id) = recording(); + s.stop(id).unwrap(); + s.accept(Transcript::new(id, 1, "Hola 世界", true)).unwrap(); + (s, id) +} + +#[test] +fn partials_replace_and_final_requires_stop() { + let (mut s, id) = recording(); + s.accept(Transcript::new(id, 1, "Hello", false)).unwrap(); + s.accept(Transcript::new(id, 2, "Hola mundo", false)).unwrap(); + assert_eq!(s.draft(), "Hola mundo"); + assert_eq!(s.accept(Transcript::new(id, 1, "old", false)), Err(VoiceError::StaleRevision)); + assert_eq!(s.accept(Transcript::new(id, 3, "final", true)), Err(VoiceError::InvalidState)); + s.stop(id).unwrap(); + s.accept(Transcript::new(id, 3, "Hola mundo.", true)).unwrap(); + assert_eq!(s.state(), State::Review); + assert_eq!(s.draft(), "Hola mundo."); +} + +#[test] +fn edits_and_retarget_invalidate_previous_approval() { + let (mut s, id) = review(); + s.set_draft(id, 1, "reviewed text").unwrap(); + assert_eq!(s.begin_apply(id, 1), Err(VoiceError::StaleRevision)); + s.retarget(id, "other-position").unwrap(); + assert_eq!(s.begin_apply(id, 2), Err(VoiceError::StaleRevision)); + assert_eq!(s.begin_apply(id, 3).unwrap(), "reviewed text"); +} + +#[test] +fn apply_is_one_shot_and_target_failure_keeps_draft() { + let (mut s, id) = review(); + assert_eq!(s.begin_apply(id, 1).unwrap(), "Hola 世界"); + assert_eq!(s.begin_apply(id, 1), Err(VoiceError::InvalidState)); + s.applied(id, Err("target moved".into())).unwrap(); + assert_eq!(s.state(), State::Review); + assert_eq!(s.draft(), "Hola 世界"); + s.begin_apply(id, 1).unwrap(); + s.applied(id, Ok(())).unwrap(); + assert_eq!(s.state(), State::Complete); + assert!(s.draft().is_empty()); + assert_eq!(s.begin_apply(id, 1), Err(VoiceError::InvalidState)); +} + +#[test] +fn cancellation_rejects_late_results_and_new_session_has_new_id() { + let (mut s, id) = recording(); + s.accept(Transcript::new(id, 1, "private words", false)).unwrap(); + s.cancel(id).unwrap(); + assert!(s.draft().is_empty()); + let next = s.start("new-target").unwrap(); + assert!(next > id); + assert_eq!(s.accept(Transcript::new(id, 99, "late", false)), Err(VoiceError::StaleSession)); + assert_eq!(s.state(), State::Preparing); +} + +#[test] +fn oversized_transcript_is_rejected_without_changing_draft() { + let (mut s, id) = review(); + assert_eq!(s.set_draft(id, 1, &"x".repeat(MAX_TRANSCRIPT_BYTES + 1)), Err(VoiceError::TooLong)); + assert_eq!(s.draft(), "Hola 世界"); + assert_eq!(s.revision(), 1); + s.set_draft(id, 1, "").unwrap(); + assert_eq!(s.begin_apply(id, 2), Err(VoiceError::Empty)); +} + +#[test] +fn failure_retains_partial_and_debug_redacts_transcript() { + let (mut s, id) = recording(); + let t = Transcript::new(id, 1, "secret personal text", false); + assert!(!format!("{t:?}").contains("secret personal text")); + s.accept(t).unwrap(); + s.fail(id, "device disconnected").unwrap(); + assert_eq!(s.state(), State::Review); + assert_eq!(s.error(), Some("device disconnected")); + assert_eq!(s.draft(), "secret personal text"); + assert!(!format!("{s:?}").contains("secret personal text")); +} diff --git a/crates/voice-runtime/Cargo.toml b/crates/voice-runtime/Cargo.toml new file mode 100644 index 00000000..5a88ec4d --- /dev/null +++ b/crates/voice-runtime/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "craft-voice-runtime" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true +description = "Host-independent local voice capture and transcription runtime" + +[lints] +workspace = true + +[features] +default = [] +microphone = ["dep:cpal"] + +[dependencies] +craft-voice-core = { workspace = true } +serde_json = { workspace = true } +tempfile = "3" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +cpal = { version = "0.16", optional = true } diff --git a/crates/voice-runtime/examples/transcribe.rs b/crates/voice-runtime/examples/transcribe.rs new file mode 100644 index 00000000..7972e5f6 --- /dev/null +++ b/crates/voice-runtime/examples/transcribe.rs @@ -0,0 +1,32 @@ +//! A standalone library consumer. Supply your own whisper-cli, model and PCM16 WAV. +use craft_voice_runtime::{ + Runtime, RuntimeEvent, WhisperConfig, + audio::{MAX_SAMPLES, PcmAudio}, +}; +use std::{io::Read, path::PathBuf, time::Duration}; + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args().skip(1).collect(); + if args.len() != 4 { + return Err("usage: transcribe WHISPER_CLI MODEL WAV PRIVATE_TEMP_ROOT".into()); + } + let file = std::fs::File::open(&args[2])?; + let mut bytes = Vec::new(); + file.take((MAX_SAMPLES * 2 + 65537) as u64).read_to_end(&mut bytes)?; + let audio = PcmAudio::read_wav(&bytes)?; + let config = WhisperConfig::new(PathBuf::from(&args[0]), PathBuf::from(&args[1]), PathBuf::from(&args[3])); + let mut runtime = Runtime::from_audio(1, Box::new(config), audio)?; + loop { + for event in runtime.poll() { + match event { + RuntimeEvent::Transcript(t) if t.is_final => { + println!("{}", t.text); + return Ok(()); + } + RuntimeEvent::Error { message, .. } => return Err(message.into()), + _ => {} + } + } + std::thread::sleep(Duration::from_millis(20)); + } +} diff --git a/crates/voice-runtime/src/audio.rs b/crates/voice-runtime/src/audio.rs new file mode 100644 index 00000000..44a87681 --- /dev/null +++ b/crates/voice-runtime/src/audio.rs @@ -0,0 +1,138 @@ +//! Bounded speech audio and its portable PCM16 WAV representation. + +pub const SAMPLE_RATE: usize = 16000; +pub const MAX_SAMPLES: usize = SAMPLE_RATE * craft_voice_core::MAX_SECONDS; + +#[derive(Clone, Debug, Default)] +pub struct PcmAudio { + samples: Vec, +} + +impl PcmAudio { + pub fn new(samples: Vec) -> Result { + if samples.len() > MAX_SAMPLES { + return Err("recording exceeds 60 seconds".into()); + } + Ok(Self { samples }) + } + pub fn samples(&self) -> &[i16] { + &self.samples + } + pub fn seconds(&self) -> f32 { + self.samples.len() as f32 / SAMPLE_RATE as f32 + } + + pub fn wav(&self) -> Vec { + let bytes = (self.samples.len() * 2) as u32; + let mut out = Vec::with_capacity(44 + bytes as usize); + out.extend_from_slice(b"RIFF"); + out.extend_from_slice(&(36 + bytes).to_le_bytes()); + out.extend_from_slice(b"WAVEfmt "); + out.extend_from_slice(&16_u32.to_le_bytes()); + out.extend_from_slice(&1_u16.to_le_bytes()); + out.extend_from_slice(&1_u16.to_le_bytes()); + out.extend_from_slice(&(SAMPLE_RATE as u32).to_le_bytes()); + out.extend_from_slice(&(SAMPLE_RATE as u32 * 2).to_le_bytes()); + out.extend_from_slice(&2_u16.to_le_bytes()); + out.extend_from_slice(&16_u16.to_le_bytes()); + out.extend_from_slice(b"data"); + out.extend_from_slice(&bytes.to_le_bytes()); + for sample in &self.samples { + out.extend_from_slice(&sample.to_le_bytes()); + } + out + } + + /// Reads RIFF PCM16 mono 16 kHz, allowing extra RIFF chunks. + pub fn read_wav(bytes: &[u8]) -> Result { + if bytes.len() > MAX_SAMPLES * 2 + 65536 || bytes.get(..4) != Some(b"RIFF") || bytes.get(8..12) != Some(b"WAVE") { + return Err("expected a bounded PCM16 mono 16 kHz WAV file".into()); + } + let read_u32 = |at: usize| -> Option { Some(u32::from_le_bytes(bytes.get(at..at + 4)?.try_into().ok()?)) }; + let declared = read_u32(4).ok_or("truncated WAV header")? as usize; + if declared.checked_add(8) != Some(bytes.len()) { + return Err("truncated or inconsistent WAV length".into()); + } + let mut at = 12_usize; + let mut format_ok = false; + let mut data = None; + while at.checked_add(8).is_some_and(|end| end <= bytes.len()) { + let len = read_u32(at + 4).ok_or("truncated WAV chunk")? as usize; + let end = at.checked_add(8).and_then(|n| n.checked_add(len)).ok_or("WAV chunk is too large")?; + let chunk = bytes.get(at + 8..end).ok_or("truncated WAV chunk")?; + match bytes.get(at..at + 4) { + Some(b"fmt ") => { + format_ok = + chunk.len() >= 16 && chunk[..4] == [1, 0, 1, 0] && chunk[4..8] == 16000_u32.to_le_bytes() && chunk[12..16] == [2, 0, 16, 0]; + } + Some(b"data") => { + if data.is_some() { + return Err("multiple WAV data chunks are not supported".into()); + } + data = Some(chunk); + } + _ => {} + } + at = end.checked_add(len % 2).ok_or("WAV chunk is too large")?; + } + if at != bytes.len() { + return Err("truncated WAV chunk header or missing chunk padding".into()); + } + if !format_ok { + return Err("WAV must be PCM16, mono, 16 kHz".into()); + } + let data = data.ok_or("WAV has no audio data")?; + if !data.len().is_multiple_of(2) { + return Err("WAV ends in a partial sample".into()); + } + Self::new(data.as_chunks::<2>().0.iter().copied().map(i16::from_le_bytes).collect()) + } +} + +/// The loudest sample in `samples`, from 0 (silence) to 1 (full scale), for an input level meter. +pub fn peak(samples: &[f32]) -> f32 { + samples.iter().filter(|s| s.is_finite()).fold(0.0_f32, |m, s| m.max(s.abs())).min(1.0) +} + +/// Stateful linear resampling across callback boundaries, with channel averaging. +pub struct AudioBuffer { + rate: u32, + channels: usize, + frame: u64, + next: f64, + last: f32, + audio: PcmAudio, +} + +impl AudioBuffer { + pub fn new(rate: u32, channels: usize) -> Result { + if !(8000..=384000).contains(&rate) || channels == 0 || channels > 64 { + return Err("unsupported microphone sample rate or channel count".into()); + } + Ok(Self { rate, channels, frame: 0, next: 0.0, last: 0.0, audio: PcmAudio::default() }) + } + pub fn audio(&self) -> &PcmAudio { + &self.audio + } + /// True means the 60-second limit was reached. Invalid input is rejected before mutation. + pub fn push(&mut self, input: &[f32]) -> Result { + if !input.len().is_multiple_of(self.channels) || input.iter().any(|s| !s.is_finite()) { + return Err("invalid microphone sample data".into()); + } + for frame in input.chunks_exact(self.channels) { + let mono = frame.iter().map(|s| s.clamp(-1.0, 1.0)).sum::() / self.channels as f32; + while self.next <= self.frame as f64 { + if self.audio.samples.len() >= MAX_SAMPLES { + return Ok(true); + } + let fraction = if self.frame == 0 { 1.0 } else { (self.next - (self.frame - 1) as f64) as f32 }; + let value = self.last + (mono - self.last) * fraction; + self.audio.samples.push((value.clamp(-1.0, 1.0) * 32767.0).round() as i16); + self.next += self.rate as f64 / SAMPLE_RATE as f64; + } + self.last = mono; + self.frame += 1; + } + Ok(self.audio.samples.len() >= MAX_SAMPLES) + } +} diff --git a/crates/voice-runtime/src/capture.rs b/crates/voice-runtime/src/capture.rs new file mode 100644 index 00000000..e71391fb --- /dev/null +++ b/crates/voice-runtime/src/capture.rs @@ -0,0 +1,164 @@ +//! CPAL owns native platform APIs. The realtime callback only copies into a bounded queue. + +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use cpal::{FromSample, SizedSample}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + mpsc::{self, Receiver, SyncSender}, +}; +use std::time::Duration; + +const CHUNK: usize = 4096; +pub(crate) struct AudioChunk { + pub samples: [f32; CHUNK], + pub len: usize, +} + +pub(crate) struct Capture { + _stream: cpal::Stream, + rx: Receiver, + overflow: Arc, + error: Arc>>, + pub rate: u32, + pub channels: usize, +} + +pub fn devices() -> Result, String> { + cpal::default_host() + .input_devices() + .map_err(|e| format!("cannot list microphones: {e}"))? + .map(|d| d.name().map_err(|e| format!("cannot read microphone name: {e}"))) + .collect() +} + +impl Capture { + pub fn start(name: Option<&str>) -> Result { + let host = cpal::default_host(); + let device = match name { + Some(name) => host.input_devices().map_err(|e| e.to_string())?.find(|d| d.name().is_ok_and(|n| n == name)), + None => host.default_input_device(), + } + .ok_or("no matching input device; check microphone permissions and device selection")?; + let config = device.default_input_config().map_err(|e| format!("cannot open microphone: {e}"))?; + let rate = config.sample_rate().0; + let channels = config.channels() as usize; + if channels == 0 || channels > 64 { + return Err("unsupported microphone channel count".into()); + } + let (tx, rx) = mpsc::sync_channel(32); + let overflow = Arc::new(AtomicBool::new(false)); + let error = Arc::new(Mutex::new(None)); + let stream_config: cpal::StreamConfig = config.clone().into(); + macro_rules! stream { + ($ty:ty) => { + build::<$ty>(&device, &stream_config, channels, tx, overflow.clone(), error.clone())? + }; + } + let stream = match config.sample_format() { + cpal::SampleFormat::I8 => stream!(i8), + cpal::SampleFormat::I16 => stream!(i16), + cpal::SampleFormat::I32 => stream!(i32), + cpal::SampleFormat::I64 => stream!(i64), + cpal::SampleFormat::U8 => stream!(u8), + cpal::SampleFormat::U16 => stream!(u16), + cpal::SampleFormat::U32 => stream!(u32), + cpal::SampleFormat::U64 => stream!(u64), + cpal::SampleFormat::F32 => stream!(f32), + cpal::SampleFormat::F64 => stream!(f64), + _ => return Err("unsupported microphone sample format".into()), + }; + stream.play().map_err(|e| format!("cannot start microphone: {e}"))?; + Ok(Self { _stream: stream, rx, overflow, error, rate, channels }) + } + + pub fn read(&self) -> Result, String> { + if self.overflow.load(Ordering::Acquire) { + return Err("microphone buffer overflow; recording stopped".into()); + } + if let Some(error) = self.error.lock().map_err(|_| "microphone error state unavailable")?.take() { + return Err(error); + } + match self.rx.recv_timeout(Duration::from_millis(20)) { + Ok(chunk) => Ok(Some(chunk)), + Err(mpsc::RecvTimeoutError::Timeout) => Ok(None), + Err(mpsc::RecvTimeoutError::Disconnected) => Err("microphone disconnected".into()), + } + } +} + +fn build( + device: &cpal::Device, + config: &cpal::StreamConfig, + channels: usize, + tx: SyncSender, + overflow: Arc, + error: Arc>>, +) -> Result +where + T: SizedSample, + f32: FromSample, +{ + device + .build_input_stream( + config, + move |data: &[T], _| { + let width = CHUNK / channels * channels; + for samples in data.chunks(width) { + let mut chunk = AudioChunk { samples: [0.0; CHUNK], len: samples.len() }; + for (out, &sample) in chunk.samples.iter_mut().zip(samples) { + *out = sample.to_sample::(); + } + if tx.try_send(chunk).is_err() { + overflow.store(true, Ordering::Release); + break; + } + } + }, + move |e| { + if let Ok(mut slot) = error.lock() { + *slot = Some(format!("microphone failed: {e}")); + } + }, + None, + ) + .map_err(|e| format!("cannot create microphone stream: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::audio::{AudioBuffer, SAMPLE_RATE, peak}; + use std::time::Instant; + /// Real CPAL host/device. Speak for two seconds. A virtual native device may serve recorded + /// input; label that distinction in acceptance evidence rather than claiming hardware coverage. + #[test] + #[ignore = "requires a native audio input device; speak into the selected microphone"] + fn actual_microphone_delivers_samples_and_releases_the_device() { + let device = std::env::var("VOICE_CAPTURE_DEVICE").ok(); + let capture = Capture::start(device.as_deref()).expect("open native input device"); + let mut audio = AudioBuffer::new(capture.rate, capture.channels).expect("capture format"); + let deadline = Instant::now() + Duration::from_secs(8); + let mut callbacks = 0; + let mut loudest = 0.0_f32; + while audio.audio().samples().len() < SAMPLE_RATE * 2 && Instant::now() < deadline { + if let Some(chunk) = capture.read().expect("read native callbacks") { + callbacks += 1; + loudest = loudest.max(peak(&chunk.samples[..chunk.len])); + audio.push(&chunk.samples[..chunk.len]).expect("convert native samples"); + } + } + eprintln!( + "native capture: callbacks={callbacks}, source_rate={}, source_channels={}, pcm16_samples={}, peak={loudest}", + capture.rate, + capture.channels, + audio.audio().samples().len() + ); + assert!(callbacks > 0, "device must deliver callbacks"); + assert!(audio.audio().samples().len() >= SAMPLE_RATE * 2, "device must deliver two seconds of sample data"); + assert!(loudest > 0.001, "speak or supply non-silent recorded input"); + drop(capture); + let reopened = Capture::start(device.as_deref()).expect("device reopens after cleanup"); + drop(reopened); + } +} diff --git a/crates/voice-runtime/src/lib.rs b/crates/voice-runtime/src/lib.rs new file mode 100644 index 00000000..8faed702 --- /dev/null +++ b/crates/voice-runtime/src/lib.rs @@ -0,0 +1,52 @@ +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +//! Host-independent capture and transcription. Providers never receive notebook access. + +pub mod audio; +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +mod capture; +mod runtime; +mod temporary; +pub mod whisper; + +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + +pub use runtime::{Runtime, RuntimeEvent, microphone_devices}; +pub use whisper::WhisperConfig; + +/// A process/provider can observe cancellation without blocking its owner. +#[derive(Clone, Default)] +pub struct Cancellation { + cancelled: Arc, + interrupts: Vec>, +} + +impl Cancellation { + pub fn cancel(&self) { + self.cancelled.store(true, Ordering::Release); + } + pub fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Acquire) || self.interrupts.iter().any(|x| x.load(Ordering::Acquire)) + } + #[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] + pub(crate) fn interrupted_by(&self, flag: Arc) -> Self { + let mut next = self.clone(); + next.interrupts.push(flag); + next + } +} + +/// Implementations recognize audio only. Authorization and edits stay in the host. +/// +/// Operations must observe cancellation promptly and bound blocking I/O and output. Runtime +/// drops join workers to release capture and process files. A provider that ignores +/// cancellation can block its owner on drop. The bundled Whisper provider polls cancellation +/// every 10 ms and enforces a timeout. +pub trait Transcriber: Send + 'static { + fn capabilities(&self) -> craft_voice_core::Capabilities; + fn prepare(&mut self, cancel: &Cancellation) -> Result<(), String>; + fn transcribe(&mut self, audio: &audio::PcmAudio, cancel: &Cancellation) -> Result; +} diff --git a/crates/voice-runtime/src/runtime.rs b/crates/voice-runtime/src/runtime.rs new file mode 100644 index 00000000..9bab00cb --- /dev/null +++ b/crates/voice-runtime/src/runtime.rs @@ -0,0 +1,383 @@ +//! Worker ownership and a one-slot, coalescing preview queue. + +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +use crate::audio::SAMPLE_RATE; +use crate::{Cancellation, Transcriber, audio::PcmAudio}; +use craft_voice_core::Transcript; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + mpsc::{self, Receiver, SyncSender}, +}; +use std::thread::JoinHandle; +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +use std::{ + sync::{Condvar, Mutex}, + time::Duration, +}; + +#[derive(Debug)] +pub enum RuntimeEvent { + Ready { + session: u64, + }, + Stopped { + session: u64, + }, + /// Recorded length so far, and the loudest input since the last report (0 to 1). + Progress { + session: u64, + seconds: f32, + level: f32, + }, + Transcript(Transcript), + Error { + session: u64, + message: String, + }, +} + +pub struct Runtime { + cancel: Cancellation, + stop: Arc, + events: Option>, + worker: Option>, +} + +impl Runtime { + pub fn from_audio(session: u64, mut provider: Box, audio: PcmAudio) -> Result { + Self::spawn( + move |cancel, _, tx| { + provider.prepare(&cancel)?; + let _ = tx.send(RuntimeEvent::Ready { session }); + let _ = tx.send(RuntimeEvent::Stopped { session }); + let text = provider.transcribe(&audio, &cancel)?; + if !cancel.is_cancelled() { + let _ = tx.send(RuntimeEvent::Transcript(Transcript::new(session, 1, text, true))); + } + Ok(()) + }, + session, + ) + } + + pub fn start(session: u64, provider: Box, device: Option) -> Result { + #[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] + { + Self::spawn(move |cancel, stop, tx| record(session, provider, device, cancel, stop, tx), session) + } + #[cfg(not(all(feature = "microphone", not(target_arch = "wasm32"))))] + { + let _ = (session, provider, device); + Err("microphone support is not enabled in this build".into()) + } + } + + fn spawn( + work: impl FnOnce(Cancellation, Arc, SyncSender) -> Result<(), String> + Send + 'static, + session: u64, + ) -> Result { + let cancel = Cancellation::default(); + let stop = Arc::new(AtomicBool::new(false)); + let (tx, rx) = mpsc::sync_channel(16); + let token = cancel.clone(); + let stop_flag = stop.clone(); + let worker = std::thread::Builder::new() + .name("craft-voice".into()) + .spawn(move || { + if let Err(message) = work(token.clone(), stop_flag, tx.clone()) + && !token.is_cancelled() + { + let _ = tx.send(RuntimeEvent::Error { session, message }); + } + }) + .map_err(|e| format!("cannot start voice worker: {e}"))?; + Ok(Self { cancel, stop, events: Some(rx), worker: Some(worker) }) + } + + pub fn stop(&self) { + self.stop.store(true, Ordering::Release); + } + pub fn cancel(&self) { + self.cancel.cancel(); + self.stop(); + } + pub fn poll(&mut self) -> Vec { + self.events.as_ref().map(|rx| rx.try_iter().take(32).collect()).unwrap_or_default() + } +} + +impl Drop for Runtime { + fn drop(&mut self) { + self.cancel(); + self.events.take(); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +pub fn microphone_devices() -> Result, String> { + #[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] + { + crate::capture::devices() + } + #[cfg(not(all(feature = "microphone", not(target_arch = "wasm32"))))] + { + Err("microphone support is not enabled in this build".into()) + } +} + +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +const NO_AUDIO_LIMIT: Duration = Duration::from_secs(5); +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +const NO_AUDIO: &str = "The microphone opened but sent no audio for 5 seconds. Check that the right microphone is selected, that it is not disabled, and that your system lets desktop apps use it."; + +/// Tracks actual callback delivery, separately from opening a stream or decoding speech. +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +struct CaptureClock { + last_audio: std::time::Instant, + started: Option, +} +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +impl CaptureClock { + fn new(now: std::time::Instant) -> Self { + Self { last_audio: now, started: None } + } + fn received(&mut self, now: std::time::Instant) -> bool { + self.last_audio = now; + if self.started.is_none() { + self.started = Some(now); + true + } else { + false + } + } + fn check(&self, now: std::time::Instant) -> Result { + if now.duration_since(self.last_audio) >= NO_AUDIO_LIMIT { + return Err(NO_AUDIO.into()); + } + Ok(self.started.is_some_and(|at| now.duration_since(at) >= Duration::from_secs(craft_voice_core::MAX_SECONDS as u64))) + } +} + +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +struct Work { + audio: PcmAudio, + final_pass: bool, +} +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +#[derive(Default)] +struct Queue { + work: Option, + done: bool, + error: Option, +} +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +type SharedQueue = Arc<(Mutex, Condvar)>; + +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +fn record( + session: u64, + mut provider: Box, + device: Option, + cancel: Cancellation, + stop: Arc, + tx: SyncSender, +) -> Result<(), String> { + // Open the microphone first: words spoken while the recognizer is being checked are kept. + let queue: SharedQueue = Arc::new((Mutex::new(Queue::default()), Condvar::new())); + let capture_failed = Arc::new(AtomicBool::new(false)); + let failure_signal = capture_failed.clone(); + let shared = queue.clone(); + let token = cancel.clone(); + let stopped = stop.clone(); + let events = tx.clone(); + let capture = std::thread::Builder::new() + .name("craft-microphone".into()) + .spawn(move || { + let result = capture_audio(session, device, &token, &stopped, &events, &shared); + finish_capture(&shared, &failure_signal, result); + }) + .map_err(|e| format!("cannot start microphone worker: {e}"))?; + let decoder_cancel = cancel.interrupted_by(capture_failed); + let result = provider.prepare(&decoder_cancel).and_then(|()| decode(session, provider.as_mut(), &decoder_cancel, &stop, &tx, &queue)); + // Stop capture on all decode exits, including a provider failure. Cancellation of a + // preview must not suppress the failure event sent by the outer worker. + stop.store(true, Ordering::Release); + let _ = capture.join(); + decode_outcome(result, &queue) +} + +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +fn finish_capture(queue: &SharedQueue, failed: &AtomicBool, result: Result<(), String>) { + if let Ok(mut q) = queue.0.lock() { + q.done = true; + q.error = result.err(); + if q.error.is_some() { + failed.store(true, Ordering::Release); + } + queue.1.notify_all(); + } else { + failed.store(true, Ordering::Release); + } +} + +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +fn decode_outcome(result: Result, queue: &SharedQueue) -> Result { + let mut q = queue.0.lock().map_err(|_| "voice preview queue unavailable")?; + match q.error.take() { + Some(error) => Err(error), + None => result, + } +} + +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +fn capture_audio( + session: u64, + device: Option, + cancel: &Cancellation, + stop: &AtomicBool, + tx: &SyncSender, + queue: &SharedQueue, +) -> Result<(), String> { + if cancel.is_cancelled() { + return Ok(()); + } + let capture = crate::capture::Capture::start(device.as_deref())?; + let mut buffer = crate::audio::AudioBuffer::new(capture.rate, capture.channels)?; + let mut preview_at = SAMPLE_RATE * 5; + let mut reported_at = 0; + let mut clock = CaptureClock::new(std::time::Instant::now()); + let mut loudest = 0.0_f32; + while !cancel.is_cancelled() && !stop.load(Ordering::Acquire) { + if clock.check(std::time::Instant::now())? { + stop.store(true, Ordering::Release); + break; + } + let Some(chunk) = capture.read()? else { continue }; + let samples = chunk.samples.get(..chunk.len).unwrap_or_default(); + if samples.is_empty() { + continue; + } + if clock.received(std::time::Instant::now()) { + let _ = tx.send(RuntimeEvent::Ready { session }); + } + loudest = loudest.max(crate::audio::peak(samples)); + let full = buffer.push(samples)?; + let count = buffer.audio().samples().len(); + if count >= reported_at + SAMPLE_RATE / 5 { + let _ = tx.try_send(RuntimeEvent::Progress { session, seconds: buffer.audio().seconds(), level: loudest }); + loudest = 0.0; + reported_at = count; + } + if count >= preview_at { + let mut q = queue.0.lock().map_err(|_| "voice preview queue unavailable")?; + q.work = Some(Work { audio: buffer.audio().clone(), final_pass: false }); + queue.1.notify_one(); + preview_at = count + SAMPLE_RATE * 5; + } + if full { + stop.store(true, Ordering::Release); + } + } + drop(capture); // Release the device before the final decode, not after it. + if !cancel.is_cancelled() { + let _ = tx.send(RuntimeEvent::Stopped { session }); + let mut q = queue.0.lock().map_err(|_| "voice preview queue unavailable")?; + q.work = Some(Work { audio: buffer.audio().clone(), final_pass: true }); + queue.1.notify_one(); + } + Ok(()) +} + +#[cfg(all(feature = "microphone", not(target_arch = "wasm32")))] +fn decode( + session: u64, + provider: &mut dyn Transcriber, + cancel: &Cancellation, + stop: &Arc, + tx: &SyncSender, + queue: &SharedQueue, +) -> Result<(), String> { + let mut revision = 0_u64; + loop { + if cancel.is_cancelled() { + return Ok(()); + } + let mut q = queue.0.lock().map_err(|_| "voice preview queue unavailable")?; + if let Some(error) = q.error.take() { + return Err(error); + } + let Some(work) = q.work.take() else { + if q.done { + return Ok(()); + } + let _guard = queue.1.wait_timeout(q, Duration::from_millis(20)).map_err(|_| "voice preview queue unavailable")?; + continue; + }; + drop(q); + let token = if work.final_pass { cancel.clone() } else { cancel.interrupted_by(stop.clone()) }; + let result = provider.transcribe(&work.audio, &token); + if token.is_cancelled() { + if cancel.is_cancelled() { + return Ok(()); + } else { + continue; + } + } + // A preview is a courtesy: one that fails is skipped and recording goes on. Only the + // final pass decides whether the recording produced words. + let text = match result { + Ok(text) => text, + Err(_) if !work.final_pass => continue, + Err(error) => return Err(error), + }; + revision += 1; + let _ = tx.send(RuntimeEvent::Transcript(Transcript::new(session, revision, text, work.final_pass))); + if work.final_pass { + return Ok(()); + } + } +} + +#[cfg(all(test, feature = "microphone", not(target_arch = "wasm32")))] +mod capture_failure_tests { + use super::*; + #[cfg(unix)] + use std::time::Instant; + + #[cfg(unix)] + #[test] + fn capture_failure_interrupts_real_process_and_wins_over_decoder_error() { + let queue: SharedQueue = Arc::new((Mutex::new(Queue::default()), Condvar::new())); + let failed = Arc::new(AtomicBool::new(false)); + let token = Cancellation::default().interrupted_by(failed.clone()); + let mut child = std::process::Command::new("/bin/sleep").arg("10").spawn().unwrap(); + let q = queue.clone(); + let flag = failed.clone(); + let reporter = std::thread::spawn(move || finish_capture(&q, &flag, Err("microphone disconnected".into()))); + let started = Instant::now(); + let result = crate::whisper::wait_child(&mut child, &token, Duration::from_secs(3)); + reporter.join().unwrap(); + assert_eq!(decode_outcome(result, &queue), Err("microphone disconnected".to_string())); + assert!(started.elapsed() < Duration::from_secs(2)); + assert!(child.try_wait().unwrap().is_some()); + } + + #[test] + fn recording_requires_samples_and_stalled_audio_fails_after_five_seconds() { + let start = std::time::Instant::now(); + let mut clock = CaptureClock::new(start); + assert!(clock.started.is_none()); + assert_eq!(clock.check(start + Duration::from_secs(4)), Ok(false)); + assert!(clock.check(start + Duration::from_secs(5)).is_err()); + assert!(clock.received(start + Duration::from_secs(4))); + assert!(!clock.received(start + Duration::from_secs(5))); + assert_eq!(clock.started, Some(start + Duration::from_secs(4))); + assert_eq!(clock.check(start + Duration::from_secs(9)), Ok(false)); + assert!(clock.check(start + Duration::from_secs(10)).is_err()); + clock.received(start + Duration::from_secs(64)); + assert_eq!(clock.check(start + Duration::from_secs(64)), Ok(true)); + } +} diff --git a/crates/voice-runtime/src/temporary.rs b/crates/voice-runtime/src/temporary.rs new file mode 100644 index 00000000..fd03eeb1 --- /dev/null +++ b/crates/voice-runtime/src/temporary.rs @@ -0,0 +1,77 @@ +//! Private process files, outside the host's documents, with an active lease. + +use std::fs::{self, File}; +use std::path::Path; +use std::time::{Duration, SystemTime}; + +pub(crate) struct JobFiles { + _lease: File, + pub dir: tempfile::TempDir, +} + +impl JobFiles { + pub fn new(root: &Path) -> Result { + fs::create_dir_all(root).map_err(|e| format!("cannot create voice temporary folder: {e}"))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(root, fs::Permissions::from_mode(0o700)).map_err(|e| format!("cannot protect voice temporary folder: {e}"))?; + } + cleanup(root); + let dir = tempfile::Builder::new().prefix("job-").tempdir_in(root).map_err(|e| format!("cannot create voice process files: {e}"))?; + let lease = File::create(dir.path().join("lease")).map_err(|e| e.to_string())?; + lease.lock().map_err(|e| e.to_string())?; + Ok(Self { dir, _lease: lease }) + } +} + +fn cleanup(root: &Path) { + let Ok(entries) = fs::read_dir(root) else { return }; + for entry in entries.flatten() { + if !entry.file_name().to_string_lossy().starts_with("job-") || !entry.file_type().is_ok_and(|t| t.is_dir()) { + continue; + } + let path = entry.path(); + let Ok(file) = File::options().read(true).write(true).open(path.join("lease")) else { continue }; + let old = file + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|t| SystemTime::now().duration_since(t).ok()) + .is_some_and(|d| d > Duration::from_secs(60)); + if old && file.try_lock().is_ok() { + let _ = fs::remove_dir_all(path); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn process_files_are_removed_on_drop() { + let root = tempfile::tempdir().expect("root"); + let job = JobFiles::new(root.path()).expect("job"); + let path = job.dir.path().to_path_buf(); + fs::write(path.join("input.wav"), b"private audio fixture").expect("audio"); + drop(job); + assert!(!path.exists()); + } + + #[test] + fn cleanup_removes_abandoned_jobs_but_not_active_jobs() { + let root = tempfile::tempdir().expect("root"); + let active = JobFiles::new(root.path()).expect("active"); + let old = SystemTime::now() - Duration::from_secs(120); + active._lease.set_modified(old).expect("age active lease"); + let abandoned = root.path().join("job-abandoned"); + fs::create_dir(&abandoned).expect("abandoned folder"); + let file = File::create(abandoned.join("lease")).expect("lease"); + file.set_modified(old).expect("age abandoned lease"); + drop(file); + cleanup(root.path()); + assert!(active.dir.path().exists()); + assert!(!abandoned.exists()); + } +} diff --git a/crates/voice-runtime/src/whisper.rs b/crates/voice-runtime/src/whisper.rs new file mode 100644 index 00000000..329ab9c5 --- /dev/null +++ b/crates/voice-runtime/src/whisper.rs @@ -0,0 +1,243 @@ +//! A user-selected whisper-cli process. No shell, downloads, or provider accounts. + +use crate::{Cancellation, Transcriber, audio::PcmAudio, temporary::JobFiles}; +use craft_voice_core::{Capabilities, MAX_TRANSCRIPT_BYTES}; +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::{Mutex, OnceLock}; +use std::time::{Duration, Instant, SystemTime}; + +const MAX_OUTPUT: usize = 1024 * 1024; +/// whisper.cpp skips input shorter than one second; a short press is padded with silence. +const MIN_SAMPLES: usize = crate::audio::SAMPLE_RATE * 11 / 10; + +/// An executable that passed the `--help` check, by path, size and modification time, and +/// whether it can suppress non-speech tokens. Checked once per executable, not per recording, +/// so pressing the talk key does not wait for a process start before the microphone opens. +type Checked = HashMap, Option<&'static str>)>; +static CHECKED: OnceLock> = OnceLock::new(); + +fn identity(path: &Path) -> Option<(u64, Option)> { + let meta = path.metadata().ok()?; + Some((meta.len(), meta.modified().ok())) +} + +#[cfg(windows)] +#[path = "whisper.win.rs"] +mod platform; +#[cfg(windows)] +use platform::suppress_console; + +#[cfg(not(windows))] +fn suppress_console(_command: &mut Command) {} + +/// The option names a `--help` text advertises, as whole words (`-f` is not found inside `-of`). +fn advertised(help: &str) -> std::collections::HashSet<&str> { + help.split(|c: char| c.is_whitespace() || c == ',' || c == '[' || c == ']').filter(|w| w.starts_with('-')).collect() +} + +#[derive(Clone)] +pub struct WhisperConfig { + pub executable: PathBuf, + pub model: PathBuf, + pub language: String, + pub temporary_root: PathBuf, + pub timeout: Duration, + suppress_non_speech: Option<&'static str>, +} + +impl WhisperConfig { + pub fn new(executable: PathBuf, model: PathBuf, temporary_root: PathBuf) -> Self { + Self { executable, model, temporary_root, language: "auto".into(), timeout: Duration::from_secs(120), suppress_non_speech: None } + } + + pub fn validate(&mut self, cancel: &Cancellation) -> Result<(), String> { + if !self.executable.is_absolute() || !self.executable.is_file() { + return Err("choose an existing absolute path to whisper-cli".into()); + } + if !self.model.is_absolute() || !self.model.metadata().is_ok_and(|m| m.is_file() && m.len() > 0) { + return Err("choose an existing local Whisper model file".into()); + } + if self.language.len() > 16 || self.language.is_empty() || !self.language.chars().all(|c| c.is_ascii_alphabetic() || c == '-') { + return Err("choose auto or a language code such as en or es".into()); + } + let id = identity(&self.executable); + let cache = CHECKED.get_or_init(Default::default); + if let (Some((len, modified)), Ok(known)) = (id, cache.lock()) + && let Some(&(l, m, sns)) = known.get(&self.executable) + && l == len + && m == modified + { + self.suppress_non_speech = sns; + return Ok(()); + } + let files = JobFiles::new(&self.temporary_root)?; + let help_path = files.dir.path().join("help"); + let out = File::create(&help_path).map_err(|e| e.to_string())?; + let err = out.try_clone().map_err(|e| e.to_string())?; + let mut command = Command::new(&self.executable); + command.arg("--help").stdin(Stdio::null()).stdout(out).stderr(err); + suppress_console(&mut command); + let mut child = command.spawn().map_err(|e| format!("cannot start whisper-cli: {e}"))?; + // Some versions use a nonzero exit status for --help. Flags still must be present. + let result = wait_child(&mut child, cancel, Duration::from_secs(10)); + if cancel.is_cancelled() { + return Err("voice operation cancelled".into()); + } + if result.as_ref().is_err_and(|e| e == "recognizer timed out") { + return result; + } + let bytes = read_bounded(&help_path)?; + let help = String::from_utf8_lossy(&bytes); + let flags = advertised(&help); + if !["-oj", "-of", "-m", "-f", "-l"].iter().all(|flag| flags.contains(flag)) { + return Err("this whisper-cli does not advertise the required JSON/model/file/language options".into()); + } + self.suppress_non_speech = non_speech_option(&flags); + if let (Some((len, modified)), Ok(mut known)) = (id, cache.lock()) { + if known.len() >= 32 { + known.clear(); + } + known.insert(self.executable.clone(), (len, modified, self.suppress_non_speech)); + } + Ok(()) + } + + pub fn recognize(&self, audio: &PcmAudio, cancel: &Cancellation) -> Result { + if audio.samples().is_empty() { + return Ok(String::new()); + } + if cancel.is_cancelled() { + return Err("voice operation cancelled".into()); + } + let files = JobFiles::new(&self.temporary_root)?; + let wav = files.dir.path().join("input.wav"); + let output = files.dir.path().join("result"); + let bytes = if audio.samples().len() < MIN_SAMPLES { + let mut samples = audio.samples().to_vec(); + samples.resize(MIN_SAMPLES, 0); + PcmAudio::new(samples)?.wav() + } else { + audio.wav() + }; + fs::write(&wav, bytes).map_err(|e| format!("cannot stage voice audio: {e}"))?; + let mut command = Command::new(&self.executable); + command.arg("-m").arg(&self.model).arg("-f").arg(&wav).arg("-l").arg(&self.language).arg("-oj").arg("-of").arg(&output); + if let Some(flag) = self.suppress_non_speech { + command.arg(flag); + } + command.stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()); + suppress_console(&mut command); + let mut child = command.spawn().map_err(|e| format!("cannot start whisper-cli: {e}"))?; + wait_child(&mut child, cancel, self.timeout)?; + parse_transcript(&read_bounded(&output.with_extension("json"))?) + } +} + +impl Transcriber for WhisperConfig { + fn capabilities(&self) -> Capabilities { + Capabilities { batch_transcription: true, partial_transcripts: true, language_selection: true, ..Capabilities::default() } + } + fn prepare(&mut self, cancel: &Cancellation) -> Result<(), String> { + self.validate(cancel) + } + fn transcribe(&mut self, audio: &PcmAudio, cancel: &Cancellation) -> Result { + self.recognize(audio, cancel) + } +} + +/// Always reaps on cancellation/timeout. Diagnostics do not include speech or subprocess output. +pub fn wait_child(child: &mut Child, cancel: &Cancellation, timeout: Duration) -> Result<(), String> { + let started = Instant::now(); + loop { + if cancel.is_cancelled() || started.elapsed() >= timeout { + let _ = child.kill(); + let _ = child.wait(); + return Err(if cancel.is_cancelled() { "voice operation cancelled" } else { "recognizer timed out" }.into()); + } + match child.try_wait() { + Ok(Some(status)) => { + return if status.success() { + Ok(()) + } else { + Err("recognizer exited unsuccessfully; check the selected model and executable".into()) + }; + } + Ok(None) => std::thread::sleep(Duration::from_millis(10)), + Err(e) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(format!("cannot observe recognizer: {e}")); + } + } + } +} + +fn read_bounded(path: &Path) -> Result, String> { + let file = File::open(path).map_err(|e| format!("cannot read recognizer result: {e}"))?; + let mut bytes = Vec::new(); + file.take((MAX_OUTPUT + 1) as u64).read_to_end(&mut bytes).map_err(|e| e.to_string())?; + if bytes.len() > MAX_OUTPUT { + return Err("recognizer result exceeds 1 MiB".into()); + } + Ok(bytes) +} + +pub fn parse_transcript(bytes: &[u8]) -> Result { + if bytes.len() > MAX_OUTPUT { + return Err("recognizer result exceeds 1 MiB".into()); + } + let value: serde_json::Value = serde_json::from_slice(bytes).map_err(|_| "recognizer returned invalid JSON")?; + let segments = value.get("transcription").and_then(|v| v.as_array()).ok_or("recognizer JSON has no transcription array")?; + if segments.len() > 4096 { + return Err("recognizer returned too many segments".into()); + } + let mut text = String::new(); + for segment in segments { + let words = segment.get("text").and_then(|t| t.as_str()).ok_or("recognizer segment has no text")?; + if non_speech(words) { + continue; + } + if text.len().saturating_add(words.len()) > MAX_TRANSCRIPT_BYTES { + return Err("transcript exceeds 64 KiB".into()); + } + text.push_str(words); + } + Ok(text.trim().into()) +} + +/// whisper.cpp labels silence and noise as whole segments such as `[BLANK_AUDIO]` or `(music)`. +/// Only those known labels are dropped: a spoken aside such as `(double-check the date)` is the +/// person's words and goes to review like the rest. +fn non_speech(segment: &str) -> bool { + let t = segment.trim(); + if t.is_empty() { + return true; + } + let wrapped = [('[', ']'), ('(', ')'), ('*', '*')].iter().any(|&(open, close)| t.len() >= 2 && t.starts_with(open) && t.ends_with(close)); + let inner = t.get(1..t.len().saturating_sub(1)).unwrap_or_default().trim().to_ascii_lowercase().replace(['_', '-'], " "); + wrapped + && matches!( + inner.as_str(), + "blank audio" | "silence" | "music" | "noise" | "background noise" | "no speech" | "inaudible" | "static" | "sound" + ) +} + +fn non_speech_option(flags: &std::collections::HashSet<&str>) -> Option<&'static str> { + ["-sns", "--suppress-nst"].into_iter().find(|flag| flags.contains(flag)) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn uses_the_exact_suppression_option_advertised_by_the_program() { + assert_eq!(non_speech_option(&advertised("-m MODEL -f FILE --suppress-nst")), Some("--suppress-nst")); + assert_eq!(non_speech_option(&advertised("[-sns, --suppress-nst]")), Some("-sns")); + assert_eq!(non_speech_option(&advertised("--something-else")), None); + assert!(!advertised("-of FNAME").contains("-f")); + } +} diff --git a/crates/voice-runtime/src/whisper.win.rs b/crates/voice-runtime/src/whisper.win.rs new file mode 100644 index 00000000..6cf12f7a --- /dev/null +++ b/crates/voice-runtime/src/whisper.win.rs @@ -0,0 +1,7 @@ +//! Start the console recognizer without opening a new console window. + +pub(super) fn suppress_console(command: &mut std::process::Command) { + use std::os::windows::process::CommandExt; + const CREATE_NO_WINDOW: u32 = 0x0800_0000; + command.creation_flags(CREATE_NO_WINDOW); +} diff --git a/crates/voice-runtime/tests/audio.rs b/crates/voice-runtime/tests/audio.rs new file mode 100644 index 00000000..66853644 --- /dev/null +++ b/crates/voice-runtime/tests/audio.rs @@ -0,0 +1,57 @@ +use craft_voice_runtime::audio::{AudioBuffer, MAX_SAMPLES, PcmAudio, peak}; + +#[test] +fn the_level_meter_reads_the_loudest_sample() { + assert_eq!(peak(&[]), 0.0); + assert_eq!(peak(&[0.0; 64]), 0.0); + assert_eq!(peak(&[0.1, -0.5, 0.25]), 0.5); + assert_eq!(peak(&[2.0, f32::NAN]), 1.0); +} + +#[test] +fn stereo_48khz_is_mixed_and_resampled_across_chunks() { + let mut buffer = AudioBuffer::new(48000, 2).unwrap(); + buffer.push(&[0.0, 0.0, 1.0, 1.0]).unwrap(); + buffer.push(&[1.0, 1.0, 1.0, 1.0, 0.0, 0.0, -1.0, -1.0]).unwrap(); + assert_eq!(buffer.audio().samples(), &[0, 32767]); + let before = buffer.audio().samples().to_vec(); + assert!(buffer.push(&[f32::NAN, 0.0]).is_err()); + assert_eq!(buffer.audio().samples(), before); +} + +#[test] +fn wav_round_trip_keeps_signed_pcm_and_rejects_truncation() { + let audio = PcmAudio::new(vec![-32768, -1, 0, 1, 32767]).unwrap(); + let wav = audio.wav(); + assert_eq!(PcmAudio::read_wav(&wav).unwrap().samples(), &[-32768, -1, 0, 1, 32767]); + assert!(PcmAudio::read_wav(&wav[..wav.len() - 1]).is_err()); + let mut wrong_rate = wav; + wrong_rate[24..28].copy_from_slice(&48000u32.to_le_bytes()); + assert!(PcmAudio::read_wav(&wrong_rate).is_err()); +} + +#[test] +fn duration_limit_stops_capture_and_never_grows_beyond_sixty_seconds() { + let mut buffer = AudioBuffer::new(16000, 1).unwrap(); + assert!(buffer.push(&vec![0.5; MAX_SAMPLES + 20]).unwrap()); + assert_eq!(buffer.audio().samples().len(), MAX_SAMPLES); + assert!(buffer.push(&[1.0]).unwrap()); + assert_eq!(buffer.audio().samples().len(), MAX_SAMPLES); + assert!(PcmAudio::new(vec![0; MAX_SAMPLES + 1]).is_err()); +} + +#[test] +fn wav_rejects_unparsed_trailing_bytes_and_missing_padding() { + let mut trailing = PcmAudio::new(vec![0]).unwrap().wav(); + trailing.extend_from_slice(b"x"); + let length = (trailing.len() - 8) as u32; + trailing[4..8].copy_from_slice(&length.to_le_bytes()); + assert!(PcmAudio::read_wav(&trailing).is_err()); + let mut odd = PcmAudio::default().wav(); + odd.extend_from_slice(b"JUNK"); + odd.extend_from_slice(&1_u32.to_le_bytes()); + odd.push(0); + let length = (odd.len() - 8) as u32; + odd[4..8].copy_from_slice(&length.to_le_bytes()); + assert!(PcmAudio::read_wav(&odd).is_err()); +} diff --git a/crates/voice-runtime/tests/process.rs b/crates/voice-runtime/tests/process.rs new file mode 100644 index 00000000..e9997a69 --- /dev/null +++ b/crates/voice-runtime/tests/process.rs @@ -0,0 +1,53 @@ +use craft_voice_runtime::whisper::{parse_transcript, wait_child}; +use craft_voice_runtime::{Cancellation, WhisperConfig}; +use std::time::{Duration, Instant}; + +#[test] +fn whisper_json_fixture_is_text_not_commands_or_logs() { + let json = br#"{"transcription":[{"text":" Hola"},{"text":" mundo."}]}"#; + assert_eq!(parse_transcript(json).expect("transcript"), "Hola mundo."); + assert!(parse_transcript(br#"{"wrong":[]}"#).is_err()); + assert!(parse_transcript(b"not json").is_err()); + let huge = format!("{{\"transcription\":[{{\"text\":\"{}\"}}]}}", "x".repeat(65537)); + assert!(parse_transcript(huge.as_bytes()).is_err()); +} + +#[test] +fn missing_recognizer_is_a_setup_error() { + let t = tempfile::tempdir().expect("temporary root"); + let mut c = WhisperConfig::new(t.path().join("missing"), t.path().join("model"), t.path().into()); + assert!(c.validate(&Cancellation::default()).is_err()); +} + +#[cfg(unix)] +#[test] +fn cancellation_kills_and_reaps_a_real_child() { + let mut child = std::process::Command::new("/bin/sleep").arg("10").spawn().expect("sleep"); + let cancel = Cancellation::default(); + cancel.cancel(); + let start = Instant::now(); + assert!(wait_child(&mut child, &cancel, Duration::from_secs(20)).is_err()); + assert!(start.elapsed() < Duration::from_secs(2)); + assert!(child.try_wait().expect("reaped status").is_some()); +} + +#[cfg(unix)] +#[test] +fn timeout_reaps_without_waiting_for_the_childs_normal_exit() { + let mut child = std::process::Command::new("/bin/sleep").arg("10").spawn().expect("sleep"); + assert!(wait_child(&mut child, &Cancellation::default(), Duration::from_millis(20)).is_err()); + assert!(child.try_wait().expect("status").is_some()); +} + +#[test] +fn words_in_brackets_are_still_words() { + let json = br#"{"transcription":[{"text":" Call Ana"},{"text":" (double-check the date)"},{"text":" [laughs] tomorrow."}]}"#; + assert_eq!(parse_transcript(json).expect("transcript"), "Call Ana (double-check the date) [laughs] tomorrow."); +} + +#[test] +fn silence_labels_are_not_words() { + let json = br#"{"transcription":[{"text":" [BLANK_AUDIO]"},{"text":" Hello"},{"text":" (music)"},{"text":" world."}]}"#; + assert_eq!(parse_transcript(json).expect("transcript"), "Hello world."); + assert_eq!(parse_transcript(br#"{"transcription":[{"text":" [BLANK_AUDIO]"}]}"#).expect("silence"), ""); +} diff --git a/crates/voice-runtime/tests/runtime.rs b/crates/voice-runtime/tests/runtime.rs new file mode 100644 index 00000000..31c38e64 --- /dev/null +++ b/crates/voice-runtime/tests/runtime.rs @@ -0,0 +1,24 @@ +use craft_voice_runtime::audio::PcmAudio; +use craft_voice_runtime::{Runtime, RuntimeEvent}; + +#[test] +fn a_missing_local_provider_fails_without_starting_a_microphone() { + let t = tempfile::tempdir().expect("temp"); + let config = craft_voice_runtime::WhisperConfig::new(t.path().join("missing"), t.path().join("model"), t.path().into()); + let mut runtime = Runtime::from_audio(7, Box::new(config), PcmAudio::new(vec![0; 160]).expect("audio")).expect("worker"); + let end = std::time::Instant::now() + std::time::Duration::from_secs(2); + let mut failed = false; + while std::time::Instant::now() < end { + for event in runtime.poll() { + if matches!(event, RuntimeEvent::Error { session: 7, .. }) { + failed = true; + } + assert!(!matches!(event, RuntimeEvent::Ready { .. })); + } + if failed { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert!(failed, "invalid setup must be reported"); +} diff --git a/docs/architecture.md b/docs/architecture.md index 3f2548ea..ac4347b4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -173,3 +173,22 @@ folder rules it proved are now in `crates/store`. `docs/research/` holds the notes this design rests on: Microsoft's published `.one` file format specifications, paraphrased. They are notes, not Microsoft text. The feature list this project was designed against is kept out of this repository. + +## Optional reusable voice libraries + +`craft-voice-core` (layer 1) owns pure session transitions, full-replacement +transcript revisions, bounds and provider capabilities. It has no dependencies. +`craft-voice-runtime` (layer 3) owns bounded audio conversion, optional CPAL capture, +background decoding, private job files and the local whisper-cli adapter. Neither +library depends on NoteCraft or egui. Its `transcribe` example is a second consumer. + +The engine owns pure voice state and emits typed effects. Voice commands require a +code-only LocalUi origin, except metadata-only status. CLI and MCP parameters cannot +select that context. Provider updates enter through a native controller method. + +The optional host controller validates an opaque insertion ticket against editor +identity/generation, selected text, original body and source fingerprint. Insert +uses existing Block/Span editing, history and normal page.save/autosave. Failed +validation retains the draft for review or explicit retargeting. Candidates must round-trip +semantically through the existing page format before reaching editor history. No page +serializer changes are required. See [dictation](how-to/dictate.md). diff --git a/docs/commands.md b/docs/commands.md index 5aff689f..e282089f 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -276,3 +276,61 @@ Reload the tree from disk. No parameters. +## `voice.start` + +Start local reviewed dictation. + +Parameters: + +- target (required): opaque insertion ticket + +## `voice.stop` + +Stop capture and finalize dictation. + +Parameters: + +- session (required): voice session id + +## `voice.cancel` + +Discard the voice session. + +Parameters: + +- session (required): voice session id + +## `voice.status` + +Read voice lifecycle status without transcript content. + +No parameters. + +## `voice.draft.set` + +Review and edit the local transcript. + +Parameters: + +- session (required): voice session id +- revision (required): expected transcript revision +- text (required): reviewed transcript, at most 64 KiB + +## `voice.retarget` + +Choose a new insertion position. + +Parameters: + +- session (required): voice session id +- target (required): opaque insertion ticket + +## `voice.commit` + +Insert the reviewed local transcript. + +Parameters: + +- session (required): voice session id +- revision (required): approved transcript revision + diff --git a/docs/feature-parity.md b/docs/feature-parity.md index c4d0ba85..db51e89f 100644 --- a/docs/feature-parity.md +++ b/docs/feature-parity.md @@ -36,6 +36,7 @@ side by side. Status words: | Collaboration | Sharing | By design | A shared folder. Live co-authoring is Later. | | Output | Export | Later | Export scope is undecided. | | Output | Import | In progress | Required, import only (owner's decision, 2026-10-17). `notecraft-cli import onenote` reads `.one` sections and optional `.onetoc2` color metadata, then writes notebook folders in either page format. Still missing: TOC section order, ink, tags, `.onepkg`, and an import command in the window. | +| Input | Local dictation | Partial | Optional `voice` build, reviewed text insert, Undo and save. See [dictation](how-to/dictate.md) for format and hardware acceptance limits. | | Output | Add-ins, AI | Later | | | Look | Themes | Done | 14 themes, contrast checked. `themes.md`. | | Look | Ruled and grid page backgrounds | Done | View, Rule Lines: none, lines or squares behind the page, in colours each theme checks for contrast. Remembered between runs. | diff --git a/docs/gaps.md b/docs/gaps.md index f326a5d1..fadb1ed1 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -5,6 +5,18 @@ it is fixed and tested, not when it is promised. ## Product +- **Optional dictation needs real speech and device acceptance.** The `voice` build + supports reviewed text insertion through a local whisper-cli/model. + Real recognition, device permissions/disconnects, and Windows/macOS voice paths + remain unverified. Inline code/math targets, tables, source view, and cross-block + selections are unsupported. Model language capability and transcript language + metadata are not detected. Literal punctuation that the format cannot preserve + is rejected without changing the page, retaining the draft. New dictation messages + fall back to English in real locales. Faulty recognizer output can grow on disk until timeout; + memory reads are bounded. Crash leftovers are cleaned on the next job, not launch. + Conversation, hosted providers, synthesis and browser capture are later work. + See [dictation](how-to/dictate.md). + - **Ctrl+A, Ctrl+C, Ctrl+X and Ctrl+V reach the words**, from 2026-10-19. None of them was bound anywhere in the window before that: a left-to-right paragraph is an egui text box and egui answered them by itself, so the gap only showed where egui was not drawing. That diff --git a/docs/how-to/README.md b/docs/how-to/README.md index 49de64da..38e0b48d 100644 --- a/docs/how-to/README.md +++ b/docs/how-to/README.md @@ -4,6 +4,7 @@ Short guides for getting work done. Each one is a page; none is longer than a sc | Guide | What it covers | |---|---| +| [Dictate into a page](dictate.md) | Optional local speech, transcript review, supported targets and verification limits | | [Start a notebook](start-a-notebook.md) | The first notebook, sections, pages and sub-pages | | [Arrange a page in boxes](arrange-a-page.md) | Containers: moving and resizing them, and where the arrangement is kept | | [Draw diagrams and charts](draw-diagrams.md) | Mermaid diagrams and charts: where they are kept, how to see them drawn, and what happens when one cannot be | diff --git a/docs/how-to/dictate.md b/docs/how-to/dictate.md new file mode 100644 index 00000000..94a738ce --- /dev/null +++ b/docs/how-to/dictate.md @@ -0,0 +1,153 @@ +# Dictate into a page + +Dictation is an optional native feature. Build with: + +```sh +cargo build -p notecraft --features voice +``` + +Choose **Dictate** (the microphone beside the ribbon tabs) or press **F8**. The first +time, **Dictation setup** opens: choose a local `whisper.cpp` `whisper-cli` executable +and a model, then use **Test microphone** and **Test recognizer** to check both halves. +The executable and model must already exist on your computer. NoteCraft does not +download them, contact a hosted provider, or require an account. Setup stays +available from the `...` button beside **Dictate** and from **File > Options**. +Linux builds need the CPAL/ALSA development dependencies. On macOS, +`bash packaging/OSX/package.sh --voice` builds an optional voice-enabled app bundle +with the microphone usage description and hardened runtime audio-input entitlement. + +Choose the language in setup: **Detect automatically**, English, Spanish, and others. +Use a multilingual model for Spanish and other languages; models whose name ends in +`.en` understand English only. This adapter does not inspect the model's languages. + +1. Click where the words should go. With no cursor (on a new page, say), or with the + cursor in a table, code or an equation, the words become a new paragraph at the end + of the page. +2. Choose **Dictate** or press **F8**, and speak. A bar at the bottom shows **Listening**, + a level meter and the time. Tap F8 to start and again to finish, or hold it while you + talk. **Enter** or **Done** finishes; **Esc** or **Cancel** stops and discards. +3. A preview of the words appears under the line they will go into, revised every few + seconds, then replaced by the final pass when you finish. +4. Review the words, choose **Edit** to correct them, then **Insert here**, or + **Discard**. Lines become separate paragraphs. Undo takes the insertion back. + Turn off **Show the words for review before inserting them** in setup to insert as + soon as the final pass is done. + +One recording lasts at most 60 seconds. The transcript is limited to 64 KiB. +Source view and read-only pages cannot take dictation. Dictation is literal text; it +does not interpret spoken commands or reformat notes. + +If you edit the page while the words are being spoken or reviewed, **Insert here** puts +them where your cursor is at that moment and says so. If you open a different page +before inserting, the words are never moved there on their own, not even with review +turned off: the preview names the page they were for and offers **Insert on this page**. +If another program changes the page's file, reload the page and dictate again. + +With review turned off, only a finished final pass for an unchanged page and selection +goes in by itself. Moving the cursor or editing the page keeps the words for explicit +review; Insert can then choose the current cursor position. Words left from a +recording that failed (a disconnected microphone, say) wait for review. A held shortcut +ends the recording when the key comes up or when NoteCraft loses focus. + +Dictation inserts literal Span text and checks that the page format can save and reload +it unchanged before applying an edit. Existing readers and writers do not preserve +every punctuation combination. Backslashes, paired emphasis markers and tag-like +paragraph starts may be rejected. The draft stays visible for review and copying. +Edit the draft to remove unsupported punctuation before inserting. Dictation never +adds hidden characters or silently interprets the draft as formatting. + +Problems are explained in the bar with something to do about them: **Microphone +settings** opens the system's microphone privacy page (Windows and macOS), and +**Dictation setup** opens setup. + +Audio and transcripts are session-only until you choose Insert. The process +adapter temporarily writes WAV and JSON files in a private `voice-cache` folder +beside local settings, outside notebooks. It deletes job files on completion, +cancellation and failure. A later job cleans abandoned inactive files after an +interrupted process. Cleanup is best effort and is not secure erasure. Do not +put the settings/cache folder in a synced location. Raw speech and note contents +are not written to diagnostics by this feature. + +Recognition runs in background workers, one process at a time, with a default +120-second timeout per inference. Preview recognition starts after five seconds +and reloads the model for each pass. This is chunked preview, not token streaming. +A preview that fails is skipped; only the final pass can end a recording with an +error. + +The microphone opens as soon as you press the talk key. The recognizer's `--help` +check runs alongside it, once per executable while the app is open, so the first +words are not lost. On Windows the recognizer runs without a console window. +When the recognizer advertises `-sns`, non-speech tokens are suppressed, and +known silence labels such as `[BLANK_AUDIO]` or `(music)` are dropped; other words +in brackets are kept for review. + +The panel shows RECORDING only once the microphone delivers audio, with an input +level meter. If the device sends nothing for five seconds, including a stream that stalls after +recording started, recording stops with a +message. If audio arrives but stays silent for two seconds, the panel says so: check +the selected microphone, mute switches, and the system's microphone permission. Clips shorter +than a second are padded with silence; if nothing is recognized, the panel says so. +Inserted words get a space where they meet a word already on the line. +Multiple lines cannot be inserted before indexed table widths or paragraph justification +would move: use one line there, or click below the page to append all the words. +A trusted but faulty recognizer can grow its output file until timeout; reads into +memory are bounded. Only select an executable you trust. + +The engine registers `voice.start`, `voice.stop`, `voice.cancel`, +`voice.draft.set`, `voice.retarget`, `voice.commit`, and `voice.status`. +Microphone and edit commands require the local UI invocation context. CLI and MCP +cannot grant themselves that context using parameters. Status exposes lifecycle +metadata only. Existing trusted UI automation can still click native controls; +this does not add a separate security boundary around an already trusted operator. + +Conversation, notebook retrieval through speech, live hosted voice adapters, +speech synthesis, and browser capture are later milestones. An MCP server alone +does not establish compatibility with a proprietary application's voice mode. + +## Verification limits + +Automated tests cover session revisions, literal insertion, both page formats, +selection, normal history/save integration, stale targets, canonical cache paths, +audio conversion and real process cancellation. Those tests do not establish +microphone permission behavior, recognition accuracy, or language support. +Real microphone/model acceptance and Windows/macOS device testing remain required. + +## Manual native acceptance + +On Linux, Windows and macOS, test a real microphone and a real local whisper-cli/model. +Record the OS, device, recognizer version, model and language with the result. + +1. Test microphone in setup. Speak, mute it and select another device. Check the live + meter, silence warning, disconnect error and retry after reconnecting. +2. Deny system microphone access. Start dictation and check a permission or no-audio + error within five seconds after stream opening. Re-enable access and retry. On + macOS, test the packaged app; terminal consent is a separate check. +3. Dictate short sentences in two model-supported languages. Check previews, final + review, edits, Undo/Redo, save and reload in HTML and CMGFM. Try a very short press + and the 60-second limit. Try punctuation that the format cannot preserve. +4. Cancel during startup, listening, preview inference and final inference. Check the + microphone indicator turns off, recognizer exits and job files disappear. Retry + after each cancellation and after an invalid-model error. +5. Move the caret, select other text, edit the page, switch pages and change the file + externally while decoding. Check automatic insertion never retargets silently, + rejected edits retain review and explicit insertion requires a valid target. +6. Tap and hold the shortcut; release it while another window has focus. Check repeat + keys and modifiers. Setup and dialogs must not start a microphone behind them. + +An opt-in test checks actual CPAL callbacks, two seconds of non-silent sample data, +and reopening after cleanup. Speak during the test: + +```sh +cargo test -p craft-voice-runtime --features microphone actual_microphone_delivers_samples_and_releases_the_device -- --ignored --nocapture --test-threads=1 +``` + +`VOICE_CAPTURE_DEVICE` can select an exact CPAL device name. Native virtual devices +may serve recorded or synthetic audio; label that evidence as virtual input. It does +not verify a physical microphone, acoustic fidelity or OS consent prompts. + +New interface messages use English and pseudo-language catalogs. Real locales fall +back to English until reviewed translations are supplied. + +Multiple paragraphs are inserted only at a paragraph end or below the page. Inserting +several lines before existing words is refused so those words cannot move ahead of the +new transcript. The words stay in review. diff --git a/i18n/ar-XB.tsv b/i18n/ar-XB.tsv index 9fb39f04..13b69b6b 100644 --- a/i18n/ar-XB.tsv +++ b/i18n/ar-XB.tsv @@ -800,3 +800,118 @@ @id ui.welcome_wizard_finish [[RLI]]$$$Finish@@@[[PDI]] @id ui.welcome_wizard_done_title [[RLI]]$$$Thank you@@@[[PDI]] @id ui.welcome_wizard_done_body [[RLI]]$$$NoteCraft is ready to use. Your choices are saved. This setup appears again only after you reset the preferences.@@@[[PDI]] +@id voice.another_program_changed_this_page_reload_it_then_d_cc71cb4e [[RLI]]$$$Another program changed this page. Reload it, then dictate again.@@@[[PDI]] +@id voice.at_the_end_of_the_page_24302b0e [[RLI]]$$$at the end of the page@@@[[PDI]] +@id voice.at_your_cursor_e3b613bb [[RLI]]$$$at your cursor@@@[[PDI]] +@id voice.browse_c1b20f52 [[RLI]]$$$Browse…@@@[[PDI]] +@id voice.cannot_create_the_voice_cache_error_05777284 [[RLI]]$$$Cannot create the voice cache: {error}@@@[[PDI]] +@id voice.cannot_locate_the_local_settings_folder_6346b636 [[RLI]]$$$Cannot locate the local settings folder.@@@[[PDI]] +@id voice.cannot_open_the_microphone_settings_error_b3a9cb13 [[RLI]]$$$Cannot open the microphone settings: {error}@@@[[PDI]] +@id voice.cannot_read_the_page_from_disk_reload_it_then_dict_b17aa77a [[RLI]]$$$Cannot read the page from disk. Reload it, then dictate again.@@@[[PDI]] +@id voice.cannot_read_the_page_source_error_3c53fd4d [[RLI]]$$$Cannot read the page source: {error}@@@[[PDI]] +@id voice.cannot_resolve_the_voice_cache_error_902cca25 [[RLI]]$$$Cannot resolve the voice cache: {error}@@@[[PDI]] +@id voice.cannot_save_dictation_settings_error_fa8703c7 [[RLI]]$$$Cannot save dictation settings: {error}@@@[[PDI]] +@id voice.check_your_setup_b61b421a [[RLI]]$$$Check your setup@@@[[PDI]] +@id voice.chinese_15dc1886 [[RLI]]$$$Chinese@@@[[PDI]] +@id voice.choose_a_cursor_position_in_the_paragraph_s_words_6538d9ab [[RLI]]$$$Choose a cursor position in the paragraph's words, outside formatting marks.@@@[[PDI]] +@id voice.choose_a_cursor_position_outside_formatting_marks_54a40d44 [[RLI]]$$$Choose a cursor position outside formatting marks.@@@[[PDI]] +@id voice.choose_a_paragraph_before_inserting_3f802b0c [[RLI]]$$$Choose a paragraph before inserting.@@@[[PDI]] +@id voice.choose_a_paragraph_without_inline_code_or_equation_bf91894f [[RLI]]$$$Choose a paragraph without inline code or equations before dictating.@@@[[PDI]] +@id voice.choose_a_program_and_a_model_to_finish_82dc257d [[RLI]]$$$Choose a program and a model to finish.@@@[[PDI]] +@id voice.choose_a_speech_model_f18c2ce5 [[RLI]]$$$Choose a speech model@@@[[PDI]] +@id voice.choose_the_speech_recognizer_and_a_model_to_start_4a44889b [[RLI]]$$$Choose the speech recognizer and a model to start dictating.@@@[[PDI]] +@id voice.choose_whisper_cli_db09c1c2 [[RLI]]$$$Choose whisper-cli@@@[[PDI]] +@id voice.click_where_the_words_should_go_and_choose_insert_473a713a [[RLI]]$$$Click where the words should go and choose Insert again.@@@[[PDI]] +@id voice.detect_automatically_25a36033 [[RLI]]$$$Detect automatically@@@[[PDI]] +@id voice.dictate_3eaff69c [[RLI]]$$$Dictate@@@[[PDI]] +@id voice.dictate_at_the_cursor_or_at_the_end_of_the_page_12635b46 [[RLI]]$$$Dictate at the cursor, or at the end of the page@@@[[PDI]] +@id voice.dictated_words_8c96e44a [[RLI]]$$$Dictated words@@@[[PDI]] +@id voice.dictation_2a7fe421 [[RLI]]$$$Dictation@@@[[PDI]] +@id voice.dictation_currently_supports_paragraphs_headings_q_4db15ad2 [[RLI]]$$$Dictation currently supports paragraphs, headings, quotes, and list text.@@@[[PDI]] +@id voice.dictation_discarded_2159761f [[RLI]]$$$Dictation discarded.@@@[[PDI]] +@id voice.dictation_inserted_ctrl_z_takes_it_back_a6eb393d [[RLI]]$$$Dictation inserted. Ctrl+Z takes it back.@@@[[PDI]] +@id voice.dictation_runs_on_this_computer_with_whisper_cpp_c_fe751c46 [[RLI]]$$$Dictation runs on this computer with whisper.cpp. Choose its whisper-cli program and a speech model you have downloaded. NoteCraft never downloads them and never sends audio anywhere.@@@[[PDI]] +@id voice.dictation_setup_6be1e51d [[RLI]]$$$Dictation setup…@@@[[PDI]] +@id voice.dictation_setup_b70a0aa8 [[RLI]]$$$Dictation setup@@@[[PDI]] +@id voice.discard_36fff63c [[RLI]]$$$Discard@@@[[PDI]] +@id voice.dismiss_70afe9ef [[RLI]]$$$Dismiss@@@[[PDI]] +@id voice.done_e9b450d1 [[RLI]]$$$Done@@@[[PDI]] +@id voice.done_editing_b4938456 [[RLI]]$$$Done editing@@@[[PDI]] +@id voice.dutch_334a85f1 [[RLI]]$$$Dutch@@@[[PDI]] +@id voice.enter_or_shortcut_25ef81f2 [[RLI]]$$$Enter or {shortcut}@@@[[PDI]] +@id voice.esc_1f7a4f9e [[RLI]]$$$Esc@@@[[PDI]] +@id voice.finish_dictating_0f7d9f4b [[RLI]]$$$Finish dictating@@@[[PDI]] +@id voice.finish_dictating_enter_esc_cancels_2151a686 [[RLI]]$$$Finish dictating (Enter). Esc cancels.@@@[[PDI]] +@id voice.finishing_90218af7 [[RLI]]$$$Finishing…@@@[[PDI]] +@id voice.found_bbba8413 [[RLI]]$$$Found@@@[[PDI]] +@id voice.french_44389f6a [[RLI]]$$$French@@@[[PDI]] +@id voice.german_da91388c [[RLI]]$$$German@@@[[PDI]] +@id voice.heard_you_loudest_sound_decibels_db_ce75d644 [[RLI]]$$$Heard you. Loudest sound: {decibels} dB.@@@[[PDI]] +@id voice.insert_here_1fe34d33 [[RLI]]$$$Insert here@@@[[PDI]] +@id voice.insert_on_this_page_f7f93d5b [[RLI]]$$$Insert on this page@@@[[PDI]] +@id voice.insert_or_discard_the_dictated_words_first_df8983ce [[RLI]]$$$Insert or discard the dictated words first@@@[[PDI]] +@id voice.italian_485b57ad [[RLI]]$$$Italian@@@[[PDI]] +@id voice.japanese_9239c22d [[RLI]]$$$Japanese@@@[[PDI]] +@id voice.keep_the_settings_folder_outside_notebooks_before_a55ba6bc [[RLI]]$$$Keep the settings folder outside notebooks before enabling dictation.@@@[[PDI]] +@id voice.korean_dec66c5b [[RLI]]$$$Korean@@@[[PDI]] +@id voice.language_89b86ab0 [[RLI]]$$$Language@@@[[PDI]] +@id voice.listening_dc353487 [[RLI]]$$$Listening@@@[[PDI]] +@id voice.loading_the_model_45a4f448 [[RLI]]$$$Loading the model…@@@[[PDI]] +@id voice.microphone_24280537 [[RLI]]$$$Microphone@@@[[PDI]] +@id voice.microphone_level_7b9bf79a [[RLI]]$$$Microphone level@@@[[PDI]] +@id voice.microphone_settings_2a04af24 [[RLI]]$$$Microphone settings@@@[[PDI]] +@id voice.models_whose_name_ends_in_en_understand_english_on_41b266d0 [[RLI]]$$$Models whose name ends in .en understand English only.@@@[[PDI]] +@id voice.no_audio_arrived_from_the_microphone_check_the_sel_68498934 [[RLI]]$$$No audio arrived from the microphone. Check the selected microphone and the system's microphone permission.@@@[[PDI]] +@id voice.no_sound_from_the_microphone_yet_check_that_the_ri_84320e7b [[RLI]]$$$No sound from the microphone yet. Check that the right microphone is selected and not muted.@@@[[PDI]] +@id voice.no_speech_was_recognized_7ff9a913 [[RLI]]$$$No speech was recognized.@@@[[PDI]] +@id voice.not_found_475c8486 [[RLI]]$$$Not found@@@[[PDI]] +@id voice.notecraft_can_t_hear_your_microphone_check_that_it_5f82d9f8 [[RLI]]$$$NoteCraft can't hear your microphone. Check that it is connected and not muted, that the right one is chosen in Dictation setup, and that {system} lets desktop apps use it.@@@[[PDI]] +@id voice.notecraft_couldn_t_save_the_page_so_it_won_t_add_d_5c189933 [[RLI]]$$$NoteCraft couldn't save the page, so it won't add dictated words to it yet.@@@[[PDI]] +@id voice.open_a_page_to_dictate_into_0300e9a6 [[RLI]]$$$Open a page to dictate into.@@@[[PDI]] +@id voice.open_system_microphone_settings_11e3ebb5 [[RLI]]$$$Open {system} microphone settings@@@[[PDI]] +@id voice.page_view_6a49efac [[RLI]]$$$Page view@@@[[PDI]] +@id voice.portuguese_23882c57 [[RLI]]$$$Portuguese@@@[[PDI]] +@id voice.preview_going_place_2eeaa180 [[RLI]]$$$Preview, going {place}@@@[[PDI]] +@id voice.ready_to_insert_place_7f3f4d09 [[RLI]]$$$Ready to insert {place}@@@[[PDI]] +@id voice.recognition_took_too_long_a_smaller_model_such_as_fff73af3 [[RLI]]$$$Recognition took too long. A smaller model, such as base or tiny, is faster.@@@[[PDI]] +@id voice.recognizer_program_whisper_cli_eb53d9da [[RLI]]$$$Recognizer program (whisper-cli)@@@[[PDI]] +@id voice.refresh_56e3badc [[RLI]]$$$Refresh@@@[[PDI]] +@id voice.reopen_the_app_to_start_a_new_target_sequence_907696fc [[RLI]]$$$Reopen the app to start a new target sequence.@@@[[PDI]] +@id voice.return_to_the_page_view_before_dictating_2c363339 [[RLI]]$$$Return to the page view before dictating.@@@[[PDI]] +@id voice.say_something_9aa1505d [[RLI]]$$$Say something…@@@[[PDI]] +@id voice.shortcut_e2012dea [[RLI]]$$$Shortcut@@@[[PDI]] +@id voice.show_the_words_for_review_before_inserting_them_d313939a [[RLI]]$$$Show the words for review before inserting them@@@[[PDI]] +@id voice.spanish_8df7f1b3 [[RLI]]$$$Spanish@@@[[PDI]] +@id voice.speech_model_3c5f0bfe [[RLI]]$$$Speech model@@@[[PDI]] +@id voice.starting_the_microphone_26a22bd2 [[RLI]]$$$Starting the microphone…@@@[[PDI]] +@id voice.switch_back_to_the_page_view_to_dictate_e3deb7f8 [[RLI]]$$$Switch back to the page view to dictate.@@@[[PDI]] +@id voice.system_default_9d8d3808 [[RLI]]$$$System default@@@[[PDI]] +@id voice.tap_shortcut_to_start_and_again_to_finish_or_hold_6f592a2d [[RLI]]$$$Tap {shortcut} to start and again to finish, or hold it while you talk.@@@[[PDI]] +@id voice.test_microphone_49e76fed [[RLI]]$$$Test microphone@@@[[PDI]] +@id voice.test_recognizer_f042c45f [[RLI]]$$$Test recognizer@@@[[PDI]] +@id voice.the_computer_fell_behind_while_recording_try_again_e30d7359 [[RLI]]$$$The computer fell behind while recording. Try again; closing busy apps helps if it keeps happening.@@@[[PDI]] +@id voice.the_displayed_draft_is_not_the_accepted_revision_r_3002e242 [[RLI]]$$$The displayed draft is not the accepted revision. Review it again before inserting.@@@[[PDI]] +@id voice.the_microphone_sends_audio_but_it_is_silent_check_d7cf0faa [[RLI]]$$$The microphone sends audio, but it is silent. Check mute switches and that the right microphone is chosen.@@@[[PDI]] +@id voice.the_page_changed_while_you_were_talking_click_wher_2d22a184 [[RLI]]$$$The page changed while you were talking. Click where the words should go and choose Insert again.@@@[[PDI]] +@id voice.the_page_changed_while_you_were_talking_so_the_wor_41829e1a [[RLI]]$$$The page or cursor changed, so the words went where your cursor is now. Ctrl+Z takes them back.@@@[[PDI]] +@id voice.the_page_closed_fc708395 [[RLI]]$$$The page closed.@@@[[PDI]] +@id voice.the_paragraph_closed_868b6d17 [[RLI]]$$$The paragraph closed.@@@[[PDI]] +@id voice.the_recognizer_and_the_model_work_11e481ee [[RLI]]$$$The recognizer and the model work.@@@[[PDI]] +@id voice.the_speech_recognizer_isn_t_set_up_correctly_open_57ac2283 [[RLI]]$$$The speech recognizer isn't set up correctly. Open Dictation setup to check the program and the model.@@@[[PDI]] +@id voice.the_speech_recognizer_model_language_microphone_an_3fece2c0 [[RLI]]$$$The speech recognizer, model, language, microphone and shortcut used to dictate into pages.@@@[[PDI]] +@id voice.the_voice_cache_resolves_into_a_notebook_choose_a_1f9a56e8 [[RLI]]$$$The voice cache resolves into a notebook. Choose a local settings folder outside notebooks.@@@[[PDI]] +@id voice.the_words_are_longer_than_64_kib_shorten_them_befo_74a29916 [[RLI]]$$$The words are longer than 64 KiB. Shorten them before inserting.@@@[[PDI]] +@id voice.there_are_no_words_to_insert_d397d6a5 [[RLI]]$$$There are no words to insert.@@@[[PDI]] +@id voice.these_words_were_for_the_page_title_c7a50f95 [[RLI]]$$$These words were for the page "{title}"@@@[[PDI]] +@id voice.these_words_were_for_the_page_title_choose_insert_27760264 [[RLI]]$$$These words were for the page "{title}". Choose Insert on this page to put them here instead.@@@[[PDI]] +@id voice.these_words_would_become_a_page_tag_edit_the_tag_m_bfed5fa8 [[RLI]]$$$These words would become a page tag. Edit the tag marker before inserting.@@@[[PDI]] +@id voice.this_cursor_position_cannot_preserve_the_existing_ccbd81ed [[RLI]]$$$This cursor position cannot preserve the existing text. Click below the page to append the words instead.@@@[[PDI]] +@id voice.this_page_format_cannot_preserve_these_literal_wor_fc42673c [[RLI]]$$$This page format cannot preserve these literal words yet. Edit punctuation in the draft, or copy the draft before discarding it.@@@[[PDI]] +@id voice.this_page_is_read_only_here_so_dictation_can_t_wri_47a8e613 [[RLI]]$$$This page is read-only here, so dictation can't write in it.@@@[[PDI]] +@id voice.try_again_042c862e [[RLI]]$$$Try again@@@[[PDI]] +@id voice.try_again_a_little_closer_to_the_microphone_30f96ada [[RLI]]$$$Try again, a little closer to the microphone.@@@[[PDI]] +@id voice.where_to_get_whisper_cli_and_a_model_3b6c0fcb [[RLI]]$$$Where to get whisper-cli and a model@@@[[PDI]] +@id voice.your_system_3a1356ff [[RLI]]$$$your system@@@[[PDI]] +@id voice.layout_would_shift [[RLI]]$$$This insertion would move paragraph or table layout. Insert one line, or click below the page to append the words.@@@[[PDI]] +@id voice.target_changed_review_required [[RLI]]$$$The page or cursor changed. Review the words and choose Insert to use the current position.@@@[[PDI]] +@id voice.multiline_needs_line_end [[RLI]]$$$Multiple lines need the cursor at the end of a paragraph. Move there, or click below the page to append the words.@@@[[PDI]] diff --git a/i18n/en-XA.tsv b/i18n/en-XA.tsv index adb16e8d..81739005 100644 --- a/i18n/en-XA.tsv +++ b/i18n/en-XA.tsv @@ -800,3 +800,118 @@ @id ui.welcome_wizard_finish $$$Finish@@@ @id ui.welcome_wizard_done_title $$$Thank you@@@ @id ui.welcome_wizard_done_body $$$NoteCraft is ready to use. Your choices are saved. This setup appears again only after you reset the preferences.@@@ +@id voice.another_program_changed_this_page_reload_it_then_d_cc71cb4e $$$Another program changed this page. Reload it, then dictate again.@@@ +@id voice.at_the_end_of_the_page_24302b0e $$$at the end of the page@@@ +@id voice.at_your_cursor_e3b613bb $$$at your cursor@@@ +@id voice.browse_c1b20f52 $$$Browse…@@@ +@id voice.cannot_create_the_voice_cache_error_05777284 $$$Cannot create the voice cache: {error}@@@ +@id voice.cannot_locate_the_local_settings_folder_6346b636 $$$Cannot locate the local settings folder.@@@ +@id voice.cannot_open_the_microphone_settings_error_b3a9cb13 $$$Cannot open the microphone settings: {error}@@@ +@id voice.cannot_read_the_page_from_disk_reload_it_then_dict_b17aa77a $$$Cannot read the page from disk. Reload it, then dictate again.@@@ +@id voice.cannot_read_the_page_source_error_3c53fd4d $$$Cannot read the page source: {error}@@@ +@id voice.cannot_resolve_the_voice_cache_error_902cca25 $$$Cannot resolve the voice cache: {error}@@@ +@id voice.cannot_save_dictation_settings_error_fa8703c7 $$$Cannot save dictation settings: {error}@@@ +@id voice.check_your_setup_b61b421a $$$Check your setup@@@ +@id voice.chinese_15dc1886 $$$Chinese@@@ +@id voice.choose_a_cursor_position_in_the_paragraph_s_words_6538d9ab $$$Choose a cursor position in the paragraph's words, outside formatting marks.@@@ +@id voice.choose_a_cursor_position_outside_formatting_marks_54a40d44 $$$Choose a cursor position outside formatting marks.@@@ +@id voice.choose_a_paragraph_before_inserting_3f802b0c $$$Choose a paragraph before inserting.@@@ +@id voice.choose_a_paragraph_without_inline_code_or_equation_bf91894f $$$Choose a paragraph without inline code or equations before dictating.@@@ +@id voice.choose_a_program_and_a_model_to_finish_82dc257d $$$Choose a program and a model to finish.@@@ +@id voice.choose_a_speech_model_f18c2ce5 $$$Choose a speech model@@@ +@id voice.choose_the_speech_recognizer_and_a_model_to_start_4a44889b $$$Choose the speech recognizer and a model to start dictating.@@@ +@id voice.choose_whisper_cli_db09c1c2 $$$Choose whisper-cli@@@ +@id voice.click_where_the_words_should_go_and_choose_insert_473a713a $$$Click where the words should go and choose Insert again.@@@ +@id voice.detect_automatically_25a36033 $$$Detect automatically@@@ +@id voice.dictate_3eaff69c $$$Dictate@@@ +@id voice.dictate_at_the_cursor_or_at_the_end_of_the_page_12635b46 $$$Dictate at the cursor, or at the end of the page@@@ +@id voice.dictated_words_8c96e44a $$$Dictated words@@@ +@id voice.dictation_2a7fe421 $$$Dictation@@@ +@id voice.dictation_currently_supports_paragraphs_headings_q_4db15ad2 $$$Dictation currently supports paragraphs, headings, quotes, and list text.@@@ +@id voice.dictation_discarded_2159761f $$$Dictation discarded.@@@ +@id voice.dictation_inserted_ctrl_z_takes_it_back_a6eb393d $$$Dictation inserted. Ctrl+Z takes it back.@@@ +@id voice.dictation_runs_on_this_computer_with_whisper_cpp_c_fe751c46 $$$Dictation runs on this computer with whisper.cpp. Choose its whisper-cli program and a speech model you have downloaded. NoteCraft never downloads them and never sends audio anywhere.@@@ +@id voice.dictation_setup_6be1e51d $$$Dictation setup…@@@ +@id voice.dictation_setup_b70a0aa8 $$$Dictation setup@@@ +@id voice.discard_36fff63c $$$Discard@@@ +@id voice.dismiss_70afe9ef $$$Dismiss@@@ +@id voice.done_e9b450d1 $$$Done@@@ +@id voice.done_editing_b4938456 $$$Done editing@@@ +@id voice.dutch_334a85f1 $$$Dutch@@@ +@id voice.enter_or_shortcut_25ef81f2 $$$Enter or {shortcut}@@@ +@id voice.esc_1f7a4f9e $$$Esc@@@ +@id voice.finish_dictating_0f7d9f4b $$$Finish dictating@@@ +@id voice.finish_dictating_enter_esc_cancels_2151a686 $$$Finish dictating (Enter). Esc cancels.@@@ +@id voice.finishing_90218af7 $$$Finishing…@@@ +@id voice.found_bbba8413 $$$Found@@@ +@id voice.french_44389f6a $$$French@@@ +@id voice.german_da91388c $$$German@@@ +@id voice.heard_you_loudest_sound_decibels_db_ce75d644 $$$Heard you. Loudest sound: {decibels} dB.@@@ +@id voice.insert_here_1fe34d33 $$$Insert here@@@ +@id voice.insert_on_this_page_f7f93d5b $$$Insert on this page@@@ +@id voice.insert_or_discard_the_dictated_words_first_df8983ce $$$Insert or discard the dictated words first@@@ +@id voice.italian_485b57ad $$$Italian@@@ +@id voice.japanese_9239c22d $$$Japanese@@@ +@id voice.keep_the_settings_folder_outside_notebooks_before_a55ba6bc $$$Keep the settings folder outside notebooks before enabling dictation.@@@ +@id voice.korean_dec66c5b $$$Korean@@@ +@id voice.language_89b86ab0 $$$Language@@@ +@id voice.listening_dc353487 $$$Listening@@@ +@id voice.loading_the_model_45a4f448 $$$Loading the model…@@@ +@id voice.microphone_24280537 $$$Microphone@@@ +@id voice.microphone_level_7b9bf79a $$$Microphone level@@@ +@id voice.microphone_settings_2a04af24 $$$Microphone settings@@@ +@id voice.models_whose_name_ends_in_en_understand_english_on_41b266d0 $$$Models whose name ends in .en understand English only.@@@ +@id voice.no_audio_arrived_from_the_microphone_check_the_sel_68498934 $$$No audio arrived from the microphone. Check the selected microphone and the system's microphone permission.@@@ +@id voice.no_sound_from_the_microphone_yet_check_that_the_ri_84320e7b $$$No sound from the microphone yet. Check that the right microphone is selected and not muted.@@@ +@id voice.no_speech_was_recognized_7ff9a913 $$$No speech was recognized.@@@ +@id voice.not_found_475c8486 $$$Not found@@@ +@id voice.notecraft_can_t_hear_your_microphone_check_that_it_5f82d9f8 $$$NoteCraft can't hear your microphone. Check that it is connected and not muted, that the right one is chosen in Dictation setup, and that {system} lets desktop apps use it.@@@ +@id voice.notecraft_couldn_t_save_the_page_so_it_won_t_add_d_5c189933 $$$NoteCraft couldn't save the page, so it won't add dictated words to it yet.@@@ +@id voice.open_a_page_to_dictate_into_0300e9a6 $$$Open a page to dictate into.@@@ +@id voice.open_system_microphone_settings_11e3ebb5 $$$Open {system} microphone settings@@@ +@id voice.page_view_6a49efac $$$Page view@@@ +@id voice.portuguese_23882c57 $$$Portuguese@@@ +@id voice.preview_going_place_2eeaa180 $$$Preview, going {place}@@@ +@id voice.ready_to_insert_place_7f3f4d09 $$$Ready to insert {place}@@@ +@id voice.recognition_took_too_long_a_smaller_model_such_as_fff73af3 $$$Recognition took too long. A smaller model, such as base or tiny, is faster.@@@ +@id voice.recognizer_program_whisper_cli_eb53d9da $$$Recognizer program (whisper-cli)@@@ +@id voice.refresh_56e3badc $$$Refresh@@@ +@id voice.reopen_the_app_to_start_a_new_target_sequence_907696fc $$$Reopen the app to start a new target sequence.@@@ +@id voice.return_to_the_page_view_before_dictating_2c363339 $$$Return to the page view before dictating.@@@ +@id voice.say_something_9aa1505d $$$Say something…@@@ +@id voice.shortcut_e2012dea $$$Shortcut@@@ +@id voice.show_the_words_for_review_before_inserting_them_d313939a $$$Show the words for review before inserting them@@@ +@id voice.spanish_8df7f1b3 $$$Spanish@@@ +@id voice.speech_model_3c5f0bfe $$$Speech model@@@ +@id voice.starting_the_microphone_26a22bd2 $$$Starting the microphone…@@@ +@id voice.switch_back_to_the_page_view_to_dictate_e3deb7f8 $$$Switch back to the page view to dictate.@@@ +@id voice.system_default_9d8d3808 $$$System default@@@ +@id voice.tap_shortcut_to_start_and_again_to_finish_or_hold_6f592a2d $$$Tap {shortcut} to start and again to finish, or hold it while you talk.@@@ +@id voice.test_microphone_49e76fed $$$Test microphone@@@ +@id voice.test_recognizer_f042c45f $$$Test recognizer@@@ +@id voice.the_computer_fell_behind_while_recording_try_again_e30d7359 $$$The computer fell behind while recording. Try again; closing busy apps helps if it keeps happening.@@@ +@id voice.the_displayed_draft_is_not_the_accepted_revision_r_3002e242 $$$The displayed draft is not the accepted revision. Review it again before inserting.@@@ +@id voice.the_microphone_sends_audio_but_it_is_silent_check_d7cf0faa $$$The microphone sends audio, but it is silent. Check mute switches and that the right microphone is chosen.@@@ +@id voice.the_page_changed_while_you_were_talking_click_wher_2d22a184 $$$The page changed while you were talking. Click where the words should go and choose Insert again.@@@ +@id voice.the_page_changed_while_you_were_talking_so_the_wor_41829e1a $$$The page or cursor changed, so the words went where your cursor is now. Ctrl+Z takes them back.@@@ +@id voice.the_page_closed_fc708395 $$$The page closed.@@@ +@id voice.the_paragraph_closed_868b6d17 $$$The paragraph closed.@@@ +@id voice.the_recognizer_and_the_model_work_11e481ee $$$The recognizer and the model work.@@@ +@id voice.the_speech_recognizer_isn_t_set_up_correctly_open_57ac2283 $$$The speech recognizer isn't set up correctly. Open Dictation setup to check the program and the model.@@@ +@id voice.the_speech_recognizer_model_language_microphone_an_3fece2c0 $$$The speech recognizer, model, language, microphone and shortcut used to dictate into pages.@@@ +@id voice.the_voice_cache_resolves_into_a_notebook_choose_a_1f9a56e8 $$$The voice cache resolves into a notebook. Choose a local settings folder outside notebooks.@@@ +@id voice.the_words_are_longer_than_64_kib_shorten_them_befo_74a29916 $$$The words are longer than 64 KiB. Shorten them before inserting.@@@ +@id voice.there_are_no_words_to_insert_d397d6a5 $$$There are no words to insert.@@@ +@id voice.these_words_were_for_the_page_title_c7a50f95 $$$These words were for the page "{title}"@@@ +@id voice.these_words_were_for_the_page_title_choose_insert_27760264 $$$These words were for the page "{title}". Choose Insert on this page to put them here instead.@@@ +@id voice.these_words_would_become_a_page_tag_edit_the_tag_m_bfed5fa8 $$$These words would become a page tag. Edit the tag marker before inserting.@@@ +@id voice.this_cursor_position_cannot_preserve_the_existing_ccbd81ed $$$This cursor position cannot preserve the existing text. Click below the page to append the words instead.@@@ +@id voice.this_page_format_cannot_preserve_these_literal_wor_fc42673c $$$This page format cannot preserve these literal words yet. Edit punctuation in the draft, or copy the draft before discarding it.@@@ +@id voice.this_page_is_read_only_here_so_dictation_can_t_wri_47a8e613 $$$This page is read-only here, so dictation can't write in it.@@@ +@id voice.try_again_042c862e $$$Try again@@@ +@id voice.try_again_a_little_closer_to_the_microphone_30f96ada $$$Try again, a little closer to the microphone.@@@ +@id voice.where_to_get_whisper_cli_and_a_model_3b6c0fcb $$$Where to get whisper-cli and a model@@@ +@id voice.your_system_3a1356ff $$$your system@@@ +@id voice.layout_would_shift $$$This insertion would move paragraph or table layout. Insert one line, or click below the page to append the words.@@@ +@id voice.target_changed_review_required $$$The page or cursor changed. Review the words and choose Insert to use the current position.@@@ +@id voice.multiline_needs_line_end $$$Multiple lines need the cursor at the end of a paragraph. Move there, or click below the page to append the words.@@@ diff --git a/i18n/en.tsv b/i18n/en.tsv index d9e26e3d..c0ff03df 100644 --- a/i18n/en.tsv +++ b/i18n/en.tsv @@ -800,3 +800,118 @@ @id ui.welcome_wizard_finish Finish @id ui.welcome_wizard_done_title Thank you @id ui.welcome_wizard_done_body NoteCraft is ready to use. Your choices are saved. This setup appears again only after you reset the preferences. +@id voice.another_program_changed_this_page_reload_it_then_d_cc71cb4e Another program changed this page. Reload it, then dictate again. +@id voice.at_the_end_of_the_page_24302b0e at the end of the page +@id voice.at_your_cursor_e3b613bb at your cursor +@id voice.browse_c1b20f52 Browse… +@id voice.cannot_create_the_voice_cache_error_05777284 Cannot create the voice cache: {error} +@id voice.cannot_locate_the_local_settings_folder_6346b636 Cannot locate the local settings folder. +@id voice.cannot_open_the_microphone_settings_error_b3a9cb13 Cannot open the microphone settings: {error} +@id voice.cannot_read_the_page_from_disk_reload_it_then_dict_b17aa77a Cannot read the page from disk. Reload it, then dictate again. +@id voice.cannot_read_the_page_source_error_3c53fd4d Cannot read the page source: {error} +@id voice.cannot_resolve_the_voice_cache_error_902cca25 Cannot resolve the voice cache: {error} +@id voice.cannot_save_dictation_settings_error_fa8703c7 Cannot save dictation settings: {error} +@id voice.check_your_setup_b61b421a Check your setup +@id voice.chinese_15dc1886 Chinese +@id voice.choose_a_cursor_position_in_the_paragraph_s_words_6538d9ab Choose a cursor position in the paragraph's words, outside formatting marks. +@id voice.choose_a_cursor_position_outside_formatting_marks_54a40d44 Choose a cursor position outside formatting marks. +@id voice.choose_a_paragraph_before_inserting_3f802b0c Choose a paragraph before inserting. +@id voice.choose_a_paragraph_without_inline_code_or_equation_bf91894f Choose a paragraph without inline code or equations before dictating. +@id voice.choose_a_program_and_a_model_to_finish_82dc257d Choose a program and a model to finish. +@id voice.choose_a_speech_model_f18c2ce5 Choose a speech model +@id voice.choose_the_speech_recognizer_and_a_model_to_start_4a44889b Choose the speech recognizer and a model to start dictating. +@id voice.choose_whisper_cli_db09c1c2 Choose whisper-cli +@id voice.click_where_the_words_should_go_and_choose_insert_473a713a Click where the words should go and choose Insert again. +@id voice.detect_automatically_25a36033 Detect automatically +@id voice.dictate_3eaff69c Dictate +@id voice.dictate_at_the_cursor_or_at_the_end_of_the_page_12635b46 Dictate at the cursor, or at the end of the page +@id voice.dictated_words_8c96e44a Dictated words +@id voice.dictation_2a7fe421 Dictation +@id voice.dictation_currently_supports_paragraphs_headings_q_4db15ad2 Dictation currently supports paragraphs, headings, quotes, and list text. +@id voice.dictation_discarded_2159761f Dictation discarded. +@id voice.dictation_inserted_ctrl_z_takes_it_back_a6eb393d Dictation inserted. Ctrl+Z takes it back. +@id voice.dictation_runs_on_this_computer_with_whisper_cpp_c_fe751c46 Dictation runs on this computer with whisper.cpp. Choose its whisper-cli program and a speech model you have downloaded. NoteCraft never downloads them and never sends audio anywhere. +@id voice.dictation_setup_6be1e51d Dictation setup… +@id voice.dictation_setup_b70a0aa8 Dictation setup +@id voice.discard_36fff63c Discard +@id voice.dismiss_70afe9ef Dismiss +@id voice.done_e9b450d1 Done +@id voice.done_editing_b4938456 Done editing +@id voice.dutch_334a85f1 Dutch +@id voice.enter_or_shortcut_25ef81f2 Enter or {shortcut} +@id voice.esc_1f7a4f9e Esc +@id voice.finish_dictating_0f7d9f4b Finish dictating +@id voice.finish_dictating_enter_esc_cancels_2151a686 Finish dictating (Enter). Esc cancels. +@id voice.finishing_90218af7 Finishing… +@id voice.found_bbba8413 Found +@id voice.french_44389f6a French +@id voice.german_da91388c German +@id voice.heard_you_loudest_sound_decibels_db_ce75d644 Heard you. Loudest sound: {decibels} dB. +@id voice.insert_here_1fe34d33 Insert here +@id voice.insert_on_this_page_f7f93d5b Insert on this page +@id voice.insert_or_discard_the_dictated_words_first_df8983ce Insert or discard the dictated words first +@id voice.italian_485b57ad Italian +@id voice.japanese_9239c22d Japanese +@id voice.keep_the_settings_folder_outside_notebooks_before_a55ba6bc Keep the settings folder outside notebooks before enabling dictation. +@id voice.korean_dec66c5b Korean +@id voice.language_89b86ab0 Language +@id voice.listening_dc353487 Listening +@id voice.loading_the_model_45a4f448 Loading the model… +@id voice.microphone_24280537 Microphone +@id voice.microphone_level_7b9bf79a Microphone level +@id voice.microphone_settings_2a04af24 Microphone settings +@id voice.models_whose_name_ends_in_en_understand_english_on_41b266d0 Models whose name ends in .en understand English only. +@id voice.no_audio_arrived_from_the_microphone_check_the_sel_68498934 No audio arrived from the microphone. Check the selected microphone and the system's microphone permission. +@id voice.no_sound_from_the_microphone_yet_check_that_the_ri_84320e7b No sound from the microphone yet. Check that the right microphone is selected and not muted. +@id voice.no_speech_was_recognized_7ff9a913 No speech was recognized. +@id voice.not_found_475c8486 Not found +@id voice.notecraft_can_t_hear_your_microphone_check_that_it_5f82d9f8 NoteCraft can't hear your microphone. Check that it is connected and not muted, that the right one is chosen in Dictation setup, and that {system} lets desktop apps use it. +@id voice.notecraft_couldn_t_save_the_page_so_it_won_t_add_d_5c189933 NoteCraft couldn't save the page, so it won't add dictated words to it yet. +@id voice.open_a_page_to_dictate_into_0300e9a6 Open a page to dictate into. +@id voice.open_system_microphone_settings_11e3ebb5 Open {system} microphone settings +@id voice.page_view_6a49efac Page view +@id voice.portuguese_23882c57 Portuguese +@id voice.preview_going_place_2eeaa180 Preview, going {place} +@id voice.ready_to_insert_place_7f3f4d09 Ready to insert {place} +@id voice.recognition_took_too_long_a_smaller_model_such_as_fff73af3 Recognition took too long. A smaller model, such as base or tiny, is faster. +@id voice.recognizer_program_whisper_cli_eb53d9da Recognizer program (whisper-cli) +@id voice.refresh_56e3badc Refresh +@id voice.reopen_the_app_to_start_a_new_target_sequence_907696fc Reopen the app to start a new target sequence. +@id voice.return_to_the_page_view_before_dictating_2c363339 Return to the page view before dictating. +@id voice.say_something_9aa1505d Say something… +@id voice.shortcut_e2012dea Shortcut +@id voice.show_the_words_for_review_before_inserting_them_d313939a Show the words for review before inserting them +@id voice.spanish_8df7f1b3 Spanish +@id voice.speech_model_3c5f0bfe Speech model +@id voice.starting_the_microphone_26a22bd2 Starting the microphone… +@id voice.switch_back_to_the_page_view_to_dictate_e3deb7f8 Switch back to the page view to dictate. +@id voice.system_default_9d8d3808 System default +@id voice.tap_shortcut_to_start_and_again_to_finish_or_hold_6f592a2d Tap {shortcut} to start and again to finish, or hold it while you talk. +@id voice.test_microphone_49e76fed Test microphone +@id voice.test_recognizer_f042c45f Test recognizer +@id voice.the_computer_fell_behind_while_recording_try_again_e30d7359 The computer fell behind while recording. Try again; closing busy apps helps if it keeps happening. +@id voice.the_displayed_draft_is_not_the_accepted_revision_r_3002e242 The displayed draft is not the accepted revision. Review it again before inserting. +@id voice.the_microphone_sends_audio_but_it_is_silent_check_d7cf0faa The microphone sends audio, but it is silent. Check mute switches and that the right microphone is chosen. +@id voice.the_page_changed_while_you_were_talking_click_wher_2d22a184 The page changed while you were talking. Click where the words should go and choose Insert again. +@id voice.the_page_changed_while_you_were_talking_so_the_wor_41829e1a The page or cursor changed, so the words went where your cursor is now. Ctrl+Z takes them back. +@id voice.the_page_closed_fc708395 The page closed. +@id voice.the_paragraph_closed_868b6d17 The paragraph closed. +@id voice.the_recognizer_and_the_model_work_11e481ee The recognizer and the model work. +@id voice.the_speech_recognizer_isn_t_set_up_correctly_open_57ac2283 The speech recognizer isn't set up correctly. Open Dictation setup to check the program and the model. +@id voice.the_speech_recognizer_model_language_microphone_an_3fece2c0 The speech recognizer, model, language, microphone and shortcut used to dictate into pages. +@id voice.the_voice_cache_resolves_into_a_notebook_choose_a_1f9a56e8 The voice cache resolves into a notebook. Choose a local settings folder outside notebooks. +@id voice.the_words_are_longer_than_64_kib_shorten_them_befo_74a29916 The words are longer than 64 KiB. Shorten them before inserting. +@id voice.there_are_no_words_to_insert_d397d6a5 There are no words to insert. +@id voice.these_words_were_for_the_page_title_c7a50f95 These words were for the page "{title}" +@id voice.these_words_were_for_the_page_title_choose_insert_27760264 These words were for the page "{title}". Choose Insert on this page to put them here instead. +@id voice.these_words_would_become_a_page_tag_edit_the_tag_m_bfed5fa8 These words would become a page tag. Edit the tag marker before inserting. +@id voice.this_cursor_position_cannot_preserve_the_existing_ccbd81ed This cursor position cannot preserve the existing text. Click below the page to append the words instead. +@id voice.this_page_format_cannot_preserve_these_literal_wor_fc42673c This page format cannot preserve these literal words yet. Edit punctuation in the draft, or copy the draft before discarding it. +@id voice.this_page_is_read_only_here_so_dictation_can_t_wri_47a8e613 This page is read-only here, so dictation can't write in it. +@id voice.try_again_042c862e Try again +@id voice.try_again_a_little_closer_to_the_microphone_30f96ada Try again, a little closer to the microphone. +@id voice.where_to_get_whisper_cli_and_a_model_3b6c0fcb Where to get whisper-cli and a model +@id voice.your_system_3a1356ff your system +@id voice.layout_would_shift This insertion would move paragraph or table layout. Insert one line, or click below the page to append the words. +@id voice.target_changed_review_required The page or cursor changed. Review the words and choose Insert to use the current position. +@id voice.multiline_needs_line_end Multiple lines need the cursor at the end of a paragraph. Move there, or click below the page to append the words. diff --git a/packaging/OSX/Info.plist.in b/packaging/OSX/Info.plist.in index 3c89005d..12256925 100644 --- a/packaging/OSX/Info.plist.in +++ b/packaging/OSX/Info.plist.in @@ -16,6 +16,7 @@ NoteCraftBuildCommit@BUILD_SHA@ LSApplicationCategoryTypepublic.app-category.productivity LSMinimumSystemVersion11.0 + NSMicrophoneUsageDescriptionDictation uses your microphone to transcribe speech locally into a page. NSHighResolutionCapable NSSupportsAutomaticGraphicsSwitching NSHumanReadableCopyrightNoteCraft contributors. MIT or Apache-2.0. diff --git a/packaging/OSX/entitlements.plist b/packaging/OSX/entitlements.plist index 899d535f..c984ce5a 100644 --- a/packaging/OSX/entitlements.plist +++ b/packaging/OSX/entitlements.plist @@ -1,3 +1,7 @@ - + + + com.apple.security.device.audio-input + + diff --git a/packaging/OSX/package.sh b/packaging/OSX/package.sh index 1ddd7790..e977c549 100755 --- a/packaging/OSX/package.sh +++ b/packaging/OSX/package.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Build a NoteCraft.app, a separate notebook setup app, DMG and CLI archive on macOS. -# Default: universal. Options: --arch aarch64|x86_64|universal, --out DIR, --no-build. +# Default: universal. Options: --arch aarch64|x86_64|universal, --out DIR, --no-build, --voice. # CARGO_TARGET_DIR and DIST may override the build and distribution directories. set -euo pipefail @@ -8,11 +8,13 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" ARCH=universal OUT="${DIST:-$ROOT/dist/macos}" BUILD=1 +VOICE_FEATURE=() while [[ $# -gt 0 ]]; do case "$1" in --arch) [[ $# -ge 2 ]] || { echo "--arch needs a value" >&2; exit 2; }; ARCH="$2"; shift 2 ;; --out) [[ $# -ge 2 ]] || { echo "--out needs a value" >&2; exit 2; }; OUT="$2"; shift 2 ;; --no-build) BUILD=0; shift ;; + --voice) VOICE_FEATURE=(--features notecraft/voice); shift ;; -h|--help) sed -n '2,4p' "$0"; exit 0 ;; *) echo "unknown option: $1" >&2; exit 2 ;; esac @@ -58,7 +60,7 @@ echo "==> NoteCraft $VERSION ($ARCH)" if [[ "$BUILD" = 1 ]]; then rustup target add "${targets[@]}" for target in "${targets[@]}"; do - (cd "$ROOT" && CARGO_TARGET_DIR="$TARGET_ROOT" cargo build --locked --release --target "$target" -p notecraft -p notecraft-cli) + (cd "$ROOT" && CARGO_TARGET_DIR="$TARGET_ROOT" cargo build --locked --release --target "$target" -p notecraft -p notecraft-cli "${VOICE_FEATURE[@]}") done fi # Both slices must be built from the same inputs, including local uncommitted work. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index acd5b054..d93ea248 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -36,6 +36,8 @@ mod version; /// The layer of each workspace crate. A crate may depend only on crates in lower layers. const LAYERS: &[(&str, u8)] = &[ ("notecraft-model", 0), + ("craft-voice-core", 1), + ("craft-voice-runtime", 3), ("notecraft-math", 1), ("notecraft-diagram", 1), ("notecraft-ink", 1),