feat: dictate from any app, with a widget that sends to a session (core 0.43.0)

PRIVACY.md

@@ -71,6 +71,23 @@ into the Claude session where you asked for the report, which, like anything in
 session, goes to Anthropic. It sends nothing anywhere else, and writes only its own cursor
 (`catch-up.json`) beside the list. It runs only when you ask for it.
 
+## Dictating from any app
+
+With dictation on and Settings > Dictation > "Dictate from any app" on (the default when dictation
+is on), a small helper watches the keyboard so the dictation key works with another app in front.
+It reads only the dictation key's own keys, notes only THAT another key was pressed while they are
+held (never which), and, only while the widget's list of sessions is open, the digits 1 to 9 and
+Esc (which it keeps from the app in front). It writes nothing anywhere and sends nothing. What you
+say is handled as all dictation is: transcribed on your PC, then sent to the Claude session you
+pick, or to the orchestrator tab.
+
+With more than one copy of Prism Terminal open (for example a stable copy beside the installed
+app), one copy runs the dictation for all of them. The copies talk over a local named pipe that
+only your Windows account uses: the running copy asks the others for their tab list (the tab's
+name, folder and whether Claude runs there) so the list of sessions shows every copy's, and hands a
+copy the words to type into one of its tabs. Nothing on the pipe leaves your PC, and nothing is
+written to disk.
+
 ## What programs in a tab can do
 
 A program running in a tab (Claude Code's /copy, or anything you reach over ssh) can put text ON

core/dictation-host/parakeetHost.ps1

@@ -0,0 +1,129 @@
+# THE RESIDENT PARAKEET HOST (owner, 2026-10-17: "It needs to be ready at all
+# times"; the startup is "the important bit"). parakeet-cli loads the model for
+# every pass: MEASURED on the owner's RTX 5090, 0.72 to 0.86 s a pass whatever
+# the clip's length, nearly all of it start-up. This keeps ONE model loaded in
+# a long-lived Windows PowerShell (the app's helper pattern, no new binary) and
+# calls the engine's own parakeet.dll for each pass: 24 ms (1 s clip) to 170 ms
+# (60 s clip) on the GPU, the text identical to parakeet-cli's, MEASURED.
+# Report: research/prism-terminal/2026-10-17-dictation-performance.md.
+#
+# args:   <engine folder> <model> <gpu 1|0> <threads> [<cache folder>]
+# stdin:  "pass <n>" + LF, then n bytes of a 16 kHz mono PCM16 WAV; "quit" ends it.
+# stdout: "ready" once the model is loaded; per pass "ok <base64 of the UTF-8
+#         text>" or "err <why>". Nothing else is ever written there.
+#
+# The C# is compiled ONCE into <cache folder> (keyed by its own hash) and
+# loaded from there afterwards, so a start pays no compiler.
+param([string]$dir, [string]$model, [int]$gpu = 1, [int]$threads = 4, [string]$cache = '')
+$ErrorActionPreference = 'Stop'
+$src = @'
+using System;
+using System.IO;
+using System.Text;
+using System.Runtime.InteropServices;
+public static class PTParakeetHost {
+  [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] static extern IntPtr LoadLibraryW(string path);
+  [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] static extern bool SetDllDirectoryW(string path);
+  [DllImport("ggml.dll", CallingConvention = CallingConvention.Cdecl)] static extern void ggml_backend_load_all();
+  [StructLayout(LayoutKind.Sequential)] public struct CtxParams { [MarshalAs(UnmanagedType.I1)] public bool use_gpu; public int gpu_device; }
+  // parakeet.h's parakeet_full_params, field for field (passed BY VALUE).
+  [StructLayout(LayoutKind.Sequential)] public struct FullParams {
+    public int strategy; public int n_threads; public int offset_ms; public int duration_ms;
+    [MarshalAs(UnmanagedType.I1)] public bool no_context; public int audio_ctx;
+    public IntPtr seg_cb, seg_ud, tok_cb, tok_ud, prog_cb, prog_ud, enc_cb, enc_ud, abort_cb, abort_ud;
+  }
+  [DllImport("parakeet.dll", CallingConvention = CallingConvention.Cdecl)] static extern IntPtr parakeet_init_from_file_with_params([MarshalAs(UnmanagedType.LPStr)] string path, CtxParams p);
+  [DllImport("parakeet.dll", CallingConvention = CallingConvention.Cdecl)] static extern IntPtr parakeet_full_default_params_by_ref(int strategy);
+  [DllImport("parakeet.dll", CallingConvention = CallingConvention.Cdecl)] static extern void parakeet_free_params(IntPtr p);
+  [DllImport("parakeet.dll", CallingConvention = CallingConvention.Cdecl)] static extern int parakeet_full(IntPtr ctx, FullParams p, float[] samples, int n);
+  [DllImport("parakeet.dll", CallingConvention = CallingConvention.Cdecl)] static extern int parakeet_full_n_segments(IntPtr ctx);
+  [DllImport("parakeet.dll", CallingConvention = CallingConvention.Cdecl)] static extern IntPtr parakeet_full_get_segment_text(IntPtr ctx, int i);
+  [UnmanagedFunctionPointer(CallingConvention.Cdecl)] delegate void LogCb(int level, IntPtr text, IntPtr ud);
+  [DllImport("parakeet.dll", CallingConvention = CallingConvention.Cdecl)] static extern void parakeet_log_set(LogCb cb, IntPtr ud);
+  static readonly LogCb quiet = (l, t, u) => { };
+
+  static string Utf8(IntPtr p) {
+    if (p == IntPtr.Zero) return "";
+    int n = 0; while (Marshal.ReadByte(p, n) != 0) n++;
+    var b = new byte[n]; Marshal.Copy(p, b, 0, n); return Encoding.UTF8.GetString(b);
+  }
+  // The app's own WAV (16-bit mono, 16 kHz): the samples of its "data" chunk.
+  static float[] Pcm(byte[] wav) {
+    int i = 12;
+    while (i + 8 <= wav.Length) {
+      int len = BitConverter.ToInt32(wav, i + 4);
+      if (len < 0) break;
+      if (wav[i] == 'd' && wav[i + 1] == 'a' && wav[i + 2] == 't' && wav[i + 3] == 'a') {
+        int n = Math.Min(len, wav.Length - i - 8) / 2;
+        var f = new float[n];
+        for (int k = 0; k < n; k++) f[k] = BitConverter.ToInt16(wav, i + 8 + k * 2) / 32768f;
+        return f;
+      }
+      i += 8 + len + (len & 1);
+    }
+    return new float[0];
+  }
+  static string ReadLine(Stream s) {
+    var b = new StringBuilder();
+    while (true) {
+      int c = s.ReadByte();
+      if (c < 0) return null;
+      if (c == '\n') return b.ToString().TrimEnd('\r');
+      if (b.Length > 64) return "";
+      b.Append((char)c);
+    }
+  }
+  public static void Run(string dir, string model, bool gpu, int threads) {
+    var output = Console.Out;
+    SetDllDirectoryW(dir);
+    LoadLibraryW(Path.Combine(dir, "ggml.dll"));
+    LoadLibraryW(Path.Combine(dir, "parakeet.dll"));
+    parakeet_log_set(quiet, IntPtr.Zero);
+    ggml_backend_load_all();
+    IntPtr ctx = parakeet_init_from_file_with_params(model, new CtxParams { use_gpu = gpu, gpu_device = 0 });
+    if (ctx == IntPtr.Zero) { output.WriteLine("err the model did not load"); output.Flush(); return; }
+    IntPtr dp = parakeet_full_default_params_by_ref(0);
+    var fp = (FullParams)Marshal.PtrToStructure(dp, typeof(FullParams));
+    parakeet_free_params(dp);
+    fp.n_threads = threads;
+    output.WriteLine("ready"); output.Flush();
+    var input = Console.OpenStandardInput();
+    while (true) {
+      string head = ReadLine(input);
+      if (head == null || head == "quit") return;
+      int n;
+      if (!head.StartsWith("pass ") || !int.TryParse(head.Substring(5), out n) || n < 0 || n > 64 * 1024 * 1024) { output.WriteLine("err a bad request"); output.Flush(); return; }
+      var wav = new byte[n]; int got = 0;
+      while (got < n) { int r = input.Read(wav, got, n - got); if (r <= 0) return; got += r; }
+      var pcm = Pcm(wav);
+      if (pcm.Length == 0) { output.WriteLine("ok "); output.Flush(); continue; }
+      int rc = parakeet_full(ctx, fp, pcm, pcm.Length);
+      if (rc != 0) { output.WriteLine("err the pass failed (" + rc + ")"); output.Flush(); continue; }
+      var sb = new StringBuilder();
+      int segs = parakeet_full_n_segments(ctx);
+      for (int s = 0; s < segs; s++) sb.Append(Utf8(parakeet_full_get_segment_text(ctx, s)));
+      output.WriteLine("ok " + Convert.ToBase64String(Encoding.UTF8.GetBytes(sb.ToString().Trim())));
+      output.Flush();
+    }
+  }
+}
+'@
+$loaded = $false
+if ($cache) {
+  try {
+    $sha = [Security.Cryptography.SHA256]::Create()
+    $hash = ([BitConverter]::ToString($sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($src))) -replace '-', '').Substring(0, 16)
+    $dll = Join-Path $cache "pt-parakeet-host-$hash.dll"
+    if (-not (Test-Path -LiteralPath $dll)) {
+      New-Item -ItemType Directory -Force -Path $cache | Out-Null
+      $tmp = Join-Path $cache "pt-parakeet-host-$hash-$PID.dll"
+      Add-Type -TypeDefinition $src -OutputAssembly $tmp -OutputType Library
+      Move-Item -LiteralPath $tmp -Destination $dll -Force -ErrorAction SilentlyContinue
+      if (Test-Path -LiteralPath $tmp) { Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue }
+    }
+    Add-Type -LiteralPath $dll
+    $loaded = $true
+  } catch { $loaded = $false }
+}
+if (-not $loaded) { Add-Type -TypeDefinition $src }
+[PTParakeetHost]::Run($dir, $model, $gpu -eq 1, $threads)

core/main/__fixtures__/fakeParakeetHost.mjs

@@ -0,0 +1,37 @@
+// A stand-in for core/dictation-host/parakeetHost.ps1: the same protocol on
+// stdin and stdout, run by Node. argv: <engine folder> <model> <gpu 1|0>
+// <threads> [<cache>]. FAKE_HOST=never: it never says "ready".
+// Each answer's text is JSON of what it saw: the pass number, the bytes, its pid.
+// The globals by name, as fakeParakeetCli.mjs does (the eslint config).
+import { Buffer } from 'node:buffer'
+import process from 'node:process'
+import { setInterval } from 'node:timers'
+const [dir, model, gpu] = process.argv.slice(2)
+if (process.env.FAKE_HOST === 'never') {
+  setInterval(() => {}, 1000)
+} else {
+  process.stdout.write('ready\n')
+  let buf = Buffer.alloc(0)
+  let need = -1
+  let n = 0
+  process.stdin.on('data', (d) => {
+    buf = Buffer.concat([buf, d])
+    for (;;) {
+      if (need < 0) {
+        const i = buf.indexOf(10)
+        if (i < 0) return
+        const head = buf.subarray(0, i).toString()
+        buf = buf.subarray(i + 1)
+        if (head === 'quit') process.exit(0)
+        need = Number(head.slice(5))
+      }
+      if (buf.length < need) return
+      const bytes = need
+      buf = buf.subarray(need)
+      need = -1
+      n += 1
+      const text = JSON.stringify({ n, bytes, pid: process.pid, gpu: gpu === '1', model, dir })
+      process.stdout.write(`ok ${Buffer.from(text, 'utf8').toString('base64')}\n`)
+    }
+  })
+}

core/main/dictationEngine.test.ts

@@ -406,6 +406,17 @@ describe('standing down', () => {
     expect(w.launched).toHaveLength(2)
   })
 
+  // Prism Terminal's dictation service (owner, 2026-10-17: "It needs to be
+  // ready at all times"): Infinity never stands down; stop() still ends it.
+  it('never stands down with an idleMs of Infinity', async () => {
+    const w = world({}, { idleMs: Infinity })
+    await w.engine.transcribe(req())
+    await sleep(300)
+    expect(w.engine.info().running).toBe(true)
+    w.engine.stop()
+    await until(() => gone(w.children[0]))
+  })
+
   it('counts the idle time from the LAST request, not the first', async () => {
     const w = world({}, { idleMs: 500 })
     await w.engine.transcribe(req())
@@ -631,3 +642,85 @@ describe('a Parakeet model', () => {
     await until(() => gone(w.children[0]))
   })
 })
+
+// THE RESIDENT PARAKEET HOST (2026-10-17): the model kept loaded in one
+// process (core/dictation-host/parakeetHost.ps1), parakeet-cli only when it
+// cannot start. The stand-in speaks its protocol, run by this Node.
+const HOST_FIXTURE = join(dirname(fileURLToPath(import.meta.url)), '__fixtures__', 'fakeParakeetHost.mjs')
+function hostWorld(env: Record<string, string> = {}, knobs: Partial<Knobs> = {}): World & { hosts: ChildProcess[] } {
+  const hosts: ChildProcess[] = []
+  const w = world(knobs, {
+    powershell: 'FAKE-POWERSHELL',
+    parakeetHost: () => ({ script: 'parakeetHost.ps1', cache: null }),
+    spawnImpl: ((file: string, args: string[], options: { env?: NodeJS.ProcessEnv }) => {
+      let child: ChildProcess
+      if (file === 'FAKE-POWERSHELL') {
+        // The host's own argv after `-File <script>`.
+        const rest = args.slice(args.indexOf('-File') + 2)
+        child = spawn(process.execPath, [HOST_FIXTURE, ...rest], { ...options, env: { ...options.env, ...env } })
+        hosts.push(child)
+      } else child = spawn(file, args, options)
+      allChildren.push(child)
+      return child
+    }) as typeof spawn
+  })
+  return { ...w, hosts }
+}
+const hostSaw = (r: TranscribeResult): { n: number; bytes: number; pid: number; gpu: boolean; model: string } => {
+  if (!r.ok) throw new Error(`expected an answer, got ${r.reason}: ${r.detail ?? ''}`)
+  return JSON.parse(r.text)
+}
+
+describe('the resident Parakeet host', () => {
+  it('answers every pass from ONE process that keeps the model, and runs no parakeet-cli', async () => {
+    const w = hostWorld()
+    const a = hostSaw(await w.engine.transcribe(req({ modelId: 'pk' })))
+    const b = hostSaw(await w.engine.transcribe(req({ modelId: 'pk' })))
+    expect([a.n, b.n]).toEqual([1, 2])
+    expect(b.pid).toBe(a.pid)
+    expect(a.bytes).toBe(WAV.byteLength)
+    expect(a.model).toBe(MODELS.pk)
+    expect(w.launched).toHaveLength(0)
+    expect(w.hosts).toHaveLength(1)
+  })
+
+  it('falls back to parakeet-cli when the host never gets ready, and does not try it again', async () => {
+    const w = hostWorld({ FAKE_HOST: 'never' }, {})
+    const eng = createDictationEngine({
+      cpuDir: () => w.cpuDir,
+      store: { modelPath: (id) => MODELS[id] ?? null, gpuEngineDir: () => null },
+      hasNvidia: async () => false,
+      engineFor: (id) => (id === 'pk' ? 'parakeet' : 'whisper'),
+      startTimeoutMs: 300,
+      powershell: 'FAKE-POWERSHELL',
+      parakeetHost: () => ({ script: 'parakeetHost.ps1', cache: null }),
+      spawnImpl: ((file: string, args: string[], options: { env?: NodeJS.ProcessEnv }) => {
+        const child =
+          file === 'FAKE-POWERSHELL'
+            ? spawn(process.execPath, [HOST_FIXTURE, ...args.slice(args.indexOf('-File') + 2)], { ...options, env: { ...options.env, FAKE_HOST: 'never' } })
+            : spawn(file, args, options)
+        if (file === 'FAKE-POWERSHELL') w.hosts.push(child)
+        allChildren.push(child)
+        return child
+      }) as typeof spawn,
+      command: (_exePath, args) => ({ file: process.execPath, args: [PARAKEET_FIXTURE, ...args] })
+    })
+    engines.push(eng)
+    expect((await eng.transcribe(req({ modelId: 'pk' }))).ok).toBe(true)
+    expect((await eng.transcribe(req({ modelId: 'pk' }))).ok).toBe(true)
+    expect(w.hosts).toHaveLength(1)
+    await until(() => w.hosts.every(gone))
+  })
+
+  it('is ended by stop(), and by a Whisper model taking over', async () => {
+    const w = hostWorld()
+    await w.engine.transcribe(req({ modelId: 'pk' }))
+    expect(w.engine.info().running).toBe(true)
+    w.engine.stop()
+    await until(() => w.hosts.every(gone))
+    await w.engine.transcribe(req({ modelId: 'pk' }))
+    expect(w.hosts).toHaveLength(2)
+    seen(await w.engine.transcribe(req({ modelId: 'base' })))
+    await until(() => w.hosts.every(gone))
+  })
+})