From 53e0096759c127a61e2eeaa86f95bd749bb24143 Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 30 Jul 2026 14:45:54 -0400 Subject: [PATCH 1/3] feat(read-aloud): speak selected text on web and desktop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Select text in the web or Electron app and a speaker button floats above the selection; press it and the daemon speaks the selection back. Press again, or clear the selection, to stop. Uses the existing voice-mode TTS provider (local Kokoro by default, or OpenAI) rather than adding a second provider stack. Synthesis is segmented with a 2-segment prefetch because local TTS returns raw 24 kHz PCM (~48 KB/s) — a long selection in one message would be megabytes. - protocol: speech.tts.read_aloud.request -> N .response segments, plus a cancel request - server: read-aloud-controller sanitizes markup, splits, synthesizes and streams; the sentence splitter is extracted from tts-manager and now shared with voice mode - client: startReadAloud() returns a handle with cancel() - app: selection anchoring from endpoint rects plus the clipping ancestor's visible box, a pure above/below/park placement decision, and playback state in a small store reusing the voice AudioEngine Web and Electron only. React Native exposes no JS API for the current text selection, so the native bubble is a deliberate no-op. Capability-gated on server_info.features.readAloud — hosts without it show no button at all, no fallback path. --- docs/refactors/read-aloud-handoff.md | 149 +++++++ .../read-aloud-selection-anchoring-plan.md | 275 ++++++++++++ packages/app/src/app/_layout.tsx | 2 + packages/app/src/i18n/resources/ar.ts | 12 + packages/app/src/i18n/resources/en.ts | 12 + packages/app/src/i18n/resources/es.ts | 12 + packages/app/src/i18n/resources/fr.ts | 12 + packages/app/src/i18n/resources/ja.ts | 12 + packages/app/src/i18n/resources/pt-BR.ts | 12 + packages/app/src/i18n/resources/ru.ts | 12 + packages/app/src/i18n/resources/zh-CN.ts | 12 + .../app/src/read-aloud/read-aloud-audio.ts | 25 ++ .../src/read-aloud/read-aloud-audio.web.ts | 75 ++++ .../read-aloud/read-aloud-placement.test.ts | 152 +++++++ .../src/read-aloud/read-aloud-placement.ts | 166 +++++++ .../read-aloud-selection-bubble.tsx | 12 + .../read-aloud-selection-bubble.web.tsx | 400 +++++++++++++++++ .../src/read-aloud/read-aloud-store.test.ts | 84 ++++ .../app/src/read-aloud/read-aloud-store.ts | 185 ++++++++ .../read-aloud/use-selection-anchor.web.ts | 410 ++++++++++++++++++ packages/app/src/voice/audio-engine-types.ts | 6 + packages/app/src/voice/audio-engine.native.ts | 7 + packages/app/src/voice/audio-engine.web.ts | 19 +- packages/app/src/voice/voice-runtime.test.ts | 1 + packages/client/src/daemon-client.ts | 93 ++++ packages/client/src/index.ts | 3 + packages/protocol/src/messages.ts | 50 +++ .../server/src/server/agent/tts-manager.ts | 100 +---- packages/server/src/server/session.ts | 10 + .../voice/read-aloud-controller.test.ts | 204 +++++++++ .../session/voice/read-aloud-controller.ts | 275 ++++++++++++ .../src/server/session/voice/voice-session.ts | 21 + .../src/server/speech/read-aloud-text.test.ts | 60 +++ .../src/server/speech/read-aloud-text.ts | 48 ++ .../src/server/speech/tts-text-splitter.ts | 104 +++++ .../server/src/server/websocket-server.ts | 4 + public-docs/voice.md | 13 + 37 files changed, 2949 insertions(+), 100 deletions(-) create mode 100644 docs/refactors/read-aloud-handoff.md create mode 100644 docs/refactors/read-aloud-selection-anchoring-plan.md create mode 100644 packages/app/src/read-aloud/read-aloud-audio.ts create mode 100644 packages/app/src/read-aloud/read-aloud-audio.web.ts create mode 100644 packages/app/src/read-aloud/read-aloud-placement.test.ts create mode 100644 packages/app/src/read-aloud/read-aloud-placement.ts create mode 100644 packages/app/src/read-aloud/read-aloud-selection-bubble.tsx create mode 100644 packages/app/src/read-aloud/read-aloud-selection-bubble.web.tsx create mode 100644 packages/app/src/read-aloud/read-aloud-store.test.ts create mode 100644 packages/app/src/read-aloud/read-aloud-store.ts create mode 100644 packages/app/src/read-aloud/use-selection-anchor.web.ts create mode 100644 packages/server/src/server/session/voice/read-aloud-controller.test.ts create mode 100644 packages/server/src/server/session/voice/read-aloud-controller.ts create mode 100644 packages/server/src/server/speech/read-aloud-text.test.ts create mode 100644 packages/server/src/server/speech/read-aloud-text.ts create mode 100644 packages/server/src/server/speech/tts-text-splitter.ts diff --git a/docs/refactors/read-aloud-handoff.md b/docs/refactors/read-aloud-handoff.md new file mode 100644 index 0000000000..6f9afe5545 --- /dev/null +++ b/docs/refactors/read-aloud-handoff.md @@ -0,0 +1,149 @@ +# Read Aloud — Session Handoff + +Picking this up cold? Read this, then +[read-aloud-selection-anchoring-plan.md](./read-aloud-selection-anchoring-plan.md) — all +three of its steps have since landed, and its Outcome section records what shipped. + +- **Branch:** `text-selection-speech-modal`, branched from `d1ce2b77f`. + +## What the feature is + +Select text on web/desktop → a 32×32 speaker button floats above the selection → press it +and the daemon speaks the selection back. Press again, or clear the selection, to stop. + +**Web and Electron only.** React Native exposes no JS API for the current text selection — +iOS hands it to the system edit menu, Android to an ActionMode — so there is nothing to +anchor to on the mobile apps. `read-aloud-selection-bubble.tsx` (the non-`.web` file) is a +deliberate `return null`. Closing that gap would mean a turn-level "Read aloud" button next +to the copy button, reading the whole turn instead of a selection. Not built; not started. + +Uses the **existing voice-mode TTS provider** (local Kokoro by default, or OpenAI). No +ElevenLabs — the existing provider already has config, env vars, and docs plumbing. + +## State: what works, verified how + +Green as of handoff: `npm run typecheck`, `npm run lint`, `npm run format`, 42 server tests +(`packages/server/src/server/speech/read-aloud-text.test.ts`, +`src/server/session/voice/`, `src/server/agent/tts-manager.test.ts`), 50 app tests +(`src/i18n/resources.test.ts`, `src/voice/voice-runtime.test.ts`). + +Verified in a real browser via Playwright against the dev stack: button appears above the +selection; press → stop square + selection survives the press; press again → idle; click +away → button gone and playback stopped. Audio confirmed reaching the output graph by +instrumenting `AudioContext` — two segments, non-silent buffers (peak 0.27 / 0.35), context +`running`. + +Since then, anchoring was rewritten. Also green: 14 new tests +(`packages/app/src/read-aloud/read-aloud-placement.test.ts`) and nine browser cases measured +off real `getBoundingClientRect()` values — see the plan's Outcome section for the table. + +### Architecture + +| Piece | What it does | +| -------------------------------------------------------- | -------------------------------------------------------------------------------------- | +| `protocol/src/messages.ts` | `speech.tts.read_aloud.request` → N `.response` segments; `…cancel_read_aloud.request` | +| `server/session/voice/read-aloud-controller.ts` | Sanitize → split → synthesize with 2-segment prefetch → stream | +| `server/speech/read-aloud-text.ts` | Strips markup before synthesis | +| `server/speech/tts-text-splitter.ts` | Extracted from `tts-manager.ts`; **shared with voice mode** | +| `client/src/daemon-client.ts` → `startReadAloud()` | Returns a handle with `cancel()`; streams via `on(…response)` | +| `app/src/read-aloud/use-selection-anchor.web.ts` | Endpoint rects + the clipping-ancestor `visibleBox`; retains during playback | +| `app/src/read-aloud/read-aloud-placement.ts` | Pure above/below/park decision and clamping; unit-tested | +| `app/src/read-aloud/read-aloud-selection-bubble.web.tsx` | Renders into `#overlay-root`; owns the widths and stop intent | +| `app/src/read-aloud/read-aloud-store.ts` | Playback state machine, generation counter, speed | +| `app/src/read-aloud/read-aloud-audio.web.ts` | Reuses the voice-mode `AudioEngine` (playback context only, no mic) | + +Segmented streaming is not gold-plating: local TTS returns raw 24 kHz PCM (~48 KB/s), so a +long selection in one message would be megabytes. + +**Capability-gated** on `server_info.features.readAloud` (`COMPAT(readAloud)`, v0.2.5). +Hosts without it get no button at all — an upgrade prompt on every text selection would be +worse than the feature being absent. No fallback path. + +## Environment gotchas — every one of these cost real time to rediscover + +**1. Node 22+ is required for the dev daemon.** In dev the daemon runs from TS source and +forks the local speech worker with `--experimental-strip-types`, a Node 22+ flag, using +`process.execPath`. On Node 20 the worker dies instantly (`exit code 9`) and every +synthesis fails. nvm default here is v20.19.6; v22.20.0 and v23.3.0 are installed. + +```bash +nvm use 22 +``` + +This is pre-existing and not read-aloud specific — voice mode and dictation break the same +way. + +**2. The daemon does NOT hot-reload.** `packages/server/scripts/dev-runner.ts` has no +watcher. The Expo app hot-reloads; the daemon does not. **Restart `npm run dev` after any +server change** or you will test stale code and draw wrong conclusions. + +**2b. `nvm use 22` does not work from a non-interactive shell here.** The zsh profile +installs an nvm lazy-load shim, and outside an interactive shell the shim recurses until zsh +gives up with `maximum nested function level reached` — `node`, `npm`, and `npx` all fail, +and prepending to `PATH` does not help because the shell _function_ shadows the binary. Call +the binary by absolute path instead: + +```bash +~/.nvm/versions/node/v22.20.0/bin/node ~/.nvm/versions/node/v22.20.0/bin/npm run typecheck +``` + +**3. `Buffer` is not a browser global.** Every consumer imports it explicitly +(`voice/voice-runtime.ts:1`). `@types/node` makes a bare `Buffer` reference typecheck while +failing at runtime in the bundle. This already caused one silent no-audio bug. + +## Running it + +```bash +nvm use 22 +npm run dev # terminal 1 — daemon on 127.0.0.1:6768 +npm run dev:app # terminal 2 — Expo web on http://localhost:8081 +``` + +No extra env needed: `dev-app.sh` derives the daemon endpoint from `PASEO_LISTEN`, and +`dev-daemon.sh` defaults `PASEO_LOCAL_MODELS_DIR` to `~/.paseo/models/local-speech`, which +already has Kokoro — so no ~1 GB model download. + +Runs alongside the user's normal Paseo on **6767** — different port, different `PASEO_HOME` +(`.dev/paseo-home`). **Never restart the 6767 daemon**; it manages live agents. The dev +home already has a project and workspace registered from earlier testing. + +To try it: open the workspace → diff-stat button in the top bar → expand a file → drag-select +a couple of diff lines. Agent output in a chat works too. + +### Probing the daemon directly + +Faster than the UI for server-side questions. Connect to `ws://localhost:6768/ws`, send a +`hello` frame **first** (session messages before hello are rejected), then wrap requests as +`{type: "session", message: {…}}`. Run the script from the repo root so `ws` resolves. + +## Open questions for the next session + +- **Segment gaps.** Local Kokoro runs ~1× realtime with a ~6 s cold start. The splitter + emits one segment per sentence, so a short first sentence ("Are you working?" → 1.79 s) + drains before the next finishes synthesizing — measured **6.1 s of silence** mid-read. + Three options, none implemented, user has not chosen: + 1. Pack short sentences into ~250-char chunks, **read-aloud only** — do not touch the + shared splitter, voice mode wants the fast short first segment. Recommended. + 2. Buffer two segments before starting. Gapless, but start moves ~6 s → ~13 s. Worse. + 3. Switch to OpenAI TTS — far faster than realtime, gaps vanish, needs an API key. +- **Error detail is dropped.** Unknown failure codes render the generic "Couldn't read that + aloud"; the daemon's real message sits unused in `failure.message`. Worth surfacing on + hover so the next failure is self-diagnosing. +- **Markup sanitization is unit-tested only.** It was written after the last daemon restart, + so it has never run against a live daemon. First thing to confirm after restarting: + selecting a `` block should speak only the inner text. + +## Things I got wrong — don't repeat them + +- **Claimed "verified end-to-end" when I had only verified the state machine.** The UI went + idle → Stop → idle exactly as it would on success, but that transition was driven by a + swallowed error, and no audio sample ever reached the output. Two bugs hid in that gap. + For anything audio- or layout-related, assert on the real artifact — buffer peaks, + `getBoundingClientRect()` — not on labels. +- **Forced `/opt/homebrew/bin/node` (v24) onto PATH** so my runs worked, which masked the + Node 20 worker crash the user hit immediately. Verify in the environment the user + actually runs, not one bent to work. +- **Swallowed errors in a `.catch(() => {})`.** Turned a hard `ReferenceError` into "no + error, no sound", the worst possible failure mode. That catch now reports. +- **Assumed `getClientRects()` was one rect per line.** It is not — see the plan's traps + section. The sub-agent caught this before it was written. diff --git a/docs/refactors/read-aloud-selection-anchoring-plan.md b/docs/refactors/read-aloud-selection-anchoring-plan.md new file mode 100644 index 0000000000..cc64527570 --- /dev/null +++ b/docs/refactors/read-aloud-selection-anchoring-plan.md @@ -0,0 +1,275 @@ +# Read-Aloud Selection Anchoring Plan + +> **Status: all three steps landed.** See [Outcome](#outcome) at the bottom for what shipped, +> what changed from this plan, and the browser measurements behind each case. + +The read-aloud button (`packages/app/src/read-aloud/`) anchors to +`range.getBoundingClientRect()` — the box around the **entire** selection. Select more +than a screenful, or scroll after selecting, and the selection's top edge is off-screen; +the button clamps to the top of the window, nowhere near what the user is looking at. + +Goal: anchor to the part of the selection that is **actually visible**, in the pane it +actually lives in. + +## The three bugs + +They are independent. 1 is the reported symptom; 2 and 3 were found while investigating it. + +### 1. Anchors to the whole selection, not the visible part + +``` + ▓▓▓▓▓▓ selection starts up here ▓▓▓▓▓▓▓▓▓ ⇠ scrolled out of view + ┌─ viewport ─────────────────────────────────┐ + │ ( 🔊 ) ← clamped to y=8, on top of text │ ✗ not where you're looking + │ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ │ + │ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ …and ends here. │ ← eyes here + └────────────────────────────────────────────┘ +``` + +Wanted: anchor below the visible **end** when the start is clipped; keep today's +above-the-start placement otherwise. + +### 2. "Visible" is measured against the window, not the scroll pane + +Client rects ignore ancestor `overflow` entirely. The timeline scrolls inside +`div[data-testid="agent-chat-scroll"]` (`packages/app/src/agent-stream/strategy-web.tsx`, +~line 618), which occupies only part of the window. Text scrolled out of _that pane_ still +reports a rect inside the window: + +``` + ┌─ window ───────────────────────────────────┐ + │ ┌ sidebar ┐ ┌─ agent-chat-scroll ────────┐ │ + │ │ │ │ ▓▓▓ selection ▓▓▓ │ │ + │ │ │ │ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ │ │ ← the real visible box + │ │ │ └────────────────────────────┘ │ + │ │ │ ░░░ scrolled out of the pane │ ← still "in the window" + │ │ │ ░░░ but invisible │ per getBoundingClientRect + │ └─────────┘ ┌─ composer ─────────────────┐ │ + └─────────────┴────────────────────────────┴─┘ +``` + +The same mistake exists on the x-axis today: the clamp uses `window.innerWidth` and +ignores the sidebar. Fixing 2 fixes that for free. + +### 3. Scrolling away kills playback + +`packages/app/src/agent-stream/strategy-web.tsx` sets `overscan: 8`; rows outside that +window unmount. The range's boundary nodes detach → `isReadAloudEligible`'s `isConnected` +check fails → the anchor goes null → the `useEffect` in the bubble calls +`stopReadAloud()`. **Scrolling a long conversation stops audio mid-sentence.** No amount +of positioning work fixes this. + +## Traps (verified during investigation — do not rediscover) + +- **`getClientRects()` is not one rect per line.** It also returns rects for element boxes + fully contained in the range, so a selection spanning a `

` contributes that + container's full-height rect. `rects[0]` and `rects[N-1]` can each be the entire + selection box — taking first/last reproduces bug 1. +- **Use collapsed clones instead.** `range.cloneRange()` collapsed to start and to end + gives two caret rects carrying each endpoint's line height. O(1) per frame rather than + scanning thousands of fragments on every scroll tick, and immune to container + contamination. +- **Caret rects are zero-width.** The existing `rect.width === 0 && rect.height === 0` + guard in `readSelectionAnchor` must become height-only, or every anchor is rejected. +- **`clampToViewport` inverts on a small box.** `Math.max(PADDING, Math.min(viewport - +size - PADDING, value))` — when `viewport < size + 2 * PADDING` the `Math.max` wins and + pushes the bubble past the far edge. Never fires against `window.innerHeight`; fires + readily against a short diff pane once the basis becomes `visibleBox`. +- **`centerX` must come from the anchored rect**, not the bounding box. Anchor to the + bottom while centering on the whole selection and the bubble drifts sideways on wide + code blocks and diffs. +- **Neither `scroll` nor `resize` fires on reflow** — streaming output, virtualizer row + remeasure, dragging `components/resize-handle.tsx`. A `ResizeObserver` on the scroll + container is the repo-consistent fix (precedent: `strategy-web.tsx` ~415-440). +- **Coordinate space and the scroll listener are already right.** `#overlay-root` is + `position: fixed; inset: 0` (`packages/app/src/lib/overlay-root.ts`), so absolute coords + inside it are viewport coords — no conversion needed. `window.addEventListener("scroll", +…, true)` already catches nested scrollers via capture phase. Don't "fix" either. + +## Nothing in the repo already does this + +`computeHoverCardPosition` (`components/workspace-hover-card.tsx`) and `computePosition` +(`components/ui/tooltip.tsx`) both take a `displayArea` and clamp into it — the right +_shape_, but they know nothing about target visibility and are `measureInWindow`/RN-flavored. +`components/ui/combobox.tsx` uses `@floating-ui/react-native`; only that package is a +dependency, so `@floating-ui/dom`'s clipping-ancestor detection is **not** available and is +not worth pulling in for a 32px button. `docs/floating-panels.md` is explicit: copy the +closest file and trim rather than inventing a shared primitive. + +The reusable idea is the `displayArea` parameter. `visibleBox` slots into that role. + +## Algorithm + +Per frame: `firstRect` (start-endpoint caret rect), `lastRect` (end-endpoint caret rect), +`visibleBox`. + +``` +visibleBox = intersect(windowRect, every clipping ancestor's rect) +visible(r) = r.bottom > visibleBox.top && r.top < visibleBox.bottom +``` + +| Case | Anchor rect | Placement | +| ---------------------------------- | ------------ | --------------------------------------------- | +| Both endpoints visible | `firstRect` | above; flip below if it doesn't fit (today's) | +| Top clipped, bottom visible | `lastRect` | **below**; flip above if it doesn't fit | +| Bottom clipped, top visible | `firstRect` | above; flip below if it doesn't fit (today's) | +| Both clipped (taller than the box) | `visibleBox` | near its **bottom** edge, inside the box | +| Entirely outside `visibleBox` | idle → hide | speaking → park at nearest `visibleBox` edge | + +Both-clipped anchors to `visibleBox`, not to a scanned mid-selection rect: the property +that matters is stability under scroll, and a mid-selection rect jitters as you scroll +while costing O(N) exactly when N is largest. **Accepted imperfection, document it:** a +selection with a large unselected gap in the middle can put the button over unselected +content. + +The entirely-outside case must split on playback state — while speaking, that button is +the only stop control. + +## Sequencing + +Land 1 and 2 together (2 is a prerequisite for 1 being correct). 3 is separable and can +ship independently, before or after. + +### Step 1 — `visibleBox` + +`packages/app/src/read-aloud/use-selection-anchor.web.ts` + +Walk `parentElement` from the range's start element, collecting elements whose computed +`overflow-x`/`overflow-y` is `auto | scroll | hidden | clip`. **Compute the chain once when +the selection settles and cache it** — the chain doesn't change on scroll, only its rects +do — so per-frame cost is a handful of `getBoundingClientRect()` calls. Invalidate on +`selectionchange` and on node disconnect. Add a `ResizeObserver` on the nearest scroll +container. + +### Step 2 — anchor to the visible edge + +`use-selection-anchor.web.ts` returns geometry (`text`, `firstRect`, `lastRect`, +`visibleBox`); `read-aloud-selection-bubble.web.tsx` decides above/below and clamps, since +it alone owns the width constants. Note the bubble has **three** widths — `BUBBLE_WIDTH`, +`SPEAKING_BUBBLE_WIDTH` (icon + speed chips), `FAILED_BUBBLE_WIDTH` — so the anchor math +keys off whichever pill is showing. Fix the `clampToViewport` degenerate-box inversion +(centre in the box when it is too small to clamp into). + +### Step 3 — decouple playback lifetime from anchor liveness + +The text is captured at settle time and playback needs no live range. While +`status !== "idle"`, retain the last-known anchor parked at the nearest `visibleBox` edge +instead of nulling. Stop only on explicit intent: pressing stop, Escape, or a new non-empty +selection. This changes the `useEffect` on `anchor` in the bubble, not the geometry. + +Keep `handlePointerDown` nulling the anchor on outside mousedown — that is defensible UX +and not part of this problem. + +## Verification + +Vitest cannot see any of this — it is layout behavior. Verify in a real browser via +Playwright MCP against the dev stack (see the handoff doc for how to bring it up). + +Cases to drive, all with a real mouse drag (programmatic `Range` selection skips the +pointer path): + +1. Short selection, fully visible → button above it. Regression check on today's behavior. +2. Select > 1 screenful in the timeline, scroll so the start is off-screen → button below + the visible end, not pinned to the top. +3. Same, scrolled so the end is off-screen → button above the visible start. +4. Selection taller than the pane, scrolled to the middle → button at the bottom edge of + the pane. +5. Select in the timeline, scroll until the selection leaves the pane entirely while + **idle** → button hides. While **speaking** → button parks at the pane edge and still + stops playback. +6. Selection in the Changes panel (a short pane) → button stays inside the panel and does + not invert past its far edge. +7. Step 3 only: start playback, scroll far enough to unmount the selected rows, confirm + audio keeps playing and the button still stops it. + +Assert on real geometry, not just presence: read the button's `getBoundingClientRect()` and +check it against the pane's box and the selection's caret rects. + +## Outcome + +### What shipped + +| File | Role | +| ---------------------------------------- | --------------------------------------------------------------------------- | +| `read-aloud-placement.ts` **(new)** | `decidePlacement()` — the table above as pure arithmetic, plus `AnchorRect` | +| `read-aloud-placement.test.ts` **(new)** | 14 vitest cases over that arithmetic | +| `use-selection-anchor.web.ts` | Geometry only: `text`, `firstRect`, `lastRect`, `visibleBox` | +| `read-aloud-selection-bubble.web.tsx` | Owns the widths, calls `decidePlacement`, owns stop intent | + +Three deliberate departures from the plan as written: + +- **The placement table is a pure function, unit-tested.** "Vitest cannot see any of this" + was only true while the math was tangled with the DOM. Playwright then verifies the + wiring, not the arithmetic. Five of the seven cases below are also vitest cases. +- **The hook takes a `retainWhileDetached` flag** rather than the bubble retaining a stale + anchor. Parking at the pane edge needs a _live_ `visibleBox` after the rows unmount, and + only the hook has the clipping chain. The chain's elements outlive the rows, so it + re-measures fine with `firstRect`/`lastRect` null. +- **Stop intent is `anchor.text` changing, not `anchor` going null.** The plan's step 3 said + to keep `handlePointerDown` nulling the anchor _and_ to stop nulling while speaking — + those collide, and the collision leaves audio playing with no stop control. Resolution: + outside-mousedown stays an explicit stop, and the bubble's effect keys on the text so a + replacement selection also stops the previous read. Escape stops and clears the selection; + it had no handler before. + +### The trap the plan missed + +Collapsed endpoint clones are the right idea, but **a mouse drag routinely ends at +`(DIV, 0)`** — an element boundary with nothing just inside it — and Chrome returns an empty +rect there. Falling back to `range.getBoundingClientRect()` reintroduces exactly the +container contamination the collapsed clone was avoiding: measured live, that fallback put +the bubble at the pane's _top_ edge for a selection whose visible end was 235px lower. + +Fix: when the collapsed clone is empty, resolve the endpoint to the nearest text node the +range actually covers and measure a one-character range there. That walk is O(N), so it runs +once when the selection settles; the resulting `Range` objects are cached and re-measured per +frame. Ranges track DOM mutation, so they stay correct as the virtualizer works, and a +detached one reports zeros — which is precisely the "rects are null" signal step 3 needs. + +### Verified in Chrome against the dev stack + +Every number below is `getBoundingClientRect()` off the live bubble, checked against the +pane's box and an independent measure of the selection's visible extent. + +| Case | Pane | Measured | +| ------------------------------------ | ------ | ------------------------------------------------------------- | +| 1. Fully visible | 84–765 | bubble bottom 254.5 = selection top 262.5 − 8 | +| 2. Start scrolled out of the pane | 84–765 | bubble top 308 = visible end 300 + 8 (was pinned to y=8) | +| 3. End scrolled out | 84–765 | bubble bottom 491.5 = visible start 499.5 − 8 | +| 3b. Tracking under scroll | 84–284 | 227/167/107 across three scroll steps, each `visBottom + 8` | +| 4. Selection taller than the pane | 84–244 | bubble top 204 = paneBottom − 40, **stable across 5 scrolls** | +| 5. Left the pane, idle | 84–284 | bubble gone | +| 5b. Left the pane, speaking | 84–764 | parked at 92 = paneTop + 8, still "Stop", press stopped it | +| 6. No room above → flip below | 84–764 | flipped to 147.5 = firstRect bottom + 8, never past the edge | +| 7. Selected subtree removed mid-read | 84–764 | selection gone, bubble stays at 724 = paneBottom − 40, stops | + +Plus the stop-intent rules: a new selection while speaking stops the previous read, Escape +stops and dismisses, and an outside click stops. + +**The Changes panel is the only surface with a multi-level chain**, and it is the one that +proves `visibleBox` is doing real work. Selecting a diff line resolves **seven** clipping +ancestors — the horizontally scrollable code column (`overflow-x: auto`, 356–800), the file +body, `git-diff-scroll` (`overflow-y: auto`, 120–901), and three more `hidden` wrappers up to +the root — intersecting to `{top 155, bottom 901, left 356, right 800}`. + +| Changes-panel case | Measured | +| ---------------------------- | ---------------------------------------------------------------------- | +| Select a diff line | no room above → below at 197 = selection bottom 189 + 8 | +| x-clamp basis | left 364 = **code column** left 356 + 8 — not the window, not the pane | +| Scroll the code column right | selection left ran 378 → 298 → 178; bubble held at 364, never left it | +| Scroll the diff pane past it | bubble gone (idle) | + +The horizontal-scroll row is the clearest evidence for bug 2's x-axis half: the old +`window.innerWidth` clamp would have let the bubble follow the text out of the code column +and over the sidebar. + +Case 7 was driven by removing the selected subtree from the DOM rather than by scrolling far +enough to trip the virtualizer's `overscan: 8` — the test conversation is not long enough to +unmount rows. The mechanism is the same one the virtualizer exercises (the range's nodes +detach), but a long-conversation run is still worth doing once. + +### Accepted, documented + +A selection with a large unselected gap in its middle can put the bubble over unselected +content in case 4. Anchoring to `visibleBox` buys stability under scroll and O(1) cost +exactly when the selection is largest; a mid-selection rect would jitter and cost O(N). diff --git a/packages/app/src/app/_layout.tsx b/packages/app/src/app/_layout.tsx index 037592558c..25070eb5a0 100644 --- a/packages/app/src/app/_layout.tsx +++ b/packages/app/src/app/_layout.tsx @@ -39,6 +39,7 @@ import { RootErrorBoundary } from "@/components/root-error-boundary"; import { WorkspaceSetupDialog } from "@/components/workspace-setup-dialog"; import { WorkspaceShortcutTargetsSubscriber } from "@/components/workspace-shortcut-targets-subscriber"; import { FloatingPanelPortalHost } from "@/components/ui/floating-panel-portal"; +import { ReadAloudSelectionBubble } from "@/read-aloud/read-aloud-selection-bubble"; import { HostChooserModal, useHostChooser } from "@/hosts/host-chooser"; import { getIsElectronRuntime, @@ -547,6 +548,7 @@ function AppContainer({ children, chromeEnabled: chromeEnabledOverride }: AppCon ) : null} {isCompactLayout ? sidebarChrome : null} + diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index 48d82e4257..08febc4ab1 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -1419,6 +1419,18 @@ export const ar: TranslationResources = { copied: "منقول", }, }, + readAloud: { + action: "القراءة بصوت عالٍ", + stop: "إيقاف", + speed: "سرعة التشغيل {{rate}}", + errors: { + ttsUnavailable: "لم يتم إعداد تحويل النص إلى كلام على هذا المضيف", + tooLong: "التحديد أطول من أن يُقرأ بصوت عالٍ", + empty: "لا يوجد شيء لقراءته بصوت عالٍ", + unsupported: "القراءة بصوت عالٍ غير متوفرة هنا", + failed: "تعذّرت قراءة ذلك بصوت عالٍ", + }, + }, realtimeVoice: { actions: { mute: "كتم صوت الوقت الحقيقي", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index a7fd2d1aaa..3556ffe0b2 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -1430,6 +1430,18 @@ export const en = { copied: "Copied", }, }, + readAloud: { + action: "Read aloud", + stop: "Stop", + speed: "{{rate}} playback speed", + errors: { + ttsUnavailable: "Text-to-speech isn't set up on this host", + tooLong: "Selection is too long to read aloud", + empty: "Nothing to read aloud", + unsupported: "Read aloud isn't available here", + failed: "Couldn't read that aloud", + }, + }, realtimeVoice: { actions: { mute: "Mute realtime voice", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index 1abd7cc059..2c310b69a2 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -1462,6 +1462,18 @@ export const es: TranslationResources = { copied: "Copiado", }, }, + readAloud: { + action: "Leer en voz alta", + stop: "Detener", + speed: "Velocidad de reproducción {{rate}}", + errors: { + ttsUnavailable: "La conversión de texto a voz no está configurada en este host", + tooLong: "La selección es demasiado larga para leerla en voz alta", + empty: "No hay nada que leer en voz alta", + unsupported: "Leer en voz alta no está disponible aquí", + failed: "No se pudo leer eso en voz alta", + }, + }, realtimeVoice: { actions: { mute: "Silenciar voz en tiempo real", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 58dc392cf1..92b2eec869 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -1465,6 +1465,18 @@ export const fr: TranslationResources = { copied: "Copié", }, }, + readAloud: { + action: "Lire à voix haute", + stop: "Arrêter", + speed: "Vitesse de lecture {{rate}}", + errors: { + ttsUnavailable: "La synthèse vocale n'est pas configurée sur cet hôte", + tooLong: "La sélection est trop longue pour être lue à voix haute", + empty: "Rien à lire à voix haute", + unsupported: "La lecture à voix haute n'est pas disponible ici", + failed: "Impossible de lire ce texte à voix haute", + }, + }, realtimeVoice: { actions: { mute: "Couper la voix en temps réel", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index 0be4e6e982..cb780395ed 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -1435,6 +1435,18 @@ export const ja: TranslationResources = { copied: "コピーしました", }, }, + readAloud: { + action: "読み上げ", + stop: "停止", + speed: "再生速度 {{rate}}", + errors: { + ttsUnavailable: "このホストでは音声合成が設定されていません", + tooLong: "選択範囲が長すぎて読み上げできません", + empty: "読み上げる内容がありません", + unsupported: "ここでは読み上げを利用できません", + failed: "読み上げできませんでした", + }, + }, realtimeVoice: { actions: { mute: "リアルタイム音声をミュート", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 51c9553fac..e76c99b8b5 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -1448,6 +1448,18 @@ export const ptBR: TranslationResources = { copied: "Copiado", }, }, + readAloud: { + action: "Ler em voz alta", + stop: "Parar", + speed: "Velocidade de reprodução {{rate}}", + errors: { + ttsUnavailable: "A conversão de texto em fala não está configurada neste host", + tooLong: "A seleção é longa demais para ser lida em voz alta", + empty: "Nada para ler em voz alta", + unsupported: "Ler em voz alta não está disponível aqui", + failed: "Não foi possível ler isso em voz alta", + }, + }, realtimeVoice: { actions: { mute: "Silenciar voz em tempo real", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index d546a92181..b016818c41 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -1453,6 +1453,18 @@ export const ru: TranslationResources = { copied: "Скопировано", }, }, + readAloud: { + action: "Прочитать вслух", + stop: "Остановить", + speed: "Скорость воспроизведения {{rate}}", + errors: { + ttsUnavailable: "Синтез речи не настроен на этом хосте", + tooLong: "Выделенный текст слишком длинный для чтения вслух", + empty: "Нечего читать вслух", + unsupported: "Чтение вслух здесь недоступно", + failed: "Не удалось прочитать это вслух", + }, + }, realtimeVoice: { actions: { mute: "Отключить звук в реальном времени", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index eb22734bdf..97178ffda7 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -1400,6 +1400,18 @@ export const zhCN: TranslationResources = { copied: "已复制", }, }, + readAloud: { + action: "朗读", + stop: "停止", + speed: "播放速度 {{rate}}", + errors: { + ttsUnavailable: "此主机未配置文字转语音", + tooLong: "选中的文本过长，无法朗读", + empty: "没有可朗读的内容", + unsupported: "此处无法使用朗读", + failed: "无法朗读该内容", + }, + }, realtimeVoice: { actions: { mute: "静音 realtime voice", diff --git a/packages/app/src/read-aloud/read-aloud-audio.ts b/packages/app/src/read-aloud/read-aloud-audio.ts new file mode 100644 index 0000000000..8a1ab5bf8d --- /dev/null +++ b/packages/app/src/read-aloud/read-aloud-audio.ts @@ -0,0 +1,25 @@ +/** + * Read-aloud playback, native fallback. + * + * Read aloud is driven by a text selection, and native has no JS-reachable + * selection API for `` — iOS hands selection to the system edit + * menu, Android to an ActionMode, and neither is reachable without a native + * module. So there is no entry point for it on native and playback is a no-op. + * The real implementation lives in `read-aloud-audio.web.ts`. + */ +export const isReadAloudAudioSupported = false; + +export async function playReadAloudSegment(_params: { + audioBase64: string; + format: string; +}): Promise { + throw new Error("Read aloud is not supported on this platform"); +} + +export function setReadAloudPlaybackRate(_rate: number): void { + // No playback to speed up. +} + +export function stopReadAloudAudio(): void { + // No playback to stop. +} diff --git a/packages/app/src/read-aloud/read-aloud-audio.web.ts b/packages/app/src/read-aloud/read-aloud-audio.web.ts new file mode 100644 index 0000000000..7698108fda --- /dev/null +++ b/packages/app/src/read-aloud/read-aloud-audio.web.ts @@ -0,0 +1,75 @@ +// Not a browser global — the bundle has no `Buffer` unless it is imported, and +// `@types/node` makes the bare reference typecheck while failing at runtime. +import { Buffer } from "buffer"; + +import { createAudioEngine } from "@/voice/audio-engine"; +import type { AudioEngine } from "@/voice/audio-engine-types"; + +/** + * Read-aloud playback on web. + * + * Reuses the voice-mode audio engine because on web its `initialize()` only + * opens a playback `AudioContext` — capture (and therefore the microphone + * permission prompt) is behind the separate `startCapture()` call, which read + * aloud never makes. `play()` already queues sequentially, which is exactly the + * ordering read-aloud segments need. + */ +export const isReadAloudAudioSupported = true; + +let engine: AudioEngine | null = null; + +function getEngine(): AudioEngine { + if (!engine) { + engine = createAudioEngine( + { + onCaptureData: () => undefined, + onVolumeLevel: () => undefined, + }, + { traceLabel: "read-aloud" }, + ); + } + return engine; +} + +function toMimeType(format: string): string { + if (format === "mp3") { + return "audio/mpeg"; + } + return format.startsWith("audio/") ? format : `audio/${format}`; +} + +/** + * Speed multiplier applied to every segment, including the one in flight. + * + * Kept here rather than passed per-segment so it survives the gap between + * segments and applies to audio the daemon has already streamed. + */ +export function setReadAloudPlaybackRate(rate: number): void { + getEngine().setPlaybackRate(rate); +} + +export async function playReadAloudSegment(params: { + audioBase64: string; + format: string; +}): Promise { + const bytes = Buffer.from(params.audioBase64, "base64"); + const active = getEngine(); + await active.initialize(); + await active.play({ + size: bytes.byteLength, + type: toMimeType(params.format), + async arrayBuffer() { + return Uint8Array.from(bytes).buffer; + }, + }); +} + +export function stopReadAloudAudio(): void { + if (!engine) { + return; + } + // Order matters: drain the queue first so `stop()` does not let a queued + // segment start playing after the user asked for silence. + engine.clearQueue(); + engine.stop(); +} diff --git a/packages/app/src/read-aloud/read-aloud-placement.test.ts b/packages/app/src/read-aloud/read-aloud-placement.test.ts new file mode 100644 index 0000000000..3c573e5e88 --- /dev/null +++ b/packages/app/src/read-aloud/read-aloud-placement.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "vitest"; + +import { + ANCHOR_OFFSET, + BOX_PADDING, + decidePlacement, + intersectRects, + type AnchorRect, +} from "@/read-aloud/read-aloud-placement"; + +const WIDTH = 32; +const HEIGHT = 32; + +/** A scroll pane occupying the right side of a 1200x900 window. */ +const PANE: AnchorRect = { top: 100, bottom: 700, left: 300, right: 1200 }; + +function lineRect(top: number, left: number, right: number): AnchorRect { + return { top, bottom: top + 19, left, right }; +} + +/** Caret rects are zero-width, which is what the geometry actually produces. */ +function caret(top: number, x: number): AnchorRect { + return { top, bottom: top + 19, left: x, right: x }; +} + +function place(firstRect: AnchorRect | null, lastRect: AnchorRect | null, box = PANE) { + return decidePlacement({ firstRect, lastRect, visibleBox: box, width: WIDTH, height: HEIGHT }); +} + +describe("intersectRects", () => { + it("intersects overlapping boxes", () => { + expect(intersectRects({ top: 0, bottom: 900, left: 0, right: 1200 }, PANE)).toEqual(PANE); + }); + + it("collapses instead of inverting when the boxes are disjoint", () => { + const result = intersectRects( + { top: 0, bottom: 100, left: 0, right: 100 }, + { top: 400, bottom: 500, left: 400, right: 500 }, + ); + expect(result.bottom).toBeGreaterThanOrEqual(result.top); + expect(result.right).toBeGreaterThanOrEqual(result.left); + }); +}); + +describe("decidePlacement", () => { + it("anchors above the start when the whole selection is visible", () => { + const first = lineRect(300, 400, 700); + const result = place(first, lineRect(340, 400, 620)); + + expect(result.isOffscreen).toBe(false); + expect(result.top).toBe(300 - HEIGHT - ANCHOR_OFFSET); + // Centred on the anchored rect, not on the whole selection. + expect(result.left).toBe(550 - WIDTH / 2); + }); + + it("anchors below the visible end when the start is scrolled out of the pane", () => { + // The start is above the pane but still inside the window — the case + // window-relative measurement gets wrong. + const first = lineRect(20, 400, 700); + const last = lineRect(400, 400, 620); + const result = place(first, last); + + expect(result.isOffscreen).toBe(false); + expect(result.top).toBe(last.bottom + ANCHOR_OFFSET); + expect(result.left).toBe(510 - WIDTH / 2); + }); + + it("keeps the above-the-start placement when only the end is clipped", () => { + const first = lineRect(400, 400, 700); + const result = place(first, lineRect(900, 400, 620)); + + expect(result.isOffscreen).toBe(false); + expect(result.top).toBe(400 - HEIGHT - ANCHOR_OFFSET); + }); + + it("anchors to the pane's bottom edge when the selection is taller than the pane", () => { + const result = place(lineRect(-500, 400, 700), lineRect(1400, 400, 620)); + + expect(result.isOffscreen).toBe(false); + expect(result.top).toBe(PANE.bottom - HEIGHT - ANCHOR_OFFSET); + expect(result.left).toBe((PANE.left + PANE.right) / 2 - WIDTH / 2); + }); + + it("reports offscreen and parks at the top edge when the selection scrolled above the pane", () => { + const result = place(lineRect(-200, 400, 700), lineRect(-100, 400, 620)); + + expect(result.isOffscreen).toBe(true); + expect(result.top).toBe(PANE.top + BOX_PADDING); + }); + + it("reports offscreen and parks at the bottom edge when the selection scrolled below the pane", () => { + const result = place(lineRect(900, 400, 700), lineRect(1000, 400, 620)); + + expect(result.isOffscreen).toBe(true); + expect(result.top).toBe(PANE.bottom - HEIGHT - BOX_PADDING); + }); + + it("parks at the bottom edge when the range's nodes have detached", () => { + const result = place(null, null); + + expect(result.isOffscreen).toBe(true); + expect(result.top).toBe(PANE.bottom - HEIGHT - BOX_PADDING); + expect(result.left).toBe((PANE.left + PANE.right) / 2 - WIDTH / 2); + }); + + it("flips below when there is no room above the start", () => { + const first = lineRect(PANE.top + 2, 400, 700); + const result = place(first, lineRect(PANE.top + 40, 400, 620)); + + expect(result.top).toBe(first.bottom + ANCHOR_OFFSET); + }); + + it("flips above when there is no room below the visible end", () => { + const first = lineRect(20, 400, 700); + const last = lineRect(PANE.bottom - 20, 400, 620); + const result = place(first, last); + + expect(result.top).toBe(last.top - HEIGHT - ANCHOR_OFFSET); + }); + + it("clamps into the pane, not the window", () => { + // A selection hugging the pane's left edge would otherwise centre the + // bubble over the sidebar. + const first = caret(300, PANE.left + 2); + const result = place(first, caret(320, PANE.left + 40)); + + expect(result.left).toBe(PANE.left + BOX_PADDING); + }); + + it("centres rather than inverting when the box is too short to hold the bubble", () => { + const shortPane: AnchorRect = { top: 200, bottom: 236, left: 300, right: 1200 }; + const result = place(lineRect(210, 400, 700), lineRect(215, 400, 620), shortPane); + + // The naive clamp would push the bubble to `bottom - HEIGHT - PADDING`, + // which is above the box's top. + expect(result.top).toBe((shortPane.top + shortPane.bottom) / 2 - HEIGHT / 2); + expect(result.top).toBeGreaterThanOrEqual(shortPane.top); + }); + + it("keeps a wide failure pill inside the pane", () => { + const first = caret(300, PANE.right - 10); + const result = decidePlacement({ + firstRect: first, + lastRect: caret(300, PANE.right - 4), + visibleBox: PANE, + width: 220, + height: HEIGHT, + }); + + expect(result.left + 220).toBeLessThanOrEqual(PANE.right - BOX_PADDING); + }); +}); diff --git a/packages/app/src/read-aloud/read-aloud-placement.ts b/packages/app/src/read-aloud/read-aloud-placement.ts new file mode 100644 index 0000000000..21429a6608 --- /dev/null +++ b/packages/app/src/read-aloud/read-aloud-placement.ts @@ -0,0 +1,166 @@ +/** + * Where the read-aloud bubble goes, given the selection's endpoints and the box + * it is actually visible in. + * + * Pure arithmetic, no DOM: the geometry is read in + * `use-selection-anchor.web.ts` and the widths are owned by + * `read-aloud-selection-bubble.web.tsx`, so the decision in between is testable + * on its own. + */ + +/** A viewport-space box. Structurally a subset of `DOMRect`. */ +export interface AnchorRect { + top: number; + bottom: number; + left: number; + right: number; +} + +export interface PlacementInput { + /** Caret rect at the selection's start, or null once its nodes detached. */ + firstRect: AnchorRect | null; + /** Caret rect at the selection's end, or null once its nodes detached. */ + lastRect: AnchorRect | null; + /** Window intersected with every clipping ancestor — the real visible area. */ + visibleBox: AnchorRect; + width: number; + height: number; +} + +export interface Placement { + left: number; + top: number; + /** + * The selection is not in view. Idle bubbles hide; a speaking bubble stays + * parked at the nearest edge, because it is the only stop control. + */ + isOffscreen: boolean; +} + +/** Gap between the selection edge and the bubble. */ +export const ANCHOR_OFFSET = 8; +/** Breathing room between the bubble and the edge of the visible box. */ +export const BOX_PADDING = 8; + +export function intersectRects(a: AnchorRect, b: AnchorRect): AnchorRect { + const top = Math.max(a.top, b.top); + const left = Math.max(a.left, b.left); + // Disjoint boxes would invert; collapse to a degenerate box at the overlap + // edge so downstream math never sees `bottom < top`. + return { + top, + left, + bottom: Math.max(top, Math.min(a.bottom, b.bottom)), + right: Math.max(left, Math.min(a.right, b.right)), + }; +} + +/** + * Vertical-only: a rect is "in view" when any part of its line band overlaps + * the box. Horizontal overlap does not decide which endpoint to anchor to — + * the final clamp handles the x-axis. + */ +function isVisible(rect: AnchorRect, box: AnchorRect): boolean { + return rect.bottom > box.top && rect.top < box.bottom; +} + +/** + * Clamp one axis into `[min, max]`. + * + * A box shorter than the bubble plus its padding has no valid slot, and the + * naive `max(pad, min(limit, value))` inverts there — the `max` wins and pushes + * the bubble out past the far edge. Centring is the sane degenerate answer. + */ +function clampIntoRange(value: number, size: number, min: number, max: number): number { + const lo = min + BOX_PADDING; + const hi = max - size - BOX_PADDING; + if (hi < lo) { + return (min + max) / 2 - size / 2; + } + return Math.min(Math.max(value, lo), hi); +} + +function centerXOf(rect: AnchorRect): number { + return (rect.left + rect.right) / 2; +} + +interface AnchorChoice { + centerX: number; + top: number; + isOffscreen: boolean; +} + +/** Above the rect, flipped below when there is no room above. */ +function above(rect: AnchorRect, box: AnchorRect, height: number): number { + const top = rect.top - height - ANCHOR_OFFSET; + return top < box.top + BOX_PADDING ? rect.bottom + ANCHOR_OFFSET : top; +} + +/** Below the rect, flipped above when there is no room below. */ +function below(rect: AnchorRect, box: AnchorRect, height: number): number { + const top = rect.bottom + ANCHOR_OFFSET; + return top + height > box.bottom - BOX_PADDING ? rect.top - height - ANCHOR_OFFSET : top; +} + +function pickAnchor( + firstRect: AnchorRect | null, + lastRect: AnchorRect | null, + box: AnchorRect, + height: number, +): AnchorChoice { + // Start endpoint in view — including "both visible" and "end clipped" — keeps + // the original above-the-start placement. + if (firstRect && isVisible(firstRect, box)) { + return { + centerX: centerXOf(firstRect), + top: above(firstRect, box, height), + isOffscreen: false, + }; + } + // Start scrolled out, end still in view: anchor below what the user can see. + if (lastRect && isVisible(lastRect, box)) { + return { centerX: centerXOf(lastRect), top: below(lastRect, box, height), isOffscreen: false }; + } + + // Neither endpoint is in view. A selection taller than the box still has its + // middle on screen, so anchor to the box itself rather than scanning for a + // mid-selection rect: stable under scroll, and O(1) exactly when the + // selection is largest. Accepted imperfection — a selection with a big + // unselected gap in the middle can put the bubble over unselected content. + const spansBox = + firstRect !== null && + lastRect !== null && + firstRect.top <= box.top && + lastRect.bottom >= box.bottom; + if (spansBox) { + return { + centerX: centerXOf(box), + top: box.bottom - height - ANCHOR_OFFSET, + isOffscreen: false, + }; + } + + // Entirely out of view: park at the edge it went out through. + const scrolledAbove = lastRect !== null && lastRect.bottom <= box.top; + const parkRect = scrolledAbove ? lastRect : (firstRect ?? lastRect); + return { + centerX: parkRect ? centerXOf(parkRect) : centerXOf(box), + top: scrolledAbove ? box.top + BOX_PADDING : box.bottom - height - BOX_PADDING, + isOffscreen: true, + }; +} + +export function decidePlacement({ + firstRect, + lastRect, + visibleBox, + width, + height, +}: PlacementInput): Placement { + const anchor = pickAnchor(firstRect, lastRect, visibleBox, height); + return { + left: clampIntoRange(anchor.centerX - width / 2, width, visibleBox.left, visibleBox.right), + top: clampIntoRange(anchor.top, height, visibleBox.top, visibleBox.bottom), + isOffscreen: anchor.isOffscreen, + }; +} diff --git a/packages/app/src/read-aloud/read-aloud-selection-bubble.tsx b/packages/app/src/read-aloud/read-aloud-selection-bubble.tsx new file mode 100644 index 0000000000..31a6bdc9af --- /dev/null +++ b/packages/app/src/read-aloud/read-aloud-selection-bubble.tsx @@ -0,0 +1,12 @@ +/** + * Read-aloud selection bubble, native fallback. + * + * The bubble is anchored to a live text selection, and native gives JS no way to + * read one: `` hands selection to the iOS edit menu or the + * Android ActionMode, neither of which is reachable without a native module. So + * there is nothing to anchor to and the component renders nothing. The real + * implementation lives in `read-aloud-selection-bubble.web.tsx`. + */ +export function ReadAloudSelectionBubble(): null { + return null; +} diff --git a/packages/app/src/read-aloud/read-aloud-selection-bubble.web.tsx b/packages/app/src/read-aloud/read-aloud-selection-bubble.web.tsx new file mode 100644 index 0000000000..1b77dc85af --- /dev/null +++ b/packages/app/src/read-aloud/read-aloud-selection-bubble.web.tsx @@ -0,0 +1,400 @@ +import { useCallback, useEffect, useMemo, type ReactElement, type ReactNode } from "react"; +import { createPortal } from "react-dom"; +import { Pressable, Text, View } from "react-native"; +import { useTranslation } from "react-i18next"; +import { StyleSheet } from "react-native-unistyles"; +import { Square, Volume2 } from "lucide-react-native"; +import { useLocalSearchParams, usePathname } from "expo-router"; + +import { LoadingSpinner } from "@/components/ui/loading-spinner"; +import { getOverlayRoot, OVERLAY_Z } from "@/lib/overlay-root"; +import { decidePlacement } from "@/read-aloud/read-aloud-placement"; +import { + READ_ALOUD_RATES, + setReadAloudRate, + startReadAloud, + stopReadAloud, + useReadAloudSnapshot, + type ReadAloudFailure, + type ReadAloudRate, + type ReadAloudSnapshot, +} from "@/read-aloud/read-aloud-store"; +import { + READ_ALOUD_BUBBLE_DATASET, + useSelectionAnchor, +} from "@/read-aloud/use-selection-anchor.web"; +import { useHostFeatureMap } from "@/runtime/host-features"; +import { useHostRuntimeClient, useHosts } from "@/runtime/host-runtime"; +import { parseActiveWorkspaceSelection } from "@/stores/navigation-active-workspace-store/navigation"; + +const BUBBLE_HEIGHT = 32; +/** Icon-only: the speaker says "read this" without a caption. */ +const BUBBLE_WIDTH = 32; +/** A failure is the one case that earns words, so it can explain itself. */ +const FAILED_BUBBLE_WIDTH = 220; +/** The pill draws a 1px border, so its content box is 2px narrower than the pill. */ +const BUBBLE_BORDER = 1; +const ICON_BUTTON_WIDTH = BUBBLE_WIDTH - BUBBLE_BORDER * 2; +/** Width of one speed chip, and the gap the row adds around them. */ +const RATE_CHIP_WIDTH = 34; +const RATE_ROW_GAP = 2; +const RATE_ROW_PADDING = 4; +/** + * Total width once the speed chips are showing. They sit in the same pill as + * the stop button, so the whole thing has to be measured for the anchor math. + */ +const SPEAKING_BUBBLE_WIDTH = + BUBBLE_BORDER * 2 + + ICON_BUTTON_WIDTH + + READ_ALOUD_RATES.length * (RATE_CHIP_WIDTH + RATE_ROW_GAP) + + RATE_ROW_PADDING; + +function formatRate(rate: ReadAloudRate): string { + return `${rate}x`; +} + +function useRouteServerId(): string | null { + const params = useLocalSearchParams<{ + serverId?: string | string[]; + workspaceId?: string | string[]; + }>(); + const pathname = usePathname(); + // Read-only: unlike `useActiveWorkspaceSelection`, this must not also record + // the workspace as "last visited" — that stays owned by the workspace screen. + return parseActiveWorkspaceSelection({ pathname, params })?.serverId ?? null; +} + +/** + * Which host synthesizes the speech. + * + * Read aloud carries no workspace context — it is pure text-to-speech — so any + * host that advertises the capability will do. The host owning the current + * workspace is preferred so the work lands where the content came from, but + * selections on routes without a workspace (settings, history) still get a + * voice. Reachability is not checked here: the daemon client resolves to `null` + * before it has a transport, and a host that drops mid-request surfaces the + * failure on the bubble. + */ +function useReadAloudServerId(): string | null { + const routeServerId = useRouteServerId(); + const hosts = useHosts(); + const serverIds = useMemo(() => hosts.map((host) => host.serverId), [hosts]); + const featureByServerId = useHostFeatureMap(serverIds, "readAloud"); + + return useMemo(() => { + const supportsReadAloud = (serverId: string) => featureByServerId.get(serverId) === true; + + if (routeServerId && supportsReadAloud(routeServerId)) { + return routeServerId; + } + return serverIds.find(supportsReadAloud) ?? null; + }, [featureByServerId, routeServerId, serverIds]); +} + +function renderStatusIcon(status: ReadAloudSnapshot["status"], color: string): ReactNode { + if (status === "loading") { + return ; + } + if (status === "speaking") { + return ; + } + return ; +} + +function useFailureLabel(failure: ReadAloudFailure | null): string | null { + const { t } = useTranslation(); + if (!failure) { + return null; + } + switch (failure.code) { + case "tts_unavailable": + return t("readAloud.errors.ttsUnavailable"); + case "text_too_long": + return t("readAloud.errors.tooLong"); + case "empty_text": + return t("readAloud.errors.empty"); + case "unsupported_platform": + return t("readAloud.errors.unsupported"); + default: + return t("readAloud.errors.failed"); + } +} + +/** + * The pill takes one of three shapes: a round icon, a wide pill with the speed + * chips, or a labelled failure. A function rather than a reassigned variable — + * the three styles have different shapes, so the first assignment would pin the + * variable's type and reject the rest. + */ +function pickContainerStyle(hasFailure: boolean, showRates: boolean) { + if (hasFailure) { + return styles.failedBubble; + } + return showRates ? styles.speakingBubble : styles.bubble; +} + +/** + * One speed button. Split out of the bubble so the press handler and the + * accessibility state are stable per rate instead of rebuilt on every render. + */ +function RateChip({ rate, isActive }: { rate: ReadAloudRate; isActive: boolean }): ReactElement { + const { t } = useTranslation(); + const handlePress = useCallback(() => setReadAloudRate(rate), [rate]); + const accessibilityState = useMemo(() => ({ selected: isActive }), [isActive]); + + return ( + + {formatRate(rate)} + + ); +} + +export function ReadAloudSelectionBubble(): ReactElement | null { + const { t } = useTranslation(); + // COMPAT(readAloud): added in v0.2.5, drop the gate when floor >= v0.2.5. + // `useReadAloudServerId` only returns hosts that advertise the RPC, so hosts + // without it simply get no bubble — an upgrade prompt on every text selection + // would be worse than the feature being absent. + const serverId = useReadAloudServerId(); + const client = useHostRuntimeClient(serverId ?? ""); + const enabled = serverId !== null && client !== null; + + const snapshot = useReadAloudSnapshot(); + const failureLabel = useFailureLabel(snapshot.failure); + const isBusy = snapshot.status !== "idle"; + + // While something is playing the anchor outlives its range, so scrolling the + // selected rows out of the virtualizer no longer kills the read. + const anchor = useSelectionAnchor(enabled, isBusy); + const anchorText = anchor?.text ?? null; + + // Playback stops on intent, not on geometry: the selection was replaced, or + // the user clicked away (which drops the anchor even while retaining). A + // selection merely scrolled out of view keeps reading. + useEffect(() => { + stopReadAloud(); + }, [anchorText]); + + useEffect(() => { + if (!enabled) { + stopReadAloud(); + } + }, [enabled]); + + // A boolean, not the anchor itself: the anchor is a fresh object on every + // scroll frame, which would re-register this listener 60 times a second. + const hasAnchor = anchor !== null; + useEffect(() => { + if (!hasAnchor) { + return; + } + const handleKeyDown = (event: KeyboardEvent) => { + if (event.key !== "Escape") { + return; + } + stopReadAloud(); + // Dropping the selection dismisses the bubble too — Escape means "done + // with this", not just "be quiet". + window.getSelection()?.removeAllRanges(); + }; + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [hasAnchor]); + + const setContainerRef = useCallback((node: View | null) => { + const element = node as unknown as HTMLElement | null; + if (!element) { + return; + } + // Web-only file: a mousedown that reaches the browser collapses the + // selection before the press handler runs, and the selection is what the + // bubble is anchored to. + const preventSelectionCollapse = (event: MouseEvent) => event.preventDefault(); + element.addEventListener("mousedown", preventSelectionCollapse); + return () => element.removeEventListener("mousedown", preventSelectionCollapse); + }, []); + + const handlePress = useCallback(() => { + if (!client || !anchor) { + return; + } + if (isBusy) { + stopReadAloud(); + return; + } + // A press after a failure is a retry; `startReadAloud` clears the failure. + startReadAloud({ client, text: anchor.text }); + }, [anchor, client, isBusy]); + + // The speed chips only appear once there is something to speed up, so an + // ordinary text selection still gets the small icon-only bubble. + const showRates = failureLabel === null && isBusy; + let width = BUBBLE_WIDTH; + if (failureLabel !== null) { + width = FAILED_BUBBLE_WIDTH; + } else if (showRates) { + width = SPEAKING_BUBBLE_WIDTH; + } + + const position = useMemo(() => { + if (!anchor) { + return null; + } + return decidePlacement({ + firstRect: anchor.firstRect, + lastRect: anchor.lastRect, + visibleBox: anchor.visibleBox, + width, + height: BUBBLE_HEIGHT, + }); + }, [anchor, width]); + + if (!enabled || !anchor || !position) { + return null; + } + // Out of view and nothing playing: there is nothing to point at and nothing + // to stop. Mid-read the bubble stays, parked at the edge of its pane. + if (position.isOffscreen && !isBusy) { + return null; + } + + const label = failureLabel ?? (isBusy ? t("readAloud.stop") : t("readAloud.action")); + const containerStyle = pickContainerStyle(failureLabel !== null, showRates); + + return createPortal( + + + + {renderStatusIcon( + snapshot.status, + failureLabel === null ? styles.icon.color : styles.failedIcon.color, + )} + {/* The icon carries the action on its own; only a failure needs words. */} + {failureLabel === null ? null : ( + + {failureLabel} + + )} + + {!showRates + ? null + : READ_ALOUD_RATES.map((rate) => ( + + ))} + + , + getOverlayRoot(), + ); +} + +const styles = StyleSheet.create((theme) => ({ + anchorLayer: { + position: "absolute", + zIndex: OVERLAY_Z.toast, + // The shared overlay root is `pointer-events: none`; opt this subtree back in. + pointerEvents: "auto", + }, + bubble: { + alignItems: "center", + justifyContent: "center", + height: BUBBLE_HEIGHT, + width: BUBBLE_WIDTH, + borderRadius: theme.borderRadius.full, + backgroundColor: theme.colors.popover, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.borderAccent, + ...theme.shadow.md, + }, + speakingBubble: { + flexDirection: "row", + alignItems: "center", + gap: RATE_ROW_GAP, + height: BUBBLE_HEIGHT, + // The stop button keeps the pill's full round end; only the chip side needs + // breathing room, so the padding is asymmetric. + paddingRight: RATE_ROW_PADDING, + borderRadius: theme.borderRadius.full, + backgroundColor: theme.colors.popover, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.borderAccent, + ...theme.shadow.md, + }, + failedBubble: { + flexDirection: "row", + alignItems: "center", + height: BUBBLE_HEIGHT, + maxWidth: FAILED_BUBBLE_WIDTH, + paddingHorizontal: theme.spacing[2], + borderRadius: theme.borderRadius.xl, + backgroundColor: theme.colors.popover, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.borderAccent, + ...theme.shadow.md, + }, + iconButton: { + alignItems: "center", + justifyContent: "center", + // Stretch rather than a fixed height: the pill owns the border, so the + // content box is shorter than BUBBLE_HEIGHT and a fixed child would spill. + alignSelf: "stretch", + width: ICON_BUTTON_WIDTH, + flexShrink: 0, + }, + failedContent: { + flexDirection: "row", + alignItems: "center", + alignSelf: "stretch", + gap: theme.spacing[1], + }, + rateChip: { + alignItems: "center", + justifyContent: "center", + width: RATE_CHIP_WIDTH, + height: BUBBLE_HEIGHT - RATE_ROW_PADDING * 2, + borderRadius: theme.borderRadius.full, + }, + rateChipActive: { + alignItems: "center", + justifyContent: "center", + width: RATE_CHIP_WIDTH, + height: BUBBLE_HEIGHT - RATE_ROW_PADDING * 2, + borderRadius: theme.borderRadius.full, + backgroundColor: theme.colors.accent, + }, + rateLabel: { + color: theme.colors.mutedForeground, + fontSize: theme.fontSize.xs, + fontWeight: "500", + }, + rateLabelActive: { + color: theme.colors.accentForeground, + fontSize: theme.fontSize.xs, + fontWeight: "600", + }, + icon: { + color: theme.colors.foreground, + }, + failedIcon: { + color: theme.colors.destructive, + }, + label: { + color: theme.colors.foreground, + fontSize: theme.fontSize.sm, + fontWeight: "500", + }, +})); diff --git a/packages/app/src/read-aloud/read-aloud-store.test.ts b/packages/app/src/read-aloud/read-aloud-store.test.ts new file mode 100644 index 0000000000..623967aec6 --- /dev/null +++ b/packages/app/src/read-aloud/read-aloud-store.test.ts @@ -0,0 +1,84 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { DaemonClient } from "@getpaseo/client"; + +const playReadAloudSegment = vi.fn().mockResolvedValue(undefined); +const setReadAloudPlaybackRate = vi.fn(); +const stopReadAloudAudio = vi.fn(); + +vi.mock("@/read-aloud/read-aloud-audio", () => ({ + isReadAloudAudioSupported: true, + playReadAloudSegment: (params: { audioBase64: string; format: string }) => + playReadAloudSegment(params), + setReadAloudPlaybackRate: (rate: number) => setReadAloudPlaybackRate(rate), + stopReadAloudAudio: () => stopReadAloudAudio(), +})); + +const { getReadAloudSnapshot, READ_ALOUD_RATES, setReadAloudRate, startReadAloud, stopReadAloud } = + await import("@/read-aloud/read-aloud-store"); + +/** Minimal stand-in: the store only ever calls `startReadAloud` on the client. */ +function createClient(): DaemonClient { + return { + startReadAloud: vi.fn().mockReturnValue({ requestId: "req-1", cancel: vi.fn() }), + } as unknown as DaemonClient; +} + +describe("read aloud playback rate", () => { + beforeEach(() => { + stopReadAloud(); + setReadAloudRate(1); + vi.clearAllMocks(); + }); + + it("offers speeds capped at 2x, because playback is not pitch-corrected", () => { + expect(READ_ALOUD_RATES).toEqual([1, 1.5, 2]); + }); + + it("applies a rate change to audio that is already playing", () => { + startReadAloud({ client: createClient(), text: "hello" }); + setReadAloudRate(2); + + expect(setReadAloudPlaybackRate).toHaveBeenCalledWith(2); + expect(getReadAloudSnapshot().rate).toBe(2); + }); + + it("keeps the chosen rate after stopping, and re-applies it on the next read", () => { + setReadAloudRate(1.5); + stopReadAloud(); + + // The rate outlives the playback it was chosen during: picking 1.5x once + // should still be 1.5x for the next selection. + expect(getReadAloudSnapshot().rate).toBe(1.5); + + setReadAloudPlaybackRate.mockClear(); + startReadAloud({ client: createClient(), text: "hello again" }); + + // Re-applied rather than assumed: the audio engine is lazily created and + // starts at 1x, so a stale engine would silently play at the wrong speed. + expect(setReadAloudPlaybackRate).toHaveBeenCalledWith(1.5); + }); + + it("ignores a tap on the rate that is already selected", () => { + setReadAloudRate(2); + setReadAloudPlaybackRate.mockClear(); + + setReadAloudRate(2); + + expect(setReadAloudPlaybackRate).not.toHaveBeenCalled(); + }); + + it("reports the rate on every snapshot, including failures", () => { + setReadAloudRate(2); + const client = createClient(); + vi.mocked(client.startReadAloud).mockImplementation((params) => { + params.onError({ code: "tts_unavailable", message: "no tts" }); + return { requestId: "req-1", cancel: vi.fn() }; + }); + + startReadAloud({ client, text: "hello" }); + + const snapshot = getReadAloudSnapshot(); + expect(snapshot.failure?.code).toBe("tts_unavailable"); + expect(snapshot.rate).toBe(2); + }); +}); diff --git a/packages/app/src/read-aloud/read-aloud-store.ts b/packages/app/src/read-aloud/read-aloud-store.ts new file mode 100644 index 0000000000..9e7b4540c5 --- /dev/null +++ b/packages/app/src/read-aloud/read-aloud-store.ts @@ -0,0 +1,185 @@ +import { useSyncExternalStore } from "react"; +import type { DaemonClient, ReadAloudHandle } from "@getpaseo/client"; + +import { + isReadAloudAudioSupported, + playReadAloudSegment, + setReadAloudPlaybackRate, + stopReadAloudAudio, +} from "@/read-aloud/read-aloud-audio"; + +/** + * Selectable speeds. + * + * Capped at 2x because playback is a raw resampling rate on the shared voice + * audio engine — pitch is not preserved, and past 2x the voice stops sounding + * like a voice. 2x matches what podcast and article readers offer anyway. + */ +export const READ_ALOUD_RATES = [1, 1.5, 2] as const; + +export type ReadAloudRate = (typeof READ_ALOUD_RATES)[number]; + +const DEFAULT_RATE: ReadAloudRate = 1; + +export interface ReadAloudFailure { + /** Daemon-side code (`tts_unavailable`, `text_too_long`, …) or a client code. */ + code: string; + /** Raw daemon message, used only when the code has no translated copy. */ + message: string; +} + +export interface ReadAloudSnapshot { + /** `loading` = synthesizing, no audio yet. `speaking` = audio is playing. */ + status: "idle" | "loading" | "speaking"; + failure: ReadAloudFailure | null; + rate: ReadAloudRate; +} + +/** + * Outlives any single playback: picking 2x once should hold for the next + * selection too, so it deliberately survives `stopReadAloud`. + */ +let playbackRate: ReadAloudRate = DEFAULT_RATE; + +let snapshot: ReadAloudSnapshot = { status: "idle", failure: null, rate: playbackRate }; +const listeners = new Set<() => void>(); + +/** + * Bumped by every start/stop so callbacks from a superseded request are ignored + * instead of resurrecting state for audio the user already dismissed. + */ +let generation = 0; +let handle: ReadAloudHandle | null = null; +let pendingSegmentPlaybacks = 0; +let streamEnded = false; + +// Takes everything but `rate`: the rate is owned by module state, so folding it +// in here keeps every call site from having to remember to carry it forward. +function setSnapshot(next: Omit): void { + snapshot = { ...next, rate: playbackRate }; + for (const listener of listeners) { + listener(); + } +} + +function finishIfDone(token: number): void { + if (token !== generation) { + return; + } + if (!streamEnded || pendingSegmentPlaybacks > 0) { + return; + } + handle = null; + setSnapshot({ status: "idle", failure: snapshot.failure }); +} + +export function getReadAloudSnapshot(): ReadAloudSnapshot { + return snapshot; +} + +export function subscribeReadAloud(listener: () => void): () => void { + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +export function useReadAloudSnapshot(): ReadAloudSnapshot { + return useSyncExternalStore(subscribeReadAloud, getReadAloudSnapshot, getReadAloudSnapshot); +} + +/** Stop playback and cancel daemon-side synthesis. Safe to call when idle. */ +export function stopReadAloud(): void { + generation += 1; + handle?.cancel(); + handle = null; + pendingSegmentPlaybacks = 0; + streamEnded = false; + stopReadAloudAudio(); + setSnapshot({ status: "idle", failure: null }); +} + +/** + * Change speed. Takes effect immediately on audio that is already playing, so + * the user hears the result of the tap without restarting the selection. + */ +export function setReadAloudRate(rate: ReadAloudRate): void { + if (rate === playbackRate) { + return; + } + playbackRate = rate; + setReadAloudPlaybackRate(rate); + setSnapshot({ status: snapshot.status, failure: snapshot.failure }); +} + +export function startReadAloud(params: { client: DaemonClient; text: string }): void { + stopReadAloud(); + + if (!isReadAloudAudioSupported) { + setSnapshot({ + status: "idle", + failure: { code: "unsupported_platform", message: "Read aloud is not available here" }, + }); + return; + } + + // The engine is created lazily and defaults to 1x, so a rate chosen during an + // earlier selection has to be re-applied rather than assumed to still be set. + setReadAloudPlaybackRate(playbackRate); + + const token = generation; + setSnapshot({ status: "loading", failure: null }); + + handle = params.client.startReadAloud({ + text: params.text, + onSegment: (segment) => { + if (token !== generation) { + return; + } + pendingSegmentPlaybacks += 1; + if (snapshot.status === "loading") { + setSnapshot({ status: "speaking", failure: null }); + } + const settleSegment = (error?: unknown) => { + if (token !== generation) { + return; + } + pendingSegmentPlaybacks = Math.max(0, pendingSegmentPlaybacks - 1); + // A stop is not a failure — `stopReadAloud` already bumped `generation`, + // so anything reaching here is a genuine decode/playback problem. Report + // it: silently swallowing meant a broken segment looked like success + // with no audio, which is indistinguishable from "the feature is dead". + if (error !== undefined) { + setSnapshot({ + status: snapshot.status, + failure: { + code: "playback_failed", + message: error instanceof Error ? error.message : String(error), + }, + }); + } + finishIfDone(token); + }; + void playReadAloudSegment({ + audioBase64: segment.audioBase64, + format: segment.format, + }).then( + () => settleSegment(), + (error: unknown) => settleSegment(error ?? new Error("Playback failed")), + ); + }, + onError: (error) => { + if (token !== generation) { + return; + } + setSnapshot({ status: snapshot.status, failure: { ...error } }); + }, + onEnd: () => { + if (token !== generation) { + return; + } + streamEnded = true; + finishIfDone(token); + }, + }); +} diff --git a/packages/app/src/read-aloud/use-selection-anchor.web.ts b/packages/app/src/read-aloud/use-selection-anchor.web.ts new file mode 100644 index 0000000000..b00030f120 --- /dev/null +++ b/packages/app/src/read-aloud/use-selection-anchor.web.ts @@ -0,0 +1,410 @@ +import { useEffect, useRef, useState } from "react"; + +import { intersectRects, type AnchorRect } from "@/read-aloud/read-aloud-placement"; + +export interface SelectionAnchor { + /** The selected text, captured when the selection settles. */ + text: string; + /** + * Caret rect at the selection's start. Null once the range's nodes have + * detached — the timeline virtualizer unmounts rows outside its overscan — + * while the anchor is being retained for an in-flight read. + */ + firstRect: AnchorRect | null; + /** Caret rect at the selection's end. Null under the same conditions. */ + lastRect: AnchorRect | null; + /** + * Window intersected with every clipping ancestor. Client rects ignore + * ancestor `overflow` entirely, so text scrolled out of the timeline pane + * still reports a rect inside the window; this is the box that decides what + * "visible" means. + */ + visibleBox: AnchorRect; +} + +/** Below this a selection is almost certainly a stray click-drag, not a phrase. */ +const MIN_SELECTION_CHARS = 2; + +/** + * Marks the bubble subtree so selection tracking can tell "the user pressed the + * bubble" apart from "the user clicked away". Applied via `dataSet`, which + * react-native-web renders as `data-read-aloud-bubble`. + */ +export const READ_ALOUD_BUBBLE_DATASET = { readAloudBubble: "" } as const; + +const READ_ALOUD_BUBBLE_SELECTOR = "[data-read-aloud-bubble]"; + +function isInsideReadAloudBubble(target: EventTarget | null): boolean { + const element = resolveElement(target instanceof Node ? target : null); + return element?.closest(READ_ALOUD_BUBBLE_SELECTOR) != null; +} + +/** How long the selection must hold still (keyboard selection) before we anchor. */ +const SETTLE_MS = 180; + +/** Computed `overflow` values that clip descendants out of view. */ +const CLIPPING_OVERFLOW = new Set(["auto", "scroll", "hidden", "clip"]); + +function resolveElement(node: Node | null): HTMLElement | null { + if (!node) { + return null; + } + if (node instanceof HTMLElement) { + return node; + } + return node.parentElement; +} + +/** + * Read aloud covers agent output, diffs, and file contents — everything except + * surfaces that own their own selection semantics: form fields (the composer) + * and the terminal, which already implements copy-on-select over xterm. + */ +function isReadAloudEligible(range: Range): boolean { + const element = resolveElement(range.commonAncestorContainer); + if (!element || !element.isConnected) { + return false; + } + return !element.closest( + 'input, textarea, select, [contenteditable=""], [contenteditable="true"], .xterm', + ); +} + +function windowRect(): AnchorRect { + return { top: 0, left: 0, bottom: window.innerHeight, right: window.innerWidth }; +} + +/** + * Every ancestor that clips its descendants, nearest first. + * + * Recomputed only when the selection settles: the chain itself does not change + * on scroll, only its rects do, so per-frame cost stays a handful of + * `getBoundingClientRect()` calls. + */ +function collectClipChain(start: HTMLElement | null): HTMLElement[] { + const chain: HTMLElement[] = []; + for (let node = start; node; node = node.parentElement) { + const style = window.getComputedStyle(node); + if (CLIPPING_OVERFLOW.has(style.overflowX) || CLIPPING_OVERFLOW.has(style.overflowY)) { + chain.push(node); + } + } + return chain; +} + +function computeVisibleBox(chain: HTMLElement[]): AnchorRect { + let box = windowRect(); + for (const element of chain) { + // A clipping ancestor can unmount under a retained anchor; skip it rather + // than intersecting with the zero rect a detached node reports. + if (!element.isConnected) { + continue; + } + box = intersectRects(box, element.getBoundingClientRect()); + } + return box; +} + +function toAnchorRect(rect: DOMRect): AnchorRect { + return { top: rect.top, bottom: rect.bottom, left: rect.left, right: rect.right }; +} + +function intersectsRange(range: Range, node: Text): boolean { + // `comparePoint` is -1 before the range, 0 inside, 1 after. + return range.comparePoint(node, 0) <= 0 && range.comparePoint(node, node.length) >= 0; +} + +/** The first (or last) non-empty text node the range actually covers. */ +function findEdgeText(range: Range, atStart: boolean): { node: Text; offset: number } | null { + const root = range.commonAncestorContainer; + const scope = root.nodeType === Node.ELEMENT_NODE ? root : root.parentNode; + if (!scope) { + return null; + } + + const walker = document.createTreeWalker(scope, NodeFilter.SHOW_TEXT); + let found: Text | null = null; + while (walker.nextNode()) { + const node = walker.currentNode as Text; + if (node.length === 0 || !intersectsRange(range, node)) { + continue; + } + found = node; + if (atStart) { + break; + } + } + if (!found) { + return null; + } + + const boundary = atStart ? range.startContainer : range.endContainer; + if (found === boundary) { + return { node: found, offset: atStart ? range.startOffset : range.endOffset }; + } + return { node: found, offset: atStart ? 0 : found.length }; +} + +/** + * A range positioned at one end of the selection, kept for re-measuring. + * + * A collapsed clone rather than `getClientRects()` indexing: that list is not + * one rect per line — it also carries a rect for every element box fully + * contained in the range, so a selection spanning a `

` contributes that + * container's full-height box and `rects[0]` can be the entire selection. + * + * Chrome reports an empty rect for a collapsed range parked at an element + * boundary, and a mouse drag that ends between elements produces exactly that: + * `endOffset === 0` on a `

`, with nothing "just inside" to widen toward. + * Resolving to the nearest covered text node costs a walk, so it happens once + * when the selection settles and the resulting range is re-measured per frame. + * Falling back to the whole-selection box instead would reintroduce the + * container contamination this is avoiding. + */ +function buildEndpointRange(range: Range, atStart: boolean): Range | null { + const clone = range.cloneRange(); + clone.collapse(atStart); + // Caret rects are zero-width by construction, so only height is meaningful. + if (clone.getBoundingClientRect().height > 0) { + return clone; + } + + const edge = findEdgeText(range, atStart); + if (!edge) { + return null; + } + // One character wide: a collapsed range at the same spot would report the + // same empty rect the clone just did. + const start = atStart + ? Math.min(edge.offset, edge.node.length - 1) + : Math.max(0, edge.offset - 1); + const sliver = document.createRange(); + sliver.setStart(edge.node, start); + sliver.setEnd(edge.node, start + 1); + return sliver; +} + +function measureEndpoint(range: Range | null): AnchorRect | null { + if (!range) { + return null; + } + // A detached range — the virtualizer unmounted its rows — reports zeros. + const rect = range.getBoundingClientRect(); + return rect.height > 0 ? toAnchorRect(rect) : null; +} + +/** + * Tracks the current text selection as viewport-anchored geometry, web only. + * + * The text is captured when the selection settles rather than when the caller + * acts on it: pressing anything can collapse the selection first, at which point + * `window.getSelection()` is already empty. + * + * `retainWhileDetached` decouples playback lifetime from anchor liveness. Speech + * needs no live range — the text was captured at settle time — so while + * something is playing, a selection whose rows the virtualizer unmounted keeps + * its anchor (with null rects) instead of vanishing and taking the stop control + * with it. + */ +export function useSelectionAnchor( + enabled: boolean, + retainWhileDetached: boolean, +): SelectionAnchor | null { + const [anchor, setAnchor] = useState(null); + const pointerDownRef = useRef(false); + const settleTimerRef = useRef | null>(null); + const frameRef = useRef(null); + const clipChainRef = useRef(null); + const endpointsRef = useRef<{ start: Range | null; end: Range | null } | null>(null); + const retainedRef = useRef(null); + // A ref, not a dependency: flipping retain must not tear down and re-register + // every document listener mid-playback. + const retainRef = useRef(retainWhileDetached); + retainRef.current = retainWhileDetached; + + useEffect(() => { + if (!enabled) { + retainedRef.current = null; + clipChainRef.current = null; + endpointsRef.current = null; + setAnchor(null); + return; + } + + let observer: ResizeObserver | null = null; + + const clearSettleTimer = () => { + if (settleTimerRef.current !== null) { + clearTimeout(settleTimerRef.current); + settleTimerRef.current = null; + } + }; + + const clearFrame = () => { + if (frameRef.current !== null) { + window.cancelAnimationFrame(frameRef.current); + frameRef.current = null; + } + }; + + const publish = (next: SelectionAnchor | null) => { + retainedRef.current = next; + setAnchor(next); + }; + + const observeChain = (chain: HTMLElement[]) => { + if (typeof ResizeObserver === "undefined") { + return; + } + observer?.disconnect(); + // Neither `scroll` nor `resize` fires on reflow — streaming output, + // virtualizer row remeasure, dragging the pane resize handle — and all + // three move the selection under a viewport-positioned bubble. + observer ??= new ResizeObserver(() => scheduleFrame()); + for (const element of chain) { + observer.observe(element); + } + }; + + const readSelection = (resolveEndpoints: boolean): SelectionAnchor | null => { + const selection = window.getSelection(); + if (!selection || selection.isCollapsed || selection.rangeCount === 0) { + return null; + } + + const text = selection.toString().trim(); + if (text.length < MIN_SELECTION_CHARS) { + return null; + } + + const range = selection.getRangeAt(0); + if (!isReadAloudEligible(range)) { + return null; + } + + // Both the clipping chain and the endpoint ranges are properties of the + // selection, not of the scroll position, so they are resolved once when it + // settles and only re-measured on subsequent frames. + if (resolveEndpoints || clipChainRef.current === null || endpointsRef.current === null) { + const chain = collectClipChain(resolveElement(range.startContainer)); + clipChainRef.current = chain; + observeChain(chain); + endpointsRef.current = { + start: buildEndpointRange(range, true), + end: buildEndpointRange(range, false), + }; + } + + const firstRect = measureEndpoint(endpointsRef.current.start); + const lastRect = measureEndpoint(endpointsRef.current.end); + if (!firstRect && !lastRect) { + return null; + } + + return { + text, + firstRect, + lastRect, + visibleBox: computeVisibleBox(clipChainRef.current), + }; + }; + + const commit = (resolveEndpoints: boolean) => { + const next = readSelection(resolveEndpoints); + if (next) { + publish(next); + return; + } + + const retained = retainedRef.current; + if (retainRef.current && retained) { + // The range is gone but the read is not. Keep the text and re-measure + // the box its pane still occupies so the bubble can park at an edge. + publish({ + ...retained, + firstRect: null, + lastRect: null, + visibleBox: computeVisibleBox(clipChainRef.current ?? []), + }); + return; + } + + clipChainRef.current = null; + endpointsRef.current = null; + publish(null); + }; + + const scheduleSettled = () => { + clearSettleTimer(); + settleTimerRef.current = setTimeout(() => { + settleTimerRef.current = null; + commit(true); + }, SETTLE_MS); + }; + + const scheduleFrame = () => { + if (frameRef.current !== null) { + return; + } + frameRef.current = window.requestAnimationFrame(() => { + frameRef.current = null; + commit(false); + }); + }; + + const handleSelectionChange = () => { + // Mid-drag the selection is still moving, so drop the stale anchor and + // wait for the release rather than flashing a bubble at a moving edge. + if (pointerDownRef.current) { + clearSettleTimer(); + publish(null); + return; + } + scheduleSettled(); + }; + + const handlePointerDown = (event: Event) => { + // Pressing the bubble must not dismiss it — that press is the whole point, + // and the bubble also preventDefaults so the selection itself survives. + if (isInsideReadAloudBubble(event.target)) { + return; + } + // Explicit intent to leave: this is the one path that drops a retained + // anchor, and the bubble reads that as "stop". Without it, retention + // would keep audio playing with no visible way to stop it. + pointerDownRef.current = true; + clearSettleTimer(); + publish(null); + }; + + const handlePointerUp = (event: Event) => { + if (isInsideReadAloudBubble(event.target)) { + return; + } + pointerDownRef.current = false; + scheduleSettled(); + }; + + document.addEventListener("selectionchange", handleSelectionChange); + document.addEventListener("mousedown", handlePointerDown, true); + document.addEventListener("mouseup", handlePointerUp, true); + document.addEventListener("touchend", handlePointerUp, true); + // Capture phase so nested scrollers are caught too. + window.addEventListener("scroll", scheduleFrame, true); + window.addEventListener("resize", scheduleFrame); + + return () => { + clearSettleTimer(); + clearFrame(); + observer?.disconnect(); + document.removeEventListener("selectionchange", handleSelectionChange); + document.removeEventListener("mousedown", handlePointerDown, true); + document.removeEventListener("mouseup", handlePointerUp, true); + document.removeEventListener("touchend", handlePointerUp, true); + window.removeEventListener("scroll", scheduleFrame, true); + window.removeEventListener("resize", scheduleFrame); + }; + }, [enabled]); + + return anchor; +} diff --git a/packages/app/src/voice/audio-engine-types.ts b/packages/app/src/voice/audio-engine-types.ts index 03108b5cf6..32356d130a 100644 --- a/packages/app/src/voice/audio-engine-types.ts +++ b/packages/app/src/voice/audio-engine-types.ts @@ -21,6 +21,12 @@ export interface AudioEngine { isMuted(): boolean; play(audio: AudioPlaybackSource): Promise; + /** + * Speed multiplier for playback. Applies to the segment currently playing and + * to everything queued behind it. Pitch is NOT preserved — this is a raw + * resampling rate, so keep it near 1 (read aloud caps at 2x). + */ + setPlaybackRate(rate: number): void; stop(): void; clearQueue(): void; isPlaying(): boolean; diff --git a/packages/app/src/voice/audio-engine.native.ts b/packages/app/src/voice/audio-engine.native.ts index 81f0fda85e..0bc0fed5ec 100644 --- a/packages/app/src/voice/audio-engine.native.ts +++ b/packages/app/src/voice/audio-engine.native.ts @@ -304,6 +304,13 @@ export function createAudioEngine( }); }, + setPlaybackRate(_rate: number) { + // Not supported: native playback hands raw PCM to the audio module at a + // fixed sample rate with no resampling hook. The only caller is read + // aloud, which is web-only (see `read-aloud-audio.ts`), so nothing on + // native asks for a rate today. + }, + stop() { native.stopPlayback(); clearPlaybackTimeout(); diff --git a/packages/app/src/voice/audio-engine.web.ts b/packages/app/src/voice/audio-engine.web.ts index 0453c0ae33..50e1ec4288 100644 --- a/packages/app/src/voice/audio-engine.web.ts +++ b/packages/app/src/voice/audio-engine.web.ts @@ -90,6 +90,7 @@ export function createAudioEngine( _options?: { traceLabel?: string }, ): AudioEngine { const refs: { + playbackRate: number; playbackContext: AudioContext | null; captureContext: AudioContext | null; stream: MediaStream | null; @@ -107,6 +108,7 @@ export function createAudioEngine( settled: boolean; } | null; } = { + playbackRate: 1, playbackContext: null, captureContext: null, stream: null, @@ -174,10 +176,13 @@ export function createAudioEngine( ) : await decodeAudioData(context, arrayBuffer); - const durationSec = audioBuffer.duration; const source = context.createBufferSource(); source.buffer = audioBuffer; + source.playbackRate.value = refs.playbackRate; source.connect(context.destination); + // Wall-clock duration, not buffer duration: at 2x the audio is done in half + // the time, and callers use this to know how long playback actually takes. + const durationSec = audioBuffer.duration / refs.playbackRate; return await new Promise((resolve, reject) => { refs.activePlayback = { source, resolve, reject, settled: false }; @@ -379,6 +384,18 @@ export function createAudioEngine( }); }, + setPlaybackRate(rate: number) { + if (!Number.isFinite(rate) || rate <= 0) { + return; + } + refs.playbackRate = rate; + // Retune the segment already in flight so the change is audible now + // rather than at the next segment boundary. + if (refs.activePlayback) { + refs.activePlayback.source.playbackRate.value = rate; + } + }, + stop() { if (refs.activePlayback) { const active = refs.activePlayback; diff --git a/packages/app/src/voice/voice-runtime.test.ts b/packages/app/src/voice/voice-runtime.test.ts index aa79d6b245..237a490e93 100644 --- a/packages/app/src/voice/voice-runtime.test.ts +++ b/packages/app/src/voice/voice-runtime.test.ts @@ -13,6 +13,7 @@ function createAudioEngineMock(): AudioEngine { toggleMute: vi.fn().mockReturnValue(true), isMuted: vi.fn().mockReturnValue(false), play: vi.fn().mockResolvedValue(0.1), + setPlaybackRate: vi.fn(), stop: vi.fn(), clearQueue: vi.fn(), isPlaying: vi.fn().mockReturnValue(false), diff --git a/packages/client/src/daemon-client.ts b/packages/client/src/daemon-client.ts index 8b99449f25..d5a482985c 100644 --- a/packages/client/src/daemon-client.ts +++ b/packages/client/src/daemon-client.ts @@ -428,6 +428,24 @@ export interface FileUploadInput { requestId?: string; chunkSize?: number; } + +export interface ReadAloudSegment { + segmentIndex: number; + segmentCount: number; + audioBase64: string; + /** Provider-reported format, e.g. `pcm;rate=24000` or `mp3`. */ + format: string; +} + +export interface ReadAloudError { + code: string; + message: string; +} + +export interface ReadAloudHandle { + requestId: string; + cancel(): void; +} export type FileUploadResult = FileUploadResponse["payload"]; type FileDownloadTokenPayload = FileDownloadTokenResponse["payload"]; type ListProviderFeaturesPayload = ListProviderFeaturesResponseMessage["payload"]; @@ -3423,6 +3441,81 @@ export class DaemonClient { this.sendSessionMessage({ type: "audio_played", id }); } + /** + * Synthesize `text` on the daemon and stream the audio back segment by + * segment, in order. Segments arrive as they finish synthesizing so playback + * can start on the first sentence; the caller is responsible for queueing + * them. Returns a handle whose `cancel()` stops synthesis and detaches the + * listener. + */ + startReadAloud(params: { + text: string; + onSegment: (segment: ReadAloudSegment) => void; + onError: (error: ReadAloudError) => void; + onEnd: () => void; + }): ReadAloudHandle { + const requestId = this.createRequestId(); + let settled = false; + + const unsubscribe = this.on("speech.tts.read_aloud.response", (message) => { + const payload = message.payload; + if (payload.requestId !== requestId || settled) { + return; + } + + if (payload.error) { + settled = true; + unsubscribe(); + params.onError(payload.error); + params.onEnd(); + return; + } + + if (payload.audio !== undefined && payload.format !== undefined) { + params.onSegment({ + segmentIndex: payload.segmentIndex, + segmentCount: payload.segmentCount, + audioBase64: payload.audio, + format: payload.format, + }); + } + + if (payload.isLast) { + settled = true; + unsubscribe(); + params.onEnd(); + } + }); + + try { + this.sendSessionMessageStrict({ + type: "speech.tts.read_aloud.request", + requestId, + text: params.text, + }); + } catch (error) { + settled = true; + unsubscribe(); + params.onError({ + code: "transport_unavailable", + message: error instanceof Error ? error.message : String(error), + }); + params.onEnd(); + } + + return { + requestId, + cancel: () => { + if (settled) { + return; + } + settled = true; + unsubscribe(); + this.sendSessionMessage({ type: "speech.tts.cancel_read_aloud.request", requestId }); + }, + }; + } + // ============================================================================ // Git Operations // ============================================================================ diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index d7f21f1246..915acf9f3f 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -32,6 +32,9 @@ export type { DaemonEvent, BrowserAutomationExecuteRequestMessage, BrowserAutomationExecuteResponseMessage, + ReadAloudError, + ReadAloudHandle, + ReadAloudSegment, WebSocketFactory, WebSocketLike, } from "./daemon-client.js"; diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index af44e09f90..a1364baa5d 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -1217,6 +1217,24 @@ export const DictationStreamCancelMessageSchema = z.object({ dictationId: z.string(), }); +// ============================================================================ +// Read aloud (on-demand TTS for arbitrary client-selected text) +// ============================================================================ + +export const ReadAloudRequestMessageSchema = z.object({ + type: z.literal("speech.tts.read_aloud.request"), + requestId: z.string(), + text: z.string(), +}); + +// Fire-and-forget: cancelling read-aloud has no `.response`. The daemon stops +// synthesizing and stops emitting `speech.tts.read_aloud.response` segments for +// this requestId; the client has already torn down its own playback. +export const ReadAloudCancelRequestMessageSchema = z.object({ + type: z.literal("speech.tts.cancel_read_aloud.request"), + requestId: z.string(), +}); + const GitSetupOptionsSchema = z.object({ baseBranch: z.string().optional(), createNewBranch: z.boolean().optional(), @@ -2444,6 +2462,8 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ HubExecutionControlRequestSchema, BrowserAutomationExecuteResponseSchema, VoiceAudioChunkMessageSchema, + ReadAloudRequestMessageSchema, + ReadAloudCancelRequestMessageSchema, AbortRequestMessageSchema, AudioPlayedMessageSchema, FetchAgentsRequestMessageSchema, @@ -2705,6 +2725,29 @@ export const DictationStreamErrorMessageSchema = z.object({ }), }); +// One request fans out to N of these, in segment order. Audio is segmented so a +// long selection starts playing after the first sentence instead of after the +// whole synthesis, and so no single message carries minutes of raw PCM. +export const ReadAloudResponseMessageSchema = z.object({ + type: z.literal("speech.tts.read_aloud.response"), + payload: z.object({ + requestId: z.string(), + segmentIndex: z.number().int().nonnegative(), + segmentCount: z.number().int().nonnegative(), + isLast: z.boolean(), + // Base64 audio for this segment. Absent on the terminal error message. + audio: z.string().optional(), + // Provider-reported audio format, e.g. "pcm;rate=24000" or "mp3". + format: z.string().optional(), + error: z + .object({ + code: z.string(), + message: z.string(), + }) + .optional(), + }), +}); + export const ServerCapabilityStateSchema = z.object({ enabled: z.boolean(), reason: z.string(), @@ -2800,6 +2843,8 @@ export const ServerInfoStatusPayloadSchema = z agentDetach: z.boolean().optional(), // COMPAT(agentThinkingUpdate): added in v0.2.4, remove gate after 2027-01-28. agentThinkingUpdate: z.boolean().optional(), + // COMPAT(readAloud): added in v0.2.5, drop the gate when floor >= v0.2.5. + readAloud: z.boolean().optional(), // COMPAT(daemonDiagnostics): added in v0.1.100, remove gate after 2026-12-25 once daemon floor >= v0.1.100. daemonDiagnostics: z.boolean().optional(), // COMPAT(daemonSelfUpdate): added in v0.1.93, remove gate after 2026-12-13. @@ -5189,6 +5234,7 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ ActivityLogMessageSchema, AssistantChunkMessageSchema, AudioOutputMessageSchema, + ReadAloudResponseMessageSchema, TranscriptionResultMessageSchema, VoiceInputStateMessageSchema, DictationStreamAckMessageSchema, @@ -5537,6 +5583,10 @@ export type DictationStreamStartMessage = z.infer; export type DictationStreamFinishMessage = z.infer; export type DictationStreamCancelMessage = z.infer; +export type ReadAloudRequestMessage = z.infer; +export type ReadAloudCancelRequestMessage = z.infer; +export type ReadAloudResponseMessage = z.infer; +export type ReadAloudResponsePayload = ReadAloudResponseMessage["payload"]; export type CreateAgentRequestMessage = z.infer; export type AgentAttachment = z.infer; export type ForgeChangeRequestAttachment = z.infer; diff --git a/packages/server/src/server/agent/tts-manager.ts b/packages/server/src/server/agent/tts-manager.ts index ee6c747019..b343cf6751 100644 --- a/packages/server/src/server/agent/tts-manager.ts +++ b/packages/server/src/server/agent/tts-manager.ts @@ -3,6 +3,7 @@ import type { Readable } from "node:stream"; import { v4 as uuidv4 } from "uuid"; import type { TextToSpeechProvider } from "../speech/speech-provider.js"; import { toResolver, type Resolvable } from "../speech/provider-resolver.js"; +import { splitTextForTts, type TtsSegment } from "../speech/tts-text-splitter.js"; import type { SessionOutboundMessage } from "../messages.js"; interface PendingPlayback { @@ -12,11 +13,6 @@ interface PendingPlayback { streamEnded: boolean; } -interface TtsSegment { - index: number; - text: string; -} - type PreparedTtsSegment = TtsSegment & { format: string; stream: Readable; @@ -27,103 +23,9 @@ type PreparedSegmentResult = | { kind: "aborted" } | { kind: "error"; error: unknown }; -const MAX_TTS_SEGMENT_CHARS = 260; const TTS_PREFETCH_SEGMENTS = 2; const CLOSED_AUDIO_ID_TTL_MS = 10_000; -function splitOversizedFragment(fragment: string, maxChars: number): string[] { - const trimmed = fragment.trim(); - if (!trimmed) { - return []; - } - - if (trimmed.length <= maxChars) { - return [trimmed]; - } - - const clauseChunks = trimmed.split(/(?<=[,;:])\s+/); - if (clauseChunks.length > 1) { - const parts: string[] = []; - let current = ""; - - const pushCurrent = () => { - const value = current.trim(); - if (value) { - parts.push(value); - } - current = ""; - }; - - for (const clause of clauseChunks) { - const clauseText = clause.trim(); - if (!clauseText) { - continue; - } - - if (clauseText.length > maxChars) { - pushCurrent(); - parts.push(...splitOversizedFragment(clauseText, maxChars)); - continue; - } - - if (!current) { - current = clauseText; - continue; - } - - const candidate = `${current} ${clauseText}`; - if (candidate.length <= maxChars) { - current = candidate; - continue; - } - - pushCurrent(); - current = clauseText; - } - - pushCurrent(); - if (parts.length > 1 || parts[0] !== trimmed) { - return parts; - } - } - - const parts: string[] = []; - let remaining = trimmed; - while (remaining.length > maxChars) { - let idx = remaining.lastIndexOf(" ", maxChars); - if (idx < Math.floor(maxChars * 0.5)) { - idx = maxChars; - } - parts.push(remaining.slice(0, idx).trim()); - remaining = remaining.slice(idx).trim(); - } - if (remaining.length > 0) { - parts.push(remaining); - } - return parts; -} - -function splitTextForTts(text: string): TtsSegment[] { - const normalized = text.trim().replace(/\s+/g, " "); - if (!normalized) { - throw new Error("Cannot synthesize empty text"); - } - - const sentences = normalized.split(/(?<=[.!?])\s+/); - const parts: TtsSegment[] = []; - let segmentIndex = 0; - - for (const sentence of sentences) { - const fragments = splitOversizedFragment(sentence, MAX_TTS_SEGMENT_CHARS); - for (const fragment of fragments) { - parts.push({ index: segmentIndex, text: fragment }); - segmentIndex += 1; - } - } - - return parts; -} - /** * Per-session TTS manager * Handles TTS audio generation and playback confirmation tracking diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index a87aa42e6c..e261b60dfd 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -1826,6 +1826,16 @@ export class Session { case "dictation_stream_cancel": this.voiceSession.handleDictationCancel(msg.dictationId); return undefined; + case "speech.tts.read_aloud.request": + // Not awaited: this spans the whole synthesis (seconds to a minute for a + // long selection), and the caller measures the awaited duration as + // request latency. Read aloud is a stream of `.response` segments, not a + // request/response, and the controller reports its own failures. + void this.voiceSession.handleReadAloudRequest(msg); + return undefined; + case "speech.tts.cancel_read_aloud.request": + this.voiceSession.handleReadAloudCancel(msg); + return undefined; case "restart_server_request": return this.handleRestartServerRequest(msg.requestId, msg.reason); case "shutdown_server_request": diff --git a/packages/server/src/server/session/voice/read-aloud-controller.test.ts b/packages/server/src/server/session/voice/read-aloud-controller.test.ts new file mode 100644 index 0000000000..1c488caf3c --- /dev/null +++ b/packages/server/src/server/session/voice/read-aloud-controller.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from "vitest"; +import pino from "pino"; +import { Readable } from "node:stream"; + +import { MAX_READ_ALOUD_CHARS, ReadAloudController } from "./read-aloud-controller.js"; +import type { SessionOutboundMessage } from "../../messages.js"; +import type { TextToSpeechProvider } from "../../speech/speech-provider.js"; + +type ReadAloudResponse = Extract< + SessionOutboundMessage, + { type: "speech.tts.read_aloud.response" } +>; + +interface Harness { + controller: ReadAloudController; + responses: ReadAloudResponse[]; + synthesized: string[]; +} + +function createHarness( + tts: TextToSpeechProvider | null, + hooks?: { + onResponse?: (response: ReadAloudResponse, harness: Harness) => void; + }, +): Harness { + const responses: ReadAloudResponse[] = []; + const synthesized: string[] = []; + + const wrapped: TextToSpeechProvider | null = tts + ? { + async synthesizeSpeech(text: string) { + synthesized.push(text); + return tts.synthesizeSpeech(text); + }, + } + : null; + + const harness: Harness = { + controller: new ReadAloudController({ + sessionId: "session-1", + logger: pino({ level: "silent" }), + tts: wrapped, + emit: (message) => { + if (message.type !== "speech.tts.read_aloud.response") { + return; + } + responses.push(message); + hooks?.onResponse?.(message, harness); + }, + }), + responses, + synthesized, + }; + + return harness; +} + +function fakeTts(audio = "sound"): TextToSpeechProvider { + return { + async synthesizeSpeech() { + return { stream: Readable.from([Buffer.from(audio)]), format: "pcm;rate=24000" }; + }, + }; +} + +describe("ReadAloudController", () => { + it("streams one audio segment per sentence, in order, marking only the last", async () => { + const { controller, responses, synthesized } = createHarness(fakeTts("abc")); + + await controller.handleRequest({ + requestId: "req-1", + text: "First sentence. Second sentence. Third sentence.", + }); + + expect(synthesized).toEqual(["First sentence.", "Second sentence.", "Third sentence."]); + expect(responses.map((message) => message.payload.segmentIndex)).toEqual([0, 1, 2]); + expect(responses.map((message) => message.payload.isLast)).toEqual([false, false, true]); + + for (const message of responses) { + expect(message.payload.requestId).toBe("req-1"); + expect(message.payload.segmentCount).toBe(3); + expect(message.payload.format).toBe("pcm;rate=24000"); + expect(message.payload.audio).toBe(Buffer.from("abc").toString("base64")); + expect(message.payload.error).toBeUndefined(); + } + }); + + it("never sends wrapper markup to the provider", async () => { + const { controller, synthesized } = createHarness(fakeTts()); + + await controller.handleRequest({ + requestId: "req-1", + text: [ + "", + "Are you working?", + "", + "This message was spoken by the user.", + ].join("\n"), + }); + + expect(synthesized).toEqual(["Are you working?", "This message was spoken by the user."]); + }); + + it("rejects a selection that is nothing but markup", async () => { + const { controller, responses, synthesized } = createHarness(fakeTts()); + + await controller.handleRequest({ requestId: "req-1", text: "" }); + + expect(synthesized).toEqual([]); + expect(responses[0].payload.error?.code).toBe("empty_text"); + }); + + it("reports tts_unavailable without emitting audio when no provider is configured", async () => { + const { controller, responses } = createHarness(null); + + await controller.handleRequest({ requestId: "req-1", text: "Read me." }); + + expect(responses).toHaveLength(1); + expect(responses[0].payload.error?.code).toBe("tts_unavailable"); + expect(responses[0].payload.isLast).toBe(true); + expect(responses[0].payload.audio).toBeUndefined(); + }); + + it("rejects text over the length cap instead of synthesizing it", async () => { + const { controller, responses, synthesized } = createHarness(fakeTts()); + + await controller.handleRequest({ + requestId: "req-1", + text: "word ".repeat(MAX_READ_ALOUD_CHARS), + }); + + expect(synthesized).toEqual([]); + expect(responses).toHaveLength(1); + expect(responses[0].payload.error?.code).toBe("text_too_long"); + }); + + it("rejects a whitespace-only selection", async () => { + const { controller, responses, synthesized } = createHarness(fakeTts()); + + await controller.handleRequest({ requestId: "req-1", text: " \n " }); + + expect(synthesized).toEqual([]); + expect(responses[0].payload.error?.code).toBe("empty_text"); + }); + + it("reports synth_failed as a terminal error when the provider throws", async () => { + const failing: TextToSpeechProvider = { + async synthesizeSpeech() { + throw new Error("model not loaded"); + }, + }; + const { controller, responses } = createHarness(failing); + + await controller.handleRequest({ requestId: "req-1", text: "Read me." }); + + expect(responses).toHaveLength(1); + expect(responses[0].payload.error).toEqual({ + code: "synth_failed", + message: "model not loaded", + }); + expect(responses[0].payload.isLast).toBe(true); + }); + + it("stops emitting further segments once the client cancels mid-stream", async () => { + const { controller, responses } = createHarness(fakeTts(), { + onResponse: (_response, harness) => { + if (harness.responses.length === 1) { + harness.controller.cancel("req-1"); + } + }, + }); + + await controller.handleRequest({ requestId: "req-1", text: "One. Two. Three." }); + + expect(responses).toHaveLength(1); + expect(responses[0].payload.segmentIndex).toBe(0); + expect(responses[0].payload.isLast).toBe(false); + }); + + it("supersedes an in-flight request when a new one arrives", async () => { + let releaseFirst: (() => void) | null = null; + const firstBlocked = new Promise((resolve) => { + releaseFirst = resolve; + }); + + const slowTts: TextToSpeechProvider = { + async synthesizeSpeech(text: string) { + if (text === "Slow.") { + await firstBlocked; + } + return { stream: Readable.from([Buffer.from("x")]), format: "pcm;rate=24000" }; + }, + }; + + const { controller, responses } = createHarness(slowTts); + + const first = controller.handleRequest({ requestId: "req-1", text: "Slow." }); + const second = controller.handleRequest({ requestId: "req-2", text: "Fast." }); + releaseFirst?.(); + await Promise.all([first, second]); + + expect(responses.map((message) => message.payload.requestId)).toEqual(["req-2"]); + }); +}); diff --git a/packages/server/src/server/session/voice/read-aloud-controller.ts b/packages/server/src/server/session/voice/read-aloud-controller.ts new file mode 100644 index 0000000000..5aa5f75211 --- /dev/null +++ b/packages/server/src/server/session/voice/read-aloud-controller.ts @@ -0,0 +1,275 @@ +import type pino from "pino"; +import type { Readable } from "node:stream"; + +import type { SessionOutboundMessage } from "../../messages.js"; +import { toResolver, type Resolvable } from "../../speech/provider-resolver.js"; +import type { TextToSpeechProvider } from "../../speech/speech-provider.js"; +import { hasSpeakableContent, sanitizeTextForReadAloud } from "../../speech/read-aloud-text.js"; +import { splitTextForTts, type TtsSegment } from "../../speech/tts-text-splitter.js"; + +/** + * Upper bound on a single read-aloud request. Guards the daemon against a + * "select all" that would otherwise queue minutes of synthesis work. + */ +export const MAX_READ_ALOUD_CHARS = 4000; + +/** How many segments are synthesized ahead of the one currently being emitted. */ +const READ_ALOUD_PREFETCH_SEGMENTS = 2; + +export type ReadAloudErrorCode = + | "empty_text" + | "text_too_long" + | "tts_unavailable" + | "synth_failed"; + +interface ActiveRequest { + abortController: AbortController; +} + +type PreparedSegment = TtsSegment & { + format: string; + stream: Readable; +}; + +export interface ReadAloudControllerOptions { + sessionId: string; + logger: pino.Logger; + tts: Resolvable; + emit: (message: SessionOutboundMessage) => void; +} + +/** + * On-demand text-to-speech for client-selected text ("Read aloud"). + * + * Deliberately separate from `TTSManager`: voice-mode audio carries an + * `isVoiceMode` drift flag and gates an agent turn on playback confirmation, + * neither of which applies here. Read aloud is fire-and-forget streaming — + * segments are pushed as they finish synthesizing and the client queues them. + */ +export class ReadAloudController { + private readonly logger: pino.Logger; + private readonly resolveTts: () => TextToSpeechProvider | null; + private readonly emit: (message: SessionOutboundMessage) => void; + private readonly active = new Map(); + + constructor(options: ReadAloudControllerOptions) { + this.logger = options.logger.child({ + module: "speech", + component: "read-aloud", + sessionId: options.sessionId, + }); + this.resolveTts = toResolver(options.tts); + this.emit = options.emit; + } + + public async handleRequest(params: { requestId: string; text: string }): Promise { + const { requestId } = params; + + // Only one read aloud plays at a time, so a new request supersedes whatever + // is still synthesizing rather than competing with it for the TTS worker. + this.cancelAll("superseded by a newer read aloud request"); + + // Cap on what the user selected, before sanitizing, so the limit means the + // same thing to them whether or not their selection was full of markup. + const rawLength = params.text.trim().length; + const text = sanitizeTextForReadAloud(params.text); + if (!hasSpeakableContent(text)) { + this.emitError(requestId, "empty_text", "Nothing to read aloud"); + return; + } + + if (rawLength > MAX_READ_ALOUD_CHARS) { + this.emitError( + requestId, + "text_too_long", + `Selection is too long to read aloud (${rawLength} of ${MAX_READ_ALOUD_CHARS} characters)`, + ); + return; + } + + const tts = this.resolveTts(); + if (!tts) { + this.emitError(requestId, "tts_unavailable", "Text-to-speech is not configured on this host"); + return; + } + + const abortController = new AbortController(); + this.active.set(requestId, { abortController }); + const abortSignal = abortController.signal; + + // Re-index after dropping unspeakable fragments so `segmentIndex` stays a + // dense 0..n-1 run and the last segment really is the last one. + const segments: TtsSegment[] = []; + for (const segment of splitTextForTts(text)) { + if (hasSpeakableContent(segment.text)) { + segments.push({ index: segments.length, text: segment.text }); + } + } + if (segments.length === 0) { + this.emitError(requestId, "empty_text", "Nothing to read aloud"); + return; + } + const startedAtMs = Date.now(); + this.logger.debug( + { requestId, chars: text.length, segmentCount: segments.length }, + "Read aloud started", + ); + + const inflight = new Map>(); + let nextToSchedule = 0; + + const scheduleAhead = () => { + while ( + nextToSchedule < segments.length && + inflight.size < READ_ALOUD_PREFETCH_SEGMENTS && + !abortSignal.aborted + ) { + const segment = segments[nextToSchedule]; + inflight.set(segment.index, this.synthesizeSegment(tts, segment)); + nextToSchedule += 1; + } + }; + + scheduleAhead(); + + try { + for (const segment of segments) { + const pending = inflight.get(segment.index); + if (!pending) { + break; + } + + let prepared: PreparedSegment; + try { + prepared = await pending; + } finally { + inflight.delete(segment.index); + } + + if (abortSignal.aborted) { + destroyStream(prepared.stream); + return; + } + + scheduleAhead(); + + const audio = await collectStream(prepared.stream); + if (abortSignal.aborted) { + return; + } + + this.emit({ + type: "speech.tts.read_aloud.response", + payload: { + requestId, + segmentIndex: segment.index, + segmentCount: segments.length, + isLast: segment.index === segments.length - 1, + audio: audio.toString("base64"), + format: prepared.format, + }, + }); + } + + this.logger.debug( + { requestId, totalMs: Date.now() - startedAtMs, segmentCount: segments.length }, + "Read aloud completed", + ); + } catch (error) { + if (abortSignal.aborted) { + this.logger.debug({ requestId }, "Read aloud aborted during synthesis"); + return; + } + this.logger.warn({ requestId, err: error }, "Read aloud synthesis failed"); + this.emitError( + requestId, + "synth_failed", + error instanceof Error ? error.message : String(error), + { segmentIndex: segments.length, segmentCount: segments.length }, + ); + } finally { + this.active.delete(requestId); + discardPrefetched(inflight); + } + } + + public cancel(requestId: string): void { + const request = this.active.get(requestId); + if (!request) { + return; + } + this.active.delete(requestId); + request.abortController.abort(); + this.logger.debug({ requestId }, "Read aloud cancelled by client"); + } + + public dispose(): void { + this.cancelAll("session closed"); + } + + private cancelAll(reason: string): void { + if (this.active.size === 0) { + return; + } + for (const [requestId, request] of this.active.entries()) { + this.active.delete(requestId); + request.abortController.abort(); + this.logger.debug({ requestId, reason }, "Read aloud cancelled"); + } + } + + private synthesizeSegment( + tts: TextToSpeechProvider, + segment: TtsSegment, + ): Promise { + return tts + .synthesizeSpeech(segment.text) + .then(({ stream, format }) => ({ ...segment, stream, format })); + } + + private emitError( + requestId: string, + code: ReadAloudErrorCode, + message: string, + position?: { segmentIndex: number; segmentCount: number }, + ): void { + this.emit({ + type: "speech.tts.read_aloud.response", + payload: { + requestId, + segmentIndex: position?.segmentIndex ?? 0, + segmentCount: position?.segmentCount ?? 0, + isLast: true, + error: { code, message }, + }, + }); + } +} + +function destroyStream(stream: Readable): void { + if (typeof stream.destroy === "function" && !stream.destroyed) { + stream.destroy(); + } +} + +function discardPrefetched(inflight: Map>): void { + for (const pending of inflight.values()) { + void pending.then( + (prepared) => destroyStream(prepared.stream), + () => undefined, + ); + } + inflight.clear(); +} + +async function collectStream(stream: Readable): Promise { + const chunks: Buffer[] = []; + try { + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + } finally { + destroyStream(stream); + } + return Buffer.concat(chunks); +} diff --git a/packages/server/src/server/session/voice/voice-session.ts b/packages/server/src/server/session/voice/voice-session.ts index f6331dba3f..0e52afd488 100644 --- a/packages/server/src/server/session/voice/voice-session.ts +++ b/packages/server/src/server/session/voice/voice-session.ts @@ -14,6 +14,7 @@ import { type DictationStreamOutboundMessage, } from "../../dictation/dictation-stream-manager.js"; import { createVoiceTurnController, type VoiceTurnController } from "./voice-turn-controller.js"; +import { ReadAloudController } from "./read-aloud-controller.js"; import { buildVoiceModeSystemPrompt, stripVoiceModeSystemPrompt } from "../../voice-config.js"; import type { VoiceCallerContext, VoiceSpeakHandler } from "../../voice-types.js"; import type { ManagedAgent } from "../../agent/agent-manager.js"; @@ -191,6 +192,7 @@ export class VoiceSession { private readonly ttsManager: TTSManager; private readonly sttManager: STTManager; + private readonly readAloudController: ReadAloudController; private readonly registerVoiceSpeakHandler?: ( agentId: string, @@ -224,6 +226,12 @@ export class VoiceSession { this.getSpeechReadiness = dictation?.getSpeechReadiness; this.ttsManager = new TTSManager(this.sessionId, this.sessionLogger, tts); + this.readAloudController = new ReadAloudController({ + sessionId: this.sessionId, + logger: this.sessionLogger, + tts, + emit: (msg) => this.emit(msg), + }); this.sttManager = new STTManager(this.sessionId, this.sessionLogger, stt, { language: sttLanguage, }); @@ -258,6 +266,18 @@ export class VoiceSession { this.dictationStreamManager.handleCancel(dictationId); } + handleReadAloudRequest( + msg: Extract, + ): Promise { + return this.readAloudController.handleRequest({ requestId: msg.requestId, text: msg.text }); + } + + handleReadAloudCancel( + msg: Extract, + ): void { + this.readAloudController.cancel(msg.requestId); + } + async handleDictationStreamStart( msg: Extract, ): Promise { @@ -1302,6 +1322,7 @@ export class VoiceSession { this.ttsManager.cleanup(); this.sttManager.cleanup(); this.dictationStreamManager.cleanupAll(); + this.readAloudController.dispose(); await this.disableVoiceModeForActiveAgent(true); this.isVoiceMode = false; diff --git a/packages/server/src/server/speech/read-aloud-text.test.ts b/packages/server/src/server/speech/read-aloud-text.test.ts new file mode 100644 index 0000000000..4df04fba02 --- /dev/null +++ b/packages/server/src/server/speech/read-aloud-text.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; + +import { hasSpeakableContent, sanitizeTextForReadAloud } from "./read-aloud-text.js"; + +describe("sanitizeTextForReadAloud", () => { + it("drops Paseo's spoken-input wrapper and speaks only the message", () => { + const selection = [ + "", + "Are you working?", + "", + "This message was spoken by the user.", + ].join("\n"); + + expect(sanitizeTextForReadAloud(selection)).toBe( + "Are you working? This message was spoken by the user.", + ); + }); + + it("keeps comparisons that are not tags", () => { + expect(sanitizeTextForReadAloud("if a < b and c > d")).toBe("if a < b and c > d"); + }); + + it("strips markdown emphasis and heading markers but keeps the words", () => { + expect(sanitizeTextForReadAloud("## The **bold** and _quiet_ `code` part")).toBe( + "The bold and quiet code part", + ); + }); + + it("speaks a link's label, not its URL", () => { + expect(sanitizeTextForReadAloud("See [the docs](https://example.com/a/b) for more")).toBe( + "See the docs for more", + ); + }); + + it("drops code fences while keeping the code inside", () => { + const selection = ["Run this:", "```bash", "npm run dev", "```"].join("\n"); + expect(sanitizeTextForReadAloud(selection)).toBe("Run this: npm run dev"); + }); + + it("strips list bullets and numbering", () => { + expect(sanitizeTextForReadAloud("- first\n- second\n1. third")).toBe("first second third"); + }); + + it("collapses a markup-only selection to nothing", () => { + expect(sanitizeTextForReadAloud("\n")).toBe(""); + }); +}); + +describe("hasSpeakableContent", () => { + it("accepts anything containing a letter or digit", () => { + expect(hasSpeakableContent("hi")).toBe(true); + expect(hasSpeakableContent("42")).toBe(true); + expect(hasSpeakableContent("こんにちは")).toBe(true); + }); + + it("rejects punctuation-only fragments that would synthesize to silence", () => { + expect(hasSpeakableContent("--- ...")).toBe(false); + expect(hasSpeakableContent("")).toBe(false); + }); +}); diff --git a/packages/server/src/server/speech/read-aloud-text.ts b/packages/server/src/server/speech/read-aloud-text.ts new file mode 100644 index 0000000000..74f643482d --- /dev/null +++ b/packages/server/src/server/speech/read-aloud-text.ts @@ -0,0 +1,48 @@ +/** + * Turn selected text into something worth hearing. + * + * A selection is raw screen text: it carries markdown syntax, and agent + * messages carry Paseo's own wrapper tags (``, ``). + * Synthesized verbatim those become spoken noise — "spoken input, are you + * working" — so they are stripped before they reach the TTS provider. + * + * Deliberately conservative: this removes markup, it does not reflow or + * summarize. Anything it cannot confidently classify as syntax is spoken. + */ + +/** ``, ``, `` — not bare `<` in `a < b`. */ +const HTML_LIKE_TAG = /<\/?[a-zA-Z][^<>]*>/g; + +/** Fenced code blocks: the fence markers and language tag, not the code. */ +const CODE_FENCE = /^```.*$/gm; + +/** Inline code, bold, italic, and strikethrough markers around their content. */ +const INLINE_MARKERS = /[`*_~]/g; + +/** Leading `#`, `>`, `-`, `*`, `+` and list numbering at the start of a line. */ +const LINE_LEAD_MARKERS = /^\s{0,3}(?:#{1,6}\s+|>\s?|[-*+]\s+|\d+[.)]\s+)/gm; + +/** `[label](https://…)` — the label is speakable, the URL is not. */ +const MARKDOWN_LINK = /\[([^\]]*)\]\((?:[^)]*)\)/g; + +/** Anything with a letter or a digit has something to pronounce. */ +const HAS_SPEAKABLE_CONTENT = /[\p{L}\p{N}]/u; + +export function sanitizeTextForReadAloud(text: string): string { + return text + .replace(CODE_FENCE, " ") + .replace(HTML_LIKE_TAG, " ") + .replace(MARKDOWN_LINK, "$1") + .replace(LINE_LEAD_MARKERS, "") + .replace(INLINE_MARKERS, "") + .replace(/\s+/g, " ") + .trim(); +} + +/** + * Whether a fragment is worth sending to the provider. Punctuation-only + * fragments synthesize to empty audio, which the client cannot decode. + */ +export function hasSpeakableContent(text: string): boolean { + return HAS_SPEAKABLE_CONTENT.test(text); +} diff --git a/packages/server/src/server/speech/tts-text-splitter.ts b/packages/server/src/server/speech/tts-text-splitter.ts new file mode 100644 index 0000000000..2d82eeefdc --- /dev/null +++ b/packages/server/src/server/speech/tts-text-splitter.ts @@ -0,0 +1,104 @@ +export interface TtsSegment { + index: number; + text: string; +} + +export const MAX_TTS_SEGMENT_CHARS = 260; + +function splitOversizedFragment(fragment: string, maxChars: number): string[] { + const trimmed = fragment.trim(); + if (!trimmed) { + return []; + } + + if (trimmed.length <= maxChars) { + return [trimmed]; + } + + const clauseChunks = trimmed.split(/(?<=[,;:])\s+/); + if (clauseChunks.length > 1) { + const parts: string[] = []; + let current = ""; + + const pushCurrent = () => { + const value = current.trim(); + if (value) { + parts.push(value); + } + current = ""; + }; + + for (const clause of clauseChunks) { + const clauseText = clause.trim(); + if (!clauseText) { + continue; + } + + if (clauseText.length > maxChars) { + pushCurrent(); + parts.push(...splitOversizedFragment(clauseText, maxChars)); + continue; + } + + if (!current) { + current = clauseText; + continue; + } + + const candidate = `${current} ${clauseText}`; + if (candidate.length <= maxChars) { + current = candidate; + continue; + } + + pushCurrent(); + current = clauseText; + } + + pushCurrent(); + if (parts.length > 1 || parts[0] !== trimmed) { + return parts; + } + } + + const parts: string[] = []; + let remaining = trimmed; + while (remaining.length > maxChars) { + let idx = remaining.lastIndexOf(" ", maxChars); + if (idx < Math.floor(maxChars * 0.5)) { + idx = maxChars; + } + parts.push(remaining.slice(0, idx).trim()); + remaining = remaining.slice(idx).trim(); + } + if (remaining.length > 0) { + parts.push(remaining); + } + return parts; +} + +/** + * Split text into sentence-ish segments small enough to synthesize with low + * latency. Segments are the unit of streaming for both voice mode and read + * aloud: the first segment can play while later ones are still synthesizing. + */ +export function splitTextForTts(text: string): TtsSegment[] { + const normalized = text.trim().replace(/\s+/g, " "); + if (!normalized) { + throw new Error("Cannot synthesize empty text"); + } + + const sentences = normalized.split(/(?<=[.!?])\s+/); + const parts: TtsSegment[] = []; + let segmentIndex = 0; + + for (const sentence of sentences) { + const fragments = splitOversizedFragment(sentence, MAX_TTS_SEGMENT_CHARS); + for (const fragment of fragments) { + parts.push({ index: segmentIndex, text: fragment }); + segmentIndex += 1; + } + } + + return parts; +} diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 9085d0c9b2..b46fa5f62d 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -1545,6 +1545,10 @@ export class VoiceAssistantWebSocketServer { agentDetach: true, // COMPAT(agentThinkingUpdate): added in v0.2.4, remove gate after 2027-01-28. agentThinkingUpdate: true, + // COMPAT(readAloud): added in v0.2.5, drop the gate when floor >= v0.2.5. + // Advertises the `speech.tts.read_aloud.*` RPC. Whether TTS is actually + // usable is reported separately by `capabilities.voice`. + readAloud: true, // COMPAT(daemonDiagnostics): added in v0.1.100, remove gate after 2026-12-25 once daemon floor >= v0.1.100. daemonDiagnostics: true, // COMPAT(daemonSelfUpdate): added in v0.1.93, remove gate after 2026-12-13. diff --git a/public-docs/voice.md b/public-docs/voice.md index 79022c15ad..859262c55c 100644 --- a/public-docs/voice.md +++ b/public-docs/voice.md @@ -111,6 +111,19 @@ Paseo uses these paths under the configured OpenAI base URL: - voice mode STT: `/v1/audio/transcriptions` - voice mode TTS: `/v1/audio/speech` +## Read Aloud + +Select text anywhere in the app — an agent's answer, a diff, a file — and a speaker button appears above the selection. Pressing it speaks the selection; the icon becomes a stop square while it plays, and pressing it again stops. Playback also stops as soon as the selection is cleared. + +Read aloud uses the same text-to-speech provider as voice mode (`features.voiceMode.tts`), so local Kokoro and OpenAI both work with no extra configuration. Audio is synthesized sentence by sentence and streamed to the client, so long selections start speaking on the first sentence. Selections above 4000 characters are rejected rather than queued. + +Two limits: + +- **Web and desktop only.** The bubble is anchored to a live text selection, and iOS/Android hand selection to the system edit menu, which the app cannot read. There is no read-aloud affordance on the mobile apps. +- **The composer and the terminal are excluded.** Both own their own selection behavior. + +Hosts older than v0.2.5 do not have the read-aloud RPC; against those the bubble simply never appears. + ## Environment Variables - `PASEO_VOICE_LLM_PROVIDER`, voice agent provider override From 1c05ef380e82de41f99364862bad47d16e4aa605 Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 30 Jul 2026 15:11:25 -0400 Subject: [PATCH 2/3] fix(read-aloud): speak selections only on their own host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit useReadAloudServerId preferred the route's host but fell back to any other paired host advertising the capability. The text being read is workspace content — code, agent output — so that fallback disclosed it across an independently paired daemon boundary. Bind to the route host with no fallback: a route host that doesn't advertise readAloud shows no button, and neither does a route with no host at all (settings, history). The doc comment argued the opposite — "read aloud carries no workspace context, any host will do" — which was the bug, so it's rewritten to state the invariant instead. --- .../read-aloud-selection-bubble.web.tsx | 34 +++++++++---------- public-docs/voice.md | 1 + 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/packages/app/src/read-aloud/read-aloud-selection-bubble.web.tsx b/packages/app/src/read-aloud/read-aloud-selection-bubble.web.tsx index 1b77dc85af..12dc4ccca7 100644 --- a/packages/app/src/read-aloud/read-aloud-selection-bubble.web.tsx +++ b/packages/app/src/read-aloud/read-aloud-selection-bubble.web.tsx @@ -24,7 +24,7 @@ import { useSelectionAnchor, } from "@/read-aloud/use-selection-anchor.web"; import { useHostFeatureMap } from "@/runtime/host-features"; -import { useHostRuntimeClient, useHosts } from "@/runtime/host-runtime"; +import { useHostRuntimeClient } from "@/runtime/host-runtime"; import { parseActiveWorkspaceSelection } from "@/stores/navigation-active-workspace-store/navigation"; const BUBBLE_HEIGHT = 32; @@ -67,28 +67,26 @@ function useRouteServerId(): string | null { /** * Which host synthesizes the speech. * - * Read aloud carries no workspace context — it is pure text-to-speech — so any - * host that advertises the capability will do. The host owning the current - * workspace is preferred so the work lands where the content came from, but - * selections on routes without a workspace (settings, history) still get a - * voice. Reachability is not checked here: the daemon client resolves to `null` - * before it has a transport, and a host that drops mid-request surfaces the - * failure on the bubble. + * The selection is spoken by the host that owns the route it came from, never + * another paired daemon. The text being read *is* workspace content — code, + * agent output — so sending it to a different host would disclose it across an + * independently paired daemon boundary. There is deliberately no fallback: a + * route host that doesn't advertise the capability shows no button, and so does + * a route with no host at all (settings, history). + * + * Reachability is not checked here: the daemon client resolves to `null` before + * it has a transport, and a host that drops mid-request surfaces the failure on + * the bubble. */ function useReadAloudServerId(): string | null { const routeServerId = useRouteServerId(); - const hosts = useHosts(); - const serverIds = useMemo(() => hosts.map((host) => host.serverId), [hosts]); + const serverIds = useMemo(() => (routeServerId ? [routeServerId] : []), [routeServerId]); const featureByServerId = useHostFeatureMap(serverIds, "readAloud"); - return useMemo(() => { - const supportsReadAloud = (serverId: string) => featureByServerId.get(serverId) === true; - - if (routeServerId && supportsReadAloud(routeServerId)) { - return routeServerId; - } - return serverIds.find(supportsReadAloud) ?? null; - }, [featureByServerId, routeServerId, serverIds]); + if (!routeServerId) { + return null; + } + return featureByServerId.get(routeServerId) === true ? routeServerId : null; } function renderStatusIcon(status: ReadAloudSnapshot["status"], color: string): ReactNode { diff --git a/public-docs/voice.md b/public-docs/voice.md index 859262c55c..e5c24ff8cd 100644 --- a/public-docs/voice.md +++ b/public-docs/voice.md @@ -121,6 +121,7 @@ Two limits: - **Web and desktop only.** The bubble is anchored to a live text selection, and iOS/Android hand selection to the system edit menu, which the app cannot read. There is no read-aloud affordance on the mobile apps. - **The composer and the terminal are excluded.** Both own their own selection behavior. +- **The selection is spoken by the host it came from.** Read aloud never sends text to another paired host, so a selection on a route with no host — settings, history — gets no button. Hosts older than v0.2.5 do not have the read-aloud RPC; against those the bubble simply never appears. From c85fd3ba189d9148498984d509c33e97680cec40 Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 30 Jul 2026 16:21:45 -0400 Subject: [PATCH 3/3] test(agent-manager): retry the interrupt-fixture teardown on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cancelAgentRun succeeds when the provider queues completion before rejecting the interrupt` failed on server-tests (windows-latest) with ENOTEMPTY: rmdir '...\agents'. The assertions passed — the fixture's cleanup is what threw. AgentManager can flush an agent snapshot into `agents/` while cleanup runs, and on Windows an open handle makes rmdir fail with ENOTEMPTY, which `force: true` does not cover. Retry the removal with the options already used in spawn.launch-regression.test.ts and run-git-command.windows-shell.test.ts. Only the shared fixture is changed; the other rmSync sites in this file have not failed and are left alone. This addresses one observed failure mode, not the whole job — server-tests has also failed with an unrelated timeout in execution-session.websocket.test.ts. --- packages/server/src/server/agent/agent-manager.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 045e243324..98d1d05887 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -514,7 +514,12 @@ async function createControlledInterruptFixture(options: { })(); await manager.waitForAgentRunStart(agent.id); }, - cleanup: () => rmSync(workdir, { recursive: true, force: true }), + // Retry the teardown: AgentManager can flush an agent snapshot into + // `agents/` while this runs, and on Windows an open handle makes rmdir + // fail with ENOTEMPTY, which `force` does not cover. Same options as + // spawn.launch-regression.test.ts and run-git-command.windows-shell.test.ts. + cleanup: () => + rmSync(workdir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }), }; }