From bd774d4d8aa1e44d18886f62b002a8724b0b8741 Mon Sep 17 00:00:00 2001 From: "Christopher S. Aondona" Date: Thu, 12 Nov 2026 11:29:59 +0100 Subject: [PATCH 01/11] feat(player): add optional browser narration controller and authored speech text --- src/react/narration-text.ts | 36 +++++++++ src/react/use-narration.ts | 157 ++++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 src/react/narration-text.ts create mode 100644 src/react/use-narration.ts diff --git a/src/react/narration-text.ts b/src/react/narration-text.ts new file mode 100644 index 0000000..3e45ef1 --- /dev/null +++ b/src/react/narration-text.ts @@ -0,0 +1,36 @@ +import type { Frame, Trace } from "../model.js"; +import { format } from "../layout/measure.js"; +import type { ThroughlineGuidance } from "./guidance.js"; + +/** Only authored text and authored transfers are narrated; never infer causes. */ +export function narrationText(frame: Frame, trace: Trace, guidance?: ThroughlineGuidance): string { + const segments: string[] = [frame.label, ...frame.state.notes]; + const chapter = guidance?.chapters?.find( + (item) => frame.index >= item.start && frame.index <= item.end, + ); + if (chapter) { + segments.push(chapter.title); + if (chapter.summary) segments.push(chapter.summary); + } + for (const item of guidance?.causalLinks ?? []) { + if (item.step !== frame.index) continue; + segments.push(item.title); + segments.push("Trigger: " + item.trigger); + segments.push("Observed event: " + item.observedEvent); + segments.push("State change: " + item.stateChange); + segments.push("Consequence: " + item.consequence); + } + for (const transfer of frame.state.transfers) { + const relation = frame.state.relations.find((item) => item.id === transfer.relation); + if (!relation) continue; + segments.push( + relation.from + " to " + relation.to + + (transfer.value === undefined ? "" : ": " + format(transfer.value)), + ); + } + // Source lengths are compiler-bounded but browser speech engines have + // varying per-utterance limits. Keep one spoken step at a safe size. + const text = segments.filter(Boolean).join(". ").replace(/\s+/g, " ").trim(); + if (text.length <= 3000) return text; + return text.slice(0, 2997).trimEnd() + "..."; +} diff --git a/src/react/use-narration.ts b/src/react/use-narration.ts new file mode 100644 index 0000000..5312884 --- /dev/null +++ b/src/react/use-narration.ts @@ -0,0 +1,157 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import type { Trace } from "../model.js"; +import type { SeekAction, Speed } from "../seek.js"; + +type Session = { + trace: Trace; + step: number; + utterance: SpeechSynthesisUtterance; + ended: boolean; +}; + +type NarrationInput = { + trace: Trace; + step: number; + text: string; + speed: Speed; + playing: boolean; + enabled: boolean; + setEnabled: (enabled: boolean) => void; + dispatch: (action: SeekAction) => void; + offered: boolean; +}; + +function speechEngine(): SpeechSynthesis | null { + if (typeof window === "undefined" || typeof SpeechSynthesisUtterance === "undefined") return null; + return window.speechSynthesis ?? null; +} + +/** + * A browser-owned, opt-in speech controller. The narration clock is driven by + * utterance completion, never a timer. Stale callbacks cannot advance a seek. + * The toggle speaks synchronously in its click handler to satisfy browsers + * that require a user activation for text-to-speech. + */ +export function useNarration({ + trace, step, text, speed, playing, enabled, setEnabled, dispatch, offered, +}: NarrationInput): { + available: boolean; + error: string | null; + toggle: () => void; +} { + const [available, setAvailable] = useState(false); + const [error, setError] = useState(null); + const current = useRef(null); + const latest = useRef({ trace, step, text, speed, playing, enabled, dispatch }); + latest.current = { trace, step, text, speed, playing, enabled, dispatch }; + const previous = useRef({ trace, step, playing, enabled }); + + const stop = useCallback(() => { + const voice = current.current; + current.current = null; + if (!voice) return; + // A browser cancellation may synchronously invoke onerror/onend. + voice.utterance.onend = null; + voice.utterance.onerror = null; + try { speechEngine()?.cancel(); } catch { /* Speech is optional. */ } + }, []); + + const start = useCallback(() => { + const engine = speechEngine(); + if (!engine) { + setError("Caption narration is unavailable in this browser."); + latest.current.dispatch({ type: "pause" }); + return; + } + stop(); + const state = latest.current; + if (!state.text) return; + try { + const utterance = new SpeechSynthesisUtterance(state.text); + utterance.rate = state.speed; + const session: Session = { + trace: state.trace, step: state.step, utterance, ended: false, + }; + current.current = session; + utterance.onend = () => { + if (current.current !== session) return; + session.ended = true; + const next = latest.current; + if (next.enabled && next.trace === session.trace && + next.step === session.step && next.playing) { + next.dispatch({ type: "tick" }); + } + }; + utterance.onerror = (event) => { + if (current.current !== session) return; + session.ended = true; + if (event.error === "canceled" || event.error === "interrupted") return; + setError("Speech could not play (" + event.error + "). Choose another browser voice or disable narration."); + latest.current.dispatch({ type: "pause" }); + }; + setError(null); + // Cancelled or previously paused speech must not leave a new voice + // stuck in the browser-wide synthesis queue. + if (engine.paused) engine.resume(); + engine.speak(utterance); + } catch { + stop(); + setError("Speech synthesis failed. Narration stopped without advancing the timeline."); + latest.current.dispatch({ type: "pause" }); + } + }, [stop]); + + useEffect(() => { + setAvailable(offered && Boolean(speechEngine())); + }, [offered]); + + // Frame replacement or manual seeking begins the new authored caption. + // A paused player may still narrate a step without automatic advancement. + useEffect(() => { + if (!enabled || !offered) { stop(); return; } + const voice = current.current; + if (!voice || voice.trace !== trace || voice.step !== step) start(); + }, [enabled, offered, trace, step, start, stop]); + + // Play/Pause acts on the voice, not an independent timer. Do not + // immediately pause a voice that was just started by a manual seek. + useEffect(() => { + const old = previous.current; + previous.current = { trace, step, playing, enabled }; + if (!enabled || old.trace !== trace || old.step !== step || + old.enabled !== enabled || old.playing === playing) return; + const voice = current.current; + const engine = speechEngine(); + if (!voice || !engine) return; + if (!playing && !voice.ended) { + try { engine.pause(); } catch { /* Preserve player transport. */ } + } else if (playing) { + if (voice.ended) { + dispatch({ type: "tick" }); + } else { + try { engine.resume(); } catch { + setError("Speech could not resume."); + dispatch({ type: "pause" }); + } + } + } + }, [trace, step, playing, enabled, dispatch]); + + // Stop our owned utterance on unmount or independent scenario replacement. + useEffect(() => () => stop(), [stop]); + + const toggle = useCallback(() => { + if (latest.current.enabled) { + stop(); + setError(null); + setEnabled(false); + } else if (speechEngine()) { + setEnabled(true); + start(); + } else { + setError("Caption narration is unavailable in this browser."); + } + }, [setEnabled, start, stop]); + + return { available, error, toggle }; +} From d8f365542beb6ffc76fce69b9e08cc95222c5ff8 Mon Sep 17 00:00:00 2001 From: "Christopher S. Aondona" Date: Thu, 12 Nov 2026 11:32:27 +0100 Subject: [PATCH 02/11] feat(player): expose independent speech toggle and speech-timed playback in React preview --- CHANGELOG.md | 6 ++++++ README.md | 23 +++++++++++++++++++++++ docs/architecture.md | 17 +++++++++++++++++ playground/main.tsx | 7 ++++++- src/react/controls.tsx | 18 ++++++++++++++++++ src/react/player.tsx | 20 +++++++++++++++++++- src/react/use-player.ts | 6 +++--- src/styles/throughline.css | 4 +++- 8 files changed, 95 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbe49da..e8ee834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ Semantic Versioning independently of the version-1 trace and program schemas. ## Unreleased +### Added (opt-in caption narration) + +- Offer muted-by-default browser caption narration separately from step tones, using speech-synchronized playback without new runtime dependencies. +- Pause/resume speaking with playback; cancel on seek, scenario change and unmount; preserve the original timer when narration is off. + + ### Changed (development preview sound) - Expose the existing opt-in Web Audio speaker control in every local playground example and scenario comparison, initially muted. No default audio in the published React player and no new audio dependencies. diff --git a/README.md b/README.md index 6d33681..3f69ba3 100644 --- a/README.md +++ b/README.md @@ -247,6 +247,29 @@ from the diagram. The screen-reader step announcement remains active regardless of whether the disclosure is expanded. Details remain open between steps until closed, without changing playback state. +## Opt-in caption narration + +The optional React-only `narration` prop exposes a **Narrate captions** +button next to the independent `sonification` sound-effects toggle. +The button is **off initially**, even when a host offers it. Its first +activation speaks the current authored step from a user gesture using +the browser's Speech Synthesis API. It reads the authored step label, +notes, matching chapter and causal explanation, and transfer summaries. +It does not invent responses, state changes or causal explanations. + +With narration enabled, **Play advances only after speech finishes**. +Pause pauses the current voice; Play resumes it. Manual seeking or +stepping cancels the previous voice and narrates the selected step. +Switching scenarios cancels the old voice and starts muted. Disabling +narration returns to timed playback and leaves step tones independent. +Playback speed applies to the next utterance, not the current one. + +Speech voices, available hardware, and user-activation policies depend +on the browser and OS. The button is disabled where speech synthesis is +unavailable; speech errors pause synchronized playback and are announced. +No text is uploaded by Throughline, but the browser may use a network +speech service. The core stays React- and browser-independent. + ## Persistent workflow progress The React player retains a cumulative trail through the **selected** step. diff --git a/docs/architecture.md b/docs/architecture.md index fb602bc..be2e7be 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -60,6 +60,23 @@ uses this internal boundary without adding a package export, trace schema, viewport control, or public selection API. Pan/zoom, automatic camera following and user-driven collapse are not part of TL-I12. +## Optional spoken explanation + +The React-only narration controller synthesizes authored caption text +(frame label, notes, guided chapter/causal content and explicit sends). +No new trace event or compiler schema is introduced. + +While narration is enabled, speech completion drives playback ticks +instead of a fixed interval. Only the active utterance may advance +the selected trace; seeking invalidates stale callbacks. Pause/resume +controls the voice, and unavailable or failed synthesis pauses the +narration clock with an accessible error. When narration is disabled, +the existing timed playback returns. Step tones remain independent. + +Browser speechSynthesis is a shared page resource; applications using +other text-to-speech output must coordinate ownership. Some voices +may use browser-managed remote services. + ## Footer layout and bounded explanations The React player composes its navigation, chapter selection, timeline, diff --git a/playground/main.tsx b/playground/main.tsx index 99c3031..b177187 100644 --- a/playground/main.tsx +++ b/playground/main.tsx @@ -90,6 +90,7 @@ function App() { scenarios={scenarios} initialScenarioId="retry-recovery" theme={theme} + narration sonification={{}} /> ) : current ? ( @@ -99,6 +100,7 @@ function App() { title={current.title} theme={theme} sonification={{}} + narration guidance={current.id === "distributed-checkout" ? CHECKOUT_GUIDANCE : undefined} /> ) : null} @@ -110,7 +112,10 @@ function App() {

Use the player controls to step, pan, zoom, collapse groups, inspect state - and switch outcomes. Sound is available in every example: select the speaker + and switch outcomes. Narration is available in every example: + select Narrate captions to hear the authored explanation. + Play waits for speech to finish before advancing. The separate + step-tone sound control is still available: select the speaker icon to unmute, then Play or step forward to hear short synthesized tones. It starts muted and does not narrate the workflow. Resize the browser or choose the 375px view to find layout problems. Changes to the source reload diff --git a/src/react/controls.tsx b/src/react/controls.tsx index 7194bf3..77585f5 100644 --- a/src/react/controls.tsx +++ b/src/react/controls.tsx @@ -10,6 +10,7 @@ interface PlayerControlsProps { state: SeekState; dispatch: React.Dispatch; sound?: { enabled: boolean; toggle: () => void }; + narration?: { enabled: boolean; available: boolean; toggle: () => void }; fullscreen: { active: boolean; available: boolean; toggle: () => void }; } @@ -17,6 +18,7 @@ export function PlayerControls({ state, dispatch, sound, + narration, fullscreen, }: PlayerControlsProps): React.JSX.Element { return ( @@ -121,6 +123,22 @@ export function PlayerControls({ )} + {narration && ( + + )}