From 44ec6d6e26cb8b8becc5eefc5116d364c5c28a20 Mon Sep 17 00:00:00 2001 From: luke-speechify <289678208+luke-speechify@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:44:42 +0100 Subject: [PATCH 1/2] feat(live-captions): live word-by-word captions synced to TTS playback with speech marks --- README.md | 1 + demos/live-captions/.env.example | 1 + demos/live-captions/.gitignore | 5 + demos/live-captions/README.md | 39 ++++ demos/live-captions/app/api/speak/route.ts | 59 ++++++ demos/live-captions/app/globals.css | 151 ++++++++++++++ demos/live-captions/app/layout.tsx | 21 ++ demos/live-captions/app/lib/turnstile.ts | 37 ++++ demos/live-captions/app/page.tsx | 226 +++++++++++++++++++++ demos/live-captions/demo.json | 6 + demos/live-captions/next.config.ts | 13 ++ demos/live-captions/package.json | 26 +++ demos/live-captions/tsconfig.json | 41 ++++ pnpm-lock.yaml | 28 +++ pnpm-workspace.yaml | 1 + site/public/index.html | 4 +- vercel.json | 8 + 17 files changed, 665 insertions(+), 2 deletions(-) create mode 100644 demos/live-captions/.env.example create mode 100644 demos/live-captions/.gitignore create mode 100644 demos/live-captions/README.md create mode 100644 demos/live-captions/app/api/speak/route.ts create mode 100644 demos/live-captions/app/globals.css create mode 100644 demos/live-captions/app/layout.tsx create mode 100644 demos/live-captions/app/lib/turnstile.ts create mode 100644 demos/live-captions/app/page.tsx create mode 100644 demos/live-captions/demo.json create mode 100644 demos/live-captions/next.config.ts create mode 100644 demos/live-captions/package.json create mode 100644 demos/live-captions/tsconfig.json diff --git a/README.md b/README.md index 6434db4..4d49ebd 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Demos with a **Live** link run in your browser at [demos.speechify.ai](https://d | [`demos/voice-agent-showcase/`](./demos/voice-agent-showcase) | Cloudflare Workers | | One page, ten live Voice Agents API demos: calendar booking, policy-bound support, a page copilot, form intake, US outbound calls with a 5-minute cap, a voice gallery, mid-call language handoff, cross-call memory, a grounded knowledge base, and dual-control troubleshooting. | | [`demos/vercel-ai-sdk/`](./demos/vercel-ai-sdk) | TypeScript (Vercel AI SDK) | | Speechify TTS through the Vercel AI SDK's unified `generateSpeech` interface via the official `@speechify/vercel` provider — one-line swap from OpenAI/ElevenLabs, plus word-level speech marks from `providerMetadata`. | | [`demos/puter-txt2speech/`](./demos/puter-txt2speech) | HTML (puter.js) | | Speaks with a Simba 3.2 voice via the Speechify provider in Puter's puter.ai.txt2speech() — one static page, your key configured once on the Puter instance. | +| [`demos/live-captions/`](./demos/live-captions) | Next.js | [Open](https://demos.speechify.ai/live-captions) | Synthesize text and render live, word-by-word captions in sync with playback, driven by the speech marks the API returns. | ## Get an API key diff --git a/demos/live-captions/.env.example b/demos/live-captions/.env.example new file mode 100644 index 0000000..534cec4 --- /dev/null +++ b/demos/live-captions/.env.example @@ -0,0 +1 @@ +SPEECHIFY_API_KEY=your_api_key_here diff --git a/demos/live-captions/.gitignore b/demos/live-captions/.gitignore new file mode 100644 index 0000000..95d1bcb --- /dev/null +++ b/demos/live-captions/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +.next/ +.env +next-env.d.ts +*.tsbuildinfo diff --git a/demos/live-captions/README.md b/demos/live-captions/README.md new file mode 100644 index 0000000..60e60c8 --- /dev/null +++ b/demos/live-captions/README.md @@ -0,0 +1,39 @@ +# Live captions with speech marks (Next.js) + +A small [Next.js](https://nextjs.org) app that synthesizes text with the Speechify API and renders live, word-by-word captions in sync with playback. Each word lights up the instant the voice speaks it, driven entirely by the `speech_marks` the API returns alongside the audio. The API key stays server-side in a route handler and never reaches the browser. + +This is the hostable web update of the earlier post [Building real-time captions with Speechify TTS speech marks](https://speechify.ai/blog/building-real-time-captions-with-speechify-tts-speech-marks). That original framed the captions as a browser extension; this demo ships the same speech-marks logic as a page you can host and run in a browser. + +## What you get + +- A one-page UI: type text, click **Synthesize**, press play, and watch each word highlight in real time. +- One server route holding the Speechify key server-side: + - `POST /api/speak` — synthesizes text with `client.audio.speech` (model `simba-3.2`, voice `geffen_32`, MP3) and returns `{ audio, speechMarks }`, where `speechMarks` is `response.speech_marks.chunks` — one entry per word with `start_time` / `end_time` in milliseconds and the word `value`. +- The sync loop: the client plays the base64 MP3 and, on every `requestAnimationFrame`, reads `audio.currentTime` and highlights the word whose `[start_time, end_time)` window contains the current position (found with a binary search). No forced alignment, no polling timer, no custom audio decoder. + +## Run it yourself + +```bash +cp .env.example .env # then paste your SPEECHIFY_API_KEY +pnpm install +pnpm dev # http://localhost:8771 +``` + +Open `http://localhost:8771`, edit the text if you like, click **Synthesize**, then press play on the audio control. + +## Dropping this into a real browser extension + +The original post built this as a browser extension, and the timing logic here is exactly what an extension content script needs. The `speech_marks` chunks are the whole trick: given `audio.currentTime`, `activeIndexAt()` in `app/page.tsx` returns the word to highlight. In a content script you keep that function verbatim and swap the React state update for a `classList` toggle on the words already in the page's DOM. The server route stays the same — it is where your key lives — and the extension calls it the way this page does. + +## How the key stays server-side + +Every Speechify call happens inside the `app/api/speak` route handler, which only ever runs on the server. The browser talks to that same-origin route; it never sees `SPEECHIFY_API_KEY`. `next.config.ts` marks `@speechify/api` as a server-external package so the SDK is never bundled into client JS. + +## Where the code came from + +The speech-marks-to-captions logic mirrors the [captions-speech-marks](../captions-speech-marks) demo, which turns the same `speech_marks` chunks into WebVTT cues. This folder drives live highlighting from those chunks instead, and wraps it in a Next.js UI with the key held server-side — which is how you would ship it in a real app. + +## Prerequisites + +- Node 20 or newer +- A `SPEECHIFY_API_KEY` from [platform.speechify.ai/api-keys](https://platform.speechify.ai/api-keys) diff --git a/demos/live-captions/app/api/speak/route.ts b/demos/live-captions/app/api/speak/route.ts new file mode 100644 index 0000000..fc220b8 --- /dev/null +++ b/demos/live-captions/app/api/speak/route.ts @@ -0,0 +1,59 @@ +import { NextResponse } from "next/server"; +import { SpeechifyClient, SpeechifyError } from "@speechify/api"; +import { verifyTurnstile } from "../../lib/turnstile"; + +export const runtime = "nodejs"; + +const client = new SpeechifyClient({ token: process.env.SPEECHIFY_API_KEY }); + +// One speech mark per word: when it starts and ends in the audio, in +// milliseconds, plus the word itself. This is what drives the live captions. +type SpeechMark = { + start_time: number; + end_time: number; + value: string; +}; + +export async function POST(req: Request) { + if (!(await verifyTurnstile(req))) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + + const { text } = await req.json(); + + if (typeof text !== "string" || text.trim() === "") { + return NextResponse.json({ error: "text is required" }, { status: 400 }); + } + + try { + const speech = await client.audio.speech({ + input: text, + voice_id: "geffen_32", + audio_format: "mp3", + model: "simba-3.2", + }); + + // chunks carry start_time / end_time (ms) + value for every word. Ship them + // to the client so it can highlight the current word as the audio plays. + const speechMarks: SpeechMark[] = (speech.speech_marks?.chunks ?? []).map( + (c) => ({ + start_time: c.start_time ?? 0, + end_time: c.end_time ?? 0, + value: c.value ?? "", + }), + ); + + return NextResponse.json({ audio: speech.audio_data, speechMarks }); + } catch (err) { + if (err instanceof SpeechifyError) { + return NextResponse.json( + { error: err.message }, + { status: err.statusCode ?? 500 }, + ); + } + return NextResponse.json( + { error: "Synthesis failed." }, + { status: 500 }, + ); + } +} diff --git a/demos/live-captions/app/globals.css b/demos/live-captions/app/globals.css new file mode 100644 index 0000000..f3a6835 --- /dev/null +++ b/demos/live-captions/app/globals.css @@ -0,0 +1,151 @@ +:root { + color-scheme: light dark; + --fg: #111; + --bg: #fff; + --muted: #666; + --border: #ddd; + --surface: #f6f6f6; + --accent: #111; + --accent-fg: #fff; +} + +@media (prefers-color-scheme: dark) { + :root { + --fg: #f2f2f2; + --bg: #0c0c0c; + --muted: #999; + --border: #2a2a2a; + --surface: #161616; + --accent: #f2f2f2; + --accent-fg: #0c0c0c; + } +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + padding: 2rem 1rem; + background: var(--bg); + color: var(--fg); + font-family: ui-sans-serif, system-ui, -apple-system, sans-serif; + line-height: 1.5; +} + +main { + max-width: 40rem; + margin: 0 auto; + display: flex; + flex-direction: column; + gap: 1.5rem; +} + +h1 { + font-size: 1.6rem; + font-weight: 500; + margin: 0; +} + +.lede { + margin: 0; + color: var(--muted); + font-size: 0.95rem; +} + +.step { + border: 1px solid var(--border); + border-radius: 0.6rem; + padding: 1.1rem; + background: var(--surface); +} + +.step h2 { + font-size: 0.72rem; + text-transform: uppercase; + letter-spacing: 0.12em; + color: var(--muted); + margin: 0 0 0.75rem; +} + +textarea { + font: inherit; + color: inherit; + width: 100%; + padding: 0.55rem 0.7rem; + border: 1px solid var(--border); + border-radius: 0.4rem; + background: var(--bg); + resize: vertical; +} + +button { + font: inherit; + cursor: pointer; + width: 100%; + padding: 0.55rem 0.7rem; + border-radius: 0.4rem; + background: var(--fg); + color: var(--bg); + border: 0; + font-weight: 500; + margin-top: 0.9rem; +} + +button:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.captions { + border: 1px solid var(--border); + border-radius: 0.6rem; + padding: 1.5rem 1.25rem; + background: var(--surface); +} + +.caption-line { + margin: 0; + font-size: 1.6rem; + font-weight: 300; + line-height: 1.7; + letter-spacing: -0.01em; + color: var(--muted); +} + +.caption-line .word { + transition: + background-color 80ms ease-out, + color 80ms ease-out; + padding: 0.05em 0.15em; + border-radius: 5px; +} + +.caption-line .word.active { + background: var(--accent); + color: var(--accent-fg); +} + +.status { + font-size: 0.85rem; + color: var(--muted); + min-height: 1.2rem; +} + +.status[data-tone="error"] { + color: #c0392b; +} + +audio { + width: 100%; + margin-top: 0.5rem; +} + +code { + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 0.85em; + background: var(--surface); + padding: 0.1em 0.35em; + border-radius: 3px; +} diff --git a/demos/live-captions/app/layout.tsx b/demos/live-captions/app/layout.tsx new file mode 100644 index 0000000..d9f8d25 --- /dev/null +++ b/demos/live-captions/app/layout.tsx @@ -0,0 +1,21 @@ +import type { Metadata } from "next"; +import type { ReactNode } from "react"; +import Script from "next/script"; +import "./globals.css"; + +export const metadata: Metadata = { + title: "Live captions with Speechify speech marks", + description: + "Synthesize text and render live, word-by-word captions in sync with playback, driven by the speech marks the Speechify API returns.", +}; + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + + + @@ -368,7 +368,7 @@

FAQ