diff --git a/README.md b/README.md index 1019530..3d4fdee 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Demos with a **Live** link run in your browser at [demos.speechify.ai](https://d | [`demos/discord-bot-speechify/`](./demos/discord-bot-speechify) | TypeScript (discord.js) | | A Discord slash-command bot: /speak synthesizes the text with the Speechify API and posts the MP3 into the channel. The command registers automatically on first run. | | [`demos/docs-read-aloud/`](./demos/docs-read-aloud) | TypeScript (zero-dep server) | | A documentation-style page with a Listen button that reads the article aloud. The button POSTs the text to a tiny server route, which synthesizes it with the Speechify API (key stays server-side) and returns the MP3 for the browser to play. Framework-agnostic. | | [`demos/ivr-ssml/`](./demos/ivr-ssml) | Next.js | [Open](https://demos.speechify.ai/ivr-ssml) | A phone-system playground for getting names, account numbers, and product terms right with SSML. Hear plain vs SSML side by side; the API key stays server-side. | +| [`demos/live-captions/`](./demos/live-captions) | Next.js | [Open](https://demos.speechify.ai/live-captions) | Synthesize text and render live, word-by-word captions in sync with playback, driven by the speech marks the API returns. | | [`demos/webpage-audiobook/`](./demos/webpage-audiobook) | Next.js | [Open](https://demos.speechify.ai/webpage-audiobook) | Paste a URL, get narrated audio. The server fetches the article, extracts the text, chunks it on sentence boundaries, and synthesizes each part with the Speechify TTS API. | diff --git a/demos/live-captions/.env.example b/demos/live-captions/.env.example new file mode 100644 index 0000000..534cec4 --- /dev/null +++ b/demos/live-captions/.env.example @@ -0,0 +1 @@ +SPEECHIFY_API_KEY=your_api_key_here diff --git a/demos/live-captions/.gitignore b/demos/live-captions/.gitignore new file mode 100644 index 0000000..e838375 --- /dev/null +++ b/demos/live-captions/.gitignore @@ -0,0 +1,9 @@ +node_modules/ +.next/ +.env +next-env.d.ts +*.tsbuildinfo +test-results/ +playwright-report/ +/.playwright/ +.last-run.json diff --git a/demos/live-captions/README.md b/demos/live-captions/README.md new file mode 100644 index 0000000..60e60c8 --- /dev/null +++ b/demos/live-captions/README.md @@ -0,0 +1,39 @@ +# Live captions with speech marks (Next.js) + +A small [Next.js](https://nextjs.org) app that synthesizes text with the Speechify API and renders live, word-by-word captions in sync with playback. Each word lights up the instant the voice speaks it, driven entirely by the `speech_marks` the API returns alongside the audio. The API key stays server-side in a route handler and never reaches the browser. + +This is the hostable web update of the earlier post [Building real-time captions with Speechify TTS speech marks](https://speechify.ai/blog/building-real-time-captions-with-speechify-tts-speech-marks). That original framed the captions as a browser extension; this demo ships the same speech-marks logic as a page you can host and run in a browser. + +## What you get + +- A one-page UI: type text, click **Synthesize**, press play, and watch each word highlight in real time. +- One server route holding the Speechify key server-side: + - `POST /api/speak` — synthesizes text with `client.audio.speech` (model `simba-3.2`, voice `geffen_32`, MP3) and returns `{ audio, speechMarks }`, where `speechMarks` is `response.speech_marks.chunks` — one entry per word with `start_time` / `end_time` in milliseconds and the word `value`. +- The sync loop: the client plays the base64 MP3 and, on every `requestAnimationFrame`, reads `audio.currentTime` and highlights the word whose `[start_time, end_time)` window contains the current position (found with a binary search). No forced alignment, no polling timer, no custom audio decoder. + +## Run it yourself + +```bash +cp .env.example .env # then paste your SPEECHIFY_API_KEY +pnpm install +pnpm dev # http://localhost:8771 +``` + +Open `http://localhost:8771`, edit the text if you like, click **Synthesize**, then press play on the audio control. + +## Dropping this into a real browser extension + +The original post built this as a browser extension, and the timing logic here is exactly what an extension content script needs. The `speech_marks` chunks are the whole trick: given `audio.currentTime`, `activeIndexAt()` in `app/page.tsx` returns the word to highlight. In a content script you keep that function verbatim and swap the React state update for a `classList` toggle on the words already in the page's DOM. The server route stays the same — it is where your key lives — and the extension calls it the way this page does. + +## How the key stays server-side + +Every Speechify call happens inside the `app/api/speak` route handler, which only ever runs on the server. The browser talks to that same-origin route; it never sees `SPEECHIFY_API_KEY`. `next.config.ts` marks `@speechify/api` as a server-external package so the SDK is never bundled into client JS. + +## Where the code came from + +The speech-marks-to-captions logic mirrors the [captions-speech-marks](../captions-speech-marks) demo, which turns the same `speech_marks` chunks into WebVTT cues. This folder drives live highlighting from those chunks instead, and wraps it in a Next.js UI with the key held server-side — which is how you would ship it in a real app. + +## Prerequisites + +- Node 20 or newer +- A `SPEECHIFY_API_KEY` from [platform.speechify.ai/api-keys](https://platform.speechify.ai/api-keys) diff --git a/demos/live-captions/app/api/speak/route.ts b/demos/live-captions/app/api/speak/route.ts new file mode 100644 index 0000000..fc220b8 --- /dev/null +++ b/demos/live-captions/app/api/speak/route.ts @@ -0,0 +1,59 @@ +import { NextResponse } from "next/server"; +import { SpeechifyClient, SpeechifyError } from "@speechify/api"; +import { verifyTurnstile } from "../../lib/turnstile"; + +export const runtime = "nodejs"; + +const client = new SpeechifyClient({ token: process.env.SPEECHIFY_API_KEY }); + +// One speech mark per word: when it starts and ends in the audio, in +// milliseconds, plus the word itself. This is what drives the live captions. +type SpeechMark = { + start_time: number; + end_time: number; + value: string; +}; + +export async function POST(req: Request) { + if (!(await verifyTurnstile(req))) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + + const { text } = await req.json(); + + if (typeof text !== "string" || text.trim() === "") { + return NextResponse.json({ error: "text is required" }, { status: 400 }); + } + + try { + const speech = await client.audio.speech({ + input: text, + voice_id: "geffen_32", + audio_format: "mp3", + model: "simba-3.2", + }); + + // chunks carry start_time / end_time (ms) + value for every word. Ship them + // to the client so it can highlight the current word as the audio plays. + const speechMarks: SpeechMark[] = (speech.speech_marks?.chunks ?? []).map( + (c) => ({ + start_time: c.start_time ?? 0, + end_time: c.end_time ?? 0, + value: c.value ?? "", + }), + ); + + return NextResponse.json({ audio: speech.audio_data, speechMarks }); + } catch (err) { + if (err instanceof SpeechifyError) { + return NextResponse.json( + { error: err.message }, + { status: err.statusCode ?? 500 }, + ); + } + return NextResponse.json( + { error: "Synthesis failed." }, + { status: 500 }, + ); + } +} diff --git a/demos/live-captions/app/globals.css b/demos/live-captions/app/globals.css new file mode 100644 index 0000000..e7d741f --- /dev/null +++ b/demos/live-captions/app/globals.css @@ -0,0 +1,239 @@ +/* Speechify brand base — mirrors demos.speechify.ai/site (speechify.ai/brand). + * ABC Diatype is licensed and NOT committed; it is loaded cross-origin from + * speechify.ai/fonts (served with Access-Control-Allow-Origin: *). Monochrome + * palette, thin display type, pill ink buttons, sentence-case voice. + * Paste this block at the TOP of the demo's app/globals.css, then make the + * demo-specific rules below it reference these tokens (no hardcoded colours). */ + +@font-face { font-family: "ABCDiatype"; src: url("https://speechify.ai/fonts/ABCDiatype-Thin.woff2") format("woff2"); font-weight: 100; font-style: normal; font-display: swap; } +@font-face { font-family: "ABCDiatype"; src: url("https://speechify.ai/fonts/ABCDiatype-Light.woff2") format("woff2"); font-weight: 300; font-style: normal; font-display: swap; } +@font-face { font-family: "ABCDiatype"; src: url("https://speechify.ai/fonts/ABCDiatype-Regular.woff2") format("woff2"); font-weight: 400; font-style: normal; font-display: swap; } +@font-face { font-family: "ABCDiatype"; src: url("https://speechify.ai/fonts/ABCDiatype-Medium.woff2") format("woff2"); font-weight: 500; font-style: normal; font-display: swap; } +@font-face { font-family: "ABCDiatype"; src: url("https://speechify.ai/fonts/ABCDiatype-Bold.woff2") format("woff2"); font-weight: 700; font-style: normal; font-display: swap; } + +:root { + --font-sans: "ABCDiatype", ui-sans-serif, system-ui, -apple-system, sans-serif; + --font-mono: ui-monospace, SFMono-Regular, Menlo, Monaco, "Cascadia Code", monospace; + + --surface-page: #ffffff; + --surface-card: #ffffff; + --surface-subtle: #f5f5f5; + --surface-raised: #fafafa; + --text-primary: #0a0a0a; + --text-secondary: #525252; + --text-tertiary: #666666; + --border-subtle: #e5e5e5; + --border-strong: #d1d1d4; + --action: #0a0a0a; + --action-hover: #2a2a2e; + --action-foreground: #fafafa; + --focus-ring: rgba(10, 10, 10, 0.22); + --success: #00c270; + --danger: #b42318; + --radius-md: 8px; + --radius-lg: 12px; + --radius-pill: 9999px; +} + +/* Interactive apps: keep a monochrome dark mapping so night viewers aren't + * blinded. Still monochrome, still on-brand (inverted ink/paper). */ +@media (prefers-color-scheme: dark) { + :root { + --surface-page: #0a0a0a; + --surface-card: #101010; + --surface-subtle: #161616; + --surface-raised: #141414; + --text-primary: #fafafa; + --text-secondary: #b3b3b3; + --text-tertiary: #8a8a8a; + --border-subtle: #262626; + --border-strong: #3a3a3a; + --action: #fafafa; + --action-hover: #e5e5e5; + --action-foreground: #0a0a0a; + --focus-ring: rgba(250, 250, 250, 0.28); + } +} + +* { box-sizing: border-box; } + +body { + margin: 0; + padding: 4rem 1.5rem; + background: var(--surface-page); + color: var(--text-primary); + font-family: var(--font-sans); + font-weight: 400; + line-height: 1.55; + -webkit-font-smoothing: antialiased; + text-rendering: optimizeLegibility; +} + +main { + max-width: 44rem; + margin: 0 auto; + display: flex; + flex-direction: column; + gap: 1.75rem; +} + +/* Thin display headings, tight tracking, sentence case (author copy in sentence case). */ +h1 { + font-weight: 100; + font-size: clamp(2.25rem, 6vw, 3.5rem); + line-height: 1.02; + letter-spacing: -0.03em; + margin: 0 0 0.5rem; +} +h2 { font-weight: 300; letter-spacing: -0.01em; margin: 0 0 0.5rem; } + +.eyebrow { + font-family: var(--font-mono); + font-size: 13px; + font-weight: 500; + letter-spacing: 0.02em; + color: var(--text-tertiary); + text-transform: none; + margin: 0 0 0.75rem; +} + +.lead { + max-width: 62ch; + color: var(--text-secondary); + font-size: 1.0625rem; + line-height: 1.55; + margin: 0; +} + +a { color: var(--text-primary); text-underline-offset: 2px; } + +code, kbd, samp { font-family: var(--font-mono); } +code { + background: var(--surface-subtle); + border: 1px solid var(--border-subtle); + padding: 0.08em 0.38em; + border-radius: 5px; + font-size: 0.88em; +} + +.card, .step { + background: var(--surface-subtle); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-lg); + padding: 1.25rem; +} + +label { display: block; font-size: 0.9rem; color: var(--text-secondary); margin: 0 0 0.5rem; } + +input[type="text"], input[type="email"], textarea, select { + width: 100%; + font: inherit; + color: var(--text-primary); + background: var(--surface-card); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); + padding: 0.65rem 0.8rem; +} +textarea { resize: vertical; } + +input:focus-visible, textarea:focus-visible, select:focus-visible, button:focus-visible { + outline: 3px solid var(--focus-ring); + outline-offset: 1px; +} + +/* Pill buttons — medium weight only, never 600. */ +.btn, button.btn { + font: inherit; + font-weight: 500; + cursor: pointer; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 0.4rem; + border-radius: var(--radius-pill); + padding: 0.65rem 1.5rem; + border: 1px solid transparent; + transition: background-color 0.2s ease, color 0.2s ease, border-color 0.2s ease; +} +.btn-primary { background: var(--action); color: var(--action-foreground); } +.btn-primary:hover:not(:disabled) { background: var(--action-hover); } +.btn-outline { background: transparent; color: var(--text-primary); border-color: var(--border-strong); } +.btn-outline:hover:not(:disabled) { background: var(--action); color: var(--action-foreground); border-color: var(--action); } +.btn:disabled { opacity: 0.55; cursor: default; } + +footer { + color: var(--text-tertiary); + font-size: 0.9rem; + border-top: 1px solid var(--border-subtle); + padding-top: 1.25rem; +} +footer a { color: inherit; } + +@media (prefers-reduced-motion: reduce) { + * { animation-duration: 0.001ms !important; animation-iteration-count: 1 !important; transition-duration: 0.001ms !important; } +} + +/* --- live-captions specific --- */ + +/* Section label: mono, matches the brand eyebrow rather than the old all-caps. */ +.step h2 { + font-family: var(--font-mono); + font-size: 13px; + font-weight: 500; + letter-spacing: 0.02em; + color: var(--text-tertiary); + text-transform: none; + margin: 0 0 0.75rem; +} + +.step .btn { + margin-top: 0.9rem; +} + +#turnstile-container:not(:empty) { + margin-top: 0.9rem; +} + +.captions { + background: var(--surface-subtle); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-lg); + padding: 1.5rem 1.25rem; +} + +.caption-line { + margin: 0; + font-size: 1.6rem; + font-weight: 300; + line-height: 1.7; + letter-spacing: -0.01em; + color: var(--text-secondary); +} + +.caption-line .word { + transition: + background-color 80ms ease-out, + color 80ms ease-out; + padding: 0.05em 0.15em; + border-radius: 5px; +} + +.caption-line .word.active { + background: var(--action); + color: var(--action-foreground); +} + +.status { + font-size: 0.85rem; + color: var(--text-tertiary); + min-height: 1.2rem; +} + +.status[data-tone="error"] { + color: var(--danger); +} + +audio { + width: 100%; + margin-top: 0.5rem; +} diff --git a/demos/live-captions/app/layout.tsx b/demos/live-captions/app/layout.tsx new file mode 100644 index 0000000..d9f8d25 --- /dev/null +++ b/demos/live-captions/app/layout.tsx @@ -0,0 +1,21 @@ +import type { Metadata } from "next"; +import type { ReactNode } from "react"; +import Script from "next/script"; +import "./globals.css"; + +export const metadata: Metadata = { + title: "Live captions with Speechify speech marks", + description: + "Synthesize text and render live, word-by-word captions in sync with playback, driven by the speech marks the Speechify API returns.", +}; + +export default function RootLayout({ children }: { children: ReactNode }) { + return ( + + + + @@ -368,7 +368,7 @@

FAQ