diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt
index defed32659..cf5470a0d2 100644
--- a/.cspell-wordlist.txt
+++ b/.cspell-wordlist.txt
@@ -450,3 +450,10 @@ eurollm
doclayout
netp
Uyghur
+codebook
+codebooks
+remask
+higgs
+verbalizers
+workletized
+partitioners
diff --git a/apps/speech/app/_layout.tsx b/apps/speech/app/_layout.tsx
index e98be4cd43..546f2ef6f4 100644
--- a/apps/speech/app/_layout.tsx
+++ b/apps/speech/app/_layout.tsx
@@ -55,6 +55,13 @@ export default function Layout() {
title: 'Text-to-Speech (Kokoro)',
}}
/>
+
);
}
diff --git a/apps/speech/app/index.tsx b/apps/speech/app/index.tsx
index 61cd5f6097..80d5a8a344 100644
--- a/apps/speech/app/index.tsx
+++ b/apps/speech/app/index.tsx
@@ -35,6 +35,12 @@ export default function Home() {
>
Text-to-Speech (Kokoro)
+ router.navigate('reverb-text-to-speech/')}
+ >
+ Text-to-Speech (Reverb)
+
);
diff --git a/apps/speech/app/reverb-text-to-speech/index.tsx b/apps/speech/app/reverb-text-to-speech/index.tsx
new file mode 100644
index 0000000000..ff8da83fad
--- /dev/null
+++ b/apps/speech/app/reverb-text-to-speech/index.tsx
@@ -0,0 +1,1059 @@
+import React, { useCallback, useEffect, useRef, useState } from 'react';
+import {
+ Animated,
+ Platform,
+ ScrollView,
+ StyleSheet,
+ Text,
+ TextInput,
+ TouchableOpacity,
+ View,
+} from 'react-native';
+import { AudioManager, AudioRecorder } from 'react-native-audio-api';
+import RNBlobUtil from 'react-native-blob-util';
+import DeviceInfo from 'react-native-device-info';
+import {
+ models,
+ useSpeechToText,
+ useTextToSpeech,
+ REVERB_DEFAULT_VOICE_NAMES,
+ REVERB_FRAME_RATE,
+ REVERB_SAMPLE_RATE,
+ WHISPER_SAMPLE_RATE_HZ,
+ type ReverbDefaultVoiceName,
+ type ReverbLanguage,
+ type ReverbVoice,
+} from 'react-native-executorch';
+
+import ScreenWrapper from '../../components/ScreenWrapper';
+import { ModelPicker } from '../../components/ModelPicker';
+import { ModelStatus } from '../../components/ModelStatus';
+import { Button } from '../../components/Button';
+import { useAudioPlayer } from '../../hooks/useAudioPlayer';
+import { theme } from '../../theme';
+
+type ReverbVariant = Exclude;
+
+const MODEL_OPTIONS: { label: string; value: ReverbVariant; platform: typeof Platform.OS }[] = [
+ { label: 'MLX int8', value: 'MLX_COREML_INT8', platform: 'ios' },
+ { label: 'MLX fp16', value: 'MLX_COREML_FP16', platform: 'ios' },
+ { label: 'MLX int4', value: 'MLX_COREML_INT4', platform: 'ios' },
+ { label: 'Vulkan int4', value: 'VULKAN_INT4', platform: 'android' },
+];
+
+// Transcribes the recordings a new voice is made from.
+const WHISPER_MODEL = models.speechToText.WHISPER.BASE.DEFAULT;
+
+const LANGUAGES: { label: string; value: ReverbLanguage }[] = [
+ { label: 'English', value: 'en' },
+ { label: 'German', value: 'de' },
+ { label: 'Spanish', value: 'es' },
+ { label: 'French', value: 'fr' },
+ { label: 'Italian', value: 'it' },
+ { label: 'Polish', value: 'pl' },
+ { label: 'Portuguese', value: 'pt' },
+];
+
+// cspell:disable
+const SAMPLE_TEXTS: Record = {
+ en: 'Reverb speaks in any voice, right on your phone.',
+ de: 'Reverb spricht mit jeder Stimme, direkt am Handy.',
+ es: 'Reverb habla con cualquier voz, en tu teléfono.',
+ fr: "Reverb parle avec n'importe quelle voix, ici.",
+ it: 'Reverb parla con qualsiasi voce, sul tuo telefono.',
+ pl: 'Reverb mówi dowolnym głosem, prosto w telefonie.',
+ pt: 'O Reverb fala com qualquer voz, no seu telefone.',
+};
+// cspell:enable
+
+const PRESETS: { name: ReverbDefaultVoiceName; title: string; subtitle: string }[] = [
+ { name: 'standard_male', title: 'Standard', subtitle: 'Male · calm' },
+ { name: 'angry_male', title: 'Angry', subtitle: 'Male · intense' },
+ { name: 'sensual_female', title: 'Sensual', subtitle: 'Female · soft' },
+];
+
+const SPEED_OPTIONS = [0.8, 0.9, 1.0, 1.1, 1.25].map((value) => ({ label: `${value}x`, value }));
+const STEPS_OPTIONS = [4, 6, 8, 12, 16].map((value) => ({
+ label: value === 8 ? '8 (default)' : `${value}`,
+ value,
+}));
+const VARIATION_OPTIONS = [0, 0.15, 0.25, 0.4].map((value) => ({
+ label: value === 0.25 ? '0.25 (default)' : `${value}`,
+ value,
+}));
+
+const MAX_RECORDING_S = 20;
+const MIN_RECORDING_S = 1;
+const SILENCE_DB = -45;
+const SILENCE_BELOW_PEAK_DB = 25;
+const METER_BARS = 24;
+
+type PresetInfo = { transcript: string; language: string; duration: number };
+type CustomVoice = { id: number; name: string; voice: ReverbVoice; duration: number };
+type VoiceChoice =
+ | { kind: 'preset'; name: ReverbDefaultVoiceName }
+ | { kind: 'custom'; id: number };
+
+const isSimulator = DeviceInfo.isEmulatorSync();
+
+const messageOf = (err: unknown) => (err instanceof Error ? err.message : String(err));
+
+const formatSeconds = (s: number) =>
+ `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, '0')}`;
+
+function concat(chunks: readonly Float32Array[]): Float32Array {
+ const out = new Float32Array(chunks.reduce((n, chunk) => n + chunk.length, 0));
+ let offset = 0;
+ for (const chunk of chunks) {
+ out.set(chunk, offset);
+ offset += chunk.length;
+ }
+ return out;
+}
+
+// Linear interpolation up; down, the mean of the input samples each output covers (a crude
+// anti-alias filter).
+function resample(input: Float32Array, from: number, to: number): Float32Array {
+ if (from === to) return input;
+ const ratio = from / to;
+ const out = new Float32Array(Math.floor(input.length / ratio));
+ for (let i = 0; i < out.length; i++) {
+ if (ratio > 1) {
+ const start = Math.floor(i * ratio);
+ const end = Math.min(input.length, Math.max(start + 1, Math.floor((i + 1) * ratio)));
+ let sum = 0;
+ for (let j = start; j < end; j++) sum += input[j]!;
+ out[i] = sum / (end - start);
+ } else {
+ const x = i * ratio;
+ const j = Math.floor(x);
+ out[i] = input[j]! + (input[Math.min(j + 1, input.length - 1)]! - input[j]!) * (x - j);
+ }
+ }
+ return out;
+}
+
+const rms = (audio: Float32Array, start = 0, end = audio.length) => {
+ let sum = 0;
+ for (let i = start; i < end; i++) sum += audio[i]! * audio[i]!;
+ return Math.sqrt(sum / Math.max(1, end - start));
+};
+
+// Drops the silence before the first and after the last word, keeping 0.15 s at each edge. Silence
+// is relative to the loudest 20 ms: a phone's noise floor can sit above a fixed level (-42 dBFS
+// measured on a Galaxy S24 FE).
+function trimSilence(audio: Float32Array, rate: number): Float32Array {
+ const win = Math.round(rate * 0.02);
+ const levels: number[] = [];
+ for (let start = 0; start < audio.length; start += win) {
+ levels.push(rms(audio, start, Math.min(start + win, audio.length)));
+ }
+ const peak = levels.reduce((max, level) => Math.max(max, level), 0);
+ const threshold = Math.max(10 ** (SILENCE_DB / 20), peak * 10 ** (-SILENCE_BELOW_PEAK_DB / 20));
+ const first = levels.findIndex((level) => level >= threshold);
+ if (first < 0) return audio;
+ let last = levels.length - 1;
+ while (levels[last]! < threshold) last--;
+ const pad = Math.round(rate * 0.15);
+ return audio.slice(
+ Math.max(0, first * win - pad),
+ Math.min(audio.length, (last + 1) * win + pad)
+ );
+}
+
+// Records mono audio, resampled to Reverb's 24 kHz when the device delivers another rate.
+function useVoiceRecorder() {
+ const recorderRef = useRef(null);
+ const chunksRef = useRef([]);
+ const samplesRef = useRef(0);
+ const rateRef = useRef(REVERB_SAMPLE_RATE);
+ const [isRecording, setIsRecording] = useState(false);
+ const [seconds, setSeconds] = useState(0);
+ const [level, setLevel] = useState(0);
+
+ const start = useCallback(async (): Promise => {
+ if ((await AudioManager.requestRecordingPermissions()) !== 'Granted') {
+ return 'Microphone access was denied.';
+ }
+ chunksRef.current = [];
+ samplesRef.current = 0;
+ setSeconds(0);
+ const recorder = new AudioRecorder();
+ recorder.onAudioReady(
+ { sampleRate: REVERB_SAMPLE_RATE, bufferLength: REVERB_SAMPLE_RATE / 10, channelCount: 1 },
+ (event) => {
+ const samples = new Float32Array(event.buffer.getChannelData(0));
+ rateRef.current = event.buffer.sampleRate;
+ chunksRef.current.push(samples);
+ samplesRef.current += samples.length;
+ setSeconds(samplesRef.current / rateRef.current);
+ const db = 20 * Math.log10(rms(samples) + 1e-9);
+ setLevel(Math.min(1, Math.max(0, (db + 60) / 60)));
+ }
+ );
+ const result = await recorder.start();
+ if (result.status === 'error') {
+ recorder.clearOnAudioReady();
+ return result.message;
+ }
+ recorderRef.current = recorder;
+ setIsRecording(true);
+ return null;
+ }, []);
+
+ const stop = useCallback(async (): Promise => {
+ const recorder = recorderRef.current;
+ if (!recorder) return null;
+ recorderRef.current = null;
+ recorder.clearOnAudioReady();
+ await recorder.stop().catch(() => {});
+ setIsRecording(false);
+ setLevel(0);
+ const audio = concat(chunksRef.current);
+ chunksRef.current = [];
+ return resample(audio, rateRef.current, REVERB_SAMPLE_RATE);
+ }, []);
+
+ useEffect(() => {
+ return () => {
+ recorderRef.current?.stop().catch(() => {});
+ };
+ }, []);
+
+ return { isRecording, seconds, level, start, stop };
+}
+
+function VoiceTile({
+ title,
+ subtitle,
+ selected,
+ disabled,
+ onPress,
+ onRemove,
+}: {
+ title: string;
+ subtitle: string;
+ selected: boolean;
+ disabled: boolean;
+ onPress: () => void;
+ onRemove?: () => void;
+}) {
+ return (
+
+
+
+
+ {title.charAt(0)}
+
+
+ {onRemove && !disabled && (
+
+ ✕
+
+ )}
+
+
+ {title}
+
+
+ {subtitle}
+
+
+ );
+}
+
+function RecordButton({
+ isRecording,
+ disabled,
+ onPress,
+}: {
+ isRecording: boolean;
+ disabled: boolean;
+ onPress: () => void;
+}) {
+ const pulse = useRef(new Animated.Value(0)).current;
+
+ useEffect(() => {
+ if (!isRecording) return;
+ const loop = Animated.loop(
+ Animated.timing(pulse, { toValue: 1, duration: 1200, useNativeDriver: true })
+ );
+ loop.start();
+ return () => {
+ loop.stop();
+ pulse.setValue(0);
+ };
+ }, [isRecording, pulse]);
+
+ return (
+
+ {isRecording && (
+
+ )}
+
+
+
+
+ );
+}
+
+function LevelMeter({ level, active }: { level: number; active: boolean }) {
+ const lit = active ? Math.round(level * METER_BARS) : 0;
+ return (
+
+ {Array.from({ length: METER_BARS }, (_, i) => (
+
+ ))}
+
+ );
+}
+
+function ReverbContent() {
+ const [selectedModel, setSelectedModel] = useState(
+ Platform.OS === 'android' ? 'VULKAN_INT4' : 'MLX_COREML_INT8'
+ );
+ const [language, setLanguage] = useState('en');
+ const [text, setText] = useState(SAMPLE_TEXTS.en);
+ const [choice, setChoice] = useState({ kind: 'preset', name: 'standard_male' });
+ const [presetInfo, setPresetInfo] = useState>>(
+ {}
+ );
+ const [customVoices, setCustomVoices] = useState([]);
+ const nextVoiceId = useRef(1);
+
+ const [speed, setSpeed] = useState(1.0);
+ const [showSampler, setShowSampler] = useState(false);
+ const [steps, setSteps] = useState(6);
+ const [remaskRatio, setRemaskRatio] = useState(0.25);
+ const [fixedSeed, setFixedSeed] = useState(null);
+ const [lastSeed, setLastSeed] = useState(null);
+ const [trimEdges, setTrimEdges] = useState(true);
+
+ const [isCloneOpen, setIsCloneOpen] = useState(false);
+ const [recording, setRecording] = useState(null);
+ const [transcript, setTranscript] = useState('');
+ const [recordingLanguage, setRecordingLanguage] = useState('en');
+ const [cloneStep, setCloneStep] = useState<'transcribing' | 'encoding' | null>(null);
+
+ const [isSynthesizing, setIsSynthesizing] = useState(false);
+ const [chunkProgress, setChunkProgress] = useState(null);
+ const [result, setResult] = useState(null);
+ const [runError, setRunError] = useState(null);
+
+ const { isPlaying, playStream, stop } = useAudioPlayer(REVERB_SAMPLE_RATE);
+ const recorder = useVoiceRecorder();
+
+ const { isReady, downloadProgress, error, resource, synthesize, synthesizeStop, encodeVoice } =
+ useTextToSpeech(models.textToSpeech.REVERB[selectedModel]);
+ const whisper = useSpeechToText(WHISPER_MODEL, { preventLoad: !isCloneOpen });
+
+ useEffect(() => {
+ AudioManager.setAudioSessionOptions({
+ iosCategory: 'playAndRecord',
+ iosMode: 'default',
+ iosOptions: ['defaultToSpeaker', 'allowBluetoothA2DP'],
+ });
+ }, []);
+
+ useEffect(() => {
+ return () => {
+ synthesizeStop?.();
+ stop();
+ };
+ }, [stop, synthesizeStop]);
+
+ // The preset voices' reference transcripts and lengths, read from their JSON files.
+ useEffect(() => {
+ const voices = resource?.voices;
+ if (!voices) return;
+ let cancelled = false;
+ (async () => {
+ const info: Partial> = {};
+ for (const name of REVERB_DEFAULT_VOICE_NAMES) {
+ try {
+ const json = JSON.parse(await RNBlobUtil.fs.readFile(voices[name], 'utf8'));
+ info[name] = {
+ transcript: json.transcript,
+ language: json.language,
+ duration: json.codes.length / 8 / REVERB_FRAME_RATE,
+ };
+ } catch {
+ // A preset without its description still synthesizes.
+ }
+ }
+ if (!cancelled) setPresetInfo(info);
+ })();
+ return () => {
+ cancelled = true;
+ };
+ }, [resource]);
+
+ const selectedCustom =
+ choice.kind === 'custom' ? customVoices.find((v) => v.id === choice.id) : undefined;
+ const reference =
+ choice.kind === 'preset'
+ ? presetInfo[choice.name]
+ : selectedCustom && {
+ transcript: selectedCustom.voice.transcript,
+ language: selectedCustom.voice.language,
+ duration: selectedCustom.duration,
+ };
+
+ const isBusy = isSynthesizing || isPlaying || recorder.isRecording || cloneStep !== null;
+ const recordingSeconds = recording ? recording.length / REVERB_SAMPLE_RATE : 0;
+
+ const transcribeRecording = async (audio: Float32Array, lang: ReverbLanguage) => {
+ if (!whisper.transcribe) return;
+ setCloneStep('transcribing');
+ try {
+ const heard = await whisper.transcribe(
+ resample(audio, REVERB_SAMPLE_RATE, WHISPER_SAMPLE_RATE_HZ),
+ { language: lang }
+ );
+ setTranscript(heard.trim());
+ } catch (err) {
+ setRunError(messageOf(err));
+ } finally {
+ setCloneStep(null);
+ }
+ };
+
+ const finishRecording = async () => {
+ const audio = await recorder.stop();
+ if (!audio) return;
+ const trimmed = trimSilence(audio, REVERB_SAMPLE_RATE);
+ setRecording(trimmed);
+ setTranscript('');
+ await transcribeRecording(trimmed, recordingLanguage);
+ };
+
+ const toggleRecording = async () => {
+ if (recorder.isRecording) {
+ await finishRecording();
+ return;
+ }
+ setRunError(null);
+ stop();
+ const failure = await recorder.start();
+ if (failure) setRunError(failure);
+ };
+
+ // Stops at the longest reference the encoder takes.
+ useEffect(() => {
+ if (recorder.isRecording && recorder.seconds >= MAX_RECORDING_S) finishRecording();
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [recorder.isRecording, recorder.seconds]);
+
+ const playRecording = async () => {
+ if (!recording) return;
+ async function* once() {
+ yield { audio: recording!, sampleRate: REVERB_SAMPLE_RATE };
+ }
+ await playStream(once()).catch((err) => setRunError(messageOf(err)));
+ };
+
+ const createVoice = async () => {
+ if (!encodeVoice || !recording || !transcript.trim()) return;
+ setRunError(null);
+ setCloneStep('encoding');
+ try {
+ const voice = await encodeVoice(recording, transcript.trim(), recordingLanguage);
+ const id = nextVoiceId.current++;
+ setCustomVoices((voices) => [
+ ...voices,
+ { id, name: `My voice ${id}`, voice, duration: recordingSeconds },
+ ]);
+ setChoice({ kind: 'custom', id });
+ setRecording(null);
+ setTranscript('');
+ setIsCloneOpen(false);
+ } catch (err) {
+ setRunError(messageOf(err));
+ } finally {
+ setCloneStep(null);
+ }
+ };
+
+ const removeVoice = (id: number) => {
+ setCustomVoices((voices) => voices.filter((v) => v.id !== id));
+ if (choice.kind === 'custom' && choice.id === id) {
+ setChoice({ kind: 'preset', name: 'standard_male' });
+ }
+ };
+
+ const handleSynthesize = async () => {
+ const voice = choice.kind === 'preset' ? choice.name : selectedCustom?.voice;
+ if (!synthesize || !voice || isBusy || !text.trim()) return;
+
+ setRunError(null);
+ setResult(null);
+ setChunkProgress(null);
+ setIsSynthesizing(true);
+
+ const seed = fixedSeed ?? Math.floor(Math.random() * 2 ** 31);
+ setLastSeed(seed);
+ let audioSeconds = 0;
+ let firstAudio: number | null = null;
+ const t0 = performance.now();
+
+ try {
+ const chunks = synthesize(text, {
+ voice,
+ language,
+ speed,
+ sampling: { steps, remaskRatio, seed },
+ postprocess: trimEdges ? undefined : false,
+ });
+ await playStream(
+ chunks,
+ (chunk) => {
+ audioSeconds += chunk.duration;
+ setChunkProgress(
+ `Chunk ${chunk.chunkIndex + 1}/${chunk.totalChunks} (${chunk.duration.toFixed(1)} s)`
+ );
+ },
+ () => {
+ firstAudio = (performance.now() - t0) / 1000;
+ }
+ );
+ const elapsed = (performance.now() - t0) / 1000;
+ setResult(
+ `${audioSeconds.toFixed(1)} s of audio · first audio after ${(firstAudio ?? elapsed).toFixed(1)} s`
+ );
+ setChunkProgress(null);
+ } catch (err) {
+ setRunError(messageOf(err));
+ } finally {
+ setIsSynthesizing(false);
+ }
+ };
+
+ const handleStop = () => {
+ synthesizeStop?.();
+ stop();
+ setIsSynthesizing(false);
+ };
+
+ const selectLanguage = (lang: ReverbLanguage) => {
+ if (text === SAMPLE_TEXTS[language]) setText(SAMPLE_TEXTS[lang]);
+ setLanguage(lang);
+ if (!recording) setRecordingLanguage(lang);
+ };
+
+ const whisperStatus = !whisper.isReady
+ ? whisper.error
+ ? `Whisper failed to load: ${whisper.error.message}`
+ : `Loading Whisper to transcribe… ${Math.round(whisper.downloadProgress)}%`
+ : cloneStep === 'transcribing'
+ ? 'Transcribing with Whisper…'
+ : null;
+
+ return (
+
+
+ Reverb Text-to-Speech
+
+ Speech in the voice of any short recording, in 7 languages, generated on-device. Pick a
+ preset voice or clone your own.
+
+
+ ({
+ label: m.label,
+ value: m.value,
+ disabled: isBusy || m.platform !== Platform.OS,
+ }))}
+ selectedValue={selectedModel}
+ onValueChange={setSelectedModel}
+ />
+
+
+ {runError && (
+
+ {runError}
+
+ )}
+
+
+ Voice
+
+ {PRESETS.map(({ name, title, subtitle }) => (
+ setChoice({ kind: 'preset', name })}
+ />
+ ))}
+ {customVoices.map((v) => (
+ setChoice({ kind: 'custom', id: v.id })}
+ onRemove={() => removeVoice(v.id)}
+ />
+ ))}
+ setIsCloneOpen((open) => !open)}
+ disabled={isBusy}
+ activeOpacity={0.8}
+ >
+ {isCloneOpen ? '–' : '+'}
+ {isCloneOpen ? 'Close' : 'Clone a voice'}
+
+
+
+ {reference && !isCloneOpen && (
+
+
+ Reference · {reference.language} · {reference.duration.toFixed(1)} s
+
+ “{reference.transcript}”
+
+ )}
+
+ {isCloneOpen && (
+
+ 1 · Record a reference
+
+ Read anything for 5–15 seconds in a quiet room. Reverb copies the voice, the pace and
+ the mood of the recording.
+
+
+
+
+
+ {formatSeconds(recorder.isRecording ? recorder.seconds : recordingSeconds)}
+ / {formatSeconds(MAX_RECORDING_S)}
+
+
+
+ {isSimulator
+ ? 'Recording needs a physical device.'
+ : recorder.isRecording
+ ? 'Recording… tap to stop'
+ : recording
+ ? 'Recorded'
+ : 'Tap to record'}
+
+
+
+ {recording && !recorder.isRecording && (
+
+
+ {isPlaying ? '■ Stop' : '▶ Listen'}
+
+ transcribeRecording(recording, recordingLanguage)}
+ disabled={!whisper.isReady || isBusy}
+ >
+ ↻ Transcribe
+
+
+ )}
+
+ 2 · Check the transcript
+ ({ ...l, disabled: isBusy }))}
+ selectedValue={recordingLanguage}
+ onValueChange={setRecordingLanguage}
+ />
+ {whisperStatus && {whisperStatus}}
+
+
+ 3 · Create the voice
+
+
+
+
+ )}
+
+
+
+ Text
+ ({ ...l, disabled: isBusy }))}
+ selectedValue={language}
+ onValueChange={selectLanguage}
+ />
+
+
+
+
+ ({ ...s, disabled: isBusy }))}
+ selectedValue={speed}
+ onValueChange={setSpeed}
+ />
+ setShowSampler((v) => !v)}>
+ Sampler settings
+ {showSampler ? '▾' : '▸'}
+
+ {showSampler && (
+ <>
+ ({ ...s, disabled: isBusy }))}
+ selectedValue={steps}
+ onValueChange={setSteps}
+ />
+ ({ ...s, disabled: isBusy }))}
+ selectedValue={remaskRatio}
+ onValueChange={setRemaskRatio}
+ />
+ setFixedSeed(keep ? (lastSeed ?? 42) : null)}
+ />
+
+ >
+ )}
+
+
+
+
+ {!isPlaying ? (
+
+ ) : (
+
+ )}
+
+ {chunkProgress && (
+
+ {chunkProgress}
+
+ )}
+ {result && !isSynthesizing && (
+
+ {result}
+ {lastSeed !== null && Seed {lastSeed}}
+
+ )}
+
+
+ );
+}
+
+export default function ReverbTTSScreen() {
+ return (
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ container: { flex: 1, backgroundColor: theme.colors.background },
+ content: { padding: theme.spacing.large, paddingBottom: 40 },
+ card: {
+ backgroundColor: theme.colors.cardBackground,
+ borderRadius: theme.radius.large,
+ padding: 20,
+ marginBottom: 20,
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ },
+ cardTitle: {
+ fontSize: theme.typography.title.fontSize,
+ fontWeight: theme.typography.title.fontWeight,
+ color: theme.colors.strongPrimary,
+ marginBottom: 8,
+ },
+ cardDescription: {
+ fontSize: 14,
+ color: theme.colors.textMuted,
+ lineHeight: 20,
+ marginBottom: 16,
+ },
+ sectionTitle: {
+ fontSize: 16,
+ fontWeight: '700',
+ color: '#212529',
+ marginBottom: 12,
+ },
+ dimmed: { opacity: 0.5 },
+ voiceGrid: {
+ flexDirection: 'row',
+ flexWrap: 'wrap',
+ gap: 10,
+ },
+ voiceTile: {
+ width: '31%',
+ minHeight: 96,
+ padding: 10,
+ borderRadius: theme.radius.medium,
+ borderWidth: 1.5,
+ borderColor: theme.colors.lightBorder,
+ backgroundColor: theme.colors.background,
+ },
+ voiceTileSelected: {
+ borderColor: theme.colors.primary,
+ backgroundColor: '#eef1fb',
+ },
+ voiceTileHeader: {
+ flexDirection: 'row',
+ justifyContent: 'space-between',
+ alignItems: 'flex-start',
+ marginBottom: 8,
+ },
+ avatar: {
+ width: 30,
+ height: 30,
+ borderRadius: 15,
+ backgroundColor: theme.colors.placeholderBackground,
+ alignItems: 'center',
+ justifyContent: 'center',
+ },
+ avatarSelected: { backgroundColor: theme.colors.primary },
+ avatarText: { fontSize: 14, fontWeight: '700', color: theme.colors.textMuted },
+ avatarTextSelected: { color: '#fff' },
+ removeText: { fontSize: 13, color: theme.colors.textMuted },
+ voiceTitle: { fontSize: 14, fontWeight: '700', color: theme.colors.strongPrimary },
+ voiceSubtitle: { fontSize: 11, color: theme.colors.textMuted, marginTop: 2 },
+ newVoiceTile: {
+ borderStyle: 'dashed',
+ borderColor: theme.colors.accent,
+ backgroundColor: '#f4f8fe',
+ alignItems: 'center',
+ justifyContent: 'center',
+ },
+ newVoiceTileOpen: { borderStyle: 'solid' },
+ newVoicePlus: { fontSize: 24, color: theme.colors.accent, fontWeight: '300' },
+ newVoiceText: { fontSize: 12, color: theme.colors.accent, fontWeight: '600', marginTop: 2 },
+ referenceBox: {
+ marginTop: 14,
+ padding: 12,
+ borderRadius: theme.radius.small,
+ backgroundColor: theme.colors.background,
+ borderLeftWidth: 3,
+ borderLeftColor: theme.colors.primary,
+ },
+ referenceLabel: {
+ fontSize: 11,
+ fontWeight: '700',
+ color: theme.colors.textMuted,
+ textTransform: 'uppercase',
+ marginBottom: 4,
+ },
+ referenceText: { fontSize: 13, color: '#333', fontStyle: 'italic', lineHeight: 19 },
+ clonePanel: {
+ marginTop: 16,
+ paddingTop: 16,
+ borderTopWidth: 1,
+ borderTopColor: theme.colors.lightBorder,
+ },
+ stepTitle: {
+ fontSize: 14,
+ fontWeight: '700',
+ color: theme.colors.strongPrimary,
+ marginTop: 4,
+ marginBottom: 8,
+ },
+ hint: { fontSize: 13, color: theme.colors.textMuted, lineHeight: 18, marginBottom: 14 },
+ recorderRow: { flexDirection: 'row', alignItems: 'center', marginBottom: 12 },
+ recordButtonWrapper: {
+ width: 76,
+ height: 76,
+ alignItems: 'center',
+ justifyContent: 'center',
+ marginRight: 18,
+ },
+ recordPulse: {
+ position: 'absolute',
+ width: 64,
+ height: 64,
+ borderRadius: 32,
+ backgroundColor: '#ff3b30',
+ },
+ recordButton: {
+ width: 64,
+ height: 64,
+ borderRadius: 32,
+ backgroundColor: '#fff',
+ borderWidth: 3,
+ borderColor: '#ff3b30',
+ alignItems: 'center',
+ justifyContent: 'center',
+ },
+ recordButtonDisabled: { borderColor: '#c7c7cc' },
+ recordDotIcon: { width: 26, height: 26, borderRadius: 13, backgroundColor: '#ff3b30' },
+ recordStopIcon: { width: 22, height: 22, borderRadius: 4, backgroundColor: '#ff3b30' },
+ recorderInfo: { flex: 1 },
+ timer: {
+ fontSize: 26,
+ fontWeight: '600',
+ color: theme.colors.strongPrimary,
+ fontVariant: ['tabular-nums'],
+ },
+ timerLimit: { fontSize: 15, fontWeight: '400', color: theme.colors.textMuted },
+ meter: { flexDirection: 'row', alignItems: 'center', gap: 3, height: 18, marginVertical: 6 },
+ meterBar: { width: 4, borderRadius: 2, backgroundColor: theme.colors.placeholderBackground },
+ meterBarLit: { backgroundColor: '#22c55e' },
+ meterBarHot: { backgroundColor: '#f59e0b' },
+ recorderState: { fontSize: 12, color: theme.colors.textMuted },
+ inlineButtons: { flexDirection: 'row', gap: 10, marginBottom: 14 },
+ ghostButton: {
+ paddingHorizontal: 14,
+ paddingVertical: 8,
+ borderRadius: 20,
+ borderWidth: 1,
+ borderColor: theme.colors.primary,
+ },
+ ghostButtonText: { fontSize: 13, color: theme.colors.primary, fontWeight: '600' },
+ statusLine: { fontSize: 12, color: theme.colors.accent, marginBottom: 8 },
+ textInput: {
+ height: 120,
+ borderColor: theme.colors.border,
+ borderWidth: 1,
+ borderRadius: theme.radius.small,
+ paddingHorizontal: 12,
+ paddingVertical: 10,
+ marginTop: 8,
+ color: theme.colors.textSecondary,
+ backgroundColor: theme.colors.background,
+ fontSize: 14,
+ lineHeight: 20,
+ textAlignVertical: 'top',
+ },
+ transcriptInput: { height: 84, marginBottom: 14 },
+ disclosure: {
+ flexDirection: 'row',
+ justifyContent: 'space-between',
+ alignItems: 'center',
+ paddingVertical: 12,
+ marginTop: 4,
+ borderTopWidth: 1,
+ borderTopColor: theme.colors.lightBorder,
+ },
+ disclosureText: { fontSize: 14, fontWeight: '600', color: theme.colors.strongPrimary },
+ disclosureChevron: { fontSize: 14, color: theme.colors.textMuted },
+ buttonRow: { flexDirection: 'row', gap: theme.spacing.small },
+ progressContainer: {
+ marginTop: 12,
+ padding: 10,
+ backgroundColor: '#e8f4fd',
+ borderRadius: theme.radius.small,
+ },
+ progressText: { fontSize: 13, color: '#1a73e8', fontWeight: '500', textAlign: 'center' },
+ resultContainer: {
+ marginTop: 12,
+ padding: 10,
+ backgroundColor: '#e6f9e6',
+ borderRadius: theme.radius.small,
+ },
+ resultText: { fontSize: 13, color: '#2e7d32', fontWeight: '500', textAlign: 'center' },
+ resultSubtext: { fontSize: 11, color: '#2e7d32', textAlign: 'center', marginTop: 2 },
+ errorContainer: {
+ backgroundColor: theme.colors.errorBackground,
+ padding: 12,
+ borderRadius: theme.radius.small,
+ marginBottom: 20,
+ },
+ errorText: { color: theme.colors.errorText, fontSize: 14, textAlign: 'center' },
+});
diff --git a/docs/docs/02-extensions/speech/02-text-to-speech.md b/docs/docs/02-extensions/speech/02-text-to-speech.md
index 7f9688f396..abc7291236 100644
--- a/docs/docs/02-extensions/speech/02-text-to-speech.md
+++ b/docs/docs/02-extensions/speech/02-text-to-speech.md
@@ -1,15 +1,17 @@
---
title: Text-to-Speech (TTS)
slug: /extensions/text-to-speech
-description: 'Synthesize natural, streaming speech waveforms directly on-device in React Native using multi-stage neural pipelines (Supertonic and Kokoro).'
+description: 'Synthesize natural, streaming speech waveforms directly on-device in React Native using multi-stage neural pipelines (Supertonic, Kokoro and Reverb, with voice cloning).'
keywords:
[
react native,
text to speech,
tts,
speech synthesis,
+ voice cloning,
supertonic,
kokoro,
+ reverb,
phonemizer,
vocoder,
audio streaming,
@@ -28,8 +30,9 @@ Speech synthesis operates through multi-stage neural pipelines that combine phon
- **Supertonic ([`createSupertonicTextToSpeech`](../../06-api-reference/functions/createSupertonicTextToSpeech.md))**: A 4-stage multilingual flow-matching model (44.1 kHz) that coordinates a text encoder, duration predictor, vector estimator, and vocoder with multi-speaker voice style conditioning.
- **Kokoro ([`createKokoroTextToSpeech`](../../06-api-reference/functions/createKokoroTextToSpeech.md))**: A 2-stage phoneme-driven model (24 kHz) that pairs language-specific grapheme-to-phoneme (G2P) transcription with a duration predictor, acoustic synthesizer, and voice embedding matrices.
+- **Reverb ([`createReverbTextToSpeech`](../../06-api-reference/functions/createReverbTextToSpeech.md))**: A multilingual codec language model (24 kHz) that speaks in the voice of any short recording. A backbone fills in the audio codec tokens of the speech in a few parallel passes, conditioned on the text and a reference voice, and the codec decoder turns them into a waveform. The codec encoder makes new voices on-device — see [Voice Cloning](#voice-cloning-reverb).
-Both pipelines are wrapped uniformly by the [`useTextToSpeech`](../../06-api-reference/functions/useTextToSpeech.md) React hook.
+All three pipelines are wrapped uniformly by the [`useTextToSpeech`](../../06-api-reference/functions/useTextToSpeech.md) React hook.
@@ -76,13 +79,13 @@ See [`src/app/(screens)/text-to-speech.tsx`]( {
}
});
+ // Reverb ships its programs in the model's own repo, the one exception to the
+ // `-ExecuTorch` naming.
it.each(urls)('%s is served over https from the software-mansion org', (_path, url) => {
- expect(url).toMatch(new RegExp(`^${BASE_URL}/[A-Za-z0-9._-]+-ExecuTorch/`));
+ expect(url).toMatch(new RegExp(`^${BASE_URL}/([A-Za-z0-9._-]+-ExecuTorch|Reverb-TTS)/`));
});
it.each(urls)('%s pins an explicit revision', (_path, url) => {
diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts
index 8f53d1f19a..4e6aebb943 100644
--- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts
+++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts
@@ -236,8 +236,10 @@ describe('DEFAULT variant resolution', () => {
expect(offenders).toEqual([]);
});
+ // Reverb publishes no XNNPACK export, so it has nothing to fall back to.
it('falls back to XNNPACK on the iOS simulator, which runs neither Core ML nor MLX', () => {
const offenders = defaultsOf(registryFor({ os: 'ios', isEmulator: true }))
+ .filter(({ offers }) => offers.some((key) => key.startsWith('XNNPACK')))
.filter(({ path }) => /\/(coreml|mlx)\//.test(path))
.map(({ label, path }) => `${label}: ${path}`);
@@ -246,6 +248,7 @@ describe('DEFAULT variant resolution', () => {
it('falls back to XNNPACK on iOS when the app links XNNPACK only', () => {
const offenders = defaultsOf(registryFor({ os: 'ios', backends: ['XnnpackBackend'] }))
+ .filter(({ offers }) => offers.some((key) => key.startsWith('XNNPACK')))
.filter(({ key }) => key !== undefined && !key.startsWith('XNNPACK'))
.map(({ label, key }) => `${label}: ${key}`);
diff --git a/packages/react-native-executorch/__tests__/extensions/reverbPartitioner.test.ts b/packages/react-native-executorch/__tests__/extensions/reverbPartitioner.test.ts
new file mode 100644
index 0000000000..c6aae9706e
--- /dev/null
+++ b/packages/react-native-executorch/__tests__/extensions/reverbPartitioner.test.ts
@@ -0,0 +1,117 @@
+/**
+ * Reverb's partitioner: chunks within a duration limit, cut at the coarsest
+ * boundary that allows it. Durations come from the estimate at a pace of one
+ * letter per frame.
+ */
+import { partitionReverbText } from '../../src/extensions/speech/utils/reverbPartitioner';
+import { estimateTargetFrames } from '../../src/extensions/speech/utils/reverbUtils';
+
+const seconds = (chunk: string) => estimateTargetFrames(chunk, 'abcdefghij', 10) / 25;
+
+/** `count` words of four letters, then `end`. About 1.7 s per ten words. */
+const run = (count: number, end = '.') => `${new Array(count).fill('word').join(' ')}${end}`;
+
+const LONG_TEXT = [
+ run(10),
+ run(25, ','),
+ run(12, '!'),
+ run(60),
+ run(8, '?'),
+ run(15, ';'),
+ run(5),
+].join(' ');
+
+describe('partitionReverbText', () => {
+ it('keeps a text that fits as one chunk', () => {
+ expect(partitionReverbText(run(10), seconds)).toEqual([run(10)]);
+ });
+
+ it('returns no chunks for an empty text', () => {
+ expect(partitionReverbText('', seconds)).toEqual([]);
+ expect(partitionReverbText(' ', seconds)).toEqual([]);
+ });
+
+ it('keeps every chunk within the duration limit', () => {
+ const chunks = partitionReverbText(LONG_TEXT, seconds, { maxDuration: 4 });
+
+ expect(chunks.length).toBeGreaterThan(3);
+ for (const chunk of chunks) expect(seconds(chunk)).toBeLessThanOrEqual(4);
+ });
+
+ it('gives back the text, its whitespace collapsed, when the chunks are joined', () => {
+ const text = ` ${run(30)}\n\n${run(40, ',')} ${run(50)}\t${run(3)} `;
+
+ const chunks = partitionReverbText(text, seconds, { maxDuration: 3 });
+
+ expect(chunks.join(' ')).toBe(text.split(/\s+/).filter(Boolean).join(' '));
+ });
+
+ it('packs whole sentences into a chunk while they fit', () => {
+ const [a, b, c] = [run(10), run(10, '!'), run(10, '?')];
+
+ expect(partitionReverbText(`${a} ${b} ${c}`, seconds, { maxDuration: 4 })).toEqual([
+ `${a} ${b}`,
+ c,
+ ]);
+ });
+
+ it('splits a sentence too long on its own at other punctuation', () => {
+ const sentence = `${run(10, ',')} ${run(10, ':')} ${run(10)}`;
+
+ const chunks = partitionReverbText(sentence, seconds, { maxDuration: 4 });
+
+ expect(chunks).toEqual([`${run(10, ',')} ${run(10, ':')}`, run(10)]);
+ });
+
+ it('splits a piece still too long between words', () => {
+ const chunks = partitionReverbText(run(60), seconds, { maxDuration: 4 });
+
+ expect(chunks.length).toBeGreaterThan(1);
+ for (const chunk of chunks) expect(seconds(chunk)).toBeLessThanOrEqual(4);
+ // Packed as far as it goes: one more word would not have fit.
+ for (const chunk of chunks.slice(0, -1)) expect(seconds(`${chunk} word`)).toBeGreaterThan(4);
+ });
+
+ it('starts a sentence after closing quotes, not before them', () => {
+ const text = `He said "${run(8)}" ${run(10)}`;
+
+ expect(partitionReverbText(text, seconds, { maxDuration: 3 })[0]).toBe(`He said "${run(8)}"`);
+ });
+
+ it('keeps a single word that does not fit as a chunk of its own', () => {
+ expect(partitionReverbText('a extraordinarily b', seconds, { maxDuration: 0.5 })).toEqual([
+ 'a',
+ 'extraordinarily',
+ 'b',
+ ]);
+ });
+
+ it('fits more text per chunk when chunks are estimated shorter', () => {
+ const faster = (chunk: string) => seconds(chunk) / 1.4;
+
+ const normal = partitionReverbText(LONG_TEXT, seconds, { maxDuration: 4 });
+ const fast = partitionReverbText(LONG_TEXT, faster, { maxDuration: 4 });
+
+ expect(fast.length).toBeLessThan(normal.length);
+ for (const chunk of fast) expect(faster(chunk)).toBeLessThanOrEqual(4);
+ });
+
+ it('meets the extra condition on every chunk', () => {
+ const fits = (chunk: string) => chunk.length <= 40;
+
+ const chunks = partitionReverbText(LONG_TEXT, seconds, { fits });
+
+ expect(chunks.every(fits)).toBe(true);
+ expect(chunks.join(' ')).toBe(LONG_TEXT);
+ });
+
+ it.each([
+ ['zero', 0],
+ ['negative', -1],
+ ['NaN', NaN],
+ ])('rejects a %s maxDuration', (_label, maxDuration) => {
+ expect(() => partitionReverbText('Hello.', seconds, { maxDuration })).toThrow(
+ /must be positive/
+ );
+ });
+});
diff --git a/packages/react-native-executorch/__tests__/extensions/reverbUtils.test.ts b/packages/react-native-executorch/__tests__/extensions/reverbUtils.test.ts
new file mode 100644
index 0000000000..32441a1421
--- /dev/null
+++ b/packages/react-native-executorch/__tests__/extensions/reverbUtils.test.ts
@@ -0,0 +1,216 @@
+/**
+ * Reverb's utilities. The expected normalization, estimates, and trims come
+ * from the Python they port (Echo's and Voice Studio's), run on the same inputs.
+ */
+import {
+ buildPrompt,
+ estimateTargetFrames,
+ parseVoice,
+ pauseAfter,
+ prepareText,
+ roundHalfEven,
+ trimSilence,
+} from '../../src/extensions/speech/utils/reverbUtils';
+
+describe('prepareText', () => {
+ it.each([
+ ['Wait… what?', 'Wait... what?'],
+ ['«Bonjour», dit-il.', '"Bonjour", dit-il.'],
+ ['It’s ‘fine’', "It's 'fine'"],
+ ['Szczurek–Żelazko', 'Szczurek-Żelazko'],
+ ['yes — no', 'yes - no'],
+ ['one -- two', 'one - two'],
+ ['a – — b', 'a - b'],
+ ['non breaking thin', 'non breaking thin'],
+ ['zerowidth', 'zerowidth'],
+ ['too many spaces', 'too many spaces'],
+ ['Hello , world !', 'Hello, world!'],
+ ['Hmm ... ok .', 'Hmm ... ok.'],
+ [' edges ', 'edges'],
+ ['end—', 'end -'],
+ ['--start', '- start'],
+ ['x--y--z', 'x-y-z'],
+ ['tab\there', 'tab here'],
+ ['quote„test“', 'quote"test"'],
+ ['´acute` and ʼmodifier', "'acute' and 'modifier"],
+ ])('normalizes %j as Echo did', (input, expected) => {
+ expect(prepareText(input, 'en')).toBe(expected);
+ });
+
+ it('writes numbers and symbols out as words in the given language', () => {
+ expect(prepareText('I have 3 cats & 2 dogs.', 'en')).toBe('I have three cats and two dogs.');
+ expect(prepareText('Mam 2 koty.', 'pl')).toBe('Mam dwa koty.');
+ });
+});
+
+describe('estimateTargetFrames', () => {
+ it.each([
+ ['Hello world.', 'This is a reference transcript.', 250, 97],
+ ['Mam 2 koty i psa.', 'To jest nagranie referencyjne mojego głosu.', 300, 123],
+ ['Hi', 'A long transcript of the reference clip that sets the speaking rate.', 400, 32],
+ ['Café naïve - ok!', 'Reference.', 100, 132],
+ ['Zażółć gęślą jaźń. To jest test, który ma sprawdzić długość.', 'Krótki tekst.', 150, 657],
+ ['Some text', '', 100, 819],
+ ])('estimates %j against %j over %i frames as Voice Studio does', (...args) => {
+ const [text, referenceText, referenceFrames, expected] = args;
+ expect(estimateTargetFrames(text, referenceText, referenceFrames)).toBe(expected);
+ });
+});
+
+describe('roundHalfEven', () => {
+ it.each([
+ [0.5, 0],
+ [1.5, 2],
+ [2.5, 2],
+ [2.4, 2],
+ [2.6, 3],
+ [7, 7],
+ ])('rounds %d to %d like Python', (value, expected) => {
+ expect(roundHalfEven(value)).toBe(expected);
+ });
+});
+
+describe('trimSilence', () => {
+ /** Concatenated constant segments: [seconds, amplitude] each. */
+ const segments = (...parts: [number, number][]): Float32Array => {
+ const lengths = parts.map(([seconds]) => Math.round(seconds * 24000));
+ const audio = new Float32Array(lengths.reduce((a, b) => a + b, 0));
+ let offset = 0;
+ parts.forEach(([, amplitude], i) => {
+ audio.fill(amplitude, offset, offset + lengths[i]!);
+ offset += lengths[i]!;
+ });
+ return audio;
+ };
+
+ it.each<[string, [number, number][], number, number]>([
+ [
+ 'speech',
+ [
+ [0.5, 0],
+ [1.0, 0.1],
+ [0.5, 0],
+ ],
+ 9600,
+ 38400,
+ ],
+ [
+ 'a leading blip',
+ [
+ [0.2, 0],
+ [0.1, 0.1],
+ [0.3, 0],
+ [1.0, 0.1],
+ [0.4, 0],
+ ],
+ 12000,
+ 40800,
+ ],
+ [
+ 'a trailing blip',
+ [
+ [0.3, 0],
+ [1.0, 0.1],
+ [0.25, 0],
+ [0.12, 0.1],
+ [0.3, 0],
+ ],
+ 4800,
+ 33600,
+ ],
+ [
+ 'a leading sound',
+ [
+ [0.2, 0],
+ [0.2, 0.1],
+ [0.3, 0],
+ [1.0, 0.1],
+ [0.4, 0],
+ ],
+ 2400,
+ 43200,
+ ],
+ [
+ 'a blip close to the speech',
+ [
+ [0.2, 0],
+ [0.1, 0.1],
+ [0.15, 0],
+ [1.0, 0.1],
+ [0.4, 0],
+ ],
+ 2400,
+ 37200,
+ ],
+ [
+ 'quiet speech',
+ [
+ [0.3, 0],
+ [0.5, 0.004],
+ [0.3, 0.003],
+ ],
+ 4800,
+ 21600,
+ ],
+ ['silence', [[0.5, 0]], 0, 12000],
+ ['less than a window', [[0.005, 0.5]], 0, 120],
+ [
+ 'no room for the margin',
+ [
+ [0.05, 0],
+ [1.0, 0.1],
+ [0.02, 0],
+ ],
+ 0,
+ 25680,
+ ],
+ ])('trims %s as Voice Studio does', (_label, parts, start, end) => {
+ const trimmed = trimSilence(segments(...parts), 24000, -50, 0.1);
+
+ expect(trimmed.byteOffset / 4).toBe(start);
+ expect(trimmed.byteOffset / 4 + trimmed.length).toBe(end);
+ });
+});
+
+describe('pauseAfter', () => {
+ it.each([
+ ['A sentence.', 0.375],
+ ['A question?', 0.5],
+ ['A clause,', 0.13],
+ ['He said "yes."', 0.375],
+ ['Wait...', 1],
+ ['No mark', 0],
+ ])('pauses after %j for %d s', (chunk, seconds) => {
+ expect(pauseAfter(chunk)).toBeCloseTo(seconds);
+ });
+});
+
+describe('buildPrompt', () => {
+ it('puts the language, the transcript and the text in the template', () => {
+ expect(buildPrompt('pl', 'Dzień dobry.', 'Cześć!')).toBe(
+ '<|denoise|><|lang_start|>pl<|lang_end|><|instruct_start|>None<|instruct_end|>' +
+ '<|text_start|>Dzień dobry. Cześć!<|text_end|>'
+ );
+ });
+});
+
+describe('parseVoice', () => {
+ it('parses codes, transcript and language', () => {
+ const codes = Array.from({ length: 16 }, (_, i) => i * 10);
+
+ expect(parseVoice({ codes, transcript: 'Hello.', language: 'en' })).toEqual({
+ codes: BigInt64Array.from(codes, BigInt),
+ transcript: 'Hello.',
+ language: 'en',
+ });
+ });
+
+ it.each([
+ ['no codes', { transcript: 'Hi.', language: 'en' }],
+ ['a fractional code', { codes: [0.5], transcript: 'Hi.', language: 'en' }],
+ ['no transcript', { codes: [0], language: 'en' }],
+ ['no language', { codes: [0], transcript: 'Hi.' }],
+ ])('rejects a voice with %s', (_label, json) => {
+ expect(() => parseVoice(json)).toThrow(/Invalid voice JSON/);
+ });
+});
diff --git a/packages/react-native-executorch/__tests__/tasks/reverbTextToSpeech.test.ts b/packages/react-native-executorch/__tests__/tasks/reverbTextToSpeech.test.ts
new file mode 100644
index 0000000000..0878bed73c
--- /dev/null
+++ b/packages/react-native-executorch/__tests__/tasks/reverbTextToSpeech.test.ts
@@ -0,0 +1,514 @@
+/**
+ * The Reverb pipeline on the fake runtime: the programs' contract, the calls
+ * and shapes that make a chunk, the sampler, chunking, validation, the lazy
+ * encoder, and disposal. The speech itself needs real weights.
+ */
+import { RangeDim, constraint, f16, f32, i64, method } from '../../src/core/schema';
+import {
+ REVERB_SAMPLE_RATE,
+ createReverbTextToSpeech,
+ type ReverbVoice,
+} from '../../src/extensions/speech/tasks/reverbTextToSpeech';
+import {
+ estimateTargetFrames,
+ prepareText,
+ roundHalfEven,
+} from '../../src/extensions/speech/utils/reverbUtils';
+import { fakeJsi } from '../support/fakeJsi';
+import { fakeFs } from '../support/blobUtilMock';
+import { exported } from '../support/fixtures';
+import { tracked } from '../support/lifetime';
+import type { FakeTensor } from '../support/fakeTensor';
+
+const BACKBONE_PATH = '/models/echo_mlx_fp16.pte';
+const DECODER_PATH = '/models/codec_decoder_coreml_fp16.pte';
+const ENCODER_PATH = '/models/codec_encoder_coreml_int4.pte';
+const TOKENIZER_PATH = '/models/tokenizer.json';
+const VOICE_PATH = '/models/voices/anna.json';
+
+const FRAME_SAMPLES = 960;
+const MAX_PROMPT = 64; // tokens; the fake tokenizer makes one per whitespace-separated piece
+const PREDICTED = 5; // every step predicts this code
+const POLISHED = 9; // polish fills codebooks 2-7 with this code
+const ENCODED = 17; // the encoder's codes
+
+const frames = (min: number, max: number) => RangeDim(min, max);
+const tDim = (paramSide: 'input' | 'output', tensorIdx: number, dimIdx = 1) => ({
+ paramSide,
+ tensorIdx,
+ dimIdx,
+});
+
+const backboneSchema = (cacheType = f16) => {
+ const cache = () => cacheType(2, 24, 6, frames(4, 1024), 128);
+ return exported({
+ ...method('prefill', [i64(frames(2, MAX_PROMPT)), i64(8, frames(2, 512))], [cache()]),
+ ...method(
+ 'step',
+ [i64(8, frames(4, 1024)), cache()],
+ [f32(2, frames(4, 1024)), i64(2, frames(4, 1024)), f32(2, frames(4, 1024))],
+ [
+ constraint.equality(
+ tDim('input', 0),
+ tDim('output', 0),
+ tDim('output', 1),
+ tDim('output', 2)
+ ),
+ ]
+ ),
+ ...method(
+ 'polish',
+ [i64(8, frames(4, 1024)), i64(1), cache()],
+ [i64(6, frames(4, 1024))],
+ [constraint.equality(tDim('input', 0), tDim('output', 0))]
+ ),
+ });
+};
+
+const decoderSchema = (withConstraint = true) =>
+ exported(
+ method(
+ 'forward',
+ [i64(8, frames(4, 1024))],
+ [f32(RangeDim(4 * FRAME_SAMPLES, 1024 * FRAME_SAMPLES, FRAME_SAMPLES))],
+ withConstraint
+ ? [constraint.linear(tDim('output', 0, 0), tDim('input', 0), FRAME_SAMPLES)]
+ : []
+ )
+ );
+
+const encoderSchema = (width = FRAME_SAMPLES) =>
+ exported(
+ method(
+ 'forward',
+ [f32(frames(25, 500), width)],
+ [i64(8, frames(25, 500))],
+ [constraint.equality(tDim('input', 0, 0), tDim('output', 0))]
+ )
+ );
+
+const fill = (t: FakeTensor, value: number) => {
+ for (let i = 0; i < t.numel; i++) t.setElement(i, value);
+};
+
+/** Every call's method, input shapes, and inputs. */
+let calls: { methodName: string; shapes: (readonly number[])[]; inputs: FakeTensor[] }[] = [];
+/** Masked slots of codebooks 0-1 entering each step and the polish. */
+let masked: number[] = [];
+
+const registerModels = (backbone = backboneSchema(), decoder = decoderSchema()): void => {
+ calls = [];
+ masked = [];
+ const record = (methodName: string, inputs: readonly unknown[]) => {
+ const tensors = inputs as FakeTensor[];
+ calls.push({ methodName, shapes: tensors.map((t) => t.shape), inputs: tensors });
+ };
+ fakeJsi.registerModel(BACKBONE_PATH, {
+ schema: backbone,
+ execute: (methodName, inputs, outputs) => {
+ record(methodName, inputs);
+ const tokens = inputs[0] as FakeTensor;
+ if (methodName !== 'prefill') {
+ const slots = Array.from({ length: 2 * tokens.shape[1]! }, (_, i) => tokens.getElement(i));
+ masked.push(slots.filter((code) => code === 1024).length);
+ }
+ if (methodName === 'step') {
+ fill(outputs[0]!, -1);
+ fill(outputs[1]!, PREDICTED);
+ outputs[2]!.data.forEach((_, i) => outputs[2]!.setElement(i, -i));
+ }
+ if (methodName === 'polish') fill(outputs[0]!, POLISHED);
+ },
+ });
+ fakeJsi.registerModel(DECODER_PATH, {
+ schema: decoder,
+ execute: (methodName, inputs, outputs) => {
+ record(`decoder.${methodName}`, inputs);
+ fill(outputs[0]!, 0.5); // a steady tone: nothing for the trim to cut
+ },
+ });
+ fakeJsi.registerModel(ENCODER_PATH, {
+ schema: encoderSchema(),
+ execute: (methodName, inputs, outputs) => {
+ record(`encoder.${methodName}`, inputs);
+ fill(outputs[0]!, ENCODED);
+ },
+ });
+ fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: [], open: true });
+};
+
+const PRESET = {
+ codes: new Array(8 * 30).fill(3),
+ transcript: 'This is my voice.',
+ language: 'en',
+};
+const preset: ReverbVoice = {
+ codes: BigInt64Array.from(PRESET.codes, BigInt),
+ transcript: PRESET.transcript,
+ language: 'en',
+};
+
+const config = {
+ name: 'reverb',
+ modelPaths: { backbone: BACKBONE_PATH, codecDecoder: DECODER_PATH, codecEncoder: ENCODER_PATH },
+ tokenizerPath: TOKENIZER_PATH,
+ voices: { anna: VOICE_PATH },
+} as const;
+
+const collect = async (stream: AsyncGenerator): Promise => {
+ const chunks: T[] = [];
+ for await (const chunk of stream) chunks.push(chunk);
+ return chunks;
+};
+
+const methodsCalled = () => calls.map(({ methodName }) => methodName);
+
+beforeEach(() => {
+ fakeFs.write(VOICE_PATH, JSON.stringify(PRESET));
+ registerModels();
+});
+
+describe('createReverbTextToSpeech — the model contract', () => {
+ it('accepts the exported programs and exposes the synthesis API', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+
+ expect(tts.encodeVoice).toBeInstanceOf(Function);
+ expect(tts.synthesize).toBeInstanceOf(Function);
+ expect(tts.synthesizeStop).toBeInstanceOf(Function);
+ });
+
+ it('rejects a backbone whose cache is not float16', async () => {
+ registerModels(backboneSchema(f32));
+
+ await expect(createReverbTextToSpeech(config)).rejects.toThrow(/DType mismatch/);
+ });
+
+ it('rejects a decoder that does not tie its audio length to the frames', async () => {
+ registerModels(backboneSchema(), decoderSchema(false));
+
+ await expect(createReverbTextToSpeech(config)).rejects.toThrow(/constraint/i);
+ });
+
+ it('rejects a malformed preset voice', async () => {
+ fakeFs.write(VOICE_PATH, JSON.stringify({ ...PRESET, codes: [1, 2, 3] }));
+ await expect(createReverbTextToSpeech(config)).rejects.toThrow(/Voice 'anna': A voice holds/);
+
+ fakeFs.write(VOICE_PATH, JSON.stringify({ ...PRESET, language: 'xx' }));
+ await expect(createReverbTextToSpeech(config)).rejects.toThrow(/Voice 'anna': A voice holds/);
+ });
+
+ it('releases the programs, the tokenizer and every tensor when construction fails', async () => {
+ registerModels(backboneSchema(), decoderSchema(false));
+
+ await expect(createReverbTextToSpeech(config)).rejects.toThrow();
+
+ expect(fakeJsi.liveModels()).toEqual([]);
+ expect(fakeJsi.liveTokenizers()).toEqual([]);
+ expect(fakeJsi.liveTensors()).toBe(0);
+ });
+
+ it('loads the encoder only when a voice is first encoded', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+ expect(fakeJsi.liveModels()).toEqual([BACKBONE_PATH, DECODER_PATH].sort());
+
+ await tts.encodeVoice(new Float32Array(2 * REVERB_SAMPLE_RATE), 'Hello there.', 'en');
+
+ expect(fakeJsi.liveModels()).toEqual([BACKBONE_PATH, DECODER_PATH, ENCODER_PATH].sort());
+ });
+
+ it('releases every program, the tokenizer and the tensors on dispose', async () => {
+ const tts = await createReverbTextToSpeech(config);
+ await tts.encodeVoice(new Float32Array(2 * REVERB_SAMPLE_RATE), 'Hello there.', 'en');
+
+ tts.dispose();
+
+ expect(fakeJsi.liveModels()).toEqual([]);
+ expect(fakeJsi.liveTokenizers()).toEqual([]);
+ expect(fakeJsi.liveTensors()).toBe(0);
+ });
+});
+
+describe('createReverbTextToSpeech — synthesis', () => {
+ const TEXT = 'Hello world, this is a test.';
+ const estimate = (text: string) =>
+ estimateTargetFrames(prepareText(text, 'en'), PRESET.transcript, 30);
+ const targetFrames = (text: string) => Math.max(10, estimate(text));
+
+ it('makes a chunk from a prefill, 8 steps, the polish and the decoder, in that order', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+
+ await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en' }));
+
+ expect(methodsCalled()).toEqual([
+ 'prefill',
+ ...new Array(8).fill('step'),
+ 'polish',
+ 'decoder.forward',
+ ]);
+ });
+
+ it('sizes every tensor from the prompt, the voice and the estimated length', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+
+ await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en' }));
+
+ const [prefill, step] = calls;
+ const polish = calls.find(({ methodName }) => methodName === 'polish')!;
+ const decode = calls.find(({ methodName }) => methodName === 'decoder.forward')!;
+ const promptTokens = prefill!.shapes[0]![0]!;
+ const target = targetFrames(TEXT);
+ expect(prefill!.shapes[1]).toEqual([8, 30]);
+ expect(step!.shapes).toEqual([
+ [8, target],
+ [2, 24, 6, promptTokens + 30, 128],
+ ]);
+ expect(polish.inputs[1]!.getElement(0)).toBe(30);
+ expect(decode.shapes).toEqual([[8, target]]);
+ });
+
+ it('decodes the committed codes of codebooks 0-1 and the polished codebooks 2-7', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+
+ await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en' }));
+
+ const codes = calls.find(({ methodName }) => methodName === 'decoder.forward')!.inputs[0]!;
+ const target = targetFrames(TEXT);
+ const rows = Array.from(
+ { length: 8 },
+ (_, row) =>
+ new Set(Array.from({ length: target }, (__, i) => codes.getElement(row * target + i)))
+ );
+ expect(rows.map((values) => [...values])).toEqual([
+ [PREDICTED],
+ [PREDICTED],
+ ...new Array(6).fill([POLISHED]),
+ ]);
+ });
+
+ it('cuts 0.2 s off the end and appends the pause its punctuation calls for', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+
+ const [chunk] = await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en' }));
+
+ const pause = 0.375 * REVERB_SAMPLE_RATE; // after '.'
+ expect(chunk!.audio.length).toBe(targetFrames(TEXT) * FRAME_SAMPLES - 4800 + pause);
+ expect(chunk).toMatchObject({ sampleRate: REVERB_SAMPLE_RATE, chunkIndex: 0, totalChunks: 1 });
+ expect(chunk!.duration).toBeCloseTo(chunk!.audio.length / REVERB_SAMPLE_RATE);
+ });
+
+ it('returns the raw decoder output, plus the pause, without post-processing', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+
+ const [chunk] = await collect(
+ tts.synthesize('No mark here', { voice: 'anna', language: 'en', postprocess: false })
+ );
+
+ expect(chunk!.audio.length).toBe(targetFrames('No mark here') * FRAME_SAMPLES);
+ });
+
+ it('runs as many main passes as asked', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+
+ await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en', sampling: { steps: 3 } }));
+
+ expect(methodsCalled().filter((name) => name === 'step')).toHaveLength(3);
+ });
+
+ it('stretches the target as the speed drops', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+
+ await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en', speed: 0.8 }));
+
+ expect(calls[1]!.shapes[0]).toEqual([8, roundHalfEven(estimate(TEXT) / 0.8)]);
+ });
+
+ it('prompts with the language, the voice transcript and the prepared text', async () => {
+ const vocabulary: string[] = [];
+ fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: vocabulary, open: true });
+ const tts = tracked(await createReverbTextToSpeech(config));
+
+ await collect(tts.synthesize('I have 2 cats.', { voice: 'anna', language: 'en' }));
+
+ const ids = calls[0]!.inputs[0]!;
+ const words = Array.from({ length: ids.numel }, (_, i) => vocabulary[ids.getElement(i)]);
+ expect(words.join(' ')).toBe(
+ '<|denoise|><|lang_start|>en<|lang_end|><|instruct_start|>None<|instruct_end|>' +
+ '<|text_start|>This is my voice. I have two cats.<|text_end|>'
+ );
+ });
+
+ it("commits and re-masks codebooks 0-1 on Voice Studio's schedule", async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+
+ await collect(tts.synthesize(TEXT, { voice: 'anna', language: 'en' }));
+
+ // 8 passes: each commits its share of the slots, then all but the last
+ // re-mask a quarter as many held codes. The polish gets none masked.
+ const slots = 2 * targetFrames(TEXT);
+ const share = (i: number) => (0.1 * (i / 8)) / (1 - (1 - 0.1) * (i / 8));
+ const expected = [slots];
+ for (let pass = 0; pass < 8; pass++) {
+ const open = expected[pass]!;
+ const last = pass === 7;
+ const commits = last
+ ? open
+ : Math.min(open, Math.ceil(slots * (share(pass + 1) - share(pass))));
+ const withdrawn = last ? 0 : Math.min(roundHalfEven(0.25 * commits), slots - open);
+ expected.push(open - commits + withdrawn);
+ }
+ expect(masked).toEqual(expected);
+ expect(masked.at(-1)).toBe(0);
+ });
+
+ it('speaks a text too long for one chunk in several, each from its own prefill', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+ const text = 'First sentence here. Second sentence follows it. Third one ends the text.';
+
+ const chunks = await collect(
+ tts.synthesize(text, { voice: 'anna', language: 'en', maxChunkDuration: 1.5 })
+ );
+
+ expect(chunks.length).toBeGreaterThan(1);
+ expect(chunks.map(({ chunkIndex }) => chunkIndex)).toEqual(chunks.map((_, i) => i));
+ expect(new Set(chunks.map(({ totalChunks }) => totalChunks))).toEqual(new Set([chunks.length]));
+ expect(methodsCalled().filter((name) => name === 'prefill')).toHaveLength(chunks.length);
+ });
+
+ it('keeps every prompt within the backbone token limit', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+ const text = new Array(150).fill('word').join(' ') + '.';
+
+ await collect(tts.synthesize(text, { voice: 'anna', language: 'en' }));
+
+ const prompts = calls.filter(({ methodName }) => methodName === 'prefill');
+ expect(prompts.length).toBeGreaterThan(1);
+ for (const { shapes } of prompts) expect(shapes[0]![0]).toBeLessThanOrEqual(MAX_PROMPT);
+ });
+
+ it('synthesizes in a voice it encoded', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+ const audio = new Float32Array(Math.round(2.5 * REVERB_SAMPLE_RATE));
+
+ const voice = await tts.encodeVoice(audio, 'I have 2 cats.', 'en');
+ await collect(tts.synthesize(TEXT, { voice, language: 'en' }));
+
+ expect(voice.transcript).toBe('I have two cats.');
+ expect(voice.codes).toEqual(new BigInt64Array(8 * 62).fill(BigInt(ENCODED)));
+ expect(calls.find(({ methodName }) => methodName === 'encoder.forward')!.shapes).toEqual([
+ [62, FRAME_SAMPLES],
+ ]);
+ expect(calls.find(({ methodName }) => methodName === 'prefill')!.shapes[1]).toEqual([8, 62]);
+ });
+
+ it('stops streaming when synthesizeStop is called mid-stream', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+ const text = 'First sentence here. Second sentence follows it. Third one ends the text.';
+
+ const stream = tts.synthesize(text, { voice: 'anna', language: 'en', maxChunkDuration: 1.5 });
+ const first = await stream.next();
+ tts.synthesizeStop();
+ const after = await stream.next();
+
+ expect(first.value!.totalChunks).toBeGreaterThan(1);
+ expect(after.done).toBe(true);
+ });
+
+ it('leaves no tensor behind once a stream is drained or abandoned', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+ const text = 'First sentence here. Second sentence follows it.';
+
+ await collect(tts.synthesize(text, { voice: 'anna', language: 'en', maxChunkDuration: 1.5 }));
+ const stream = tts.synthesize(text, { voice: 'anna', language: 'en', maxChunkDuration: 1.5 });
+ await stream.next();
+ await stream.return(undefined as never);
+
+ expect(fakeJsi.liveTensors()).toBe(0);
+ });
+});
+
+describe('createReverbTextToSpeech — argument validation', () => {
+ const synthesizeWith = async (options: object, text = 'Hello.') => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+ return collect(tts.synthesize(text, { voice: 'anna', language: 'en', ...options } as never));
+ };
+
+ it.each([
+ ['an empty text', {}, '', /cannot be empty/],
+ ['an unsupported language', { language: 'xx' }, 'Hello.', /Unsupported language/],
+ ['an unknown voice', { voice: 'nobody' }, 'Hello.', /Unknown voice/],
+ ['a speed that is too slow', { speed: 0.4 }, 'Hello.', /speed must be between/],
+ ['a speed that is too fast', { speed: 2.1 }, 'Hello.', /speed must be between/],
+ ['a chunk longer than the target window', { maxChunkDuration: 60 }, 'Hello.', /at most 40.96/],
+ ['a fractional step count', { sampling: { steps: 2.5 } }, 'Hello.', /positive integer/],
+ ['a remask ratio above 1', { sampling: { remaskRatio: 1.5 } }, 'Hello.', /\[0, 1\]/],
+ ['a fractional seed', { sampling: { seed: 0.5 } }, 'Hello.', /seed must be an integer/],
+ ])('rejects %s', async (_label, options, text, error) => {
+ await expect(synthesizeWith(options, text)).rejects.toThrow(error);
+ });
+
+ it.each([
+ ['codes not in rows of 8', { ...preset, codes: new BigInt64Array(9) }],
+ ['a reference shorter than 2 frames', { ...preset, codes: new BigInt64Array(8) }],
+ ['a code out of range', { ...preset, codes: new BigInt64Array(16).fill(1024n) }],
+ ['no transcript', { ...preset, transcript: ' ' }],
+ ])('rejects a voice with %s', async (_label, voice) => {
+ await expect(synthesizeWith({ voice })).rejects.toThrow(/A voice holds/);
+ });
+
+ it('rejects a voice whose transcript fills the prompt', async () => {
+ const voice = { ...preset, transcript: new Array(MAX_PROMPT).fill('word').join(' ') };
+
+ await expect(synthesizeWith({ voice })).rejects.toThrow(/leaves no room/);
+ });
+
+ it('refuses a second synthesis while one is still running, then runs it afterwards', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+ const first = tts.synthesize('One. Two. Three.', { voice: 'anna', language: 'en' });
+ await first.next();
+
+ await expect(
+ collect(tts.synthesize('Again.', { voice: 'anna', language: 'en' }))
+ ).rejects.toThrow(/already in progress/);
+ await collect(first);
+ await expect(
+ collect(tts.synthesize('Again.', { voice: 'anna', language: 'en' }))
+ ).resolves.toHaveLength(1);
+ });
+
+ it.each([
+ ['shorter than 1 s', 0.9],
+ ['longer than 20 s', 20.5],
+ ])('rejects a recording %s', async (_label, seconds) => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+ const audio = new Float32Array(Math.round(seconds * REVERB_SAMPLE_RATE));
+
+ await expect(tts.encodeVoice(audio, 'Hello.', 'en')).rejects.toThrow(/must last 1 to 20 s/);
+ });
+
+ it('rejects encoding without a transcript or in an unsupported language', async () => {
+ const tts = tracked(await createReverbTextToSpeech(config));
+ const audio = new Float32Array(2 * REVERB_SAMPLE_RATE);
+
+ await expect(tts.encodeVoice(audio, ' ', 'en')).rejects.toThrow(/transcript cannot be empty/);
+ await expect(tts.encodeVoice(audio, 'Hi.', 'xx' as never)).rejects.toThrow(/Unsupported/);
+ });
+
+ it('refuses to encode when the config names no encoder', async () => {
+ const modelPaths = { backbone: BACKBONE_PATH, codecDecoder: DECODER_PATH };
+ const tts = tracked(await createReverbTextToSpeech({ ...config, modelPaths }));
+
+ await expect(
+ tts.encodeVoice(new Float32Array(2 * REVERB_SAMPLE_RATE), 'Hello.', 'en')
+ ).rejects.toThrow(/names no codecEncoder/);
+ });
+
+ it('rejects an encoder with the wrong frame width, and releases it', async () => {
+ fakeJsi.registerModel(ENCODER_PATH, { schema: encoderSchema(480) });
+ const tts = tracked(await createReverbTextToSpeech(config));
+
+ await expect(
+ tts.encodeVoice(new Float32Array(2 * REVERB_SAMPLE_RATE), 'Hello.', 'en')
+ ).rejects.toThrow(/Constant dimension mismatch/);
+ expect(fakeJsi.liveModels()).not.toContain(ENCODER_PATH);
+ });
+});
diff --git a/packages/react-native-executorch/jest.config.js b/packages/react-native-executorch/jest.config.js
index 983f84f3bd..8b69ce99e1 100644
--- a/packages/react-native-executorch/jest.config.js
+++ b/packages/react-native-executorch/jest.config.js
@@ -24,6 +24,12 @@ module.exports = {
'^react-native-worklets$': '/__tests__/support/workletsMock.ts',
},
+ // Verbalis ships ES modules only, so it is compiled like the React Native
+ // packages the preset already lets through.
+ transformIgnorePatterns: [
+ 'node_modules/(?!((jest-)?react-native|@react-native(-community)?|verbalis)/)',
+ ],
+
// Snapshot of the public export surface — see `api/apiSurface.test.ts`.
snapshotFormat: { escapeString: false, printBasicPrototype: false },
diff --git a/packages/react-native-executorch/package.json b/packages/react-native-executorch/package.json
index ccacaad0f4..ece3cdd653 100644
--- a/packages/react-native-executorch/package.json
+++ b/packages/react-native-executorch/package.json
@@ -1,7 +1,7 @@
{
"name": "react-native-executorch",
"version": "1.0.0",
- "nativeLibsVersion": "0.11.0",
+ "nativeLibsVersion": "0.11.1",
"description": "An easy way to run AI models in React Native with ExecuTorch",
"main": "./lib/module/index.js",
"module": "./lib/module/index.js",
@@ -119,7 +119,8 @@
"registry": "https://registry.npmjs.org/"
},
"dependencies": {
- "@huggingface/jinja": "^0.5.9"
+ "@huggingface/jinja": "^0.5.9",
+ "verbalis": "^0.1.1"
},
"peerDependencies": {
"@kesha-antonov/react-native-background-downloader": ">=4.4.0",
diff --git a/packages/react-native-executorch/src/constants.ts b/packages/react-native-executorch/src/constants.ts
index 59ac9bd7ce..7a467f82f5 100644
--- a/packages/react-native-executorch/src/constants.ts
+++ b/packages/react-native-executorch/src/constants.ts
@@ -1335,6 +1335,22 @@ export const SUPERTONIC_DEFAULT_VOICE_NAMES = [
*/
export type SupertonicDefaultVoiceName = (typeof SUPERTONIC_DEFAULT_VOICE_NAMES)[number];
+/**
+ * Default Reverb voice names array.
+ * @category Speech / Constants
+ */
+export const REVERB_DEFAULT_VOICE_NAMES = [
+ 'standard_male',
+ 'angry_male',
+ 'sensual_female',
+] as const;
+
+/**
+ * Type representing a valid Reverb default voice name.
+ * @category Speech / Types
+ */
+export type ReverbDefaultVoiceName = (typeof REVERB_DEFAULT_VOICE_NAMES)[number];
+
// BIOES tag scheme for the privacy filter models: one outside tag ("O")
// followed by four prefix variants (B/I/E/S) per entity type, in the entity
// order below. The resulting arrays must match each model's id2label mapping
diff --git a/packages/react-native-executorch/src/extensions/speech/index.ts b/packages/react-native-executorch/src/extensions/speech/index.ts
index 2ada7825ea..2a31436574 100644
--- a/packages/react-native-executorch/src/extensions/speech/index.ts
+++ b/packages/react-native-executorch/src/extensions/speech/index.ts
@@ -8,3 +8,5 @@ export * from './utils/supertonicUtils';
export * from './utils/kokoroUtils';
export * from './utils/phonemizer';
export * from './utils/textPartitioner';
+export { estimateTargetFrames } from './utils/reverbUtils';
+export * from './utils/reverbPartitioner';
diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/reverbTextToSpeech.ts b/packages/react-native-executorch/src/extensions/speech/tasks/reverbTextToSpeech.ts
new file mode 100644
index 0000000000..0483249cbd
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/tasks/reverbTextToSpeech.ts
@@ -0,0 +1,689 @@
+/**
+ * Reverb Text-to-Speech (TTS) synthesis task pipeline.
+ */
+
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import RNBlobUtil from 'react-native-blob-util';
+
+import { tensor, type Tensor } from '../../../core/tensor';
+import { loadModel } from '../../../core/model';
+import {
+ validateSpec,
+ method,
+ i64,
+ f16,
+ f32,
+ DynamicDim as Dyn,
+ constraint,
+ type DimRef,
+ type Range,
+} from '../../../core/schema';
+import { wrapAsync } from '../../../core/runtime';
+import { createResourceScope } from '../../../core/lifetime';
+import { RnExecuTorchError } from '../../../core/error';
+import { mulberry32 } from '../../math';
+import { loadTokenizer } from '../../nlp/tokenizer';
+import { partitionReverbText } from '../utils/reverbPartitioner';
+import {
+ buildPrompt,
+ estimateTargetFrames,
+ parseVoice,
+ pauseAfter,
+ prepareText,
+ roundHalfEven,
+ trimSilence,
+} from '../utils/reverbUtils';
+
+/**
+ * Reverb audio sampling rate in Hz (24000).
+ * @category Speech / Constants
+ */
+export const REVERB_SAMPLE_RATE = 24000;
+
+/**
+ * Codec frames per second (25).
+ * @category Speech / Constants
+ */
+export const REVERB_FRAME_RATE = 25;
+
+/**
+ * Languages Reverb speaks.
+ * @category Speech / Constants
+ */
+export const REVERB_LANGUAGES = ['de', 'en', 'es', 'fr', 'it', 'pl', 'pt'] as const;
+
+const FRAME_SAMPLES = REVERB_SAMPLE_RATE / REVERB_FRAME_RATE; // 960
+const CODEBOOKS = 8;
+const CODES = 1024; // per codebook; the next id marks a masked slot
+const MASK_CODE = BigInt(CODES);
+// K/V cache: [keys and values, layers, KV heads, prefix, head dim]
+const CACHE_LAYERS = 24;
+const CACHE_HEADS = 6;
+const CACHE_HEAD_DIM = 128;
+
+// Voice Studio's sampler
+const T_SHIFT = 0.1; // few commits early, most in the last passes
+const CODEBOOK_PENALTY = 5; // per codebook index: codebook 0 commits before 1
+const POSITION_TEMPERATURE = 5; // scales confidence and penalty against the Gumbel noise
+const DEFAULT_STEPS = 8;
+const DEFAULT_REMASK_RATIO = 0.25;
+
+const MIN_FRAMES = 10;
+const DEFAULT_SPEED = 1.0;
+const MIN_SPEED = 0.5;
+const MAX_SPEED = 2.0;
+const DEFAULT_MAX_CHUNK_DURATION = 20;
+const DEFAULT_POSTPROCESS = { endCut: 0.2, silenceThreshold: -50, keep: 0.1 };
+
+/**
+ * A language Reverb speaks.
+ * @category Speech / Types
+ */
+export type ReverbLanguage = (typeof REVERB_LANGUAGES)[number];
+
+/**
+ * A voice to speak in: a reference clip's codec codes and transcript.
+ * @category Speech / Types
+ */
+export type ReverbVoice = {
+ /** Codec codes, `[8, frames]` row-major. */
+ readonly codes: BigInt64Array;
+ /** What the clip says, verbalized and normalized. */
+ readonly transcript: string;
+ /** The clip's language. */
+ readonly language: ReverbLanguage;
+};
+
+/**
+ * Model configuration of the Reverb Text-to-Speech pipeline.
+ * @category Speech / Types
+ * @typeParam K Preset voice keys.
+ */
+export type ReverbTtsModel = {
+ /** Discriminates this config from the other Text-to-Speech pipelines. */
+ readonly name: 'reverb';
+ /** Local or remote paths to the `.pte` programs. */
+ readonly modelPaths: {
+ /** The backbone: `prefill`, `step`, `polish`. */
+ readonly backbone: string;
+ /** The codec decoder: codes to audio. */
+ readonly codecDecoder: string;
+ /** The codec encoder: audio to codes. Only `encodeVoice` loads it. */
+ readonly codecEncoder?: string;
+ };
+ /** Local or remote path to `tokenizer.json`. */
+ readonly tokenizerPath: string;
+ /** Preset voices: name to `{ codes, transcript, language }` JSON file. */
+ readonly voices?: Record;
+};
+
+/**
+ * Sampler settings; the defaults are the measured ones.
+ * @category Speech / Types
+ */
+export type ReverbSamplingOptions = {
+ /** Main passes. Defaults to 8.
+ * NOTE: total number of steps (including polish pass) is always N + 2 */
+ readonly steps?: number;
+ /** Held codes re-masked per new commit, in `[0, 1]`. Defaults to 0.25. */
+ readonly remaskRatio?: number;
+ /** Integer seed for reproducible takes. Random by default. */
+ readonly seed?: number;
+};
+
+/**
+ * Per-chunk post-processing; the defaults are Voice Studio's.
+ * @category Speech / Types
+ */
+export type ReverbPostprocessOptions = {
+ /** Seconds cut from the end. Defaults to 0.2. */
+ readonly endCut?: number;
+ /** Level in dBFS below which an edge window is silent. Defaults to -50. */
+ readonly silenceThreshold?: number;
+ /** Seconds of silence kept at each edge. Defaults to 0.1. */
+ readonly keep?: number;
+};
+
+/**
+ * Per-call options of Reverb synthesis.
+ * @category Speech / Types
+ * @typeParam K Preset voice keys.
+ */
+export type ReverbTtsOptions = {
+ /** A preset voice key, or a {@link ReverbVoice} from `encodeVoice`. */
+ readonly voice: K | ReverbVoice;
+ /** Language of the text. */
+ readonly language: ReverbLanguage;
+ /** Speech speed factor, 0.5 to 2.0. Defaults to 1.0. */
+ readonly speed?: number;
+ /** Max seconds of speech per chunk. Defaults to 20. */
+ readonly maxChunkDuration?: number;
+ /** Post-processing, or `false` for the raw decoder output. */
+ readonly postprocess?: ReverbPostprocessOptions | false;
+ /** Sampler settings. */
+ readonly sampling?: ReverbSamplingOptions;
+};
+
+/**
+ * Audio chunk yielded by {@link ReverbTextToSpeech.synthesize}.
+ * @category Speech / Types
+ */
+export type ReverbTtsChunk = {
+ /** Float32 PCM samples in `[-1, 1]`. */
+ readonly audio: Float32Array;
+ /** Sampling rate in Hz. */
+ readonly sampleRate: number;
+ /** Duration in seconds. */
+ readonly duration: number;
+ /** Zero-based index of this chunk. */
+ readonly chunkIndex: number;
+ /** Number of chunks the text was split into. */
+ readonly totalChunks: number;
+};
+
+/**
+ * Reverb text-to-speech task runner.
+ * @category Speech / Types
+ * @typeParam K Preset voice keys.
+ */
+export type ReverbTextToSpeech = {
+ /** Releases all native resources. */
+ readonly dispose: () => void;
+
+ /**
+ * Encodes a recording into a voice, loading the codec encoder on first use.
+ * @param audio 24 kHz mono PCM, 1 to 20 s.
+ * @param transcript What the recording says.
+ * @param language The recording's language.
+ * @returns The voice.
+ * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` on a bad argument,
+ * `INVALID_STATE` without a codec encoder, or `RESOURCE_DISPOSED` if
+ * disposed.
+ */
+ readonly encodeVoice: (
+ audio: Float32Array,
+ transcript: string,
+ language: ReverbLanguage
+ ) => Promise;
+
+ /**
+ * Streams synthesized audio, one chunk of text at a time.
+ * @param text The text to speak.
+ * @param options See {@link ReverbTtsOptions}.
+ * @returns An AsyncGenerator of {@link ReverbTtsChunk}s.
+ * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` on a bad argument,
+ * `INVALID_STATE` if a synthesis is running, `RESOURCE_BUSY` if the model is
+ * in use, or `RESOURCE_DISPOSED` if disposed.
+ */
+ readonly synthesize: (
+ text: string,
+ options: ReverbTtsOptions
+ ) => AsyncGenerator;
+
+ /** Cancels an in-flight synthesis. */
+ readonly synthesizeStop: () => void;
+};
+
+const frameDim = (paramSide: 'input' | 'output', tensorIdx: number, dimIdx = 1): DimRef => ({
+ paramSide,
+ tensorIdx,
+ dimIdx,
+});
+const cache = () => f16(2, CACHE_LAYERS, CACHE_HEADS, Dyn('P'), CACHE_HEAD_DIM);
+
+// prettier-ignore
+const BACKBONE_SPEC = {
+ ...method(
+ 'prefill',
+ [i64(Dyn('L')), i64(CODEBOOKS, Dyn('R'))], // prompt tokens, reference codes
+ [cache()]
+ ),
+ ...method(
+ 'step',
+ [i64(CODEBOOKS, Dyn('T')), cache()], // tokens
+ [f32(2, Dyn('T')), i64(2, Dyn('T')), f32(2, Dyn('T'))], // confidence, prediction, held
+ [constraint.equality(frameDim('input', 0), frameDim('output', 0), frameDim('output', 1), frameDim('output', 2))]
+ ),
+ ...method(
+ 'polish',
+ [i64(CODEBOOKS, Dyn('T')), i64(1), cache()], // tokens, reference frames
+ [i64(CODEBOOKS - 2, Dyn('T'))], // codebooks 2-7
+ [constraint.equality(frameDim('input', 0), frameDim('output', 0))]
+ ),
+};
+
+const DECODER_SPEC = method(
+ 'forward',
+ [i64(CODEBOOKS, Dyn('T'))], // codes
+ [f32(Dyn('Samples'))], // audio
+ [constraint.linear(frameDim('output', 0, 0), frameDim('input', 0), FRAME_SAMPLES)]
+);
+
+const ENCODER_SPEC = method(
+ 'forward',
+ [f32(Dyn('F'), FRAME_SAMPLES)], // audio, one row per frame
+ [i64(CODEBOOKS, Dyn('F'))], // codes
+ [constraint.equality(frameDim('input', 0, 0), frameDim('output', 0))]
+);
+
+// Cumulative share of the slots committed after each main pass, from 0 to 1.
+function commitSchedule(steps: number): number[] {
+ 'worklet';
+ const schedule: number[] = [];
+ for (let i = 0; i <= steps; i++) {
+ const t = i / steps;
+ schedule.push((T_SHIFT * t) / (1 - (1 - T_SHIFT) * t));
+ }
+ return schedule;
+}
+
+// One main pass over codebooks 0-1 of `tokens`, in place: commits the pass's share of the masked slots by
+// Gumbel-perturbed confidence, then, except on the last pass, re-masks the least likely codes held before it.
+function samplePass(
+ tokens: BigInt64Array,
+ frames: number,
+ pass: { confidence: Float32Array; prediction: BigInt64Array; held: Float32Array },
+ step: number,
+ schedule: readonly number[],
+ remaskRatio: number,
+ random: () => number
+): void {
+ 'worklet';
+ const slots = 2 * frames;
+ const last = step === schedule.length - 2;
+ const score = new Float64Array(slots);
+ const masked: number[] = [];
+ const held: number[] = [];
+ for (let i = 0; i < slots; i++) {
+ const gumbel = -Math.log(-Math.log(random() + 1e-10) + 1e-10);
+ if (tokens[i] !== MASK_CODE) {
+ held.push(i);
+ continue;
+ }
+ const penalty = CODEBOOK_PENALTY * Math.floor(i / frames);
+ score[i] = (pass.confidence[i]! - penalty) / POSITION_TEMPERATURE + gumbel;
+ masked.push(i);
+ }
+
+ const share = Math.ceil(slots * (schedule[step + 1]! - schedule[step]!));
+ const count = last ? masked.length : Math.min(masked.length, share);
+ masked.sort((a, b) => score[b]! - score[a]!);
+ for (const i of masked.slice(0, count)) tokens[i] = pass.prediction[i]!;
+
+ if (last || remaskRatio <= 0) return;
+ held.sort((a, b) => pass.held[a]! - pass.held[b]!);
+ for (const i of held.slice(0, roundHalfEven(remaskRatio * count))) tokens[i] = MASK_CODE;
+}
+
+/**
+ * Creates a Reverb Text-to-Speech pipeline: speech in the voice of a short
+ * recording, in 7 languages.
+ * @category Speech / Tasks
+ * @typeParam K Preset voice keys (inferred from `config.voices`).
+ * @param config Paths to the programs, tokenizer, and voices. See
+ * {@link ReverbTtsModel}.
+ * @param runtime Optional worklet runtime to run inference on.
+ * @returns The {@link ReverbTextToSpeech} runner.
+ * @throws {RnExecuTorchError} With code `LOAD_FAILED` if an asset fails to
+ * load, or `SCHEMA_MISMATCH` if a program does not match Reverb's.
+ */
+export async function createReverbTextToSpeech(
+ config: ReverbTtsModel,
+ runtime?: WorkletRuntime
+): Promise> {
+ const scope = createResourceScope();
+ let disposed = false;
+ const dispose = (): void => {
+ disposed = true;
+ scope.dispose();
+ };
+
+ try {
+ const load = wrapAsync(loadModel, runtime);
+ const backbone = scope.track(await load(config.modelPaths.backbone));
+ const decoder = scope.track(await load(config.modelPaths.codecDecoder));
+ const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(config.tokenizerPath));
+
+ const backboneSpec = validateSpec(backbone.schema, { default: BACKBONE_SPEC });
+ const [promptTokens, referenceFrames, targetFrames] = backboneSpec.dims.range('L', 'R', 'T');
+ const [decodedFrames] = validateSpec(decoder.schema, { default: DECODER_SPEC }).dims.range('T');
+ const minFrames = Math.max(MIN_FRAMES, targetFrames.min, decodedFrames.min);
+ const maxFrames = Math.min(targetFrames.max, decodedFrames.max);
+
+ // A voice the programs take: [8, frames] codes in range, a transcript, and a supported language.
+ const validateVoice = (
+ voice: ReverbVoice,
+ code: 'INVALID_ARGUMENT' | 'LOAD_FAILED',
+ ctx: string
+ ): ReverbVoice => {
+ const frames = voice.codes.length / CODEBOOKS;
+ const valid =
+ Number.isInteger(frames) &&
+ frames >= referenceFrames.min &&
+ frames <= referenceFrames.max &&
+ voice.codes.every((c) => c >= 0n && c < MASK_CODE) &&
+ voice.transcript.trim().length > 0 &&
+ REVERB_LANGUAGES.includes(voice.language);
+ if (!valid) {
+ throw RnExecuTorchError(
+ code,
+ `${ctx}: A voice holds [${CODEBOOKS}, frames] codes below ${CODES}, with ` +
+ `${referenceFrames.min} to ${referenceFrames.max} frames, a transcript, and a ` +
+ `supported language.`
+ );
+ }
+ return voice;
+ };
+
+ const voices = {} as Record;
+ for (const [key, path] of Object.entries(config.voices ?? {}) as [K, string][]) {
+ const json = JSON.parse(await RNBlobUtil.fs.readFile(path, 'utf8'));
+ voices[key] = validateVoice(parseVoice(json), 'LOAD_FAILED', `Voice '${String(key)}'`);
+ }
+
+ const synthesizeChunkWorklet = (
+ ids: Int32Array,
+ voice: ReverbVoice,
+ frames: number,
+ chunkOpts: {
+ steps: number;
+ remaskRatio: number;
+ seed: number;
+ postprocess: Required | false;
+ pause: number;
+ }
+ ): Float32Array => {
+ 'worklet';
+
+ const { steps, remaskRatio, seed, postprocess, pause } = chunkOpts;
+ const references = voice.codes.length / CODEBOOKS;
+ const slots = 2 * frames;
+ const tokens = new BigInt64Array(CODEBOOKS * frames).fill(MASK_CODE);
+ const pass = {
+ confidence: new Float32Array(slots),
+ prediction: new BigInt64Array(slots),
+ held: new Float32Array(slots),
+ };
+ const schedule = commitSchedule(steps);
+ const random = mulberry32(seed);
+
+ // Collect execution tensors for cleanup in a single try/finally block
+ const auxTensors: Tensor[] = [];
+ const alloc = (...args: Parameters): Tensor => {
+ const t = tensor(...args);
+ auxTensors.push(t);
+ return t;
+ };
+
+ try {
+ // 1. Prefill the prompt and the reference into the K/V cache
+ const tCache = alloc('float16', [
+ 2,
+ CACHE_LAYERS,
+ CACHE_HEADS,
+ ids.length + references,
+ CACHE_HEAD_DIM,
+ ]);
+ const tIds = alloc('int64', [ids.length], BigInt64Array.from(ids, BigInt));
+ const tReference = alloc('int64', [CODEBOOKS, references], voice.codes);
+ backbone.execute('prefill', [tIds, tReference], [tCache]);
+
+ // 2. Main passes over codebooks 0-1
+ const tTokens = alloc('int64', [CODEBOOKS, frames]);
+ const tConfidence = alloc('float32', [2, frames]);
+ const tPrediction = alloc('int64', [2, frames]);
+ const tHeld = alloc('float32', [2, frames]);
+ for (let step = 0; step < steps; step++) {
+ tTokens.setData(tokens);
+ backbone.execute('step', [tTokens, tCache], [tConfidence, tPrediction, tHeld]);
+ tConfidence.getData(pass.confidence);
+ tPrediction.getData(pass.prediction);
+ tHeld.getData(pass.held);
+ samplePass(tokens, frames, pass, step, schedule, remaskRatio, random);
+ }
+
+ // 3. Polish codebooks 2-7
+ const tReferenceFrames = alloc('int64', [1], BigInt64Array.of(BigInt(references)));
+ const tFine = alloc('int64', [CODEBOOKS - 2, frames]);
+ tTokens.setData(tokens);
+ backbone.execute('polish', [tTokens, tReferenceFrames, tCache], [tFine]);
+ tokens.set(tFine.getData(new BigInt64Array((CODEBOOKS - 2) * frames)), slots);
+
+ // 4. Decode
+ const tAudio = alloc('float32', [frames * FRAME_SAMPLES]);
+ tTokens.setData(tokens);
+ decoder.execute('forward', [tTokens], [tAudio]);
+ let audio: Float32Array = tAudio.getData(new Float32Array(frames * FRAME_SAMPLES));
+
+ // 5. Cut the end, then trim the silent edges
+ if (postprocess) {
+ const { endCut, silenceThreshold, keep } = postprocess;
+ const end = Math.max(0, audio.length - Math.floor(endCut * REVERB_SAMPLE_RATE));
+ audio = trimSilence(audio.subarray(0, end), REVERB_SAMPLE_RATE, silenceThreshold, keep);
+ }
+
+ // 6. Append the pause the chunk's final punctuation calls for
+ const result = new Float32Array(audio.length + Math.round(pause * REVERB_SAMPLE_RATE));
+ result.set(audio);
+ return result;
+ } finally {
+ auxTensors.forEach((t) => t.dispose());
+ }
+ };
+
+ const synthesizeChunk = wrapAsync(synthesizeChunkWorklet, runtime);
+
+ const checkLanguage = (language: unknown, ctx: string): void => {
+ if (!REVERB_LANGUAGES.includes(language as ReverbLanguage)) {
+ throw RnExecuTorchError(
+ 'INVALID_ARGUMENT',
+ `${ctx}: Unsupported language: ${String(language)}.`
+ );
+ }
+ };
+
+ const resolveVoice = (voice: K | ReverbVoice): ReverbVoice => {
+ const resolved = typeof voice === 'object' ? voice : voices[voice];
+ if (!resolved) {
+ throw RnExecuTorchError('INVALID_ARGUMENT', `synthesize: Unknown voice: ${String(voice)}.`);
+ }
+ return validateVoice(resolved, 'INVALID_ARGUMENT', 'synthesize');
+ };
+
+ let encoder:
+ | Promise<{ encode: (audio: Float32Array) => Promise; frames: Range }>
+ | undefined;
+ const loadEncoder = async () => {
+ if (!config.modelPaths.codecEncoder) {
+ throw RnExecuTorchError('INVALID_STATE', 'encodeVoice: The config names no codecEncoder.');
+ }
+ const model = await load(config.modelPaths.codecEncoder);
+ try {
+ if (disposed) {
+ throw RnExecuTorchError('RESOURCE_DISPOSED', 'encodeVoice: The pipeline was disposed.');
+ }
+ const [frames] = validateSpec(model.schema, { default: ENCODER_SPEC }).dims.range('F');
+ scope.track(model);
+ const encodeWorklet = (audio: Float32Array): BigInt64Array => {
+ 'worklet';
+ const count = audio.length / FRAME_SAMPLES;
+ const tAudio = tensor('float32', [count, FRAME_SAMPLES], audio);
+ const tCodes = tensor('int64', [CODEBOOKS, count]);
+ try {
+ model.execute('forward', [tAudio], [tCodes]);
+ return tCodes.getData(new BigInt64Array(CODEBOOKS * count));
+ } finally {
+ tAudio.dispose();
+ tCodes.dispose();
+ }
+ };
+ return { encode: wrapAsync(encodeWorklet, runtime), frames };
+ } catch (error) {
+ model.dispose();
+ throw error;
+ }
+ };
+
+ const encodeVoice = async (
+ audio: Float32Array,
+ transcript: string,
+ language: ReverbLanguage
+ ): Promise => {
+ if (disposed) {
+ throw RnExecuTorchError('RESOURCE_DISPOSED', 'encodeVoice: The pipeline was disposed.');
+ }
+ checkLanguage(language, 'encodeVoice');
+ if (!transcript.trim()) {
+ throw RnExecuTorchError('INVALID_ARGUMENT', 'encodeVoice: The transcript cannot be empty.');
+ }
+ encoder ??= loadEncoder().catch((error) => {
+ encoder = undefined;
+ throw error;
+ });
+ const { encode, frames } = await encoder;
+
+ const count = Math.floor(audio.length / FRAME_SAMPLES);
+ const min = Math.max(frames.min, referenceFrames.min);
+ const max = Math.min(frames.max, referenceFrames.max);
+ if (count < min || count > max) {
+ throw RnExecuTorchError(
+ 'INVALID_ARGUMENT',
+ `encodeVoice: The audio must last ${min / REVERB_FRAME_RATE} to ` +
+ `${max / REVERB_FRAME_RATE} s at ${REVERB_SAMPLE_RATE} Hz ` +
+ `(got ${audio.length / REVERB_SAMPLE_RATE} s).`
+ );
+ }
+ const codes = await encode(audio.slice(0, count * FRAME_SAMPLES));
+
+ return { codes, transcript: prepareText(transcript, language), language };
+ };
+
+ let isSynthesizing = false;
+ const synthesizeStop = (): void => {
+ isSynthesizing = false;
+ };
+
+ async function* synthesize(
+ text: string,
+ options: ReverbTtsOptions
+ ): AsyncGenerator {
+ if (isSynthesizing) {
+ throw RnExecuTorchError('INVALID_STATE', 'synthesize: Synthesis is already in progress.');
+ }
+
+ if (!text || !text.trim()) {
+ throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: Input text cannot be empty.');
+ }
+
+ const { language } = options;
+ checkLanguage(language, 'synthesize');
+ const voice = resolveVoice(options.voice);
+ const references = voice.codes.length / CODEBOOKS;
+
+ const speed = options.speed ?? DEFAULT_SPEED;
+ if (!(speed >= MIN_SPEED && speed <= MAX_SPEED)) {
+ throw RnExecuTorchError(
+ 'INVALID_ARGUMENT',
+ `synthesize: speed must be between ${MIN_SPEED} and ${MAX_SPEED}.`
+ );
+ }
+
+ // A chunk at the limit must still fit the backbone's target window.
+ const maxChunkDuration = options.maxChunkDuration ?? DEFAULT_MAX_CHUNK_DURATION;
+ const maxDuration = maxFrames / REVERB_FRAME_RATE;
+ if (!(maxChunkDuration > 0 && maxChunkDuration <= maxDuration)) {
+ throw RnExecuTorchError(
+ 'INVALID_ARGUMENT',
+ `synthesize: maxChunkDuration must be above 0 and at most ${maxDuration} s.`
+ );
+ }
+
+ const steps = options.sampling?.steps ?? DEFAULT_STEPS;
+ const remaskRatio = options.sampling?.remaskRatio ?? DEFAULT_REMASK_RATIO;
+ const seed = options.sampling?.seed ?? Math.floor(Math.random() * 0x100000000);
+ if (!Number.isInteger(steps) || steps <= 0) {
+ throw RnExecuTorchError(
+ 'INVALID_ARGUMENT',
+ 'synthesize: steps must be a positive integer.'
+ );
+ }
+ if (!(remaskRatio >= 0 && remaskRatio <= 1)) {
+ throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: remaskRatio must be in [0, 1].');
+ }
+ if (!Number.isInteger(seed)) {
+ throw RnExecuTorchError('INVALID_ARGUMENT', 'synthesize: seed must be an integer.');
+ }
+ const postprocess =
+ options.postprocess === false
+ ? false
+ : {
+ endCut: options.postprocess?.endCut ?? DEFAULT_POSTPROCESS.endCut,
+ silenceThreshold:
+ options.postprocess?.silenceThreshold ?? DEFAULT_POSTPROCESS.silenceThreshold,
+ keep: options.postprocess?.keep ?? DEFAULT_POSTPROCESS.keep,
+ };
+
+ const encodePrompt = (chunk: string) =>
+ tokenizer.encode(buildPrompt(language, voice.transcript, chunk));
+ if (encodePrompt('').length >= promptTokens.max) {
+ throw RnExecuTorchError(
+ 'INVALID_ARGUMENT',
+ `synthesize: The voice's transcript leaves no room for text in the ` +
+ `${promptTokens.max}-token prompt.`
+ );
+ }
+ const targetLength = (chunk: string) =>
+ estimateTargetFrames(chunk, voice.transcript, references) / speed;
+
+ isSynthesizing = true;
+ try {
+ const chunks = partitionReverbText(
+ prepareText(text, language),
+ (chunk) => targetLength(chunk) / REVERB_FRAME_RATE,
+ {
+ maxDuration: maxChunkDuration,
+ fits: (chunk) => encodePrompt(chunk).length <= promptTokens.max,
+ }
+ );
+
+ for (const [chunkIndex, chunk] of chunks.entries()) {
+ if (!isSynthesizing) break;
+
+ const ids = encodePrompt(chunk);
+ if (ids.length > promptTokens.max) {
+ throw RnExecuTorchError(
+ 'INVALID_ARGUMENT',
+ `synthesize: "${chunk}" does not fit the ${promptTokens.max}-token prompt.`
+ );
+ }
+ const frames = Math.min(
+ Math.max(roundHalfEven(targetLength(chunk)), minFrames),
+ maxFrames
+ );
+ const audio = await synthesizeChunk(ids, voice, frames, {
+ steps,
+ remaskRatio,
+ seed: seed + chunkIndex,
+ postprocess,
+ pause: pauseAfter(chunk),
+ });
+ yield {
+ audio,
+ sampleRate: REVERB_SAMPLE_RATE,
+ duration: audio.length / REVERB_SAMPLE_RATE,
+ chunkIndex,
+ totalChunks: chunks.length,
+ };
+ }
+ } finally {
+ isSynthesizing = false;
+ }
+ }
+
+ return { dispose, encodeVoice, synthesize, synthesizeStop };
+ } catch (error) {
+ dispose();
+ throw error;
+ }
+}
diff --git a/packages/react-native-executorch/src/extensions/speech/utils/reverbPartitioner.ts b/packages/react-native-executorch/src/extensions/speech/utils/reverbPartitioner.ts
new file mode 100644
index 0000000000..0d98a8d1ba
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/utils/reverbPartitioner.ts
@@ -0,0 +1,92 @@
+/**
+ * Splits long text into chunks within a duration limit.
+ */
+
+import { RnExecuTorchError } from '../../../core/error';
+import { EOS_PATTERN, PAUSE_PATTERN } from './textPartitioner';
+
+const DEFAULT_MAX_DURATION = 20;
+const CLOSERS = /["'”’»)\]}]+$/u; // closing quotes and brackets after a word's punctuation
+
+/**
+ * Options for {@link partitionReverbText}.
+ * @category Speech / Types
+ */
+export type ReverbPartitionOptions = {
+ /** Max seconds per chunk. Defaults to 20. */
+ readonly maxDuration?: number;
+ /** A further condition every chunk must meet. */
+ readonly fits?: (chunk: string) => boolean;
+};
+
+// Splits words into runs, each ending at a word whose final mark (past closers) matches `boundary`.
+function splitAfter(words: readonly string[], boundary: RegExp): string[][] {
+ const runs: string[][] = [[]];
+ for (const word of words) {
+ runs[runs.length - 1]!.push(word);
+ if (boundary.test(word.replace(CLOSERS, '').slice(-1))) runs.push([]);
+ }
+ return runs.filter((run) => run.length > 0);
+}
+
+/**
+ * Splits text into chunks of at most `maxDuration` seconds each. Greedy,
+ * coarsest boundary first: whole sentences, then other punctuation, then words.
+ * A single word that doesn't fit is a chunk of its own.
+ * @category Speech / Functions
+ * @param text The text.
+ * @param duration Estimated seconds of speech of a chunk.
+ * @param options See {@link ReverbPartitionOptions}.
+ * @returns The chunks, which joined with spaces give back the text.
+ * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if `maxDuration` is
+ * not positive.
+ */
+export function partitionReverbText(
+ text: string,
+ duration: (chunk: string) => number,
+ options?: ReverbPartitionOptions
+): string[] {
+ const maxDuration = options?.maxDuration ?? DEFAULT_MAX_DURATION;
+ if (!(maxDuration > 0)) {
+ throw RnExecuTorchError(
+ 'INVALID_ARGUMENT',
+ 'partitionReverbText: maxDuration must be positive.'
+ );
+ }
+ const fits = (words: readonly string[]): boolean => {
+ const chunk = words.join(' ');
+ return duration(chunk) <= maxDuration && (options?.fits?.(chunk) ?? true);
+ };
+
+ // A unit too long on its own is replaced by its pieces at the next finer boundary.
+ const boundaries = [EOS_PATTERN, PAUSE_PATTERN];
+ const queue = splitAfter(text.split(/\s+/).filter(Boolean), EOS_PATTERN).map((words) => ({
+ words,
+ level: 0,
+ }));
+ const chunks: string[] = [];
+ let current: string[] = [];
+
+ while (queue.length > 0) {
+ const unit = queue.shift()!;
+ if (fits([...current, ...unit.words])) {
+ current = [...current, ...unit.words];
+ } else if (current.length > 0) {
+ chunks.push(current.join(' '));
+ current = [];
+ queue.unshift(unit);
+ } else if (unit.words.length === 1) {
+ chunks.push(unit.words[0]!);
+ } else {
+ let level = unit.level + 1;
+ let pieces = level < boundaries.length ? splitAfter(unit.words, boundaries[level]!) : [];
+ if (pieces.length < 2) {
+ level = boundaries.length;
+ pieces = unit.words.map((word) => [word]);
+ }
+ queue.unshift(...pieces.map((words) => ({ words, level })));
+ }
+ }
+ if (current.length > 0) chunks.push(current.join(' '));
+ return chunks;
+}
diff --git a/packages/react-native-executorch/src/extensions/speech/utils/reverbUtils.ts b/packages/react-native-executorch/src/extensions/speech/utils/reverbUtils.ts
new file mode 100644
index 0000000000..d89d0f96db
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/utils/reverbUtils.ts
@@ -0,0 +1,192 @@
+/**
+ * Text preparation, length estimate, and post-processing utilities for Reverb.
+ */
+
+import { Verbalizer, type LanguageHandle } from 'verbalis/core';
+import { language as de } from 'verbalis/de';
+import { language as en } from 'verbalis/en';
+import { language as es } from 'verbalis/es';
+import { language as fr } from 'verbalis/fr';
+import { language as it } from 'verbalis/it';
+import { language as pl } from 'verbalis/pl';
+import { language as pt } from 'verbalis/pt';
+
+import { RnExecuTorchError } from '../../../core/error';
+import type { ReverbLanguage, ReverbVoice } from '../tasks/reverbTextToSpeech';
+
+const BLIP_WINDOWS = 12; // an edge sound of at most 120 ms is a codec burst
+const SOUND_GAP_WINDOWS = 20; // loud windows over 200 ms apart are separate sounds
+
+const VERBALIS_LANGUAGES: Record LanguageHandle> = {
+ de,
+ en,
+ es,
+ fr,
+ it,
+ pl,
+ pt,
+};
+const verbalizers: Partial> = {};
+
+/**
+ * Parses a voice file's JSON: `{ codes, transcript, language }`, the codes as
+ * `8 × frames` integers, row-major.
+ * @param json The parsed JSON.
+ * @returns The voice.
+ * @throws {RnExecuTorchError} With code `LOAD_FAILED` on a missing or
+ * mistyped field.
+ */
+export function parseVoice(json: any): ReverbVoice {
+ const { codes, transcript, language } = json ?? {};
+ const valid =
+ Array.isArray(codes) &&
+ codes.every((code) => Number.isInteger(code)) &&
+ typeof transcript === 'string' &&
+ typeof language === 'string';
+ if (!valid) {
+ throw RnExecuTorchError('LOAD_FAILED', 'parseVoice: Invalid voice JSON format.');
+ }
+ return {
+ codes: BigInt64Array.from(codes as number[], BigInt),
+ transcript,
+ language: language as ReverbLanguage,
+ };
+}
+
+/**
+ * Writes text the way it is read aloud with Verbalis: numbers, symbols, and
+ * abbreviations as words, and quotes, dashes, ellipses, and spaces in the forms
+ * Reverb was trained on. JS thread only.
+ * @param text The text.
+ * @param language The text's language.
+ * @returns The prepared text.
+ */
+export function prepareText(text: string, language: ReverbLanguage): string {
+ const verbalizer = (verbalizers[language] ??= new Verbalizer(VERBALIS_LANGUAGES[language]()));
+ return verbalizer.process(text);
+}
+
+/**
+ * The backbone's prompt: the text's language, then the voice's transcript and
+ * the text.
+ * @param language The text's language.
+ * @param transcript The voice's transcript.
+ * @param text The text to speak.
+ * @returns The prompt.
+ */
+export function buildPrompt(language: ReverbLanguage, transcript: string, text: string): string {
+ return (
+ `<|denoise|><|lang_start|>${language}<|lang_end|><|instruct_start|>None<|instruct_end|>` +
+ `<|text_start|>${transcript} ${text}<|text_end|>`
+ );
+}
+
+// Spoken weight by Unicode class: spaces, marks, punctuation and symbols, digits, the rest.
+function spokenWeight(text: string): number {
+ 'worklet';
+ let weight = 0;
+ for (const char of text) {
+ if (/\p{Z}/u.test(char)) weight += 0.2;
+ else if (/\p{M}/u.test(char)) weight += 0;
+ else if (/[\p{P}\p{S}]/u.test(char)) weight += 0.5;
+ else if (/\p{N}/u.test(char)) weight += 3.5;
+ else weight += 1;
+ }
+ return weight;
+}
+
+/**
+ * Estimates the codec frames (25 per second) it takes to speak `text` at a
+ * reference clip's pace, as Echo's `estimate_frames` does.
+ * @category Speech / Functions
+ * @param text The text to speak.
+ * @param referenceText The reference clip's transcript.
+ * @param referenceFrames The reference clip's length in frames.
+ * @returns The estimate, at least 1.
+ */
+export function estimateTargetFrames(
+ text: string,
+ referenceText: string,
+ referenceFrames: number
+): number {
+ 'worklet';
+ let frames = (spokenWeight(text) / Math.max(spokenWeight(referenceText), 1)) * referenceFrames;
+ if (frames < 50) frames = 50 * Math.pow(frames / 50, 1 / 3); // short texts get relatively more time
+ return Math.max(1, Math.floor(frames));
+}
+
+/**
+ * Rounds half to even, like Python's `round`.
+ * @param value The value.
+ * @returns The rounded value.
+ */
+export function roundHalfEven(value: number): number {
+ 'worklet';
+ const rounded = Math.round(value);
+ return rounded - value === 0.5 && rounded % 2 !== 0 ? rounded - 1 : rounded;
+}
+
+/**
+ * Trims silent edges like Voice Studio: in 10 ms windows, dropping an isolated
+ * edge sound of at most 120 ms.
+ * @param audio The audio.
+ * @param sampleRate Its sampling rate in Hz.
+ * @param threshold Level in dBFS below which a window is silent.
+ * @param keep Seconds of silence kept at each edge.
+ * @returns A view of `audio` without the silent edges, or `audio` if all silent.
+ */
+export function trimSilence(
+ audio: Float32Array,
+ sampleRate: number,
+ threshold: number,
+ keep: number
+): Float32Array {
+ 'worklet';
+ const window = Math.floor(sampleRate / 100);
+ const windows = Math.floor(audio.length / window);
+ const sounds: { first: number; last: number; loud: number }[] = [];
+ for (let w = 0; w < windows; w++) {
+ let energy = 0;
+ for (let i = w * window; i < (w + 1) * window; i++) {
+ const sample = audio[i]!;
+ energy += sample * sample; // `** 2` makes this loop 2.5x slower in Hermes
+ }
+ if (20 * Math.log10(Math.sqrt(energy / window) + 1e-10) < threshold) continue;
+ const sound = sounds[sounds.length - 1];
+ if (sound && w - sound.last <= SOUND_GAP_WINDOWS) {
+ sound.last = w;
+ sound.loud++;
+ } else {
+ sounds.push({ first: w, last: w, loud: 1 });
+ }
+ }
+ if (sounds.length === 0) return audio;
+
+ if (sounds.length > 1 && sounds[0]!.loud <= BLIP_WINDOWS) sounds.shift();
+ if (sounds.length > 1 && sounds[sounds.length - 1]!.loud <= BLIP_WINDOWS) sounds.pop();
+ const margin = roundHalfEven(keep * 100);
+ const start = Math.max(sounds[0]!.first - margin, 0);
+ const end = Math.min(sounds[sounds.length - 1]!.last + 1 + margin, windows);
+ return audio.subarray(start * window, end * window);
+}
+
+/**
+ * Pause after a chunk by its final punctuation, in milliseconds: Kokoro's
+ * values, `...` taking the ellipsis's.
+ */
+// prettier-ignore
+export const REVERB_PAUSE_MS: Record = {
+ '.': 375, '?': 500, '!': 250, ';': 400, '...': 1000, ',': 130, ':': 250, '-': 200,
+};
+
+/**
+ * The pause after a chunk, by its final punctuation past closing quotes.
+ * @param chunk The chunk.
+ * @returns The pause in seconds.
+ */
+export function pauseAfter(chunk: string): number {
+ 'worklet';
+ const end = chunk.replace(/[\s"'”’»)\]}]+$/u, '');
+ const mark = end.endsWith('...') ? '...' : end.slice(-1);
+ return (REVERB_PAUSE_MS[mark] ?? 0) / 1000;
+}
diff --git a/packages/react-native-executorch/src/extensions/speech/utils/textPartitioner.ts b/packages/react-native-executorch/src/extensions/speech/utils/textPartitioner.ts
index 6651beaa34..720233a6cc 100644
--- a/packages/react-native-executorch/src/extensions/speech/utils/textPartitioner.ts
+++ b/packages/react-native-executorch/src/extensions/speech/utils/textPartitioner.ts
@@ -6,9 +6,17 @@
import { RnExecuTorchError } from '../../../core/error';
type Tag = 'eos' | 'pause' | 'whitespace';
-// Punctuation Regex Patterns
-const EOS_PATTERN = /[.?!;…|।॥¿¡。.!?;]/;
-const PAUSE_PATTERN = /[,:\u2014\u00AB\u00BB、,:〜~]/;
+/**
+ * Sentence-ending punctuation.
+ * @category Speech / Constants
+ */
+export const EOS_PATTERN = /[.?!;…|।॥¿¡。.!?;]/;
+
+/**
+ * Punctuation pausing within a sentence: commas, colons, dashes.
+ * @category Speech / Constants
+ */
+export const PAUSE_PATTERN = /[,:\u2014\u00AB\u00BB、,:〜~]/;
const WHITESPACE_PATTERN = /\s/;
// Standard Partitioning Constants
diff --git a/packages/react-native-executorch/src/hooks/useTextToSpeech.ts b/packages/react-native-executorch/src/hooks/useTextToSpeech.ts
index 643636ee43..fbe0409d5b 100644
--- a/packages/react-native-executorch/src/hooks/useTextToSpeech.ts
+++ b/packages/react-native-executorch/src/hooks/useTextToSpeech.ts
@@ -10,6 +10,11 @@ import {
type SupertonicTtsModel,
type SupertonicTextToSpeech,
} from '../extensions/speech/tasks/supertonicTextToSpeech';
+import {
+ createReverbTextToSpeech,
+ type ReverbTtsModel,
+ type ReverbTextToSpeech,
+} from '../extensions/speech/tasks/reverbTextToSpeech';
/**
* React hook to load and run the Kokoro Text-to-Speech pipeline.
@@ -82,20 +87,55 @@ export function useTextToSpeech(
synthesizeStop: SupertonicTextToSpeech['synthesizeStop'] | undefined;
};
+/**
+ * React hook to download, load, and run the Reverb Text-to-Speech pipeline,
+ * releasing native memory on unmount or config change.
+ *
+ * For imperative usage, see {@link createReverbTextToSpeech}.
+ * @category Hooks
+ * @typeParam K Preset voice keys.
+ * @param config The Reverb TTS model configuration. See {@link ReverbTtsModel}.
+ * @param options Load and caching options. See {@link ResourceOptions}.
+ * @returns The same object as {@link ReverbTextToSpeech} (without `dispose`),
+ * combined with loading state and download progress.
+ * @see {@link ReverbTextToSpeech}
+ */
+export function useTextToSpeech(
+ config: ReverbTtsModel,
+ options?: ResourceOptions
+): {
+ /** Whether the pipeline is loaded and ready to synthesize. */
+ isReady: boolean;
+ /** The download or load error, if any. */
+ error: Error | undefined;
+ /** Download progress across every asset, in percent. */
+ downloadProgress: number;
+ /** The config with every remote URL resolved to a local path. */
+ resource: ReverbTtsModel | undefined;
+ /** Streams synthesized audio chunks. Undefined until the pipeline is ready. */
+ synthesize: ReverbTextToSpeech['synthesize'] | undefined;
+ /** Cancels an in-flight synthesis. Undefined until the pipeline is ready. */
+ synthesizeStop: ReverbTextToSpeech['synthesizeStop'] | undefined;
+ /** Encodes a recording into a voice. Undefined until the pipeline is ready. */
+ encodeVoice: ReverbTextToSpeech['encodeVoice'] | undefined;
+};
+
export function useTextToSpeech(
- config: KokoroTtsModel | SupertonicTtsModel,
+ config: KokoroTtsModel | SupertonicTtsModel | ReverbTtsModel,
options?: ResourceOptions
) {
// Each config names the pipeline it belongs to, so the factory is resolved
// from that tag alone.
const factory = (
- ttsConfig: KokoroTtsModel | SupertonicTtsModel
- ): Promise | SupertonicTextToSpeech> => {
+ ttsConfig: KokoroTtsModel | SupertonicTtsModel | ReverbTtsModel
+ ): Promise | SupertonicTextToSpeech | ReverbTextToSpeech> => {
switch (ttsConfig.name) {
case 'kokoro':
return createKokoroTextToSpeech(ttsConfig);
case 'supertonic':
return createSupertonicTextToSpeech(ttsConfig);
+ case 'reverb':
+ return createReverbTextToSpeech(ttsConfig);
}
};
@@ -109,5 +149,7 @@ export function useTextToSpeech(
resource,
synthesize: model?.synthesize,
synthesizeStop: model?.synthesizeStop,
+ // Only Reverb encodes voices.
+ ...(model && 'encodeVoice' in model ? { encodeVoice: model.encodeVoice } : {}),
};
}
diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts
index b5c6fc885f..b71db4a83a 100644
--- a/packages/react-native-executorch/src/index.ts
+++ b/packages/react-native-executorch/src/index.ts
@@ -98,6 +98,7 @@ export * from './extensions/speech/tasks/whisperSpeechToText';
export * from './extensions/speech/tasks/qwen3AsrSpeechToText';
export * from './extensions/speech/tasks/supertonicTextToSpeech';
export * from './extensions/speech/tasks/kokoroTextToSpeech';
+export * from './extensions/speech/tasks/reverbTextToSpeech';
// Vector search over embeddings
export * from './extensions/vectorStore';
diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts
index 3637134b94..08d5ada0ea 100644
--- a/packages/react-native-executorch/src/models.ts
+++ b/packages/react-native-executorch/src/models.ts
@@ -34,6 +34,7 @@ import type { LLMTranslatorModel } from './extensions/llm/tasks/llmTranslation';
import type { FsmnVadModel } from './extensions/speech/tasks/fsmnVoiceActivityDetection';
import type { SupertonicTtsModel } from './extensions/speech/tasks/supertonicTextToSpeech';
import type { KokoroTtsModel } from './extensions/speech/tasks/kokoroTextToSpeech';
+import type { ReverbTtsModel } from './extensions/speech/tasks/reverbTextToSpeech';
import type { PhonemizerLanguage } from './extensions/speech/utils/phonemizer';
import {
type WhisperSttModel,
@@ -52,6 +53,7 @@ import {
FACEMESH_LANDMARKS,
COCO_LANDMARKS,
SUPERTONIC_DEFAULT_VOICE_NAMES,
+ REVERB_DEFAULT_VOICE_NAMES,
PRIVACY_FILTER_OPENAI_LABELS,
PRIVACY_FILTER_NEMOTRON_LABELS,
EUROLLM_LANGUAGES,
@@ -66,6 +68,7 @@ import {
type BlazeFaceLandmark,
type CocoLandmark,
type SupertonicDefaultVoiceName,
+ type ReverbDefaultVoiceName,
} from './constants';
// =============================================================================
@@ -89,8 +92,9 @@ const PLATFORM: TargetPlatform = Platform.OS === 'ios' ? 'ios' : 'android';
// Accelerators lead and XNNPACK trails: a model is only exported to Core ML,
// MLX or Vulkan once it has been shown to run better there, and XNNPACK is the
-// one backend every model exports to. Core ML sits above MLX only to make the
-// order deterministic; every group publishing both pins its winner explicitly.
+// one backend nearly every model exports to. Core ML sits above MLX only to
+// make the order deterministic; every group publishing both pins its winner
+// explicitly.
//
// The iOS simulator links the Core ML backend but cannot run it: no Neural
// Engine, and MPSGraph refuses the compiled models. MLX only ever ships a
@@ -1826,6 +1830,57 @@ const KOKORO_ZH_COREML_FP32: KokoroTtsModel = {
voices: KOKORO_ZH_VOICES,
};
+// The backbone, tokenizer and voices live in the model's own repo, the one
+// registry repo without an -ExecuTorch name; the codec in the Higgs tokenizer's.
+const REVERB_ROOT = `${BASE_URL}/Reverb-TTS/${NEXT_VERSION_TAG}`;
+const REVERB_CODEC_ROOT = `${BASE_URL}/higgs-audio-v2-tokenizer-ExecuTorch/${NEXT_VERSION_TAG}`;
+const REVERB_TOKENIZER = `${REVERB_ROOT}/model/tokenizer.json`;
+const REVERB_DEFAULT_VOICES = REVERB_DEFAULT_VOICE_NAMES.reduce(
+ (acc, name) => ({ ...acc, [name]: `${REVERB_ROOT}/voices/${name}.json` }),
+ {} as Record
+);
+
+const REVERB_MLX_COREML_FP16: ReverbTtsModel = {
+ name: 'reverb',
+ modelPaths: {
+ backbone: `${REVERB_ROOT}/model/executorch/mlx/fp16/echo_mlx_fp16.pte`,
+ codecDecoder: `${REVERB_CODEC_ROOT}/decoder/coreml/fp16/codec_decoder_coreml_fp16.pte`,
+ codecEncoder: `${REVERB_CODEC_ROOT}/encoder/coreml/int8/codec_encoder_coreml_int8.pte`,
+ },
+ tokenizerPath: REVERB_TOKENIZER,
+ voices: REVERB_DEFAULT_VOICES,
+};
+const REVERB_MLX_COREML_INT8: ReverbTtsModel = {
+ name: 'reverb',
+ modelPaths: {
+ backbone: `${REVERB_ROOT}/model/executorch/mlx/int8/echo_mlx_int8.pte`,
+ codecDecoder: `${REVERB_CODEC_ROOT}/decoder/coreml/fp16/codec_decoder_coreml_fp16.pte`,
+ codecEncoder: `${REVERB_CODEC_ROOT}/encoder/coreml/int8/codec_encoder_coreml_int8.pte`,
+ },
+ tokenizerPath: REVERB_TOKENIZER,
+ voices: REVERB_DEFAULT_VOICES,
+};
+const REVERB_MLX_COREML_INT4: ReverbTtsModel = {
+ name: 'reverb',
+ modelPaths: {
+ backbone: `${REVERB_ROOT}/model/executorch/mlx/int4/echo_mlx_int4.pte`,
+ codecDecoder: `${REVERB_CODEC_ROOT}/decoder/coreml/fp16/codec_decoder_coreml_fp16.pte`,
+ codecEncoder: `${REVERB_CODEC_ROOT}/encoder/coreml/int4/codec_encoder_coreml_int4.pte`,
+ },
+ tokenizerPath: REVERB_TOKENIZER,
+ voices: REVERB_DEFAULT_VOICES,
+};
+const REVERB_VULKAN_INT4: ReverbTtsModel = {
+ name: 'reverb',
+ modelPaths: {
+ backbone: `${REVERB_ROOT}/model/executorch/vulkan/int4/echo_vulkan_int4.pte`,
+ codecDecoder: `${REVERB_CODEC_ROOT}/decoder/vulkan/fp16/codec_decoder_vulkan_fp16.pte`,
+ codecEncoder: `${REVERB_CODEC_ROOT}/encoder/vulkan/int8/codec_encoder_vulkan_int8.pte`,
+ },
+ tokenizerPath: REVERB_TOKENIZER,
+ voices: REVERB_DEFAULT_VOICES,
+};
+
// =============================================================================
// Privacy Filter
// =============================================================================
@@ -3712,6 +3767,22 @@ export const models = {
COREML_FP32: KOKORO_ZH_COREML_FP32,
}),
},
+
+ /**
+ * Reverb — multilingual Text-to-Speech that speaks in the voice of any
+ * 1-20 s recording, given its transcript, in 7 languages. Ships three
+ * preset voices (see {@link REVERB_DEFAULT_VOICE_NAMES}).
+ */
+ // int8 leads on iOS: within ~20% of fp16's speed in the app at 260 MB
+ // less. All decode with the fp16 codec (the int8 Core ML one holds ~600 MB
+ // more on the GPU); INT4 encodes with int4 (40 MB smaller), the rest with
+ // int8. No XNNPACK export: the backbone needs MLX or Vulkan.
+ REVERB: variants({
+ MLX_COREML_INT8: REVERB_MLX_COREML_INT8,
+ MLX_COREML_FP16: REVERB_MLX_COREML_FP16,
+ MLX_COREML_INT4: REVERB_MLX_COREML_INT4,
+ VULKAN_INT4: REVERB_VULKAN_INT4,
+ }),
},
/**
diff --git a/packages/react-native-executorch/tsconfig.build.json b/packages/react-native-executorch/tsconfig.build.json
index cf63e46cf5..0f9f4ff56b 100644
--- a/packages/react-native-executorch/tsconfig.build.json
+++ b/packages/react-native-executorch/tsconfig.build.json
@@ -1,4 +1,4 @@
{
"extends": "./tsconfig",
- "exclude": ["example", "lib", "__tests__", "jest.config.js"]
+ "exclude": ["example", "lib", "__tests__", "jest.config.js", "third-party"]
}
diff --git a/packages/react-native-executorch/tsconfig.json b/packages/react-native-executorch/tsconfig.json
index f355221ebe..d6fafac7ef 100644
--- a/packages/react-native-executorch/tsconfig.json
+++ b/packages/react-native-executorch/tsconfig.json
@@ -27,5 +27,6 @@
"strict": true,
"target": "ESNext",
"verbatimModuleSyntax": true
- }
+ },
+ "exclude": ["node_modules", "lib", "third-party"]
}
diff --git a/yarn.lock b/yarn.lock
index 0d19f9e435..4cc305b2b2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -13001,6 +13001,7 @@ __metadata:
react-native-worklets: "npm:0.13.0"
test-renderer: "npm:^1.2.0"
typescript: "npm:~5.9.2"
+ verbalis: "npm:^0.1.1"
peerDependencies:
"@kesha-antonov/react-native-background-downloader": ">=4.4.0"
react: "*"
@@ -15046,6 +15047,13 @@ __metadata:
languageName: node
linkType: hard
+"verbalis@npm:^0.1.1":
+ version: 0.1.1
+ resolution: "verbalis@npm:0.1.1"
+ checksum: 10/012f4789d08a0e3ea729db7f192f2e453e525f90bc278e780e1ca17baa82e9f47ab61c8ce7307d4742142fb218f1e57556229719164d9cb25c9631584b096922
+ languageName: node
+ linkType: hard
+
"vlq@npm:^1.0.0":
version: 1.0.1
resolution: "vlq@npm:1.0.1"