diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index defed32659..cf5470a0d2 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -450,3 +450,10 @@ eurollm doclayout netp Uyghur +codebook +codebooks +remask +higgs +verbalizers +workletized +partitioners diff --git a/apps/speech/app/_layout.tsx b/apps/speech/app/_layout.tsx index e98be4cd43..546f2ef6f4 100644 --- a/apps/speech/app/_layout.tsx +++ b/apps/speech/app/_layout.tsx @@ -55,6 +55,13 @@ export default function Layout() { title: 'Text-to-Speech (Kokoro)', }} /> + ); } diff --git a/apps/speech/app/index.tsx b/apps/speech/app/index.tsx index 61cd5f6097..80d5a8a344 100644 --- a/apps/speech/app/index.tsx +++ b/apps/speech/app/index.tsx @@ -35,6 +35,12 @@ export default function Home() { > Text-to-Speech (Kokoro) + router.navigate('reverb-text-to-speech/')} + > + Text-to-Speech (Reverb) + ); diff --git a/apps/speech/app/reverb-text-to-speech/index.tsx b/apps/speech/app/reverb-text-to-speech/index.tsx new file mode 100644 index 0000000000..ff8da83fad --- /dev/null +++ b/apps/speech/app/reverb-text-to-speech/index.tsx @@ -0,0 +1,1059 @@ +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import { + Animated, + Platform, + ScrollView, + StyleSheet, + Text, + TextInput, + TouchableOpacity, + View, +} from 'react-native'; +import { AudioManager, AudioRecorder } from 'react-native-audio-api'; +import RNBlobUtil from 'react-native-blob-util'; +import DeviceInfo from 'react-native-device-info'; +import { + models, + useSpeechToText, + useTextToSpeech, + REVERB_DEFAULT_VOICE_NAMES, + REVERB_FRAME_RATE, + REVERB_SAMPLE_RATE, + WHISPER_SAMPLE_RATE_HZ, + type ReverbDefaultVoiceName, + type ReverbLanguage, + type ReverbVoice, +} from 'react-native-executorch'; + +import ScreenWrapper from '../../components/ScreenWrapper'; +import { ModelPicker } from '../../components/ModelPicker'; +import { ModelStatus } from '../../components/ModelStatus'; +import { Button } from '../../components/Button'; +import { useAudioPlayer } from '../../hooks/useAudioPlayer'; +import { theme } from '../../theme'; + +type ReverbVariant = Exclude; + +const MODEL_OPTIONS: { label: string; value: ReverbVariant; platform: typeof Platform.OS }[] = [ + { label: 'MLX int8', value: 'MLX_COREML_INT8', platform: 'ios' }, + { label: 'MLX fp16', value: 'MLX_COREML_FP16', platform: 'ios' }, + { label: 'MLX int4', value: 'MLX_COREML_INT4', platform: 'ios' }, + { label: 'Vulkan int4', value: 'VULKAN_INT4', platform: 'android' }, +]; + +// Transcribes the recordings a new voice is made from. +const WHISPER_MODEL = models.speechToText.WHISPER.BASE.DEFAULT; + +const LANGUAGES: { label: string; value: ReverbLanguage }[] = [ + { label: 'English', value: 'en' }, + { label: 'German', value: 'de' }, + { label: 'Spanish', value: 'es' }, + { label: 'French', value: 'fr' }, + { label: 'Italian', value: 'it' }, + { label: 'Polish', value: 'pl' }, + { label: 'Portuguese', value: 'pt' }, +]; + +// cspell:disable +const SAMPLE_TEXTS: Record = { + en: 'Reverb speaks in any voice, right on your phone.', + de: 'Reverb spricht mit jeder Stimme, direkt am Handy.', + es: 'Reverb habla con cualquier voz, en tu teléfono.', + fr: "Reverb parle avec n'importe quelle voix, ici.", + it: 'Reverb parla con qualsiasi voce, sul tuo telefono.', + pl: 'Reverb mówi dowolnym głosem, prosto w telefonie.', + pt: 'O Reverb fala com qualquer voz, no seu telefone.', +}; +// cspell:enable + +const PRESETS: { name: ReverbDefaultVoiceName; title: string; subtitle: string }[] = [ + { name: 'standard_male', title: 'Standard', subtitle: 'Male · calm' }, + { name: 'angry_male', title: 'Angry', subtitle: 'Male · intense' }, + { name: 'sensual_female', title: 'Sensual', subtitle: 'Female · soft' }, +]; + +const SPEED_OPTIONS = [0.8, 0.9, 1.0, 1.1, 1.25].map((value) => ({ label: `${value}x`, value })); +const STEPS_OPTIONS = [4, 6, 8, 12, 16].map((value) => ({ + label: value === 8 ? '8 (default)' : `${value}`, + value, +})); +const VARIATION_OPTIONS = [0, 0.15, 0.25, 0.4].map((value) => ({ + label: value === 0.25 ? '0.25 (default)' : `${value}`, + value, +})); + +const MAX_RECORDING_S = 20; +const MIN_RECORDING_S = 1; +const SILENCE_DB = -45; +const SILENCE_BELOW_PEAK_DB = 25; +const METER_BARS = 24; + +type PresetInfo = { transcript: string; language: string; duration: number }; +type CustomVoice = { id: number; name: string; voice: ReverbVoice; duration: number }; +type VoiceChoice = + | { kind: 'preset'; name: ReverbDefaultVoiceName } + | { kind: 'custom'; id: number }; + +const isSimulator = DeviceInfo.isEmulatorSync(); + +const messageOf = (err: unknown) => (err instanceof Error ? err.message : String(err)); + +const formatSeconds = (s: number) => + `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, '0')}`; + +function concat(chunks: readonly Float32Array[]): Float32Array { + const out = new Float32Array(chunks.reduce((n, chunk) => n + chunk.length, 0)); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; +} + +// Linear interpolation up; down, the mean of the input samples each output covers (a crude +// anti-alias filter). +function resample(input: Float32Array, from: number, to: number): Float32Array { + if (from === to) return input; + const ratio = from / to; + const out = new Float32Array(Math.floor(input.length / ratio)); + for (let i = 0; i < out.length; i++) { + if (ratio > 1) { + const start = Math.floor(i * ratio); + const end = Math.min(input.length, Math.max(start + 1, Math.floor((i + 1) * ratio))); + let sum = 0; + for (let j = start; j < end; j++) sum += input[j]!; + out[i] = sum / (end - start); + } else { + const x = i * ratio; + const j = Math.floor(x); + out[i] = input[j]! + (input[Math.min(j + 1, input.length - 1)]! - input[j]!) * (x - j); + } + } + return out; +} + +const rms = (audio: Float32Array, start = 0, end = audio.length) => { + let sum = 0; + for (let i = start; i < end; i++) sum += audio[i]! * audio[i]!; + return Math.sqrt(sum / Math.max(1, end - start)); +}; + +// Drops the silence before the first and after the last word, keeping 0.15 s at each edge. Silence +// is relative to the loudest 20 ms: a phone's noise floor can sit above a fixed level (-42 dBFS +// measured on a Galaxy S24 FE). +function trimSilence(audio: Float32Array, rate: number): Float32Array { + const win = Math.round(rate * 0.02); + const levels: number[] = []; + for (let start = 0; start < audio.length; start += win) { + levels.push(rms(audio, start, Math.min(start + win, audio.length))); + } + const peak = levels.reduce((max, level) => Math.max(max, level), 0); + const threshold = Math.max(10 ** (SILENCE_DB / 20), peak * 10 ** (-SILENCE_BELOW_PEAK_DB / 20)); + const first = levels.findIndex((level) => level >= threshold); + if (first < 0) return audio; + let last = levels.length - 1; + while (levels[last]! < threshold) last--; + const pad = Math.round(rate * 0.15); + return audio.slice( + Math.max(0, first * win - pad), + Math.min(audio.length, (last + 1) * win + pad) + ); +} + +// Records mono audio, resampled to Reverb's 24 kHz when the device delivers another rate. +function useVoiceRecorder() { + const recorderRef = useRef(null); + const chunksRef = useRef([]); + const samplesRef = useRef(0); + const rateRef = useRef(REVERB_SAMPLE_RATE); + const [isRecording, setIsRecording] = useState(false); + const [seconds, setSeconds] = useState(0); + const [level, setLevel] = useState(0); + + const start = useCallback(async (): Promise => { + if ((await AudioManager.requestRecordingPermissions()) !== 'Granted') { + return 'Microphone access was denied.'; + } + chunksRef.current = []; + samplesRef.current = 0; + setSeconds(0); + const recorder = new AudioRecorder(); + recorder.onAudioReady( + { sampleRate: REVERB_SAMPLE_RATE, bufferLength: REVERB_SAMPLE_RATE / 10, channelCount: 1 }, + (event) => { + const samples = new Float32Array(event.buffer.getChannelData(0)); + rateRef.current = event.buffer.sampleRate; + chunksRef.current.push(samples); + samplesRef.current += samples.length; + setSeconds(samplesRef.current / rateRef.current); + const db = 20 * Math.log10(rms(samples) + 1e-9); + setLevel(Math.min(1, Math.max(0, (db + 60) / 60))); + } + ); + const result = await recorder.start(); + if (result.status === 'error') { + recorder.clearOnAudioReady(); + return result.message; + } + recorderRef.current = recorder; + setIsRecording(true); + return null; + }, []); + + const stop = useCallback(async (): Promise => { + const recorder = recorderRef.current; + if (!recorder) return null; + recorderRef.current = null; + recorder.clearOnAudioReady(); + await recorder.stop().catch(() => {}); + setIsRecording(false); + setLevel(0); + const audio = concat(chunksRef.current); + chunksRef.current = []; + return resample(audio, rateRef.current, REVERB_SAMPLE_RATE); + }, []); + + useEffect(() => { + return () => { + recorderRef.current?.stop().catch(() => {}); + }; + }, []); + + return { isRecording, seconds, level, start, stop }; +} + +function VoiceTile({ + title, + subtitle, + selected, + disabled, + onPress, + onRemove, +}: { + title: string; + subtitle: string; + selected: boolean; + disabled: boolean; + onPress: () => void; + onRemove?: () => void; +}) { + return ( + + + + + {title.charAt(0)} + + + {onRemove && !disabled && ( + + ✕ + + )} + + + {title} + + + {subtitle} + + + ); +} + +function RecordButton({ + isRecording, + disabled, + onPress, +}: { + isRecording: boolean; + disabled: boolean; + onPress: () => void; +}) { + const pulse = useRef(new Animated.Value(0)).current; + + useEffect(() => { + if (!isRecording) return; + const loop = Animated.loop( + Animated.timing(pulse, { toValue: 1, duration: 1200, useNativeDriver: true }) + ); + loop.start(); + return () => { + loop.stop(); + pulse.setValue(0); + }; + }, [isRecording, pulse]); + + return ( + + {isRecording && ( + + )} + + + + + ); +} + +function LevelMeter({ level, active }: { level: number; active: boolean }) { + const lit = active ? Math.round(level * METER_BARS) : 0; + return ( + + {Array.from({ length: METER_BARS }, (_, i) => ( + + ))} + + ); +} + +function ReverbContent() { + const [selectedModel, setSelectedModel] = useState( + Platform.OS === 'android' ? 'VULKAN_INT4' : 'MLX_COREML_INT8' + ); + const [language, setLanguage] = useState('en'); + const [text, setText] = useState(SAMPLE_TEXTS.en); + const [choice, setChoice] = useState({ kind: 'preset', name: 'standard_male' }); + const [presetInfo, setPresetInfo] = useState>>( + {} + ); + const [customVoices, setCustomVoices] = useState([]); + const nextVoiceId = useRef(1); + + const [speed, setSpeed] = useState(1.0); + const [showSampler, setShowSampler] = useState(false); + const [steps, setSteps] = useState(6); + const [remaskRatio, setRemaskRatio] = useState(0.25); + const [fixedSeed, setFixedSeed] = useState(null); + const [lastSeed, setLastSeed] = useState(null); + const [trimEdges, setTrimEdges] = useState(true); + + const [isCloneOpen, setIsCloneOpen] = useState(false); + const [recording, setRecording] = useState(null); + const [transcript, setTranscript] = useState(''); + const [recordingLanguage, setRecordingLanguage] = useState('en'); + const [cloneStep, setCloneStep] = useState<'transcribing' | 'encoding' | null>(null); + + const [isSynthesizing, setIsSynthesizing] = useState(false); + const [chunkProgress, setChunkProgress] = useState(null); + const [result, setResult] = useState(null); + const [runError, setRunError] = useState(null); + + const { isPlaying, playStream, stop } = useAudioPlayer(REVERB_SAMPLE_RATE); + const recorder = useVoiceRecorder(); + + const { isReady, downloadProgress, error, resource, synthesize, synthesizeStop, encodeVoice } = + useTextToSpeech(models.textToSpeech.REVERB[selectedModel]); + const whisper = useSpeechToText(WHISPER_MODEL, { preventLoad: !isCloneOpen }); + + useEffect(() => { + AudioManager.setAudioSessionOptions({ + iosCategory: 'playAndRecord', + iosMode: 'default', + iosOptions: ['defaultToSpeaker', 'allowBluetoothA2DP'], + }); + }, []); + + useEffect(() => { + return () => { + synthesizeStop?.(); + stop(); + }; + }, [stop, synthesizeStop]); + + // The preset voices' reference transcripts and lengths, read from their JSON files. + useEffect(() => { + const voices = resource?.voices; + if (!voices) return; + let cancelled = false; + (async () => { + const info: Partial> = {}; + for (const name of REVERB_DEFAULT_VOICE_NAMES) { + try { + const json = JSON.parse(await RNBlobUtil.fs.readFile(voices[name], 'utf8')); + info[name] = { + transcript: json.transcript, + language: json.language, + duration: json.codes.length / 8 / REVERB_FRAME_RATE, + }; + } catch { + // A preset without its description still synthesizes. + } + } + if (!cancelled) setPresetInfo(info); + })(); + return () => { + cancelled = true; + }; + }, [resource]); + + const selectedCustom = + choice.kind === 'custom' ? customVoices.find((v) => v.id === choice.id) : undefined; + const reference = + choice.kind === 'preset' + ? presetInfo[choice.name] + : selectedCustom && { + transcript: selectedCustom.voice.transcript, + language: selectedCustom.voice.language, + duration: selectedCustom.duration, + }; + + const isBusy = isSynthesizing || isPlaying || recorder.isRecording || cloneStep !== null; + const recordingSeconds = recording ? recording.length / REVERB_SAMPLE_RATE : 0; + + const transcribeRecording = async (audio: Float32Array, lang: ReverbLanguage) => { + if (!whisper.transcribe) return; + setCloneStep('transcribing'); + try { + const heard = await whisper.transcribe( + resample(audio, REVERB_SAMPLE_RATE, WHISPER_SAMPLE_RATE_HZ), + { language: lang } + ); + setTranscript(heard.trim()); + } catch (err) { + setRunError(messageOf(err)); + } finally { + setCloneStep(null); + } + }; + + const finishRecording = async () => { + const audio = await recorder.stop(); + if (!audio) return; + const trimmed = trimSilence(audio, REVERB_SAMPLE_RATE); + setRecording(trimmed); + setTranscript(''); + await transcribeRecording(trimmed, recordingLanguage); + }; + + const toggleRecording = async () => { + if (recorder.isRecording) { + await finishRecording(); + return; + } + setRunError(null); + stop(); + const failure = await recorder.start(); + if (failure) setRunError(failure); + }; + + // Stops at the longest reference the encoder takes. + useEffect(() => { + if (recorder.isRecording && recorder.seconds >= MAX_RECORDING_S) finishRecording(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [recorder.isRecording, recorder.seconds]); + + const playRecording = async () => { + if (!recording) return; + async function* once() { + yield { audio: recording!, sampleRate: REVERB_SAMPLE_RATE }; + } + await playStream(once()).catch((err) => setRunError(messageOf(err))); + }; + + const createVoice = async () => { + if (!encodeVoice || !recording || !transcript.trim()) return; + setRunError(null); + setCloneStep('encoding'); + try { + const voice = await encodeVoice(recording, transcript.trim(), recordingLanguage); + const id = nextVoiceId.current++; + setCustomVoices((voices) => [ + ...voices, + { id, name: `My voice ${id}`, voice, duration: recordingSeconds }, + ]); + setChoice({ kind: 'custom', id }); + setRecording(null); + setTranscript(''); + setIsCloneOpen(false); + } catch (err) { + setRunError(messageOf(err)); + } finally { + setCloneStep(null); + } + }; + + const removeVoice = (id: number) => { + setCustomVoices((voices) => voices.filter((v) => v.id !== id)); + if (choice.kind === 'custom' && choice.id === id) { + setChoice({ kind: 'preset', name: 'standard_male' }); + } + }; + + const handleSynthesize = async () => { + const voice = choice.kind === 'preset' ? choice.name : selectedCustom?.voice; + if (!synthesize || !voice || isBusy || !text.trim()) return; + + setRunError(null); + setResult(null); + setChunkProgress(null); + setIsSynthesizing(true); + + const seed = fixedSeed ?? Math.floor(Math.random() * 2 ** 31); + setLastSeed(seed); + let audioSeconds = 0; + let firstAudio: number | null = null; + const t0 = performance.now(); + + try { + const chunks = synthesize(text, { + voice, + language, + speed, + sampling: { steps, remaskRatio, seed }, + postprocess: trimEdges ? undefined : false, + }); + await playStream( + chunks, + (chunk) => { + audioSeconds += chunk.duration; + setChunkProgress( + `Chunk ${chunk.chunkIndex + 1}/${chunk.totalChunks} (${chunk.duration.toFixed(1)} s)` + ); + }, + () => { + firstAudio = (performance.now() - t0) / 1000; + } + ); + const elapsed = (performance.now() - t0) / 1000; + setResult( + `${audioSeconds.toFixed(1)} s of audio · first audio after ${(firstAudio ?? elapsed).toFixed(1)} s` + ); + setChunkProgress(null); + } catch (err) { + setRunError(messageOf(err)); + } finally { + setIsSynthesizing(false); + } + }; + + const handleStop = () => { + synthesizeStop?.(); + stop(); + setIsSynthesizing(false); + }; + + const selectLanguage = (lang: ReverbLanguage) => { + if (text === SAMPLE_TEXTS[language]) setText(SAMPLE_TEXTS[lang]); + setLanguage(lang); + if (!recording) setRecordingLanguage(lang); + }; + + const whisperStatus = !whisper.isReady + ? whisper.error + ? `Whisper failed to load: ${whisper.error.message}` + : `Loading Whisper to transcribe… ${Math.round(whisper.downloadProgress)}%` + : cloneStep === 'transcribing' + ? 'Transcribing with Whisper…' + : null; + + return ( + + + Reverb Text-to-Speech + + Speech in the voice of any short recording, in 7 languages, generated on-device. Pick a + preset voice or clone your own. + + + ({ + label: m.label, + value: m.value, + disabled: isBusy || m.platform !== Platform.OS, + }))} + selectedValue={selectedModel} + onValueChange={setSelectedModel} + /> + + + {runError && ( + + {runError} + + )} + + + Voice + + {PRESETS.map(({ name, title, subtitle }) => ( + setChoice({ kind: 'preset', name })} + /> + ))} + {customVoices.map((v) => ( + setChoice({ kind: 'custom', id: v.id })} + onRemove={() => removeVoice(v.id)} + /> + ))} + setIsCloneOpen((open) => !open)} + disabled={isBusy} + activeOpacity={0.8} + > + {isCloneOpen ? '–' : '+'} + {isCloneOpen ? 'Close' : 'Clone a voice'} + + + + {reference && !isCloneOpen && ( + + + Reference · {reference.language} · {reference.duration.toFixed(1)} s + + “{reference.transcript}” + + )} + + {isCloneOpen && ( + + 1 · Record a reference + + Read anything for 5–15 seconds in a quiet room. Reverb copies the voice, the pace and + the mood of the recording. + + + + + + {formatSeconds(recorder.isRecording ? recorder.seconds : recordingSeconds)} + / {formatSeconds(MAX_RECORDING_S)} + + + + {isSimulator + ? 'Recording needs a physical device.' + : recorder.isRecording + ? 'Recording… tap to stop' + : recording + ? 'Recorded' + : 'Tap to record'} + + + + {recording && !recorder.isRecording && ( + + + {isPlaying ? '■ Stop' : '▶ Listen'} + + transcribeRecording(recording, recordingLanguage)} + disabled={!whisper.isReady || isBusy} + > + ↻ Transcribe + + + )} + + 2 · Check the transcript + ({ ...l, disabled: isBusy }))} + selectedValue={recordingLanguage} + onValueChange={setRecordingLanguage} + /> + {whisperStatus && {whisperStatus}} + + + 3 · Create the voice + +