diff --git a/packages/react-native-executorch/__tests__/extensions/audioSplitting.test.ts b/packages/react-native-executorch/__tests__/extensions/audioSplitting.test.ts new file mode 100644 index 0000000000..addc7664e5 --- /dev/null +++ b/packages/react-native-executorch/__tests__/extensions/audioSplitting.test.ts @@ -0,0 +1,94 @@ +/** + * How offline transcription cuts a long recording: at the last VAD gap that + * fits the model, at the quietest 100 ms when one stretch of speech is too + * long, and never through silence the VAD found nothing in. + */ +import { + halveAtSpeechGap, + splitAtSpeechGaps, +} from '../../src/extensions/speech/utils/audioSplitting'; + +const SR = 16000; +const seconds = (s: number) => new Float32Array(s * SR).fill(0.5); +const at = (s: number) => s * SR; + +describe('splitAtSpeechGaps', () => { + it('returns nothing when there is no speech', () => { + expect(splitAtSpeechGaps(seconds(10), [], at(5))).toEqual([]); + }); + + it('keeps speech that fits in one piece and drops the silence around it', () => { + const pieces = splitAtSpeechGaps(seconds(10), [{ start: 1, end: 4 }], at(5)); + expect(pieces).toEqual([[at(1), at(4)]]); + }); + + it('cuts at the last gap within the limit', () => { + const speech = [ + { start: 0, end: 2 }, + { start: 3, end: 6 }, + { start: 7, end: 9 }, + { start: 10, end: 12 }, + ]; + expect(splitAtSpeechGaps(seconds(12), speech, at(8))).toEqual([ + [at(0), at(6)], + [at(7), at(12)], + ]); + }); + + it('extends speech that reaches the last VAD frame to the end of the audio', () => { + // 10 s yields 997 VAD frames, so detected speech ends at 9.97 s. + const pieces = splitAtSpeechGaps(seconds(10), [{ start: 2, end: 9.97 }], at(10)); + expect(pieces).toEqual([[at(2), at(10)]]); + }); + + it('cuts unbroken speech at its quietest 100 ms', () => { + const audio = seconds(12); + audio.fill(0, at(6), at(6) + 1600); + const pieces = splitAtSpeechGaps(audio, [{ start: 0, end: 12 }], at(8)); + expect(pieces).toEqual([ + [0, at(6) + 800], + [at(6) + 800, at(12)], + ]); + }); + + it('falls back to the energy cut only within the segment that is too long', () => { + const audio = seconds(20); + audio.fill(0, at(13), at(13) + 1600); + const speech = [ + { start: 0, end: 3 }, + { start: 4, end: 18 }, + ]; + expect(splitAtSpeechGaps(audio, speech, at(10))).toEqual([ + [0, at(3)], + [at(4), at(13) + 800], + [at(13) + 800, at(18)], + ]); + }); +}); + +describe('halveAtSpeechGap', () => { + it('cuts in the middle of the gap nearest the center', () => { + const speech = [ + { start: 0, end: 4 }, + { start: 5, end: 13 }, + { start: 14, end: 20 }, + ]; + expect(halveAtSpeechGap(seconds(20), speech, 0, at(20))).toEqual([ + [0, at(13.5)], + [at(13.5), at(20)], + ]); + }); + + it('ignores gaps outside the middle half and cuts at the quietest 100 ms', () => { + const audio = seconds(20); + audio.fill(0, at(9), at(9) + 1600); + const speech = [ + { start: 0, end: 2 }, + { start: 3, end: 20 }, + ]; + expect(halveAtSpeechGap(audio, speech, 0, at(20))).toEqual([ + [0, at(9) + 800], + [at(9) + 800, at(20)], + ]); + }); +}); diff --git a/packages/react-native-executorch/__tests__/tasks/qwen3AsrSpeechToText.test.ts b/packages/react-native-executorch/__tests__/tasks/qwen3AsrSpeechToText.test.ts index 1d5d29a7da..b3f91fb02d 100644 --- a/packages/react-native-executorch/__tests__/tasks/qwen3AsrSpeechToText.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/qwen3AsrSpeechToText.test.ts @@ -83,6 +83,22 @@ const register = (script: string[], schema = spec()) => { }; const seconds = (s: number) => new Float32Array(s * 16000); +const speech = (s: number) => seconds(s).map((_, i) => (i % 2 ? 0.5 : -0.5)); + +/** Registers a VAD that hears speech in every frame that is not all zeros. */ +const registerVad = () => + fakeJsi.registerModel(VAD_PATH, { + schema: VAD_SCHEMA, + execute: (_, inputs, outputs) => { + const frames = inputs[0] as FakeTensor; + const [rows, width] = frames.shape as [number, number]; + for (let r = 0; r < rows; r++) { + let silent = true; + for (let c = 0; c < width && silent; c++) silent = frames.getElement(r * width + c) === 0; + outputs[0]!.setElement(r * 2, silent ? 1 : 0); + } + }, + }); describe('createQwen3AsrSpeechToText', () => { it('strips the detected-language header from the transcript', async () => { @@ -167,13 +183,65 @@ describe('createQwen3AsrSpeechToText', () => { const calls = register(['', '<|im_end|>']); const stt = await createQwen3AsrSpeechToText(config); - await stt.transcribe(new Float32Array(1000)); + await stt.transcribe(new Float32Array(4800)); // 0.5 s = 50 frames = 7 tokens. expect(calls.encoder).toEqual([{ width: 100, valid: 7 }]); stt.dispose(); }); + it('skips audio the VAD finds no speech in', async () => { + const calls = register(['hello', '<|im_end|>']); + registerVad(); + const stt = await createQwen3AsrSpeechToText(config); + + expect(await stt.transcribe(seconds(3))).toBe(''); + expect(calls.encoder).toEqual([]); + expect(calls.decoder).toEqual([]); + stt.dispose(); + }); + + it('splits long audio at the pause the VAD finds, not at the length limit', async () => { + const calls = register(['', '<|im_end|>', '', '<|im_end|>']); + registerVad(); + const stt = await createQwen3AsrSpeechToText(config); + + // 20 s of speech, a 2 s pause, then 18 s more: 40 s exceeds the 30 s limit. + const audio = speech(40); + audio.fill(0, 20 * 16000, 22 * 16000); + await stt.transcribe(audio); + + expect(calls.decoder.filter((c) => c.position === 0)).toHaveLength(2); + // Each piece is its stretch of speech padded by 30 ms: 0-20.03 s + // (2003 frames) and 21.95-40 s (1805 frames); the pause is not encoded. + expect(calls.encoder).toEqual([ + { width: 800, valid: 104 }, + { width: 800, valid: 104 }, + { width: 500, valid: 53 }, + { width: 800, valid: 104 }, + { width: 800, valid: 104 }, + { width: 300, valid: 27 }, + ]); + stt.dispose(); + }); + + it('transcribes a long piece again in halves when the decode derails', async () => { + const calls = register([ + '<|im_end|>', // the 20 s piece stops before writing anything + '', + 'hello', + '<|im_end|>', + '', + 'world', + '<|im_end|>', + ]); + const stt = await createQwen3AsrSpeechToText(config); + + expect(await stt.transcribe(seconds(20))).toBe('hello world'); + expect(calls.decoder.filter((c) => c.position === 0)).toHaveLength(3); + stt.dispose(); + }); + it('rejects an unsupported language', async () => { register(['<|im_end|>']); const stt = await createQwen3AsrSpeechToText(config); diff --git a/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts b/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts index a2ac22acc6..454e3fc245 100644 --- a/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts @@ -21,6 +21,7 @@ import { createSdxsTextToImage } from '../../src/extensions/cv/tasks/sdxsTextToI import { createFsmnVoiceActivityDetector } from '../../src/extensions/speech/tasks/fsmnVoiceActivityDetection'; import { createWhisperSpeechToText } from '../../src/extensions/speech/tasks/whisperSpeechToText'; import { fakeJsi } from '../support/fakeJsi'; +import type { FakeTensor } from '../support/fakeTensor'; import { tracked } from '../support/lifetime'; import { STRETCH_PREPROCESSING, exported, imageBuffer, writesOutputs } from '../support/fixtures'; @@ -414,6 +415,51 @@ describe('createWhisperSpeechToText', () => { await expect(createWhisperSpeechToText(config)).rejects.toThrow(/<\|endoftext\|>/); }); + + it('encodes each stretch of speech the VAD finds, skipping the pause between', async () => { + register(); + const encoded: number[] = []; + fakeJsi.registerModel(MODEL_PATH, { + schema: exported({ + ...method('encode', [f32(AUDIO_SAMPLES)], [f32(1, 1500, 384)]), + ...method('decode', [i64(1, 1), i64(1), f32(1, 1500, 384)], [f32(1, 1, 51865)]), + }), + execute: (name, inputs) => { + if (name === 'encode') encoded.push((inputs[0] as FakeTensor).numel); + }, + }); + // Speech in every frame that is not all zeros. + fakeJsi.registerModel(VAD_PATH, { + schema: VAD_SCHEMA, + execute: (_, inputs, outputs) => { + const frames = inputs[0] as FakeTensor; + const [rows, width] = frames.shape as [number, number]; + for (let r = 0; r < rows; r++) { + let silent = true; + for (let c = 0; c < width && silent; c++) silent = frames.getElement(r * width + c) === 0; + outputs[0]!.setElement(r * 2, silent ? 1 : 0); + } + }, + }); + fakeJsi.registerTokenizer(TOKENIZER_PATH, { + tokens: ['<|endoftext|>', '<|startoftranscript|>', '<|notimestamps|>'], + specialIds: [0, 1, 2], + }); + const stt = await createWhisperSpeechToText(config); + + // 40 s with a 2 s pause at 20 s: cut there, not at 29 s. Pieces carry the + // 30 ms VAD padding, plus 20 ms where a 25 ms frame overlaps the speech. + const audio = new Float32Array(40 * 16000).map((_, i) => (i % 2 ? 0.5 : -0.5)); + audio.fill(0, 20 * 16000, 22 * 16000); + await stt.transcribe(audio, { language: 'en' }); + expect(encoded).toEqual([20.03 * 16000, 18.05 * 16000]); + + // Pure silence is not transcribed at all. + encoded.length = 0; + expect(await stt.transcribe(new Float32Array(5 * 16000), { language: 'en' })).toBe(''); + expect(encoded).toEqual([]); + stt.dispose(); + }); }); // ============================================================================ diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts index d5f4cf58ab..c67d83486b 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/qwen3AsrSpeechToText.ts @@ -25,6 +25,7 @@ import { type FsmnVadModel, type VadStreamOptions, } from './fsmnVoiceActivityDetection'; +import { halveAtSpeechGap, splitAtSpeechGaps } from '../utils/audioSplitting'; import { RnExecuTorchError } from '../../../core/error'; import { createResourceScope } from '../../../core/lifetime'; @@ -40,8 +41,7 @@ const CHUNK_FRAMES = 100; // the encoder convolves 1 s chunks independently const TOKENS_PER_CHUNK = 13; // audio tokens per full chunk const WINDOW_FRAMES = 800; // the encoder attends within 8 s windows const MIN_SAMPLES = QWEN3_ASR_SAMPLE_RATE_HZ / 2; // shorter audio is zero padded, like the reference -const SPLIT_SEARCH_SAMPLES = 5 * QWEN3_ASR_SAMPLE_RATE_HZ; // how far back to look for a quiet cut -const ENERGY_WINDOW_SAMPLES = QWEN3_ASR_SAMPLE_RATE_HZ / 10; // 100 ms energy window for the cut +const RETRY_MIN_SAMPLES = 5 * QWEN3_ASR_SAMPLE_RATE_HZ; // shortest derailed piece transcribed again in halves const STRIDE_SIZE = 1 * QWEN3_ASR_SAMPLE_RATE_HZ; // streaming: new samples before re-transcribing /** @@ -125,7 +125,7 @@ export type Qwen3AsrSttModel = { readonly modelPath: string; /** Local path or remote URL of the `tokenizer.json` file. */ readonly tokenizerPath: string; - /** VAD model configuration used for speech segmentation while streaming. */ + /** VAD model configuration used for speech segmentation. */ readonly vadModel: FsmnVadModel; }; @@ -142,8 +142,8 @@ export type Qwen3AsrSpeechToText = { /** * Asynchronously transcribes a pre-recorded mono waveform sampled at * {@link QWEN3_ASR_SAMPLE_RATE_HZ}. Audio of any length is accepted; long - * recordings are split at quiet points into pieces the model can take in - * one pass. + * recordings are split at pauses the VAD finds into pieces the model can + * take in one pass, and silence is skipped. * @param audio Raw 16 kHz mono PCM audio samples (Float32Array). * @param options Transcription options. See {@link Qwen3AsrSttOptions}. * @param onToken Optional callback fired on the RN thread for each decoded @@ -222,43 +222,6 @@ function audioTokenCount(frames: number): number { return Math.floor(frames / CHUNK_FRAMES) * TOKENS_PER_CHUNK + tail; } -/** - * Splits `audio` into pieces of at most `maxSamples`, cutting each at the - * quietest 100 ms within the last 5 s before the limit, so a word is rarely - * cut in half. - * @param audio Input waveform. - * @param maxSamples Longest piece the model accepts. - * @returns `[start, end)` sample ranges covering `audio` without gaps. - */ -function splitAtQuietPoints(audio: Float32Array, maxSamples: number): [number, number][] { - 'worklet'; - const pieces: [number, number][] = []; - let start = 0; - while (audio.length - start > maxSamples) { - const limit = start + maxSamples; - const from = Math.max(start + ENERGY_WINDOW_SAMPLES, limit - SPLIT_SEARCH_SAMPLES); - let energy = 0; - for (let i = from - ENERGY_WINDOW_SAMPLES; i < from; i++) energy += audio[i]! * audio[i]!; - let best = from; - let bestEnergy = energy; - for (let end = from + 1; end <= limit; end++) { - const enter = audio[end - 1]!; - const leave = audio[end - 1 - ENERGY_WINDOW_SAMPLES]!; - energy += enter * enter - leave * leave; - if (energy < bestEnergy) { - bestEnergy = energy; - best = end; - } - } - // Cut in the middle of the quietest window. - const cut = best - ENERGY_WINDOW_SAMPLES / 2; - pieces.push([start, cut]); - start = cut; - } - pieces.push([start, audio.length]); - return pieces; -} - /** * Maps the language code in `options` to the name the model emits. * @param options Transcription options. @@ -325,6 +288,7 @@ export async function createQwen3AsrSpeechToText( const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); const voiceDetector = scope.track(await createFsmnVoiceActivityDetector(vadModel, runtime)); + const { detectVoiceWorklet } = voiceDetector; const match = validateSpec(model.schema, { dynamic: qwen3AsrSpec(Dyn('MelFrames'), Dyn('AudioTokens')), @@ -469,7 +433,7 @@ export async function createQwen3AsrSpeechToText( audio: Float32Array, forcedLanguage: string | undefined, onToken?: (token: string) => void - ): string => { + ): string | undefined => { 'worklet'; if (audio.length < MIN_SAMPLES) { const padded = new Float32Array(MIN_SAMPLES); @@ -508,6 +472,11 @@ export async function createQwen3AsrSpeechToText( nextToken = decodeStep(nextToken, position); position++; } + // Stopping at once, before the `language ` header or a forced + // transcript, is a derailed decode rather than an answer. + if (forcedLanguage ? generated.length === 0 : !inTranscript && generated.length <= 1) { + return undefined; + } // No `` tag at all: the whole output is the transcript, which // is how the reference parses it. if (!inTranscript && onToken && generated.length > 0) { @@ -528,10 +497,21 @@ export async function createQwen3AsrSpeechToText( const language = resolveLanguage(options); const parts: string[] = []; - for (const [start, end] of splitAtQuietPoints(audio, maxSamples)) { + const speech = detectVoiceWorklet(audio); + for (const [start, end] of splitAtSpeechGaps(audio, speech, maxSamples)) { if (isCancelled.getBlocking()) break; const part = transcribePiece(audio.subarray(start, end), language, onToken); - if (part !== '') parts.push(part); + if (part === undefined && end - start >= RETRY_MIN_SAMPLES) { + // The model now and then derails on a long piece the VAD heard + // speech in; its halves transcribe. + for (const [s, e] of halveAtSpeechGap(audio, speech, start, end)) { + if (isCancelled.getBlocking()) break; + const half = transcribePiece(audio.subarray(s, e), language, onToken); + if (half) parts.push(half); + } + } else if (part) { + parts.push(part); + } } return parts.join(' ').trim(); }; diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts index a042b5d338..a54b68035e 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts @@ -17,6 +17,7 @@ import { type FsmnVadModel, type VadStreamOptions, } from './fsmnVoiceActivityDetection'; +import { splitAtSpeechGaps } from '../utils/audioSplitting'; import { RnExecuTorchError } from '../../../core/error'; import { createResourceScope } from '../../../core/lifetime'; @@ -115,7 +116,9 @@ export type WhisperSpeechToText = { /** * Asynchronously transcribes a pre-recorded mono waveform sampled at - * {@link WHISPER_SAMPLE_RATE_HZ}. + * {@link WHISPER_SAMPLE_RATE_HZ}. Audio of any length is accepted; long + * recordings are split at pauses the VAD finds into pieces of at most + * 29 s, and silence is skipped. * @param audio Raw 16 kHz mono PCM audio samples (Float32Array). * @param options Transcription options. See {@link WhisperSttOptions}. * @param onToken Optional callback fired on the RN thread for each decoded @@ -205,6 +208,7 @@ export async function createWhisperSpeechToText')!; const isEnglishOnly = supportedLanguages.length === 1 && supportedLanguages[0] === 'en'; @@ -274,15 +278,13 @@ export async function createWhisperSpeechToText [ + Math.max(0, Math.round(s.start * SAMPLE_RATE_HZ)), + Math.min(audio.length, Math.round(s.end * SAMPLE_RATE_HZ)), + ]); + const last = segments[segments.length - 1]; + if (last && audio.length - last[1] <= VAD_TAIL_SAMPLES) last[1] = audio.length; + + const pieces: [number, number][] = []; + let i = 0; + while (i < segments.length) { + const start = segments[i]![0]; + const limit = start + maxSamples; + if (segments[i]![1] > limit) { + const cut = quietestCut(audio, start, limit); + pieces.push([start, cut]); + segments[i]![0] = cut; + continue; + } + while (i + 1 < segments.length && segments[i + 1]![1] <= limit) i++; + pieces.push([start, segments[i]![1]]); + i++; + } + return pieces; +} + +/** + * Splits the piece `[start, end)` in two: in the middle of the speech gap + * nearest its center, or at the quietest 100 ms around the center when no gap + * lies within its middle half. + * @param audio Input waveform, 16 kHz mono. + * @param speech Speech segments in `audio`, in seconds, as the VAD reports them. + * @param start First sample of the piece. + * @param end Sample after the last one of the piece. + * @returns The two halves as `[start, end)` sample ranges. + */ +export function halveAtSpeechGap( + audio: Float32Array, + speech: VadSegment[], + start: number, + end: number +): [number, number][] { + 'worklet'; + const middle = (start + end) / 2; + const reach = (end - start) / 4; + let cut = -1; + for (let i = 1; i < speech.length; i++) { + const gap = Math.round(((speech[i - 1]!.end + speech[i]!.start) / 2) * SAMPLE_RATE_HZ); + const distance = Math.abs(gap - middle); + if (distance <= reach && (cut < 0 || distance < Math.abs(cut - middle))) cut = gap; + } + if (cut < 0) { + cut = quietestCut(audio, start, Math.min(end, Math.round(middle + SPLIT_SEARCH_SAMPLES / 2))); + } + return [ + [start, cut], + [cut, end], + ]; +}