Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/**
* How offline transcription cuts a long recording: at the last VAD gap that
* fits the model, at the quietest 100 ms when one stretch of speech is too
* long, and never through silence the VAD found nothing in.
*/
import {
halveAtSpeechGap,
splitAtSpeechGaps,
} from '../../src/extensions/speech/utils/audioSplitting';

const SR = 16000;
const seconds = (s: number) => new Float32Array(s * SR).fill(0.5);
const at = (s: number) => s * SR;

describe('splitAtSpeechGaps', () => {
it('returns nothing when there is no speech', () => {
expect(splitAtSpeechGaps(seconds(10), [], at(5))).toEqual([]);
});

it('keeps speech that fits in one piece and drops the silence around it', () => {
const pieces = splitAtSpeechGaps(seconds(10), [{ start: 1, end: 4 }], at(5));
expect(pieces).toEqual([[at(1), at(4)]]);
});

it('cuts at the last gap within the limit', () => {
const speech = [
{ start: 0, end: 2 },
{ start: 3, end: 6 },
{ start: 7, end: 9 },
{ start: 10, end: 12 },
];
expect(splitAtSpeechGaps(seconds(12), speech, at(8))).toEqual([
[at(0), at(6)],
[at(7), at(12)],
]);
});

it('extends speech that reaches the last VAD frame to the end of the audio', () => {
// 10 s yields 997 VAD frames, so detected speech ends at 9.97 s.
const pieces = splitAtSpeechGaps(seconds(10), [{ start: 2, end: 9.97 }], at(10));
expect(pieces).toEqual([[at(2), at(10)]]);
});

it('cuts unbroken speech at its quietest 100 ms', () => {
const audio = seconds(12);
audio.fill(0, at(6), at(6) + 1600);
const pieces = splitAtSpeechGaps(audio, [{ start: 0, end: 12 }], at(8));
expect(pieces).toEqual([
[0, at(6) + 800],
[at(6) + 800, at(12)],
]);
});

it('falls back to the energy cut only within the segment that is too long', () => {
const audio = seconds(20);
audio.fill(0, at(13), at(13) + 1600);
const speech = [
{ start: 0, end: 3 },
{ start: 4, end: 18 },
];
expect(splitAtSpeechGaps(audio, speech, at(10))).toEqual([
[0, at(3)],
[at(4), at(13) + 800],
[at(13) + 800, at(18)],
]);
});
});

describe('halveAtSpeechGap', () => {
it('cuts in the middle of the gap nearest the center', () => {
const speech = [
{ start: 0, end: 4 },
{ start: 5, end: 13 },
{ start: 14, end: 20 },
];
expect(halveAtSpeechGap(seconds(20), speech, 0, at(20))).toEqual([
[0, at(13.5)],
[at(13.5), at(20)],
]);
});

it('ignores gaps outside the middle half and cuts at the quietest 100 ms', () => {
const audio = seconds(20);
audio.fill(0, at(9), at(9) + 1600);
const speech = [
{ start: 0, end: 2 },
{ start: 3, end: 20 },
];
expect(halveAtSpeechGap(audio, speech, 0, at(20))).toEqual([
[0, at(9) + 800],
[at(9) + 800, at(20)],
]);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,22 @@ const register = (script: string[], schema = spec()) => {
};

const seconds = (s: number) => new Float32Array(s * 16000);
const speech = (s: number) => seconds(s).map((_, i) => (i % 2 ? 0.5 : -0.5));

/** Registers a VAD that hears speech in every frame that is not all zeros. */
const registerVad = () =>
fakeJsi.registerModel(VAD_PATH, {
schema: VAD_SCHEMA,
execute: (_, inputs, outputs) => {
const frames = inputs[0] as FakeTensor;
const [rows, width] = frames.shape as [number, number];
for (let r = 0; r < rows; r++) {
let silent = true;
for (let c = 0; c < width && silent; c++) silent = frames.getElement(r * width + c) === 0;
outputs[0]!.setElement(r * 2, silent ? 1 : 0);
}
},
});

describe('createQwen3AsrSpeechToText', () => {
it('strips the detected-language header from the transcript', async () => {
Expand Down Expand Up @@ -167,13 +183,65 @@ describe('createQwen3AsrSpeechToText', () => {
const calls = register(['<asr_text>', '<|im_end|>']);
const stt = await createQwen3AsrSpeechToText(config);

await stt.transcribe(new Float32Array(1000));
await stt.transcribe(new Float32Array(4800));

// 0.5 s = 50 frames = 7 tokens.
expect(calls.encoder).toEqual([{ width: 100, valid: 7 }]);
stt.dispose();
});

it('skips audio the VAD finds no speech in', async () => {
const calls = register(['hello', '<|im_end|>']);
registerVad();
const stt = await createQwen3AsrSpeechToText(config);

expect(await stt.transcribe(seconds(3))).toBe('');
expect(calls.encoder).toEqual([]);
expect(calls.decoder).toEqual([]);
stt.dispose();
});

it('splits long audio at the pause the VAD finds, not at the length limit', async () => {
const calls = register(['<asr_text>', '<|im_end|>', '<asr_text>', '<|im_end|>']);
registerVad();
const stt = await createQwen3AsrSpeechToText(config);

// 20 s of speech, a 2 s pause, then 18 s more: 40 s exceeds the 30 s limit.
const audio = speech(40);
audio.fill(0, 20 * 16000, 22 * 16000);
await stt.transcribe(audio);

expect(calls.decoder.filter((c) => c.position === 0)).toHaveLength(2);
// Each piece is its stretch of speech padded by 30 ms: 0-20.03 s
// (2003 frames) and 21.95-40 s (1805 frames); the pause is not encoded.
expect(calls.encoder).toEqual([
{ width: 800, valid: 104 },
{ width: 800, valid: 104 },
{ width: 500, valid: 53 },
{ width: 800, valid: 104 },
{ width: 800, valid: 104 },
{ width: 300, valid: 27 },
]);
stt.dispose();
});

it('transcribes a long piece again in halves when the decode derails', async () => {
const calls = register([
'<|im_end|>', // the 20 s piece stops before writing anything
'<asr_text>',
'hello',
'<|im_end|>',
'<asr_text>',
'world',
'<|im_end|>',
]);
const stt = await createQwen3AsrSpeechToText(config);

expect(await stt.transcribe(seconds(20))).toBe('hello world');
expect(calls.decoder.filter((c) => c.position === 0)).toHaveLength(3);
stt.dispose();
});

it('rejects an unsupported language', async () => {
register(['<|im_end|>']);
const stt = await createQwen3AsrSpeechToText(config);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { createSdxsTextToImage } from '../../src/extensions/cv/tasks/sdxsTextToI
import { createFsmnVoiceActivityDetector } from '../../src/extensions/speech/tasks/fsmnVoiceActivityDetection';
import { createWhisperSpeechToText } from '../../src/extensions/speech/tasks/whisperSpeechToText';
import { fakeJsi } from '../support/fakeJsi';
import type { FakeTensor } from '../support/fakeTensor';
import { tracked } from '../support/lifetime';
import { STRETCH_PREPROCESSING, exported, imageBuffer, writesOutputs } from '../support/fixtures';

Expand Down Expand Up @@ -414,6 +415,51 @@ describe('createWhisperSpeechToText', () => {

await expect(createWhisperSpeechToText(config)).rejects.toThrow(/<\|endoftext\|>/);
});

it('encodes each stretch of speech the VAD finds, skipping the pause between', async () => {
register();
const encoded: number[] = [];
fakeJsi.registerModel(MODEL_PATH, {
schema: exported({
...method('encode', [f32(AUDIO_SAMPLES)], [f32(1, 1500, 384)]),
...method('decode', [i64(1, 1), i64(1), f32(1, 1500, 384)], [f32(1, 1, 51865)]),
}),
execute: (name, inputs) => {
if (name === 'encode') encoded.push((inputs[0] as FakeTensor).numel);
},
});
// Speech in every frame that is not all zeros.
fakeJsi.registerModel(VAD_PATH, {
schema: VAD_SCHEMA,
execute: (_, inputs, outputs) => {
const frames = inputs[0] as FakeTensor;
const [rows, width] = frames.shape as [number, number];
for (let r = 0; r < rows; r++) {
let silent = true;
for (let c = 0; c < width && silent; c++) silent = frames.getElement(r * width + c) === 0;
outputs[0]!.setElement(r * 2, silent ? 1 : 0);
}
},
});
fakeJsi.registerTokenizer(TOKENIZER_PATH, {
tokens: ['<|endoftext|>', '<|startoftranscript|>', '<|notimestamps|>'],
specialIds: [0, 1, 2],
});
const stt = await createWhisperSpeechToText(config);

// 40 s with a 2 s pause at 20 s: cut there, not at 29 s. Pieces carry the
// 30 ms VAD padding, plus 20 ms where a 25 ms frame overlaps the speech.
const audio = new Float32Array(40 * 16000).map((_, i) => (i % 2 ? 0.5 : -0.5));
audio.fill(0, 20 * 16000, 22 * 16000);
await stt.transcribe(audio, { language: 'en' });
expect(encoded).toEqual([20.03 * 16000, 18.05 * 16000]);

// Pure silence is not transcribed at all.
encoded.length = 0;
expect(await stt.transcribe(new Float32Array(5 * 16000), { language: 'en' })).toBe('');
expect(encoded).toEqual([]);
stt.dispose();
});
});

// ============================================================================
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
type FsmnVadModel,
type VadStreamOptions,
} from './fsmnVoiceActivityDetection';
import { halveAtSpeechGap, splitAtSpeechGaps } from '../utils/audioSplitting';
import { RnExecuTorchError } from '../../../core/error';
import { createResourceScope } from '../../../core/lifetime';

Expand All @@ -40,8 +41,7 @@ const CHUNK_FRAMES = 100; // the encoder convolves 1 s chunks independently
const TOKENS_PER_CHUNK = 13; // audio tokens per full chunk
const WINDOW_FRAMES = 800; // the encoder attends within 8 s windows
const MIN_SAMPLES = QWEN3_ASR_SAMPLE_RATE_HZ / 2; // shorter audio is zero padded, like the reference
const SPLIT_SEARCH_SAMPLES = 5 * QWEN3_ASR_SAMPLE_RATE_HZ; // how far back to look for a quiet cut
const ENERGY_WINDOW_SAMPLES = QWEN3_ASR_SAMPLE_RATE_HZ / 10; // 100 ms energy window for the cut
const RETRY_MIN_SAMPLES = 5 * QWEN3_ASR_SAMPLE_RATE_HZ; // shortest derailed piece transcribed again in halves
const STRIDE_SIZE = 1 * QWEN3_ASR_SAMPLE_RATE_HZ; // streaming: new samples before re-transcribing

/**
Expand Down Expand Up @@ -125,7 +125,7 @@ export type Qwen3AsrSttModel = {
readonly modelPath: string;
/** Local path or remote URL of the `tokenizer.json` file. */
readonly tokenizerPath: string;
/** VAD model configuration used for speech segmentation while streaming. */
/** VAD model configuration used for speech segmentation. */
readonly vadModel: FsmnVadModel;
};

Expand All @@ -142,8 +142,8 @@ export type Qwen3AsrSpeechToText = {
/**
* Asynchronously transcribes a pre-recorded mono waveform sampled at
* {@link QWEN3_ASR_SAMPLE_RATE_HZ}. Audio of any length is accepted; long
* recordings are split at quiet points into pieces the model can take in
* one pass.
* recordings are split at pauses the VAD finds into pieces the model can
* take in one pass, and silence is skipped.
* @param audio Raw 16 kHz mono PCM audio samples (Float32Array).
* @param options Transcription options. See {@link Qwen3AsrSttOptions}.
* @param onToken Optional callback fired on the RN thread for each decoded
Expand Down Expand Up @@ -222,43 +222,6 @@ function audioTokenCount(frames: number): number {
return Math.floor(frames / CHUNK_FRAMES) * TOKENS_PER_CHUNK + tail;
}

/**
* Splits `audio` into pieces of at most `maxSamples`, cutting each at the
* quietest 100 ms within the last 5 s before the limit, so a word is rarely
* cut in half.
* @param audio Input waveform.
* @param maxSamples Longest piece the model accepts.
* @returns `[start, end)` sample ranges covering `audio` without gaps.
*/
function splitAtQuietPoints(audio: Float32Array, maxSamples: number): [number, number][] {
'worklet';
const pieces: [number, number][] = [];
let start = 0;
while (audio.length - start > maxSamples) {
const limit = start + maxSamples;
const from = Math.max(start + ENERGY_WINDOW_SAMPLES, limit - SPLIT_SEARCH_SAMPLES);
let energy = 0;
for (let i = from - ENERGY_WINDOW_SAMPLES; i < from; i++) energy += audio[i]! * audio[i]!;
let best = from;
let bestEnergy = energy;
for (let end = from + 1; end <= limit; end++) {
const enter = audio[end - 1]!;
const leave = audio[end - 1 - ENERGY_WINDOW_SAMPLES]!;
energy += enter * enter - leave * leave;
if (energy < bestEnergy) {
bestEnergy = energy;
best = end;
}
}
// Cut in the middle of the quietest window.
const cut = best - ENERGY_WINDOW_SAMPLES / 2;
pieces.push([start, cut]);
start = cut;
}
pieces.push([start, audio.length]);
return pieces;
}

/**
* Maps the language code in `options` to the name the model emits.
* @param options Transcription options.
Expand Down Expand Up @@ -325,6 +288,7 @@ export async function createQwen3AsrSpeechToText(
const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath));
const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath));
const voiceDetector = scope.track(await createFsmnVoiceActivityDetector(vadModel, runtime));
const { detectVoiceWorklet } = voiceDetector;

const match = validateSpec(model.schema, {
dynamic: qwen3AsrSpec(Dyn('MelFrames'), Dyn('AudioTokens')),
Expand Down Expand Up @@ -469,7 +433,7 @@ export async function createQwen3AsrSpeechToText(
audio: Float32Array,
forcedLanguage: string | undefined,
onToken?: (token: string) => void
): string => {
): string | undefined => {
'worklet';
if (audio.length < MIN_SAMPLES) {
const padded = new Float32Array(MIN_SAMPLES);
Expand Down Expand Up @@ -508,6 +472,11 @@ export async function createQwen3AsrSpeechToText(
nextToken = decodeStep(nextToken, position);
position++;
}
// Stopping at once, before the `language <Name>` header or a forced
// transcript, is a derailed decode rather than an answer.
if (forcedLanguage ? generated.length === 0 : !inTranscript && generated.length <= 1) {
return undefined;
}
// No `<asr_text>` tag at all: the whole output is the transcript, which
// is how the reference parses it.
if (!inTranscript && onToken && generated.length > 0) {
Expand All @@ -528,10 +497,21 @@ export async function createQwen3AsrSpeechToText(
const language = resolveLanguage(options);

const parts: string[] = [];
for (const [start, end] of splitAtQuietPoints(audio, maxSamples)) {
const speech = detectVoiceWorklet(audio);
for (const [start, end] of splitAtSpeechGaps(audio, speech, maxSamples)) {
if (isCancelled.getBlocking()) break;
const part = transcribePiece(audio.subarray(start, end), language, onToken);
if (part !== '') parts.push(part);
if (part === undefined && end - start >= RETRY_MIN_SAMPLES) {
// The model now and then derails on a long piece the VAD heard
// speech in; its halves transcribe.
for (const [s, e] of halveAtSpeechGap(audio, speech, start, end)) {
if (isCancelled.getBlocking()) break;
const half = transcribePiece(audio.subarray(s, e), language, onToken);
if (half) parts.push(half);
}
} else if (part) {
parts.push(part);
}
}
return parts.join(' ').trim();
};
Expand Down
Loading
Loading