diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71f83da..c2a1ed6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,23 @@ jobs: path: verify-report/ios if-no-files-found: error + showcase: + name: Showcase source (Node) + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: showcase + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22' + cache: npm + cache-dependency-path: showcase/package-lock.json + - run: npm ci --ignore-scripts + - run: npm run check + # Not configured on purpose: real-device performance/model gates require # hardware runners. When a runner exists, trigger scripts/verify --scope # models --models-dir here with the same JSON report format. diff --git a/android/app/src/main/java/com/dialect/interpreter/MainActivity.kt b/android/app/src/main/java/com/dialect/interpreter/MainActivity.kt index 8cc959e..1734f13 100644 --- a/android/app/src/main/java/com/dialect/interpreter/MainActivity.kt +++ b/android/app/src/main/java/com/dialect/interpreter/MainActivity.kt @@ -4,6 +4,9 @@ import android.os.Bundle import androidx.activity.ComponentActivity import androidx.activity.compose.setContent import androidx.activity.enableEdgeToEdge +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.setValue import com.dialect.interpreter.ui.navigation.AppNavigation import com.dialect.interpreter.ui.theme.DialectInterpreterTheme @@ -12,21 +15,29 @@ import com.dialect.interpreter.ui.theme.DialectInterpreterTheme * - microphone permission is requested from the interpret screen when the user * presses record, never at startup (spec 03 U02); * - no session/runtime teardown here — engine handles are session-owned and - * released through SessionController.close() (spec 02 R01). + * released through SessionController.close() (spec 02 R01); + * - the system animator scale ("reduce motion") is re-read on every resume, + * so changing it in system settings applies when the user returns without + * recreating the activity. */ class MainActivity : ComponentActivity() { + private var reduceMotion by mutableStateOf(false) + + private fun readReducedMotion(): Boolean = android.provider.Settings.Global.getFloat( + contentResolver, + android.provider.Settings.Global.ANIMATOR_DURATION_SCALE, + 1f, + ) == 0f + + override fun onResume() { + super.onResume() + reduceMotion = readReducedMotion() + } override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) enableEdgeToEdge() - - // Respect the system animator scale (TalkBack/accessibility: "reduce - // motion") once per activity; decorative animations read this flag. - val reduceMotion = android.provider.Settings.Global.getFloat( - contentResolver, - android.provider.Settings.Global.ANIMATOR_DURATION_SCALE, - 1f, - ) == 0f + reduceMotion = readReducedMotion() setContent { DialectInterpreterTheme { diff --git a/android/app/src/main/java/com/dialect/interpreter/inference/TtsApi2Runtime.kt b/android/app/src/main/java/com/dialect/interpreter/inference/TtsApi2Runtime.kt index 043983a..71ead7c 100644 --- a/android/app/src/main/java/com/dialect/interpreter/inference/TtsApi2Runtime.kt +++ b/android/app/src/main/java/com/dialect/interpreter/inference/TtsApi2Runtime.kt @@ -132,17 +132,29 @@ class TtsApi2Runtime( * Conditioning is explicit in these fields — never inferred from * embedding values. * + * The conditioning package is immutable: the array-typed properties are + * defensively copied at construction AND on every read, so neither the + * producer nor any consumer can reach the stored conditioning through a + * shared array reference. [embedding], [referenceTokenIds] and + * [referenceCodes] each hand out an independent copy; mutating a + * handed-out array, or the arrays passed to the constructor, can never + * change what [synthesizePrepared] later consumes. [referenceText] is an + * immutable String. The internal [vocoderWarmState] is intentionally kept + * by reference (it is never exposed through the public surface) and is + * consumed only through the engine's per-turn state builder, which copies + * every array, so no warm-state alias escapes the engine either. + * * A prepared reference is bound to the producing [TtsApi2Runtime] instance * and its loading generation: after the engine was released (and reloaded) * it is stale and [synthesizePrepared] rejects it; passing it to another * engine instance is rejected for the same reason. */ class PreparedReference internal constructor( - val embedding: FloatArray, - val referenceText: String?, - val referenceTokenIds: IntArray?, + embedding: FloatArray, + referenceText: String?, + referenceTokenIds: IntArray?, /** Group-major [16 * frames] codec ids for the reference audio (ICL only). */ - val referenceCodes: IntArray?, + referenceCodes: IntArray?, val referenceFrames: Int, /** sha256 over PCM bytes + reference text + role file identities. */ val identity: String, @@ -150,7 +162,18 @@ class TtsApi2Runtime( internal val engineToken: Any, internal val generation: Int, ) { - val isIcl: Boolean get() = referenceCodes != null + // Private snapshots: the constructor owns its own copy of the caller's + // arrays and every read hands out a fresh copy over that snapshot. + val embedding: FloatArray = embedding.copyOf() + get() = field.copyOf() + val referenceText: String? = referenceText + val referenceTokenIds: IntArray? = referenceTokenIds?.copyOf() + get() = field?.copyOf() + val referenceCodes: IntArray? = referenceCodes?.copyOf() + get() = field?.copyOf() + + /** ICL conditioning is present exactly when reference codes were supplied. */ + val isIcl: Boolean = referenceCodes != null } // ---- Reference preparation ------------------------------------------------- @@ -187,51 +210,59 @@ class TtsApi2Runtime( val cfg = config ?: throw ModelProtocol.UnsupportedModelException("TTS bundle config missing") val roles = manifest.roles - withContext(Dispatchers.Default) { - SherpaJni.load() - val pcm24k = SherpaJni.resample(referenceAudio, inputSampleRate, Qwen3TtsProtocol.SAMPLE_RATE) - - // Speaker embedding: same raw, un-normalized value the API1 port consumes. - val embedding = runSpeakerEncoder(pcm24k) - - var refTokenIds: IntArray? = null - var refCodes: IntArray? = null - var refFrames = 0 - var warmState: VocoderWarmState? = null - if (referenceText != null) { - val tok = tokenizer ?: throw ModelProtocol.UnsupportedModelException("TTS tokenizer missing") - // Official reference wrapper; the prompt consumes ids[3:-2]. - refTokenIds = tok.encode( - "<|im_start|>assistant\n$referenceText<|im_end|>\n").toIntArray() - if (refTokenIds.size < 6) { - throw ModelProtocol.UnsupportedModelException( - "reference text wraps to only ${refTokenIds.size} tokens") + try { + withContext(Dispatchers.Default) { + SherpaJni.load() + val pcm24k = SherpaJni.resample(referenceAudio, inputSampleRate, Qwen3TtsProtocol.SAMPLE_RATE) + + // Speaker embedding: same raw, un-normalized value the API1 port consumes. + val embedding = runSpeakerEncoder(pcm24k) + + var refTokenIds: IntArray? = null + var refCodes: IntArray? = null + var refFrames = 0 + var warmState: VocoderWarmState? = null + if (referenceText != null) { + val tok = tokenizer ?: throw ModelProtocol.UnsupportedModelException("TTS tokenizer missing") + // Official reference wrapper; the prompt consumes ids[3:-2]. + refTokenIds = tok.encode( + "<|im_start|>assistant\n$referenceText<|im_end|>\n").toIntArray() + if (refTokenIds.size < 6) { + throw ModelProtocol.UnsupportedModelException( + "reference text wraps to only ${refTokenIds.size} tokens") + } + val encodeStart = System.nanoTime() + refCodes = runReferenceEncoder(pcm24k, cfg) + refFrames = refCodes.size / Qwen3TtsProtocol.NUM_CODEBOOKS + warmState = computeVocoderWarmState(refCodes, refFrames) + Log.i(TAG, "reference prepared: frames=$refFrames " + + "encodeMs=${(System.nanoTime() - encodeStart) / 1e6}") } - val encodeStart = System.nanoTime() - refCodes = runReferenceEncoder(pcm24k, cfg) - refFrames = refCodes.size / Qwen3TtsProtocol.NUM_CODEBOOKS - warmState = computeVocoderWarmState(refCodes, refFrames) - Log.i(TAG, "reference prepared: frames=$refFrames " + - "encodeMs=${(System.nanoTime() - encodeStart) / 1e6}") - } - // Identity binds the snapshot to its PCM, text and model files. - val digest = MessageDigest.getInstance("SHA-256") - fun feed(text: String) = digest.update(text.toByteArray(Charsets.UTF_8)) - feed("pcm24k:") - val pcmBytes = ByteBuffer.allocate(pcm24k.size * 4).order(ByteOrder.LITTLE_ENDIAN) - pcmBytes.asFloatBuffer().put(pcm24k) - digest.update(pcmBytes.array()) - feed("\ntext:${referenceText ?: ""}\n") - for (role in REQUIRED_ROLES) feed("$role:${roleFileIdentity(roles, role)}\n") - val identity = digest.digest().joinToString("") { "%02x".format(it) } - - PreparedReference( - embedding, referenceText, refTokenIds, refCodes, refFrames, identity, warmState, - this@TtsApi2Runtime.engineToken, generation.get()) - }.also { + // Identity binds the snapshot to its PCM, text and model files. + val digest = MessageDigest.getInstance("SHA-256") + fun feed(text: String) = digest.update(text.toByteArray(Charsets.UTF_8)) + feed("pcm24k:") + val pcmBytes = ByteBuffer.allocate(pcm24k.size * 4).order(ByteOrder.LITTLE_ENDIAN) + pcmBytes.asFloatBuffer().put(pcm24k) + digest.update(pcmBytes.array()) + feed("\ntext:${referenceText ?: ""}\n") + for (role in REQUIRED_ROLES) feed("$role:${roleFileIdentity(roles, role)}\n") + val identity = digest.digest().joinToString("") { "%02x".format(it) } + + PreparedReference( + embedding, referenceText, refTokenIds, refCodes, refFrames, identity, warmState, + this@TtsApi2Runtime.engineToken, generation.get()) + } + } finally { // The reference encoder is one-shot per preparation; release it // eagerly so an idle engine does not hold ~190 MB of weights. + // finally — not success-only — so a failed ICL preparation + // (tokenizer, reference-encoder or warm-state validation) or a + // cancellation inside the block also releases the ~190 MB session + // instead of leaking it until release(). For xvector-only + // preparations the role was never loaded and releaseRole is a + // harmless no-op (sessions.remove(key)?.close()). modelManager.releaseRole(SUB_DIR, roles, "reference_encoder") } } @@ -549,8 +580,12 @@ class TtsApi2Runtime( "past_values" to valuesTensor, )).use { cpResult -> val cpLogits = readFloatOutput(cpResult, "logits") + // sampleCodePredictor consumes only the last + // cfg.cpVocab logits itself; pass the full + // array instead of pre-slicing it (the slice + // was a redundant copy on every codebook step). val token = Qwen3TtsProtocol.sampleCodePredictor( - cpLogits.copyOfRange(cpLogits.size - cfg.cpVocab, cpLogits.size), + cpLogits, cfg, Qwen3TtsProtocol.TEMPERATURE, Qwen3TtsProtocol.TOP_K, random) frame[g] = token val shape = intArrayOf( diff --git a/android/app/src/main/java/com/dialect/interpreter/inference/TtsEngine.kt b/android/app/src/main/java/com/dialect/interpreter/inference/TtsEngine.kt index 4dfd9cd..d924c95 100644 --- a/android/app/src/main/java/com/dialect/interpreter/inference/TtsEngine.kt +++ b/android/app/src/main/java/com/dialect/interpreter/inference/TtsEngine.kt @@ -405,9 +405,16 @@ object Qwen3TtsProtocol { System.arraycopy(logitsLast, logitsLast.size - vocab, probs, 0, vocab) require(probs.all { it.isFinite() || it == Float.NEGATIVE_INFINITY }) { "Invalid talker logits" } if (suppressEos) probs[cfg.codecEosId] = Float.NEGATIVE_INFINITY - for (token in generated.toSet()) { + // Each distinct generated token is penalized exactly once; the seen + // mask replaces the per-call HashSet without changing the outcome + // (penalty application is per-index, so order is irrelevant). + val penalized = BooleanArray(vocab) + for (token in generated) { require(token in probs.indices) { "Invalid generated codec token" } - if (probs[token] > 0f) probs[token] /= repetitionPenalty else probs[token] *= repetitionPenalty + if (!penalized[token]) { + penalized[token] = true + if (probs[token] > 0f) probs[token] /= repetitionPenalty else probs[token] *= repetitionPenalty + } } for (i in cfg.cpVocab until vocab) { if (i != cfg.codecEosId) probs[i] = Float.NEGATIVE_INFINITY @@ -446,7 +453,7 @@ object Qwen3TtsProtocol { require(maxLogit.isFinite()) { "No finite sampling candidate remains" } if (temperature == 0f || topK == 1) return best if (topK in 1 until probs.size) { - val threshold = probs.copyOf().sortedDescending()[topK - 1] + val threshold = kthLargestFloat(probs, topK) for (i in probs.indices) if (probs[i] < threshold) probs[i] = Float.NEGATIVE_INFINITY } // Subtract the finite maximum before temperature scaling. Double @@ -465,6 +472,63 @@ object Qwen3TtsProtocol { return weights.indices.last { weights[it] > 0.0 } } + /** + * Exact k-th largest value of a multiset — the value + * `values.copyOf().sortedDescending()[k - 1]` selects — without copying + * or sorting [values]. Selecting the k-th largest equals selecting the + * (n-k+1)-th smallest, so a bounded heap over whichever side is smaller + * does the job in O(n log min(k, n-k+1)). Comparisons only, no arithmetic + * on the values, so the result equals the sorted reference for every + * value — except possibly the sign of a zero when the input mixes -0.0f + * and 0.0f (the heap compares IEEE; a boxed sort may total-order them). + * That sign cannot change sampling behavior: the mask test + * `x < threshold` treats both zero signs alike and exp(±0.0) are equal. + * Inputs must not contain NaN (the sampler rejects NaN logits first). + */ + internal fun kthLargestFloat(values: FloatArray, k: Int): Float { + require(values.isNotEmpty()) { "kth largest requires a nonempty array" } + require(k in 1..values.size) { "kth largest order $k outside 1..${values.size}" } + val keepLargest = k <= values.size - k + 1 + val capacity = if (keepLargest) k else values.size - k + 1 + val heap = FloatArray(capacity) + var size = 0 + for (x in values) { + if (size < capacity) { + // Sift-up insert. + var i = size++ + heap[i] = x + while (i > 0) { + val parent = (i - 1) / 2 + val ordered = if (keepLargest) heap[parent] <= heap[i] else heap[parent] >= heap[i] + if (ordered) break + val tmp = heap[parent]; heap[parent] = heap[i]; heap[i] = tmp + i = parent + } + } else if (if (keepLargest) x > heap[0] else x < heap[0]) { + // Replace the boundary entry and restore the heap invariant. + heap[0] = x + var i = 0 + while (true) { + val left = 2 * i + 1 + val right = left + 1 + var best = i + if (left < size) { + val betterLeft = if (keepLargest) heap[left] < heap[best] else heap[left] > heap[best] + if (betterLeft) best = left + } + if (right < size) { + val betterRight = if (keepLargest) heap[right] < heap[best] else heap[right] > heap[best] + if (betterRight) best = right + } + if (best == i) break + val tmp = heap[best]; heap[best] = heap[i]; heap[i] = tmp + i = best + } + } + } + return heap[0] + } + // ---- Audio frontend (exact PyTorch-style mel, verified vs librosa) ------ fun buildMelFilterbank(sr: Int, nFft: Int, nMels: Int, fmin: Double, fmax: Double): FloatArray { @@ -525,7 +589,11 @@ object Qwen3TtsProtocol { val nFft = 1024 val hop = 256 val nMels = 128 - if (audio.size < 2) { + // The reflect-padded signal must cover one full FFT window, otherwise + // the frame loop reads past the padded array (AIOOBE). Minimum: + // audio.size >= nFft - 2*pad == hop. Input shorter than one hop is + // rejected cleanly instead of crashing. + if (audio.size < hop) { throw IllegalArgumentException("reference audio too short for mel frontend") } val pad = (nFft - hop) / 2 @@ -540,6 +608,7 @@ object Qwen3TtsProtocol { val re = DoubleArray(nFft) val im = DoubleArray(nFft) val nFreqs = nFft / 2 + 1 + val mag = DoubleArray(nFreqs) for (f in 0 until frames) { val start = f * hop for (i in 0 until nFft) { @@ -547,11 +616,16 @@ object Qwen3TtsProtocol { im[i] = 0.0 } fftRadix2(re, im) + // Magnitude once per bin; every mel band reuses it. The expression + // and the band-accumulation order are unchanged, so the output is + // bit-identical to the per-band recomputation. + for (k in 0 until nFreqs) { + mag[k] = kotlin.math.sqrt(re[k] * re[k] + im[k] * im[k] + 1e-9) + } for (m in 0 until nMels) { var energy = 0.0 for (k in 0 until nFreqs) { - val mag = kotlin.math.sqrt(re[k] * re[k] + im[k] * im[k] + 1e-9) - energy += basis[m * nFreqs + k] * mag + energy += basis[m * nFreqs + k] * mag[k] } out[f * nMels + m] = max(energy, 1e-5).let(::ln).toFloat() } @@ -573,14 +647,17 @@ object Qwen3TtsProtocol { t = im[i]; im[i] = im[j]; im[j] = t } } + val twiddles = twiddlesFor(n) + val cosT = twiddles.cos + val sinT = twiddles.sin var size = 2 + var twOff = 0 while (size <= n) { val half = size / 2 - val angle = -2.0 * PI / size for (i in 0 until n step size) { for (k in 0 until half) { - val c = kotlin.math.cos(angle * k) - val s = kotlin.math.sin(angle * k) + val c = cosT[twOff + k] + val s = sinT[twOff + k] val tReal = c * re[i + k + half] - s * im[i + k + half] val tImag = s * re[i + k + half] + c * im[i + k + half] re[i + k + half] = re[i + k] - tReal @@ -589,10 +666,47 @@ object Qwen3TtsProtocol { im[i + k] += tImag } } + twOff += half size *= 2 } } + // ---- FFT twiddle cache ----------------------------------------------------- + // cos(-2*pi*k/size) / sin(...) were recomputed for every butterfly block of + // every frame; the values depend only on (n, size, k), so they are built + // once per FFT size with the exact same expressions (bit-identical to the + // inline computation) and reused. One immutable @Volatile holder keeps the + // (n, cos, sin) triple consistent for concurrent readers. + + private class TwiddleTable(val n: Int, val cos: DoubleArray, val sin: DoubleArray) + + private val twiddleLock = Any() + @Volatile private var twiddleTable = TwiddleTable(0, DoubleArray(0), DoubleArray(0)) + + private fun twiddlesFor(n: Int): TwiddleTable { + twiddleTable.let { if (it.n == n) return it } + synchronized(twiddleLock) { + if (twiddleTable.n != n) { + val cosT = DoubleArray(n - 1) + val sinT = DoubleArray(n - 1) + var off = 0 + var size = 2 + while (size <= n) { + val half = size / 2 + val angle = -2.0 * PI / size + for (k in 0 until half) { + cosT[off + k] = kotlin.math.cos(angle * k) + sinT[off + k] = kotlin.math.sin(angle * k) + } + off += half + size *= 2 + } + twiddleTable = TwiddleTable(n, cosT, sinT) + } + return twiddleTable + } + } + } @@ -1277,8 +1391,11 @@ class TtsEngine( "past_values" to valuesTensor, )).use { cpResult -> val cpLogits = readFloatOutput(cpResult, "logits") + // sampleCodePredictor copies the trailing cpVocab logits + // itself; pass the full array (the pre-slice was a + // redundant per-step copy). val token = Qwen3TtsProtocol.sampleCodePredictor( - cpLogits.copyOfRange(cpLogits.size - cfg.cpVocab, cpLogits.size), + cpLogits, cfg, Qwen3TtsProtocol.TEMPERATURE, Qwen3TtsProtocol.TOP_K, random) frame[g] = token val shape = intArrayOf( diff --git a/android/app/src/main/java/com/dialect/interpreter/ui/components/WaveformVisualizer.kt b/android/app/src/main/java/com/dialect/interpreter/ui/components/WaveformVisualizer.kt index 0c3fc8b..128cdb1 100644 --- a/android/app/src/main/java/com/dialect/interpreter/ui/components/WaveformVisualizer.kt +++ b/android/app/src/main/java/com/dialect/interpreter/ui/components/WaveformVisualizer.kt @@ -1,34 +1,34 @@ package com.dialect.interpreter.ui.components -import androidx.compose.animation.core.LinearEasing -import androidx.compose.animation.core.RepeatMode -import androidx.compose.animation.core.animateFloat -import androidx.compose.animation.core.infiniteRepeatable -import androidx.compose.animation.core.rememberInfiniteTransition -import androidx.compose.animation.core.tween -import androidx.compose.foundation.background -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.Arrangement -import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.fillMaxHeight -import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.foundation.layout.height -import androidx.compose.foundation.layout.width -import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.animation.core.Spring +import androidx.compose.animation.core.animateFloatAsState +import androidx.compose.animation.core.snap +import androidx.compose.animation.core.spring +import androidx.compose.foundation.Canvas import androidx.compose.runtime.Composable -import androidx.compose.runtime.getValue -import androidx.compose.ui.Alignment +import androidx.compose.runtime.State import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.clip +import androidx.compose.ui.geometry.CornerRadius +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.drawscope.DrawScope import androidx.compose.ui.unit.dp import com.dialect.interpreter.ui.theme.AppColors +import kotlin.math.log10 import kotlin.math.sin +import kotlin.math.PI /** - * Vertical bar waveform driven by live capture amplitude. With - * [reduceMotion] (system animator scale 0) the decorative idle/phase - * animation is skipped; bars still react to real amplitude (spec U01). + * Vertical bar waveform driven by the live capture RMS (spec motion table: + * "Input level"). One drawing surface: bar heights are computed in the draw + * phase from a single shared level spring — no per-frame layout and no + * independent per-bar springs. The level follows the real microphone RMS + * through a bounded logarithmic map ([levelFraction]) that keeps quiet + * speech readable; non-finite input falls back to the static rest outline. + * Under [reduceMotion] the level snaps instead of settling, and nothing here + * ever schedules periodic work — redraws follow real amplitude emissions. */ @Composable fun WaveformVisualizer( @@ -36,56 +36,92 @@ fun WaveformVisualizer( isActive: Boolean, modifier: Modifier = Modifier, reduceMotion: Boolean = false, + accent: Color = AppColors.accent(), ) { - val barCount = 7 - val accent = AppColors.accent() val accentFaded = accent.copy(alpha = 0.35f) + val level = animateFloatAsState( + targetValue = levelFraction(amplitude, isActive), + animationSpec = if (reduceMotion) { + snap() + } else { + spring(dampingRatio = Spring.DampingRatioNoBouncy, stiffness = Spring.StiffnessMedium) + }, + label = "waveformLevel", + ) + Canvas(modifier = modifier) { + drawWaveBars(level, accent, accentFaded) + } +} - val breathPhase = if (isActive && !reduceMotion) { - val infiniteTransition = rememberInfiniteTransition(label = "waveActive") - val phase by infiniteTransition.animateFloat( - initialValue = 0f, - targetValue = 2f * Math.PI.toFloat(), - animationSpec = infiniteRepeatable( - animation = tween(1800, easing = LinearEasing), - repeatMode = RepeatMode.Restart +private fun DrawScope.drawWaveBars(level: State, accent: Color, accentFaded: Color) { + val barWidth = 4.dp.toPx() + val spacing = 4.dp.toPx() + val total = WAVE_BAR_COUNT * barWidth + (WAVE_BAR_COUNT - 1) * spacing + val corner = CornerRadius(2.dp.toPx()) + // Deferred state read: the value is consumed here in the draw phase, so + // settling animation frames invalidate only this canvas, not composition. + val current = level.value + var x = (size.width - total) / 2f + for (index in 0 until WAVE_BAR_COUNT) { + val height = (barHeightFraction(current, index) * size.height).coerceAtLeast(barWidth) + drawRoundRect( + brush = Brush.verticalGradient( + colors = listOf(accent, accentFaded), + startY = size.height - height, + endY = size.height, ), - label = "phase" + topLeft = Offset(x, size.height - height), + size = Size(barWidth, height), + cornerRadius = corner, ) - phase - } else { - 0f + x += barWidth + spacing } +} - Row( - modifier = modifier.height(40.dp), - horizontalArrangement = Arrangement.spacedBy(4.dp, Alignment.CenterHorizontally), - verticalAlignment = Alignment.CenterVertically - ) { - repeat(barCount) { index -> - val normalizedIndex = index.toFloat() / (barCount - 1) +// --------------------------------------------------------------------------- +// Level mapping (pure; covered by WaveformLevelMappingTest) - // Target fraction: active amplitude + per-bar offset, or idle resting bar - val target = if (isActive) { - val offset = sin(normalizedIndex * Math.PI.toFloat() + breathPhase * 1.5f) - (amplitude * (0.5f + 0.5f * offset)).coerceIn(0.15f, 1f) - } else { - 0.16f - } +internal const val WAVE_BAR_COUNT = 7 - val fraction = target.coerceIn(0.08f, 1f) +/** Silence / not-listening outline as a fraction of the full height. */ +internal const val WAVE_REST_LEVEL = 0.12f - Box( - modifier = Modifier - .width(4.dp) - .fillMaxHeight(fraction) - .clip(RoundedCornerShape(2.dp)) - .background( - Brush.verticalGradient( - colors = listOf(accent, accentFaded) - ) - ) - ) - } - } +/** + * Perceptual window edges for the logarithmic map: ~0.005 RMS is near the + * noise floor, ~0.5 RMS is loud speech. Quiet conversation (0.01–0.1 RMS) + * spreads across the lower two-thirds of the bars instead of collapsing + * into the old flat 0.15 floor. + */ +internal const val WAVE_QUIET_FLOOR = 0.005f +internal const val WAVE_LOUD_CEIL = 0.5f + +/** Hard visual floor so the capsules stay visible at rest. */ +internal const val WAVE_MIN_FRACTION = 0.06f + +/** + * Maps a real RMS capture level to the shared bar level in [0, 1]: + * `log10` compression between [WAVE_QUIET_FLOOR] and [WAVE_LOUD_CEIL]. + * Inactive or non-finite input returns [WAVE_REST_LEVEL] (static decoration, + * never a fabricated activity signal); out-of-range levels clamp. + */ +internal fun levelFraction(amplitude: Float, isActive: Boolean): Float { + if (!isActive || !amplitude.isFinite()) return WAVE_REST_LEVEL + val span = log10(WAVE_LOUD_CEIL) - log10(WAVE_QUIET_FLOOR) + val t = (log10(amplitude) - log10(WAVE_QUIET_FLOOR)) / span + return if (t.isNaN()) 0f else t.coerceIn(0f, 1f) +} + +/** Static center-weighted bar window; a fixed shape, no time phase. */ +internal fun barWindow(index: Int, barCount: Int = WAVE_BAR_COUNT): Float { + val divisor = (barCount - 1).coerceAtLeast(1) + return 0.55f + 0.45f * sin(PI.toFloat() * index / divisor) } + +/** + * Height fraction of bar [index] for a shared [level] in [0, 1]: the static + * window scaled between the rest outline and full height, bounded below by + * [WAVE_MIN_FRACTION] so the bars never disappear. + */ +internal fun barHeightFraction(level: Float, index: Int, barCount: Int = WAVE_BAR_COUNT): Float = + (barWindow(index, barCount) * (WAVE_REST_LEVEL + level * (1f - WAVE_REST_LEVEL))) + .coerceIn(WAVE_MIN_FRACTION, 1f) diff --git a/android/app/src/main/java/com/dialect/interpreter/ui/navigation/AppNavigation.kt b/android/app/src/main/java/com/dialect/interpreter/ui/navigation/AppNavigation.kt index 515e993..44b92fc 100644 --- a/android/app/src/main/java/com/dialect/interpreter/ui/navigation/AppNavigation.kt +++ b/android/app/src/main/java/com/dialect/interpreter/ui/navigation/AppNavigation.kt @@ -1,5 +1,7 @@ package com.dialect.interpreter.ui.navigation +import androidx.compose.animation.EnterTransition +import androidx.compose.animation.ExitTransition import android.app.Activity import androidx.compose.foundation.isSystemInDarkTheme import androidx.compose.animation.fadeIn @@ -54,25 +56,25 @@ fun AppNavigation(reduceMotion: Boolean = false) { navController = navController, startDestination = AppRoutes.INTERPRET, enterTransition = { - if (reduceMotion) fadeIn(tween(1)) else slideInHorizontally( + if (reduceMotion) EnterTransition.None else slideInHorizontally( initialOffsetX = { it / 4 }, animationSpec = tween(TRANSITION_MS) ) + fadeIn(tween(TRANSITION_MS)) }, exitTransition = { - if (reduceMotion) fadeOut(tween(1)) else slideOutHorizontally( + if (reduceMotion) ExitTransition.None else slideOutHorizontally( targetOffsetX = { -it / 4 }, animationSpec = tween(TRANSITION_MS) ) + fadeOut(tween(TRANSITION_MS / 2)) }, popEnterTransition = { - if (reduceMotion) fadeIn(tween(1)) else slideInHorizontally( + if (reduceMotion) EnterTransition.None else slideInHorizontally( initialOffsetX = { -it / 4 }, animationSpec = tween(TRANSITION_MS) ) + fadeIn(tween(TRANSITION_MS)) }, popExitTransition = { - if (reduceMotion) fadeOut(tween(1)) else slideOutHorizontally( + if (reduceMotion) ExitTransition.None else slideOutHorizontally( targetOffsetX = { it / 4 }, animationSpec = tween(TRANSITION_MS) ) + fadeOut(tween(TRANSITION_MS / 2)) @@ -92,7 +94,10 @@ fun AppNavigation(reduceMotion: Boolean = false) { } composable(AppRoutes.VOICE_PROFILE) { - VoiceProfileScreen(onBack = { navController.popBackStack() }) + VoiceProfileScreen( + reduceMotion = reduceMotion, + onBack = { navController.popBackStack() }, + ) } composable(AppRoutes.SETTINGS) { diff --git a/android/app/src/main/java/com/dialect/interpreter/ui/screens/InterpretScreen.kt b/android/app/src/main/java/com/dialect/interpreter/ui/screens/InterpretScreen.kt index 98f369f..8e4eac8 100644 --- a/android/app/src/main/java/com/dialect/interpreter/ui/screens/InterpretScreen.kt +++ b/android/app/src/main/java/com/dialect/interpreter/ui/screens/InterpretScreen.kt @@ -8,6 +8,15 @@ import android.net.Uri import android.provider.Settings import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.animation.AnimatedContent +import androidx.compose.animation.core.Spring +import androidx.compose.animation.core.animateFloatAsState +import androidx.compose.animation.core.spring +import androidx.compose.animation.core.tween +import androidx.compose.animation.fadeIn +import androidx.compose.animation.fadeOut +import androidx.compose.animation.slideInVertically +import androidx.compose.animation.togetherWith import androidx.compose.foundation.background import androidx.compose.foundation.combinedClickable import androidx.compose.foundation.ExperimentalFoundationApi @@ -29,6 +38,8 @@ import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.statusBarsPadding import androidx.compose.foundation.layout.width import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.interaction.MutableInteractionSource +import androidx.compose.foundation.interaction.collectIsPressedAsState import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.items import androidx.compose.foundation.lazy.rememberLazyListState @@ -72,6 +83,7 @@ import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.graphicsLayer import androidx.compose.ui.hapticfeedback.HapticFeedbackType import androidx.compose.ui.platform.LocalClipboardManager import androidx.compose.ui.platform.LocalContext @@ -99,6 +111,7 @@ import com.dialect.interpreter.session.SessionPhase import com.dialect.interpreter.session.TranscriptTurn import com.dialect.interpreter.session.TurnStatus import com.dialect.interpreter.session.WorkStage +import com.dialect.interpreter.ui.theme.AccentDarkMode import com.dialect.interpreter.ui.components.WaveformVisualizer import com.dialect.interpreter.ui.theme.AppColors import com.dialect.interpreter.ui.theme.RadiusBubble @@ -223,6 +236,7 @@ fun InterpretScreen( sourceLabel = sourceLabel, targetLabel = targetLabel, micEnabled = micEnabled, + reduceMotion = reduceMotion, onMicClick = { when { uiState.isSessionActive -> viewModel.stopSession() @@ -343,13 +357,15 @@ private fun ControlHeader( color = onControl, fontWeight = FontWeight.SemiBold, ) - Text( - sessionSubtitle(uiState), - style = MaterialTheme.typography.labelMedium, - color = if (uiState.phase == SessionPhase.ACTIVE) AppColors.accent() else onControlSecondary, - maxLines = 2, - overflow = TextOverflow.Ellipsis, - ) + StatusReveal(key = sessionSubtitle(uiState), reduceMotion = reduceMotion) { subtitle -> + Text( + subtitle, + style = MaterialTheme.typography.labelMedium, + color = if (uiState.phase == SessionPhase.ACTIVE) AccentDarkMode else onControlSecondary, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } } IconButton(onClick = onLanguageClick) { Icon( @@ -373,9 +389,15 @@ private fun ControlHeader( ) } } - if (uiState.phase == SessionPhase.ACTIVE && uiState.session.captureActive) { + if (uiState.phase == SessionPhase.ACTIVE && uiState.session.captureActive && + // Half-duplex playback: while audio is going out, the microphone + // input is not being turned into a transcript (playback feeds the + // mic back), so the input level must not be presented as listening. + WorkStage.PLAYBACK !in uiState.session.activeStages + ) { Spacer(Modifier.height(8.dp)) WaveformVisualizer( + accent = AccentDarkMode, amplitude = uiState.amplitude, isActive = true, reduceMotion = reduceMotion, @@ -404,6 +426,9 @@ private fun sessionSubtitle(uiState: InterpretUiState): String { SessionPhase.IDLE -> "${uiState.sessionModeLabel} · 已停止" SessionPhase.STARTING -> "准备模型中…" SessionPhase.ACTIVE -> when { + // Half-duplex: during playback the captured input is not being + // recognized, so this is never "聆听中" even though capture stays on. + stages.contains(WorkStage.PLAYBACK) -> "播放中" stageText.isNotEmpty() && uiState.session.captureActive -> "聆听中 · $stageText" stageText.isNotEmpty() -> "处理中 · $stageText" uiState.session.captureActive -> "聆听中" @@ -414,6 +439,33 @@ private fun sessionSubtitle(uiState: InterpretUiState): String { } } +/** + * Session-status reveal (motion spec "Session status"): a short fade with a + * small vertical settle when the status text changes, snapping under reduced + * motion. Native [AnimatedContent] only — no custom layout-affecting motion. + */ +@Composable +private fun StatusReveal( + key: String, + reduceMotion: Boolean, + content: @Composable (String) -> Unit, +) { + if (reduceMotion) { + content(key) + return + } + AnimatedContent( + targetState = key, + transitionSpec = { + ((fadeIn(tween(180)) + slideInVertically(tween(180)) { it / 4 }) togetherWith + fadeOut(tween(90))).using(null) + }, + label = "statusReveal", + ) { target -> + content(target) + } +} + @Composable private fun ProblemBanner(message: String, recoverable: Boolean) { Row( @@ -529,7 +581,12 @@ private fun TurnList( lastVisible >= info.totalItemsCount - 2 } } - androidx.compose.runtime.LaunchedEffect(turns.size) { + // Keyed on the last turn's id, not the count: the snapshot list is capped + // (maxSnapshotTurns), so once the cap is reached appending also evicts the + // oldest turn and turns.size never changes — a size key would stop + // following new turns exactly when a session runs longest (motion spec + // "New conversation item": preserve reading/focus order). + androidx.compose.runtime.LaunchedEffect(turns.lastOrNull()?.id) { if (turns.isNotEmpty() && isNearBottom) { if (reduceMotion) listState.scrollToItem(turns.lastIndex) else listState.animateScrollToItem(turns.lastIndex) @@ -543,18 +600,31 @@ private fun TurnList( verticalArrangement = Arrangement.spacedBy(6.dp), ) { items(turns, key = { "${it.sessionId}-${it.id}" }) { turn -> - TurnRow(turn = turn) + // Stable turnId keys + animateItem: new turns fade in with a small + // displacement and reorders settle via placement spring; reduced + // motion disables all three (motion spec "New conversation item"). + TurnRow( + turn = turn, + modifier = Modifier.animateItem( + fadeInSpec = if (reduceMotion) null else tween(180), + placementSpec = if (reduceMotion) null else spring( + dampingRatio = Spring.DampingRatioNoBouncy, + stiffness = Spring.StiffnessMediumLow, + ), + fadeOutSpec = if (reduceMotion) null else tween(90), + ), + ) } } } @OptIn(ExperimentalFoundationApi::class) @Composable -private fun TurnRow(turn: TranscriptTurn) { +private fun TurnRow(turn: TranscriptTurn, modifier: Modifier = Modifier) { val clipboard = LocalClipboardManager.current val haptics = LocalHapticFeedback.current - Column(modifier = Modifier.fillMaxWidth()) { + Column(modifier = modifier.fillMaxWidth()) { if (turn.sourceText.isNotBlank()) { Bubble( text = turn.sourceText, @@ -707,6 +777,7 @@ private fun InterpretBottomBar( sourceLabel: String, targetLabel: String, micEnabled: Boolean, + reduceMotion: Boolean, onMicClick: () -> Unit, ) { val recording = uiState.isSessionActive @@ -733,7 +804,9 @@ private fun InterpretBottomBar( .padding(horizontal = 16.dp, vertical = 12.dp), verticalAlignment = Alignment.CenterVertically, ) { - if (recording) { + // Same half-duplex gate as the header waveform: during playback + // the red mic would claim a live listening capture. + if (recording && WorkStage.PLAYBACK !in uiState.session.activeStages) { Icon( Icons.Filled.Mic, contentDescription = null, @@ -742,20 +815,39 @@ private fun InterpretBottomBar( ) Spacer(Modifier.width(8.dp)) } - Text( - statusText, - style = MaterialTheme.typography.bodyMedium, - color = if (uiState.phase == SessionPhase.FAILED) AppColors.error() else AppColors.textSecondary(), - maxLines = 1, - overflow = TextOverflow.Ellipsis, - ) + StatusReveal(key = statusText, reduceMotion = reduceMotion) { text -> + Text( + text, + style = MaterialTheme.typography.bodyMedium, + color = if (uiState.phase == SessionPhase.FAILED) AppColors.error() else AppColors.textSecondary(), + maxLines = 1, + overflow = TextOverflow.Ellipsis, + ) + } } Spacer(Modifier.width(10.dp)) + // Small press feedback on the primary action (motion spec "Main + // action"): one scalar animated between states and applied in the + // draw phase via graphicsLayer — no layout resize, disabled under + // reduced motion (the native ripple still runs). + val micInteraction = remember { MutableInteractionSource() } + val micPressed by micInteraction.collectIsPressedAsState() + val micScale = animateFloatAsState( + targetValue = if (micPressed && !reduceMotion) 0.94f else 1f, + animationSpec = tween(120), + label = "micPressScale", + ) FilledIconButton( onClick = onMicClick, enabled = micEnabled, + interactionSource = micInteraction, modifier = Modifier .size(56.dp) + .graphicsLayer { + val scale = micScale.value // draw-phase read only + scaleX = scale + scaleY = scale + } .semantics { contentDescription = if (recording) "停止录音" else "开始录音" }, diff --git a/android/app/src/main/java/com/dialect/interpreter/ui/screens/SettingsScreen.kt b/android/app/src/main/java/com/dialect/interpreter/ui/screens/SettingsScreen.kt index c842e3c..19485f3 100644 --- a/android/app/src/main/java/com/dialect/interpreter/ui/screens/SettingsScreen.kt +++ b/android/app/src/main/java/com/dialect/interpreter/ui/screens/SettingsScreen.kt @@ -97,7 +97,7 @@ fun SettingsScreen( verticalArrangement = Arrangement.spacedBy(16.dp), ) { SettingsSection("运行环境") { - InfoRow(Icons.Filled.Memory, "推理后端", "CPU (ONNX Runtime 1.22)") + InfoRow(Icons.Filled.Memory, "运算设备", "本机 CPU") InfoRow(Icons.Filled.Smartphone, "SoC", socName.ifBlank { "—" }) InfoRow(Icons.Filled.Info, "处理方式", "全部在本机离线完成") } diff --git a/android/app/src/main/java/com/dialect/interpreter/ui/screens/VoiceProfileScreen.kt b/android/app/src/main/java/com/dialect/interpreter/ui/screens/VoiceProfileScreen.kt index 6278655..55857a1 100644 --- a/android/app/src/main/java/com/dialect/interpreter/ui/screens/VoiceProfileScreen.kt +++ b/android/app/src/main/java/com/dialect/interpreter/ui/screens/VoiceProfileScreen.kt @@ -88,7 +88,7 @@ import kotlinx.coroutines.launch */ @OptIn(ExperimentalMaterial3Api::class) @Composable -fun VoiceProfileScreen(onBack: () -> Unit) { +fun VoiceProfileScreen(reduceMotion: Boolean, onBack: () -> Unit) { val appContext = LocalContext.current.applicationContext as DialectApp val repository = appContext.container.voiceProfileRepository val scope = rememberCoroutineScope() @@ -261,6 +261,7 @@ fun VoiceProfileScreen(onBack: () -> Unit) { RecordSheet( recorder = recorder, amplitudeProvider = { amplitude }, + reduceMotion = reduceMotion, onMessage = { message -> scope.launch { snackbar.showSnackbar(message) } }, onSaved = { showRecordSheet = false @@ -276,6 +277,7 @@ fun VoiceProfileScreen(onBack: () -> Unit) { private fun RecordSheet( recorder: AudioRecorder, amplitudeProvider: () -> Float, + reduceMotion: Boolean, onMessage: (String) -> Unit, onSaved: () -> Unit, onDismiss: () -> Unit, @@ -364,6 +366,7 @@ private fun RecordSheet( WaveformVisualizer( amplitude = if (isRecording) amplitude else amplitudeProvider(), isActive = isRecording, + reduceMotion = reduceMotion, modifier = Modifier.fillMaxWidth().height(48.dp), ) Spacer(Modifier.height(16.dp)) diff --git a/android/app/src/test/java/com/dialect/interpreter/inference/Qwen3TtsProtocolTest.kt b/android/app/src/test/java/com/dialect/interpreter/inference/Qwen3TtsProtocolTest.kt index efa70c0..397b297 100644 --- a/android/app/src/test/java/com/dialect/interpreter/inference/Qwen3TtsProtocolTest.kt +++ b/android/app/src/test/java/com/dialect/interpreter/inference/Qwen3TtsProtocolTest.kt @@ -9,7 +9,14 @@ import org.junit.Test import java.io.File import java.nio.ByteBuffer import java.nio.ByteOrder +import kotlin.math.PI import kotlin.math.abs +import kotlin.math.cos +import kotlin.math.exp +import kotlin.math.ln +import kotlin.math.max +import kotlin.math.sin +import kotlin.math.sqrt import kotlin.random.Random /** @@ -439,6 +446,7 @@ class Qwen3TtsProtocolTest { assertEquals("english", Qwen3TtsProtocol.languageCodeToConfigKey["en"]) } + @Test fun `code flattening is group-major for the vocoder`() { // frames f=0..2 × groups g=0..1; value encodes (g, f) to catch swaps. val frames = listOf(intArrayOf(0, 1), intArrayOf(10, 11), intArrayOf(20, 21)) @@ -454,4 +462,485 @@ class Qwen3TtsProtocolTest { Qwen3TtsProtocol.flattenCodesGroupMajor(frames, numCodebooks = 2) } } + + // ---- Frozen 2026-09-08 baseline oracles --------------------------------- + // Bit-exact copies of the pre-rework implementations. The new code must + // reproduce these value for value (same tokens for the same RNG stream); + // they are duplicated here on purpose so a future change cannot silently + // move both sides. Baseline: v0.1.0-preview.1 (7e2cb57). + + private fun legacySampleGroup0( + logitsLast: FloatArray, + cfg: Qwen3TtsBundleConfig, + temperature: Float, + topK: Int, + repetitionPenalty: Float, + generated: List, + random: Random, + suppressEos: Boolean, + ): Int { + val vocab = cfg.talkerVocab + require(vocab > 0 && logitsLast.size >= vocab) { "Invalid talker logits shape" } + require(repetitionPenalty.isFinite() && repetitionPenalty > 0f) { "Invalid repetition penalty" } + val probs = FloatArray(vocab) + System.arraycopy(logitsLast, logitsLast.size - vocab, probs, 0, vocab) + require(probs.all { it.isFinite() || it == Float.NEGATIVE_INFINITY }) { "Invalid talker logits" } + if (suppressEos) probs[cfg.codecEosId] = Float.NEGATIVE_INFINITY + for (token in generated.toSet()) { + require(token in probs.indices) { "Invalid generated codec token" } + if (probs[token] > 0f) probs[token] /= repetitionPenalty else probs[token] *= repetitionPenalty + } + for (i in cfg.cpVocab until vocab) { + if (i != cfg.codecEosId) probs[i] = Float.NEGATIVE_INFINITY + } + return legacySampleFromLogits(probs, temperature, topK, random) + } + + private fun legacySampleCodePredictor( + logitsLast: FloatArray, + cfg: Qwen3TtsBundleConfig, + temperature: Float, + topK: Int, + random: Random, + ): Int { + val vocab = cfg.cpVocab + require(vocab > 0 && logitsLast.size >= vocab) { "Invalid code-predictor logits shape" } + val probs = FloatArray(vocab) + System.arraycopy(logitsLast, logitsLast.size - vocab, probs, 0, vocab) + return legacySampleFromLogits(probs, temperature, topK, random) + } + + private fun legacySampleFromLogits( + probs: FloatArray, + temperature: Float, + topK: Int, + random: Random, + ): Int { + require(temperature.isFinite() && temperature >= 0f) { "Invalid sampling temperature" } + require(topK >= 0) { "topK must be nonnegative (0 disables filtering)" } + require(probs.isNotEmpty() && probs.all { it.isFinite() || it == Float.NEGATIVE_INFINITY }) { + "Sampling logits contain NaN or positive infinity" + } + var best = 0 + for (i in probs.indices) if (probs[i] > probs[best]) best = i + val maxLogit = probs[best] + require(maxLogit.isFinite()) { "No finite sampling candidate remains" } + if (temperature == 0f || topK == 1) return best + if (topK in 1 until probs.size) { + val threshold = probs.copyOf().sortedDescending()[topK - 1] + for (i in probs.indices) if (probs[i] < threshold) probs[i] = Float.NEGATIVE_INFINITY + } + val weights = DoubleArray(probs.size) { + exp((probs[it].toDouble() - maxLogit.toDouble()) / temperature.toDouble()) + } + val sum = weights.sum() + check(sum.isFinite() && sum > 0.0) { "Invalid sampling probability mass" } + val r = random.nextDouble() * sum + var cum = 0.0 + for (i in weights.indices) { + cum += weights[i] + if (r < cum) return i + } + return weights.indices.last { weights[it] > 0.0 } + } + + private fun legacyLogMelSpectrogram(audio: FloatArray, sr: Int): Qwen3TtsProtocol.LogMelResult { + val nFft = 1024 + val hop = 256 + val nMels = 128 + val pad = (nFft - hop) / 2 + val padded = FloatArray(audio.size + 2 * pad) + for (i in padded.indices) { + padded[i] = audio[Qwen3TtsProtocol.reflectIndex(i - pad, audio.size)] + } + val window = FloatArray(nFft) { (0.5 - 0.5 * cos(2.0 * PI * it / nFft)).toFloat() } + val frames = 1 + (padded.size - nFft) / hop + val basis = Qwen3TtsProtocol.buildMelFilterbank(sr, nFft, nMels, 0.0, sr / 2.0) + val out = FloatArray(frames * nMels) + val re = DoubleArray(nFft) + val im = DoubleArray(nFft) + val nFreqs = nFft / 2 + 1 + for (f in 0 until frames) { + val start = f * hop + for (i in 0 until nFft) { + re[i] = (padded[start + i] * window[i]).toDouble() + im[i] = 0.0 + } + legacyFftRadix2(re, im) + for (m in 0 until nMels) { + var energy = 0.0 + for (k in 0 until nFreqs) { + val mag = sqrt(re[k] * re[k] + im[k] * im[k] + 1e-9) + energy += basis[m * nFreqs + k] * mag + } + out[f * nMels + m] = max(energy, 1e-5).let(::ln).toFloat() + } + } + return Qwen3TtsProtocol.LogMelResult(out, frames) + } + + private fun legacyFftRadix2(re: DoubleArray, im: DoubleArray) { + val n = re.size + val bits = Integer.numberOfTrailingZeros(n) + for (i in 0 until n) { + var x = i + var j = 0 + repeat(bits) { j = (j shl 1) or (x and 1); x = x shr 1 } + if (j > i) { + var t = re[i]; re[i] = re[j]; re[j] = t + t = im[i]; im[i] = im[j]; im[j] = t + } + } + var size = 2 + while (size <= n) { + val half = size / 2 + val angle = -2.0 * PI / size + for (i in 0 until n step size) { + for (k in 0 until half) { + val c = cos(angle * k) + val s = sin(angle * k) + val tReal = c * re[i + k + half] - s * im[i + k + half] + val tImag = s * re[i + k + half] + c * im[i + k + half] + re[i + k + half] = re[i + k] - tReal + im[i + k + half] = im[i + k] - tImag + re[i + k] += tReal + im[i + k] += tImag + } + } + size *= 2 + } + } + + /** Adversarial logit distributions; deterministic per (name, vocab, seed). */ + private fun distribution(name: String, vocab: Int, seed: Long): FloatArray = when (name) { + "uniform" -> FloatArray(vocab) { Random(seed + it).nextFloat() * 8f - 4f } + // Quantized values force ties at and around every topK boundary. + "quantized-duplicates" -> FloatArray(vocab) { + listOf(-3f, 0.5f, 0.5f, 2f, 2f, 2f, 7f)[Random(seed + it).nextInt(7)] + } + "few-finite" -> FloatArray(vocab) { + if (Random(seed + it).nextInt(100) < 3) Random(seed + it * 7).nextFloat() * 4f + else Float.NEGATIVE_INFINITY + } + "all-equal" -> FloatArray(vocab) { 1.5f } + else -> throw AssertionError(name) + } + + @Test + fun `group0 sampler is value-identical to frozen baseline across topK and ties`() { + val cfg = testConfig() + val vocab = cfg.talkerVocab + for (dist in listOf("uniform", "quantized-duplicates", "few-finite", "all-equal")) { + for (topK in intArrayOf(0, 1, 2, 50, 51, vocab - 1, vocab)) { + val logits = distribution(dist, vocab, seed = 1000L + topK) + val rngNew = Random(777) + val rngLegacy = Random(777) + val generatedNew = ArrayList() + val generatedLegacy = ArrayList() + for (step in 0 until 150) { + val suppress = step < 2 + val a = Qwen3TtsProtocol.sampleGroup0( + logits, cfg, 0.9f, topK, 1.05f, generatedNew, rngNew, suppress) + val b = legacySampleGroup0( + logits, cfg, 0.9f, topK, 1.05f, generatedLegacy, rngLegacy, suppress) + assertEquals("dist=$dist topK=$topK step=$step", b, a) + if (a == cfg.codecEosId) break + generatedNew.add(a) + generatedLegacy.add(b) + } + } + } + } + + @Test + fun `code predictor sampler is value-identical to frozen baseline across topK and ties`() { + val cfg = testConfig() + val vocab = cfg.cpVocab + for (dist in listOf("uniform", "quantized-duplicates", "few-finite", "all-equal")) { + for (topK in intArrayOf(0, 1, 2, 50, 51, vocab - 1, vocab)) { + val logits = distribution(dist, vocab, seed = 2000L + topK) + val rngNew = Random(555) + val rngLegacy = Random(555) + for (step in 0 until 200) { + val a = Qwen3TtsProtocol.sampleCodePredictor(logits, cfg, 0.9f, topK, rngNew) + val b = legacySampleCodePredictor(logits, cfg, 0.9f, topK, rngLegacy) + assertEquals("dist=$dist topK=$topK step=$step", b, a) + } + } + } + } + + /** Bit-exact assertion, except either zero sign may stand in for the + * other: the mask consumer `x < threshold` treats ±0.0 identically and + * the heap's IEEE comparisons do not distinguish them. */ + private fun assertSortedThreshold(expected: Float, actual: Float, message: String) { + assertTrue( + "$message expected=$expected (0x${Integer.toHexString(java.lang.Float.floatToRawIntBits(expected))}) " + + "actual=$actual (0x${Integer.toHexString(java.lang.Float.floatToRawIntBits(actual))})", + java.lang.Float.floatToRawIntBits(expected) == java.lang.Float.floatToRawIntBits(actual) || + (expected == 0f && actual == 0f)) + } + + @Test + fun `kth largest matches the sorted multiset reference exactly`() { + val rng = Random(2024) + val sizes = intArrayOf(1, 2, 3, 5, 8, 17, 64, 257) + for (size in sizes) { + for (trial in 0 until 6) { + val values = when (trial) { + 0 -> FloatArray(size) { rng.nextFloat() * 20f - 10f } + 1 -> FloatArray(size) { if (rng.nextBoolean()) 1f else 2f } + 2 -> FloatArray(size) { 0f } + 3 -> FloatArray(size).also { if (size > 0) it[rng.nextInt(size)] = 7f } + 4 -> FloatArray(size) { + if (rng.nextInt(4) == 0) Float.NEGATIVE_INFINITY else rng.nextFloat() + } + else -> FloatArray(size) { + when (rng.nextInt(5)) { + 0 -> -0f + 1 -> 0f + 2 -> 1f + 3 -> -1f + else -> Float.NEGATIVE_INFINITY + } + } + } + val sorted = values.copyOf().sortedDescending() + for (k in 1..size) { + assertSortedThreshold( + sorted[k - 1], Qwen3TtsProtocol.kthLargestFloat(values, k), + "size=$size trial=$trial k=$k") + } + } + } + // Large-vocab k sweep, sampled to keep the run fast. + for (size in intArrayOf(1024, 3072)) { + for (trial in 0 until 3) { + val values = FloatArray(size) { + if (Random(size + trial * 91 + it.toLong()).nextInt(8) == 0) { + Float.NEGATIVE_INFINITY + } else { + Random(size * 3 + trial * 17 + it.toLong()).nextFloat() * 6f - 3f + } + } + val sorted = values.copyOf().sortedDescending() + var k = 1 + while (k <= size) { + assertSortedThreshold( + sorted[k - 1], Qwen3TtsProtocol.kthLargestFloat(values, k), + "size=$size trial=$trial k=$k") + k += size / 48 + } + assertSortedThreshold( + sorted[size - 1], Qwen3TtsProtocol.kthLargestFloat(values, size), + "size=$size trial=$trial k=$size") + } + } + } + + @Test + fun `samplers read the trailing vocab slice of padded logits`() { + val cfg = testConfig() + // Production shapes: the CP graph emits [1,2,2048] at the g=1 prefill + // step (flattened to 4096 floats), so the sampler MUST use the LAST + // vocab values of a longer array. Pin the tail offset twice — against + // the frozen oracle and against the tail-only call (an offset + // mutation would otherwise keep every exactly-vocab-sized test green). + val pad = 37 + for (dist in listOf("uniform", "quantized-duplicates")) { + val cpTail = distribution(dist, cfg.cpVocab, seed = 77) + val cpPadded = FloatArray(pad + cfg.cpVocab) { + if (it < pad) 500f + it else cpTail[it - pad] + } + val g0Tail = distribution(dist, cfg.talkerVocab, seed = 78) + val g0Padded = FloatArray(pad + cfg.talkerVocab) { + if (it < pad) 500f + it else g0Tail[it - pad] + } + for (step in 0 until 50) { + val cpSeed = step * 2L + val g0Seed = step * 2L + 1 + // Oracle parity on the padded input pins the tail offset for + // both implementations. + assertEquals( + "oracle cp dist=$dist step=$step", + legacySampleCodePredictor(cpPadded, cfg, 0.9f, 50, Random(cpSeed)), + Qwen3TtsProtocol.sampleCodePredictor(cpPadded, cfg, 0.9f, 50, Random(cpSeed))) + // Prefix invisibility: padded and tail-only inputs with the + // same RNG stream must produce the same tokens. + assertEquals( + "tail cp dist=$dist step=$step", + Qwen3TtsProtocol.sampleCodePredictor(cpTail, cfg, 0.9f, 50, Random(cpSeed)), + Qwen3TtsProtocol.sampleCodePredictor(cpPadded, cfg, 0.9f, 50, Random(cpSeed))) + assertEquals( + "oracle g0 dist=$dist step=$step", + legacySampleGroup0( + g0Padded, cfg, 0.9f, 50, 1.05f, emptyList(), Random(g0Seed), false), + Qwen3TtsProtocol.sampleGroup0( + g0Padded, cfg, 0.9f, 50, 1.05f, emptyList(), Random(g0Seed), false)) + assertEquals( + "tail g0 dist=$dist step=$step", + Qwen3TtsProtocol.sampleGroup0( + g0Tail, cfg, 0.9f, 50, 1.05f, emptyList(), Random(g0Seed), false), + Qwen3TtsProtocol.sampleGroup0( + g0Padded, cfg, 0.9f, 50, 1.05f, emptyList(), Random(g0Seed), false)) + } + } + } + + @Test + fun `kth largest rejects empty arrays and out-of-range orders`() { + assertThrows(IllegalArgumentException::class.java) { + Qwen3TtsProtocol.kthLargestFloat(FloatArray(0), 1) + } + assertThrows(IllegalArgumentException::class.java) { + Qwen3TtsProtocol.kthLargestFloat(FloatArray(4) { it.toFloat() }, 0) + } + assertThrows(IllegalArgumentException::class.java) { + Qwen3TtsProtocol.kthLargestFloat(FloatArray(4) { it.toFloat() }, 5) + } + } + + @Test + fun `fft is bit-identical to frozen baseline across sizes`() { + for (n in intArrayOf(2, 4, 8, 64, 256, 1024, 4096)) { + val rng = Random(n.toLong()) + val reRef = DoubleArray(n) { rng.nextDouble() * 2 - 1 } + val imRef = DoubleArray(n) { rng.nextDouble() * 2 - 1 } + val reNew = reRef.copyOf() + val imNew = imRef.copyOf() + Qwen3TtsProtocol.fftRadix2(reNew, imNew) + legacyFftRadix2(reRef, imRef) + for (i in 0 until n) { + assertEquals("re[$i] n=$n", reRef[i], reNew[i], 0.0) + assertEquals("im[$i] n=$n", imRef[i], imNew[i], 0.0) + } + } + } + + @Test + fun `log mel is bit-identical to frozen baseline frontend`() { + val cases = linkedMapOf( + "zeros" to FloatArray(24_000), + "dc" to FloatArray(24_000) { 0.5f }, + "sine" to FloatArray(24_000) { + (0.4 * sin(2.0 * PI * 220.0 * it / 24_000.0)).toFloat() + }, + "impulse" to FloatArray(24_000).also { it[12_000] = 1f }, + "noise-2s" to FloatArray(48_000) { Random(it * 31L + 7).nextFloat() * 2f - 1f }, + "minimum-window" to FloatArray(256) { 0.25f }, + "sub-fft" to FloatArray(1023) { (it % 7) / 10f }, + "verified-8k" to FloatArray(8192).also { a -> for (i in 1000 until 6000) a[i] = 0.5f }, + ) + for ((name, audio) in cases) { + val legacy = legacyLogMelSpectrogram(audio, 24000) + val current = Qwen3TtsProtocol.logMelSpectrogram(audio, 24000) + assertEquals("frames $name", legacy.frames, current.frames) + assertTrue("bitwise $name", legacy.data.contentEquals(current.data)) + } + } + + @Test + fun `log mel rejects audio shorter than one hop cleanly`() { + // Baseline crashed with ArrayIndexOutOfBoundsException for these + // inputs; the guard now rejects them before touching the arrays. + for (size in intArrayOf(2, 100, 255)) { + assertThrows(IllegalArgumentException::class.java) { + Qwen3TtsProtocol.logMelSpectrogram(FloatArray(size) { 0.3f }, 24000) + } + } + // One full hop is exactly the minimum the frame loop can consume. + Qwen3TtsProtocol.logMelSpectrogram(FloatArray(256) { 0.3f }, 24000) + } + + // ---- Microbenchmarks (gated; JVM-only, indicative) ----------------------- + // Run with: AURALIS_PERF_BENCH=1 gradlew :app:testDebugUnitTest + // --tests "com.dialect.interpreter.inference.Qwen3TtsProtocolTest" + // Medians are JVM (JIT C2) numbers on the host; they characterize the + // algorithmic change (work and allocation per call), NOT phone or + // end-to-end synthesis latency. The legacy legs include their garbage + // (copyOf + boxed sorted list) — that is part of the measured defect. + + private var benchSink = 0 + + private inline fun benchMedian(repeats: Int, block: () -> Unit): Long { + repeat(repeats / 4 + 10) { block() } // JIT warmup + val samples = LongArray(repeats) + for (r in 0 until repeats) { + val t0 = System.nanoTime() + block() + samples[r] = System.nanoTime() - t0 + } + samples.sort() + return samples[repeats / 2] + } + + @Test + fun `benchmark sampling and mel rework`() { + Assume.assumeTrue( + "set AURALIS_PERF_BENCH=1 to run the microbenchmarks", + System.getenv("AURALIS_PERF_BENCH") == "1") + val cfg = testConfig() + val repeats = 400 + + // Code-predictor shape (vocab 2048), production topK/temperature. + val cpUniform = distribution("uniform", cfg.cpVocab, seed = 5) + val cpTies = distribution("quantized-duplicates", cfg.cpVocab, seed = 5) + val rngLegacyU = Random(1) + val rngNewU = Random(1) + val rngLegacyT = Random(1) + val rngNewT = Random(1) + val cpLegacyU = benchMedian(repeats) { + benchSink += legacySampleCodePredictor(cpUniform, cfg, 0.9f, 50, rngLegacyU) + } + val cpNewU = benchMedian(repeats) { + benchSink += Qwen3TtsProtocol.sampleCodePredictor(cpUniform, cfg, 0.9f, 50, rngNewU) + } + val cpLegacyT = benchMedian(repeats) { + benchSink += legacySampleCodePredictor(cpTies, cfg, 0.9f, 50, rngLegacyT) + } + val cpNewT = benchMedian(repeats) { + benchSink += Qwen3TtsProtocol.sampleCodePredictor(cpTies, cfg, 0.9f, 50, rngNewT) + } + + // Group-0 shape (vocab 3072) with a mid-stream generated list. + val g0Logits = distribution("uniform", cfg.talkerVocab, seed = 6) + val generated = List(512) { it % cfg.cpVocab } + val rngLegacyG = Random(1) + val rngNewG = Random(1) + val g0Legacy = benchMedian(repeats) { + benchSink += legacySampleGroup0( + g0Logits, cfg, 0.9f, 50, 1.05f, generated, rngLegacyG, false) + } + val g0New = benchMedian(repeats) { + benchSink += Qwen3TtsProtocol.sampleGroup0( + g0Logits, cfg, 0.9f, 50, 1.05f, generated, rngNewG, false) + } + + // Threshold selection alone vs full sort+copy. + val values = distribution("uniform", cfg.cpVocab, seed = 9) + val sortOnly = benchMedian(repeats) { + benchSink += values.copyOf().sortedDescending()[49].toInt() + } + val selectOnly = benchMedian(repeats) { + benchSink += Qwen3TtsProtocol.kthLargestFloat(values, 50).toInt() + } + + // Mel frontend over ~2 s of 24 kHz audio (~189 frames). + val audio = FloatArray(48_000) { Random(it * 13L + 1).nextFloat() * 2f - 1f } + val melRepeats = 10 + val melLegacy = benchMedian(melRepeats) { + benchSink += legacyLogMelSpectrogram(audio, 24000).data[0].toInt() + } + val melNew = benchMedian(melRepeats) { + benchSink += Qwen3TtsProtocol.logMelSpectrogram(audio, 24000).data[0].toInt() + } + + fun ratio(legacy: Long, new: Long) = String.format("%.2fx", legacy.toDouble() / new) + println("BENCH sampleCodePredictor[2048,uniform] legacy=${cpLegacyU}ns new=${cpNewU}ns ${ratio(cpLegacyU, cpNewU)}") + println("BENCH sampleCodePredictor[2048,ties] legacy=${cpLegacyT}ns new=${cpNewT}ns ${ratio(cpLegacyT, cpNewT)}") + println("BENCH sampleGroup0[3072,gen=512] legacy=${g0Legacy}ns new=${g0New}ns ${ratio(g0Legacy, g0New)}") + println("BENCH threshold-only[2048,k=50] sort=${sortOnly}ns select=${selectOnly}ns ${ratio(sortOnly, selectOnly)}") + println("BENCH logMelSpectrogram[2s audio] legacy=${melLegacy}ns new=${melNew}ns ${ratio(melLegacy, melNew)}") + assertTrue(benchSink != Int.MIN_VALUE) + } } diff --git a/android/app/src/test/java/com/dialect/interpreter/inference/TtsApi2PreparedReferenceTest.kt b/android/app/src/test/java/com/dialect/interpreter/inference/TtsApi2PreparedReferenceTest.kt new file mode 100644 index 0000000..b205072 --- /dev/null +++ b/android/app/src/test/java/com/dialect/interpreter/inference/TtsApi2PreparedReferenceTest.kt @@ -0,0 +1,167 @@ +package com.dialect.interpreter.inference + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotSame +import org.junit.Assert.assertNull +import org.junit.Assert.assertSame +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * JVM contract tests for [TtsApi2Runtime.PreparedReference] immutability — + * the conditioning package produced by [TtsApi2Runtime.prepareReference]. + * + * The class is internal-nested with an internal constructor, but the unit-test + * compilation is a Kotlin friend module of the app module, so the constructor + * and [TtsApi2Runtime.VocoderWarmState] are constructible here without a + * device. No model files and no ONNX runtime are involved: a prepared + * reference is pure data (exactly how the engine treats it after release()), + * so the defensive-copy contract is fully verifiable on the JVM. + * + * Coverage boundary: the engine-side release of the ~190 MB reference_encoder + * session on failed/cancelled preparations is a resource-lifecycle property + * (OnnxModelManager.release -> sessions.remove(key)?.close()) with no public + * observable, so it is covered by the try/finally restructure in + * prepareReference and static review, not by an automated test here. + */ +class TtsApi2PreparedReferenceTest { + + private fun warmState(): TtsApi2Runtime.VocoderWarmState = TtsApi2Runtime.VocoderWarmState( + convState = floatArrayOf(1f, 2f, 3f), + pastKeys = floatArrayOf(4f), + pastValues = floatArrayOf(5f, 6f), + position = 7L) + + private fun prepared( + embedding: FloatArray = floatArrayOf(0.25f, -0.5f, 1f), + referenceText: String? = "reference transcript", + referenceTokenIds: IntArray? = intArrayOf(11, 12, 13), + referenceCodes: IntArray? = intArrayOf(21, 22, 23, 24), + referenceFrames: Int = 2, + warm: TtsApi2Runtime.VocoderWarmState? = warmState(), + engineToken: Any = Any(), + generation: Int = 3, + ): TtsApi2Runtime.PreparedReference = TtsApi2Runtime.PreparedReference( + embedding, referenceText, referenceTokenIds, referenceCodes, referenceFrames, + "identity-digest", warm, engineToken, generation) + + @Test + fun `embedding is copied at construction and on every read`() { + val source = floatArrayOf(0.25f, -0.5f, 1f) + val p = prepared(embedding = source) + + // Mutating the constructor argument after construction must not leak in. + source[0] = 99f + source[1] = 99f + assertTrue(p.embedding.contentEquals(floatArrayOf(0.25f, -0.5f, 1f))) + + // Mutating a handed-out array must not leak back into the instance. + val handedOut = p.embedding + handedOut[2] = -99f + assertTrue(p.embedding.contentEquals(floatArrayOf(0.25f, -0.5f, 1f))) + + // Every read hands out an independent copy, never the stored snapshot. + assertNotSame(handedOut, p.embedding) + } + + @Test + fun `reference token ids are copied at construction and on every read`() { + val source = intArrayOf(11, 12, 13) + val p = prepared(referenceTokenIds = source) + + source[0] = -1 + assertTrue(p.referenceTokenIds!!.contentEquals(intArrayOf(11, 12, 13))) + + val handedOut = p.referenceTokenIds!! + handedOut[1] = -1 + assertTrue(p.referenceTokenIds!!.contentEquals(intArrayOf(11, 12, 13))) + assertNotSame(handedOut, p.referenceTokenIds) + } + + @Test + fun `reference codes are copied at construction and on every read`() { + val source = intArrayOf(21, 22, 23, 24) + val p = prepared(referenceCodes = source) + + source[0] = -1 + assertTrue(p.referenceCodes!!.contentEquals(intArrayOf(21, 22, 23, 24))) + + val handedOut = p.referenceCodes!! + handedOut[3] = -1 + assertTrue(p.referenceCodes!!.contentEquals(intArrayOf(21, 22, 23, 24))) + assertNotSame(handedOut, p.referenceCodes) + } + + @Test + fun `scalar and binding fields pass through unchanged`() { + val engineToken = Any() + val warm = warmState() + val p = prepared( + referenceText = "reference transcript", + referenceFrames = 2, + warm = warm, + engineToken = engineToken, + generation = 3, + ) + assertEquals("reference transcript", p.referenceText) + assertEquals(2, p.referenceFrames) + assertEquals("identity-digest", p.identity) + assertEquals(3, p.generation) + // The staleness binding keeps the exact token object the engine handed over. + assertSame(engineToken, p.engineToken) + // The warm snapshot is the exact instance the engine produced; a lost + // or copied replacement here would fail every ICL turn. + assertSame(warm, p.vocoderWarmState) + } + + @Test + fun `isIcl tracks the constructed codes and cannot be toggled through copies`() { + val p = prepared() + assertTrue(p.isIcl) + assertEquals(p.referenceCodes != null, p.isIcl) + + // Overwriting a handed-out copy's contents cannot turn ICL off or on. + p.referenceCodes!![0] = -1 + assertTrue(p.isIcl) + assertTrue(p.referenceCodes!!.contentEquals(intArrayOf(21, 22, 23, 24))) + } + + @Test + fun `xvector-only prepared reference has no ICL conditioning`() { + val p = prepared( + referenceText = null, + referenceTokenIds = null, + referenceCodes = null, + referenceFrames = 0, + warm = null, + ) + assertFalse(p.isIcl) + assertNull(p.referenceText) + assertNull(p.referenceTokenIds) + assertNull(p.referenceCodes) + assertNull(p.vocoderWarmState) + // The embedding remains fully usable for the legacy xvector port. + assertTrue(p.embedding.contentEquals(floatArrayOf(0.25f, -0.5f, 1f))) + } + + @Test + fun `vocoder warm state copies are independent of the source snapshot`() { + // The engine hands the warm snapshot to every turn through the same + // array-copy mechanism (VocoderTurnState.fromSnapshot copies each + // array); VocoderWarmState.copy() exercises that copying directly. + val snapshot = warmState() + val turn = snapshot.copy() + assertNotSame(snapshot.convState, turn.convState) + assertNotSame(snapshot.pastKeys, turn.pastKeys) + assertNotSame(snapshot.pastValues, turn.pastValues) + assertEquals(snapshot.position, turn.position) + + turn.convState[0] = -99f + turn.pastKeys[0] = -99f + turn.pastValues[0] = -99f + assertTrue(snapshot.convState.contentEquals(floatArrayOf(1f, 2f, 3f))) + assertTrue(snapshot.pastKeys.contentEquals(floatArrayOf(4f))) + assertTrue(snapshot.pastValues.contentEquals(floatArrayOf(5f, 6f))) + } +} diff --git a/android/app/src/test/java/com/dialect/interpreter/ui/WaveformLevelMappingTest.kt b/android/app/src/test/java/com/dialect/interpreter/ui/WaveformLevelMappingTest.kt new file mode 100644 index 0000000..d48d5e6 --- /dev/null +++ b/android/app/src/test/java/com/dialect/interpreter/ui/WaveformLevelMappingTest.kt @@ -0,0 +1,98 @@ +package com.dialect.interpreter.ui + +import com.dialect.interpreter.ui.components.WAVE_LOUD_CEIL +import com.dialect.interpreter.ui.components.WAVE_MIN_FRACTION +import com.dialect.interpreter.ui.components.WAVE_QUIET_FLOOR +import com.dialect.interpreter.ui.components.WAVE_REST_LEVEL +import com.dialect.interpreter.ui.components.barHeightFraction +import com.dialect.interpreter.ui.components.barWindow +import com.dialect.interpreter.ui.components.levelFraction +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import kotlin.math.nextDown +import kotlin.math.nextUp + +/** + * Pure mapping behind the waveform visualizer (motion spec: real RMS, bounded + * visual mapping readable for quiet speech, safe static fallback for + * non-finite input). Compose-free so it runs on the JVM. + */ +class WaveformLevelMappingTest { + + @Test + fun `active level is bounded and clamps out-of-range rms`() { + assertEquals(0f, levelFraction(0f, isActive = true)) + assertEquals(0f, levelFraction(WAVE_QUIET_FLOOR, isActive = true)) + assertEquals(1f, levelFraction(WAVE_LOUD_CEIL, isActive = true)) + assertEquals(1f, levelFraction(WAVE_LOUD_CEIL * 8f, isActive = true)) + for (i in 0 until 200) { + val rms = i / 50f + val level = levelFraction(rms, isActive = true) + assertTrue("rms=$rms level=$level", level in 0f..1f) + } + } + + @Test + fun `quiet speech spreads above the rest outline instead of collapsing`() { + // 0.01–0.1 RMS is ordinary conversation; it must land clearly above the + // rest outline and well before the ceiling (the old 0.15 floor made it + // indistinguishable from silence). + val quiet = levelFraction(0.01f, isActive = true) + val normal = levelFraction(0.1f, isActive = true) + assertTrue("0.01 RMS should be visible: $quiet", quiet >= WAVE_REST_LEVEL + 0.02f) + assertTrue("0.1 RMS should be strong: $normal", normal >= 0.6f) + assertTrue(normal < 1f) + } + + @Test + fun `level is monotonic in rms`() { + var previous = -1f + var rms = 1e-4f + while (rms <= 1f) { + val level = levelFraction(rms, isActive = true) + assertTrue("rms=$rms regressed: $level < $previous", level >= previous) + previous = level + rms = rms.nextUp() + } + } + + @Test + fun `inactive and non-finite inputs fall back to the static rest level`() { + for (amplitude in listOf(Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, 0.3f)) { + assertEquals( + "inactive amplitude=$amplitude", + WAVE_REST_LEVEL, levelFraction(amplitude, isActive = false)) + } + for (amplitude in listOf(Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY)) { + assertEquals( + "non-finite amplitude=$amplitude", + WAVE_REST_LEVEL, levelFraction(amplitude, isActive = true)) + } + } + + @Test + fun `bar heights stay bounded with a center-weighted static window`() { + for (level in listOf(0f, 0.15f, 0.65f, 1f)) { + for (index in 0 until 7) { + val fraction = barHeightFraction(level, index) + assertTrue("level=$level index=$index fraction=$fraction", fraction in WAVE_MIN_FRACTION..1f) + } + } + // The window is a fixed shape: center bar tallest, edges equal. + assertTrue(barWindow(3) > barWindow(0) && barWindow(3) > barWindow(6)) + assertEquals(barWindow(0), barWindow(6), 1e-6f) + // Rest outline: level 0 keeps every bar at or above the visible floor. + for (index in 0 until 7) { + assertTrue(barHeightFraction(0f, index) >= WAVE_MIN_FRACTION) + } + } + + @Test + fun `bar count of one does not divide by zero`() { + // Degenerate count: the window guard clamps the divisor instead of + // crashing; the shape stays a bounded static fraction. + assertTrue(barWindow(0, barCount = 1) in 0f..1f) + assertTrue(barHeightFraction(0.5f, 0, barCount = 1) in WAVE_MIN_FRACTION..1f) + } +} diff --git a/convert/bench_file_integrity.py b/convert/bench_file_integrity.py new file mode 100644 index 0000000..8fd2272 --- /dev/null +++ b/convert/bench_file_integrity.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Micro-benchmark for large-file acquisition shapes used by fetch_model. + +Measures the three candidate shapes for materializing one acquired file into +staging while producing its pinned SHA-256: + + A shutil.copyfile + sha256_of(target) (pre-change baseline: 2 reads) + B fused read->hash->write loop (1 read + 1 write, portable) + C APFS clonefile + read-back hash (0-copy then 1 read; darwin) + +B and C measure the shipped file_integrity primitives; A reproduces the +pre-change shape. Each measurement runs in a fresh worker subprocess so +ru_maxrss is per candidate (bytes on macOS, normalized below). Warm-cache +medians come from rotated-order repeats; evicted-cache runs stream a large +eviction file through the page cache first, then measure each candidate +against its own dedicated source file so no candidate warms another's input. +Nothing here touches real model files. + +Usage: + python3 convert/bench_file_integrity.py # warm medians + python3 convert/bench_file_integrity.py --cold # add evicted rounds + python3 convert/bench_file_integrity.py --worker A src dst # internal +""" +from __future__ import annotations + +import argparse +import json +import os +import resource +import shutil +import statistics +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +CANDIDATES = ("A", "B", "C") + +CONVERT = Path(__file__).resolve().parent +sys.path.insert(0, str(CONVERT)) +import file_integrity # noqa: E402 (the shipped primitives under test) + + +def sha256_of(path: Path) -> str: + return file_integrity.sha256_of(path) + + +def candidate_a(source: Path, target: Path) -> tuple[int, str]: + shutil.copyfile(source, target) + return target.stat().st_size, sha256_of(target) + + +def candidate_b(source: Path, target: Path) -> tuple[int, str]: + return file_integrity._stream_copy_with_digest(source, target) + + +def candidate_c(source: Path, target: Path) -> tuple[int, str]: + probe = target.with_name(target.name + ".clone-probe") + probe.unlink(missing_ok=True) + if not file_integrity._try_clonefile(source, probe): + raise RuntimeError("clonefile unavailable; C row must measure the clone path, not a fallback") + probe.unlink() + return file_integrity.copy_with_digest(source, target) + + +CANDIDATE_FUNCS = {"A": candidate_a, "B": candidate_b, "C": candidate_c} + + +def peak_rss_bytes() -> int: + rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return rss if sys.platform == "darwin" else rss * 1024 + + +def run_worker(candidate: str, source: Path, target: Path) -> dict: + start = time.perf_counter() + size, digest = CANDIDATE_FUNCS[candidate](source, target) + elapsed = time.perf_counter() - start + return {"candidate": candidate, "elapsedSeconds": elapsed, + "sizeBytes": size, "sha256": digest, + "peakRssBytes": peak_rss_bytes()} + + +def worker_main(argv: list[str]) -> int: + candidate, source, target = argv + result = run_worker(candidate, Path(source), Path(target)) + print(json.dumps(result)) + return 0 + + +def make_source(directory: Path, name: str, mebibytes: int) -> Path: + path = directory / name + pattern = bytes(range(256)) * 4096 # deterministic 1 MiB block + with path.open("wb") as writer: + block = memoryview(pattern) + for _ in range(mebibytes): + writer.write(block) + return path + + +def spawn(candidate: str, source: Path, target: Path) -> dict: + result = subprocess.run( + [sys.executable, __file__, "--worker", candidate, str(source), str(target)], + capture_output=True, text=True, check=True) + return json.loads(result.stdout) + + +def rotate(order: list[str], round_index: int) -> list[str]: + return order[round_index % len(order):] + order[:round_index % len(order)] + + +def sample(temp: Path, size: int, candidates: list[str], prefix: str, + round_index: int, samples: dict[str, list[float]], + rss: dict[str, list[int]]) -> None: + expected = sha256_of(temp / f"{prefix}-A.bin") + for candidate in candidates: + target = temp / f"{prefix}-dst-{candidate}.bin" + target.unlink(missing_ok=True) + result = spawn(candidate, temp / f"{prefix}-{candidate}.bin", target) + assert result["sha256"] == expected, f"{candidate} digest drift" + samples[candidate].append(result["elapsedSeconds"]) + rss[candidate].append(result["peakRssBytes"]) + target.unlink() + + +def summarize(rows: list[dict], size: int, cache: str, rounds: int, + samples: dict[str, list[float]], rss: dict[str, list[int]]) -> None: + for candidate in CANDIDATES: + rows.append({"sizeMiB": size, "candidate": candidate, "cache": cache, + "medianSeconds": statistics.median(samples[candidate]), + "minSeconds": min(samples[candidate]), + "stdevSeconds": statistics.stdev(samples[candidate]) if rounds > 1 else 0.0, + "rounds": rounds, "peakRssBytes": max(rss[candidate])}) + + +def warm_suite(temp: Path, sizes_mib: list[int], rounds: int) -> list[dict]: + rows = [] + for size in sizes_mib: + for candidate in CANDIDATES: + make_source(temp, f"warm-{size}-{candidate}.bin", size) + samples: dict[str, list[float]] = {c: [] for c in CANDIDATES} + rss: dict[str, list[int]] = {c: [] for c in CANDIDATES} + for round_index in range(rounds): + sample(temp, size, rotate(list(CANDIDATES), round_index), + f"warm-{size}", round_index, samples, rss) + summarize(rows, size, "warm", rounds, samples, rss) + for candidate in CANDIDATES: + (temp / f"warm-{size}-{candidate}.bin").unlink() + return rows + + +def evict_page_cache(evict_path: Path, gib: int) -> None: + """Best-effort eviction: stream gib of zeros through the page cache. + + Not positively verified (purge(8)/fs_usage need root); treat evicted rows + as directional. The clone candidate reads a freshly created vnode in both + cache states, so its warm rows are device-read-bound too. + """ + if not evict_path.exists() or evict_path.stat().st_size < gib << 30: + with evict_path.open("wb") as writer: + block = b"\0" * file_integrity.CHUNK + for _ in range(gib << 10): + writer.write(block) + writer.flush() + os.fsync(writer.fileno()) + with evict_path.open("rb") as reader: + while reader.read(file_integrity.CHUNK << 4): + pass + + +def cold_suite(temp: Path, sizes_mib: list[int], rounds: int, evict_path: Path, evict_gib: int) -> list[dict]: + rows = [] + for size in sizes_mib: + for candidate in CANDIDATES: + make_source(temp, f"cold-{size}-{candidate}.bin", size) + samples: dict[str, list[float]] = {c: [] for c in CANDIDATES} + rss: dict[str, list[int]] = {c: [] for c in CANDIDATES} + for round_index in range(rounds): + evict_page_cache(evict_path, evict_gib) + # Fixed order: dedicated per-candidate sources already prevent + # cross-candidate warming, and rotation cannot help after eviction. + sample(temp, size, list(CANDIDATES), f"cold-{size}", round_index, samples, rss) + summarize(rows, size, "evicted", rounds, samples, rss) + for candidate in CANDIDATES: + (temp / f"cold-{size}-{candidate}.bin").unlink() + return rows + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sizes", default="64,256", help="comma-separated MiB sizes") + parser.add_argument("--rounds", type=int, default=7, help="warm repetitions per candidate") + parser.add_argument("--cold-rounds", type=int, default=3) + parser.add_argument("--cold", action="store_true", help="include evicted-cache rounds") + parser.add_argument("--evict-gib", type=int, default=36, + help="eviction file size; must exceed physical RAM") + parser.add_argument("--json", action="store_true", help="print raw result rows") + parser.add_argument("--worker", nargs=3, metavar=("CAND", "SRC", "DST")) + args = parser.parse_args(argv) + if args.worker: + return worker_main(args.worker) + sizes = [int(part) for part in args.sizes.split(",") if part] + with tempfile.TemporaryDirectory(prefix="bench-integrity-") as directory: + temp = Path(directory) + evict_path = Path(temp) / "evict.bin" + rows = warm_suite(temp, sizes, args.rounds) + if args.cold: + rows += cold_suite(temp, sizes, args.cold_rounds, evict_path, args.evict_gib) + if args.json: + print(json.dumps(rows, indent=2)) + return 0 + print("| size (MiB) | cache | candidate | median s | min s | stdev s | n | peak RSS MB |") + print("|---|---|---|---|---|---|---|---|") + for row in rows: + print(f"| {row['sizeMiB']} | {row['cache']} | {row['candidate']} | " + f"{row['medianSeconds']:.3f} | {row['minSeconds']:.3f} | " + f"{row['stdevSeconds']:.3f} | {row['rounds']} | " + f"{row['peakRssBytes'] / 1048576:.1f} |") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/convert/fetch_model.py b/convert/fetch_model.py index 37e63a0..d391abb 100644 --- a/convert/fetch_model.py +++ b/convert/fetch_model.py @@ -27,26 +27,34 @@ REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT / "convert")) from manifest_contract import ArgParser, ContractError, load_manifest_v2, validate_rel_path, STQ_TRANSFORM, verify_build_record +from file_integrity import CHUNK, copy_with_digest, sha256_of EXIT_OK, EXIT_FAIL, EXIT_ENV, EXIT_CONTRACT, EXIT_ARGS = 0, 1, 2, 3, 4 DEFAULT_DEST = REPO_ROOT / "models" -CHUNK = 1024 * 1024 class IntegrityError(ValueError): pass -def apply_source_transform(manifest: dict, staged: Path, allow_derived: bool) -> None: +def apply_source_transform(manifest: dict, staged: Path, allow_derived: bool, + staged_digest: str | None = None) -> str | None: + """Run the audited STQ transform when staged bytes are the upstream input. + + staged_digest is the digest measured while the file was materialized; + only a caller that could not measure in flight pays a re-read here. + Returns the measured digest of the transformed output, or None when the + staged bytes already are what the pinned-hash gate must verify. + """ transform = manifest["source"].get("transform") if transform is None: - return + return None if transform["id"] != STQ_TRANSFORM: raise IntegrityError("unsupported source transform") expected_output = manifest["files"][0]["sha256"] - actual = sha256_of(staged) + actual = staged_digest if staged_digest is not None else sha256_of(staged) if allow_derived and actual == expected_output: - return # An already derived local bundle still has to match exactly. + return None # An already derived local bundle still has to match exactly. if actual != transform["inputSha256"]: raise IntegrityError("transform input SHA-256 differs from audited upstream artifact") script = REPO_ROOT / "android/app/src/main/cpp/hymt_jni/tools/fix_stq_type_id.py" @@ -59,9 +67,16 @@ def apply_source_transform(manifest: dict, staged: Path, allow_derived: bool) -> raise IntegrityError(f"audited STQ transform failed: {result.stderr[-1500:]}") os.replace(output, staged) print(f"derived {staged.name}: {actual} -> {expected_output}") + return expected_output + +def _download_archive(url: str, dest: Path, expected_size: int) -> str: + """Stream the pinned archive to dest, hashing the bytes in flight. -def _download_archive(url: str, dest: Path, expected_size: int) -> None: + Returns the digest of the written archive; the whole-archive + verification completes here, before any member is extracted. + """ + digest = hashlib.sha256() with urlopen(url, timeout=60) as response, dest.open("xb") as target: if response.url.split(":", 1)[0] != "https": raise IntegrityError("archive redirect left HTTPS") @@ -70,25 +85,43 @@ def _download_archive(url: str, dest: Path, expected_size: int) -> None: total += len(chunk) if total > expected_size: raise IntegrityError("archive exceeds pinned size") + digest.update(chunk) target.write(chunk) + if total != expected_size: + raise IntegrityError("archive size or SHA-256 does not match its pinned source") + return digest.hexdigest() -def extract_archive(manifest: dict, archive_path: Path, staging: Path) -> None: +def extract_archive(manifest: dict, archive_path: Path, staging: Path, + verified_digest: str | None = None) -> dict[str, str]: """Verify the whole archive, then copy only manifest-listed regular files. No extractall, links, absolute paths, traversal, or duplicate destinations. The archive's tests/docs never become part of the installed model package. + + The whole archive is authenticated before tarfile parses a single byte: + the download pass verified the digest in flight, or a local --archive is + hashed here — a mismatched archive never reaches extraction. Each + member's own bytes are hashed while it is written, and the returned + per-file digests are what install() compares against the manifest pins — + staged files are never re-read. Nothing is promoted here; the caller + only swaps staging in after every digest matches its pin. """ archive = manifest["source"]["archive"] if archive_path.is_symlink() or not archive_path.is_file(): raise IntegrityError("archive must be a regular file") - if archive_path.stat().st_size != archive["sizeBytes"] or sha256_of(archive_path) != archive["sha256"]: + if archive_path.stat().st_size != archive["sizeBytes"]: + raise IntegrityError("archive size or SHA-256 does not match its pinned source") + if verified_digest is not None and verified_digest != archive["sha256"]: + raise IntegrityError("archive size or SHA-256 does not match its pinned source") + if verified_digest is None and sha256_of(archive_path) != archive["sha256"]: raise IntegrityError("archive size or SHA-256 does not match its pinned source") prefix = archive["stripPrefix"] + "/" pkg_prefix = manifest["packageId"] + "/" wanted = {entry["path"].removeprefix(pkg_prefix): entry for entry in manifest["files"]} found = set() - with tarfile.open(archive_path, "r|*") as source: + digests: dict[str, str] = {} + with archive_path.open("rb") as raw, tarfile.open(fileobj=raw, mode="r|*") as source: for member in source: name = member.name.rstrip("/") if member.isdir() else member.name try: @@ -110,25 +143,28 @@ def extract_archive(manifest: dict, archive_path: Path, staging: Path) -> None: raise IntegrityError(f"archive member size mismatch: {name}") target = staging / rel target.parent.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + total = 0 with source.extractfile(member) as reader, target.open("xb") as writer: - shutil.copyfileobj(reader, writer, CHUNK) + while chunk := reader.read(CHUNK): + digest.update(chunk) + writer.write(chunk) + total += len(chunk) + if expected is not None and total != expected: + raise IntegrityError(f"archive member size mismatch: {name}") + digests[rel] = digest.hexdigest() + if digests[rel] != wanted[rel]["sha256"]: + raise IntegrityError(f"archive member sha256 mismatch: {name}") if found != set(wanted): raise IntegrityError(f"archive missing allow-listed files: {sorted(set(wanted) - found)}") + return digests -def sha256_of(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(CHUNK), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _download_to(repo_id: str, revision: str, repo_file: str, dest: Path) -> None: +def _download_to(repo_id: str, revision: str, repo_file: str, dest: Path) -> tuple[int, str]: from huggingface_hub import hf_hub_download cached = Path(hf_hub_download(repo_id=repo_id, filename=repo_file, revision=revision)) dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(cached, dest) + return copy_with_digest(cached, dest) def check_no_symlink_escape(dest_root: Path, package_root: Path) -> None: @@ -215,15 +251,16 @@ def install(manifest: dict, root: Path, local_source: Path | None, local_archive try: staging.mkdir(parents=True) from_archive = local_source is None and manifest["source"].get("archive") is not None + digests: dict[str, str] = {} if from_archive: if local_archive is not None: - extract_archive(manifest, local_archive, staging) + digests = extract_archive(manifest, local_archive, staging) else: with tempfile.TemporaryDirectory(prefix=f"{package_id}-archive-", dir=staging.parent) as directory: archive_path = Path(directory) / "source.tar" source = manifest["source"]["archive"] - _download_archive(source["url"], archive_path, source["sizeBytes"]) - extract_archive(manifest, archive_path, staging) + verified = _download_archive(source["url"], archive_path, source["sizeBytes"]) + digests = extract_archive(manifest, archive_path, staging, verified_digest=verified) for entry in manifest["files"]: relative = entry["path"].removeprefix(package_id + "/") staged_path = staging / relative @@ -237,11 +274,13 @@ def install(manifest: dict, root: Path, local_source: Path | None, local_archive check_no_symlink_escape(local_source, source) if not source.is_file(): raise FileNotFoundError(source) - shutil.copyfile(source, staged_path) + measured = copy_with_digest(source, staged_path) elif not from_archive: source_name = manifest["source"].get("transform", {}).get("inputPath", relative) - _download_to(manifest["source"]["repoId"], manifest["source"]["revision"], - source_name, staged_path) + measured = _download_to(manifest["source"]["repoId"], manifest["source"]["revision"], + source_name, staged_path) + else: + measured = (staged_path.stat().st_size, digests[relative]) except FileExistsError: raise except Exception as exc: @@ -250,19 +289,28 @@ def install(manifest: dict, root: Path, local_source: Path | None, local_archive if not staged_path.is_file() or staged_path.is_symlink(): print(f"acquisition did not produce a regular file: {relative}", file=sys.stderr) return EXIT_FAIL - apply_source_transform(manifest, staged_path, allow_derived=local_source is not None) + transformed = apply_source_transform(manifest, staged_path, + allow_derived=local_source is not None, + staged_digest=measured[1]) actual_size = staged_path.stat().st_size if actual_size <= 0 or (entry.get("sizeBytes") is not None and entry["sizeBytes"] != actual_size): print(f"integrity failure: {relative} size {actual_size} != {entry.get('sizeBytes')}", file=sys.stderr) return EXIT_FAIL - actual_hash = sha256_of(staged_path) + # The digest was measured while the final staged bytes were + # produced (copy, extraction, or transform); no re-read here. + actual_hash = measured[1] if transformed is None else transformed if actual_hash != entry["sha256"]: print(f"integrity failure: {relative} sha256 {actual_hash} != {entry['sha256']}", file=sys.stderr) return EXIT_FAIL entry["sizeBytes"] = actual_size with staged_path.open("rb") as source: os.fsync(source.fileno()) - verify_build_record(manifest, staging) + try: + verify_build_record(manifest, staging) + except ContractError as exc: + # The staged bundle contradicts its audited build record: an + # integrity failure (matching validate_models), not a contract error. + raise IntegrityError(f"build provenance does not match the staged package: {exc}") from exc # Manifest and files move together. The shared source manifest is # never promoted to verified and need not be writable for an install. write_json(staging / "manifest.json", manifest) @@ -336,9 +384,22 @@ def main(argv: list[str] | None = None) -> int: if args.asset_pack else args.dest) root.mkdir(parents=True, exist_ok=True) with package_lease(root, args.package): + # install() fills every entry's measured size, so detect pending + # backfill work before it runs: --update-manifest is an explicit + # request, but with nothing to backfill it is a no-op. + backfill_pending = args.update_manifest and any( + entry.get("sizeBytes") is None for entry in manifest["files"]) code = install(manifest, root, args.local_source, args.archive) - if code == EXIT_OK and args.update_manifest: - write_json(manifest_path, manifest) + if code == EXIT_OK and backfill_pending: + try: + write_json(manifest_path, manifest) + except OSError as exc: + # The verified package is already swapped in and stays; + # the requested update itself failed and must be reported + # as a failure, not as a successful install. + print(f"package installed, but the requested manifest update failed: {exc}", + file=sys.stderr) + return EXIT_ENV return code except ContractError as exc: print(f"contract error: {exc}", file=sys.stderr) diff --git a/convert/file_integrity.py b/convert/file_integrity.py new file mode 100644 index 0000000..987147a --- /dev/null +++ b/convert/file_integrity.py @@ -0,0 +1,74 @@ +"""Streaming SHA-256 and file acquisition. Callers verify pins and fsync staging.""" +from __future__ import annotations + +import hashlib +import os +import stat +import sys +from pathlib import Path + +CHUNK = 1024 * 1024 + +# hashlib.file_digest needs Python >= 3.11; the loop fallback below measured +# bit-for-bit identical and speed-parity on SHA-256 hardware (see the +# model-tooling report), so both branches are interchangeable here. +_file_digest = getattr(hashlib, "file_digest", None) + + +def sha256_of(path: Path) -> str: + with path.open("rb") as source: + if _file_digest is not None: + return _file_digest(source, "sha256").hexdigest() + digest = hashlib.sha256() + for chunk in iter(lambda: source.read(CHUNK), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _try_clonefile(source: Path, target: Path) -> bool: + """Try a copy-on-write clone, preserving deletable staging files. + + Immutable/append flags propagate and prevent cleanup or replacement. + Unsupported filesystems, flags or symbols use the streaming fallback. + """ + if sys.platform != "darwin": + return False + try: + if os.stat(source).st_flags & (stat.UF_IMMUTABLE | stat.SF_IMMUTABLE + | stat.UF_APPEND | stat.SF_APPEND): + return False + except OSError: + return False + import ctypes + libc = ctypes.CDLL(None, use_errno=True) + clone = getattr(libc, "clonefile", None) + if clone is None: + return False + clone.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int] + clone.restype = ctypes.c_int + return clone(os.fsencode(source), os.fsencode(target), 0) == 0 + + +def _stream_copy_with_digest(source: Path, target: Path) -> tuple[int, str]: + digest = hashlib.sha256() + total = 0 + buffer = bytearray(CHUNK) + view = memoryview(buffer) + with source.open("rb", buffering=0) as reader, target.open("xb") as writer: + while (count := reader.readinto(buffer)): + chunk = view[:count] + digest.update(chunk) + writer.write(chunk) + total += count + return total, digest.hexdigest() + + +def copy_with_digest(source: Path, target: Path) -> tuple[int, str]: + """Create target and return its byte count and SHA-256. + + A clone is read once for its digest; the fallback hashes while copying. + The caller fsyncs staged files and directories before promotion. + """ + if _try_clonefile(source, target): + return os.stat(target).st_size, sha256_of(target) + return _stream_copy_with_digest(source, target) diff --git a/convert/model_tasks.py b/convert/model_tasks.py index 58cc583..6407d3b 100644 --- a/convert/model_tasks.py +++ b/convert/model_tasks.py @@ -6,7 +6,6 @@ """ from __future__ import annotations -import hashlib import json import math import os @@ -19,6 +18,9 @@ from asr_runner import RunnerError, cer, edit_distance, load_wav from manifest_contract import ContractError, unique_json_keys from tts_runner import normalize_language, API2_GRAPHS, API2_HASHED_FILES +# One shared single-pass SHA-256 reader; the file_hash spelling stays for +# scripts/check_asr_swift, which imports it from here. +from fetch_model import sha256_of as file_hash # noqa: E402 CONVERT = Path(__file__).resolve().parent @@ -71,14 +73,6 @@ def __init__(self, status: str, message: str): self.status = status -def file_hash(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - def finite_number(value, name: str, minimum: float = 0) -> float: if type(value) not in (int, float) or not math.isfinite(value) or value < minimum: raise ValueError(f"{name} must be a finite number >= {minimum}") diff --git a/convert/mt_runner.py b/convert/mt_runner.py index 4483b02..8108353 100644 --- a/convert/mt_runner.py +++ b/convert/mt_runner.py @@ -34,7 +34,6 @@ import argparse import ctypes -import hashlib import json import os import platform @@ -43,22 +42,18 @@ import time from pathlib import Path +from file_integrity import sha256_of as sha256_file + PINNED = "1e411d8f5a1e23525fa3265dfb4bd76265465397" HYMT_OK, HYMT_INVALID, HYMT_ABORTED, HYMT_FAILED = 0, 1, 2, 3 STATUS_NAME = {0: "ok", 1: "invalid", 2: "aborted", 3: "failed"} -DEFAULT_LIB = Path( - os.environ.get( - "HYMT_LIB", - "/Users/arietids/Library/Caches/Auralis/mt/build-host/libhymt_core.dylib", - ) -) -DEFAULT_MODEL = Path( - os.environ.get( - "HYMT_MODEL", - "/Users/arietids/Library/Caches/Auralis/mt/models/Hy-MT1.5-1.8B-1.25bit-stq43.gguf", - ) -) +DEFAULT_CACHE = Path(os.environ.get("AURALIS_CACHE", Path.home() / "Library/Caches/Auralis")) +DEFAULT_LIB = Path(os.environ.get( + "HYMT_LIB", DEFAULT_CACHE / "mt/build-host" / + ("libhymt_core.dylib" if sys.platform == "darwin" else "libhymt_core.so"))) +DEFAULT_MODEL = Path(os.environ.get( + "HYMT_MODEL", DEFAULT_CACHE / "mt/models/Hy-MT1.5-1.8B-1.25bit-stq43.gguf")) DEFAULT_CASES = [ {"id": "zh-en-museum", "text": "今天下午我们去博物馆参观，好吗？", "sourceLanguage": "Chinese", "targetLanguage": "English", @@ -100,14 +95,6 @@ class RunnerFailError(RunnerError): exit_code = 1 -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - def other_llama_pids() -> list[int]: try: out = subprocess.check_output(["ps", "-axo", "pid=,command="], text=True) diff --git a/convert/tests/test_archive_install.py b/convert/tests/test_archive_install.py index 0767eeb..1fb76be 100644 --- a/convert/tests/test_archive_install.py +++ b/convert/tests/test_archive_install.py @@ -11,6 +11,11 @@ import fetch_model +class _FakeResponse(io.BytesIO): + """Minimal urlopen() stand-in: a readable byte stream with .url.""" + url = "https://example.com/source.tar.bz2" + + class ArchiveInstallTests(unittest.TestCase): def setUp(self): tmp = tempfile.TemporaryDirectory() @@ -71,3 +76,81 @@ def test_archive_link_is_rejected_even_when_unlisted(self): def test_duplicate_archive_member_is_rejected(self): path, _ = self.make_archive([("bundle/model.bin", b"new", "file"), ("bundle/model.bin", b"new", "file")]) self.assertEqual(self.install_archive(path), 1) + + def test_member_hash_mismatch_fails_even_when_archive_hash_matches(self): + """The archive is byte-identical to its pin, but its member payload + differs from the manifest file pin: the per-file gate must fail.""" + path, _ = self.make_archive([("bundle/model.bin", b"bad", "file")]) + pkg = self.repo / "models/asr" + pkg.mkdir(parents=True) + (pkg / "old.bin").write_bytes(b"old") + self.assertEqual(self.install_archive(path), 1) + self.assertEqual((pkg / "old.bin").read_bytes(), b"old", "old package must survive a member mismatch") + + def test_invalid_archive_never_reaches_tarfile_extraction(self): + """Whole-archive authentication precedes parsing: an archive whose + bytes differ from the pin is refused before tarfile.open runs, on + both the local --archive path (pre-hash) and the network path + (in-flight digest).""" + path, m = self.make_archive([("bundle/model.bin", b"new", "file")]) + body = bytearray(path.read_bytes()) + body[-1] ^= 0xFF # same length, different digest: passes the size gate + tampered = self.repo / "tampered.tar.bz2" + tampered.write_bytes(bytes(body)) + pkg = self.repo / "models/asr" + with patch.object(fetch_model.tarfile, "open") as opening: + self.assertEqual(self.install_archive(tampered), 1) + write_manifest(self.repo, m) + response = _FakeResponse(bytes(body)) + with patch.object(fetch_model, "urlopen", return_value=response): + self.assertEqual(fetch_model.main( + ["--package", "asr", "--dest", str(self.repo / "models")]), 1) + opening.assert_not_called() + self.assertFalse(pkg.exists()) + + def test_local_archive_size_mismatch_is_rejected_before_extraction(self): + """Local --archive path: the staged archive is size-checked against + the pin before extraction (extract_archive's own gate, distinct from + the in-flight download checks).""" + path, m = self.make_archive([("bundle/model.bin", b"new", "file")]) + m["source"]["archive"]["sizeBytes"] = m["source"]["archive"]["sizeBytes"] + 1000 + write_manifest(self.repo, m) + self.assertEqual(self.install_archive(path), 1) + self.assertFalse((self.repo / "models/asr").exists()) + + def test_download_size_mismatch_fails_in_flight_before_extraction(self): + """Network path in-flight size checks: an overlong stream trips the + running-total guard mid-download, a short one trips the EOF equality + check; either way nothing is extracted.""" + path, m = self.make_archive([("bundle/model.bin", b"new", "file")]) + body = path.read_bytes() + pkg = self.repo / "models/asr" + for label, response_body in (("overlong", body + b"trailing bytes"), + ("short", body[:-8])): + with self.subTest(label=label): + write_manifest(self.repo, m) + response = _FakeResponse(response_body) + with patch.object(fetch_model, "urlopen", return_value=response): + self.assertEqual(fetch_model.main( + ["--package", "asr", "--dest", str(self.repo / "models")]), 1) + self.assertFalse(pkg.exists(), + f"{label} download must not reach extraction") + + def test_downloaded_archive_digest_is_verified_without_a_second_read(self): + """The download pass hashes the archive; extraction receives that + digest as verified_digest instead of re-hashing the whole file.""" + path, _ = self.make_archive([("bundle/model.bin", b"new", "file"), ("bundle/docs.txt", b"d", "file")]) + observed = {} + real_extract = fetch_model.extract_archive + + def spy(manifest, archive_path, staging, **kwargs): + observed["kwargs"] = kwargs + return real_extract(manifest, archive_path, staging, **kwargs) + + response = _FakeResponse(path.read_bytes()) + with patch.object(fetch_model, "extract_archive", spy), \ + patch.object(fetch_model, "urlopen", return_value=response): + self.assertEqual(fetch_model.main( + ["--package", "asr", "--dest", str(self.repo / "models")]), 0) + self.assertEqual(observed["kwargs"].get("verified_digest"), + fetch_model.sha256_of(path)) diff --git a/convert/tests/test_fetch_model.py b/convert/tests/test_fetch_model.py index ebaadc3..6632347 100644 --- a/convert/tests/test_fetch_model.py +++ b/convert/tests/test_fetch_model.py @@ -9,6 +9,7 @@ import hashlib import json +import os import sys import tempfile import unittest @@ -80,9 +81,10 @@ def run_fetch(self, dest: Path, package_id: str = "asr") -> int: return fetch_model.main(["--package", package_id, "--dest", str(dest)]) def stub_downloader(self, blobs: dict[str, bytes]) -> None: - def fake_download(repo_id: str, revision: str, repo_file: str, dest: Path) -> None: + def fake_download(repo_id: str, revision: str, repo_file: str, dest: Path) -> tuple[int, str]: dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(blobs[repo_file]) + return len(blobs[repo_file]), sha_of(blobs[repo_file]) p = mock.patch.object(fetch_model, "_download_to", fake_download) p.start() self.addCleanup(p.stop) @@ -313,6 +315,150 @@ def inspect_swap(source, target): self.assertEqual(len(observed), 1) self.assertEqual(observed[0]["files"][0]["sha256"], sha_of(b"abc")) + def test_copy_with_digest_matches_pinned_bytes(self) -> None: + import file_integrity + source = self.repo / "single-source.bin" + source.write_bytes(b"acquired-bytes" * 1000) + for stream in (False, True): + with self.subTest(stream=stream): + target = self.repo / f"copy-{stream}.bin" + if stream: + patcher = mock.patch.object(file_integrity, "_try_clonefile", lambda a, b: False) + patcher.start() + self.addCleanup(patcher.stop) + measured = fetch_model.copy_with_digest(source, target) + self.assertEqual(measured, (len(source.read_bytes()), sha_of(source.read_bytes()))) + self.assertEqual(target.read_bytes(), source.read_bytes()) + + def test_clone_does_not_propagate_immutable_source_flags(self) -> None: + """clonefile copies BSD flags: a staged file carrying immutable or + append-only flags could not be unlinked, replaced or removed, breaking + staging cleanup and rollback. Flagged sources must therefore take the + streaming copy, and the source itself must come out untouched.""" + if sys.platform != "darwin": + self.skipTest("BSD file flags are darwin-only") + import stat as stat_module + for flag_name in ("UF_IMMUTABLE", "UF_APPEND"): + with self.subTest(flag=flag_name): + source = self.repo / f"flagged-{flag_name}.bin" + target = self.repo / f"flagged-{flag_name}-target.bin" + source.write_bytes(b"flagged-source") + original = os.stat(source).st_flags + flag = getattr(stat_module, flag_name) + if original & (stat_module.SF_IMMUTABLE | stat_module.SF_APPEND): + self.skipTest("system flags cannot be toggled safely here") + try: + os.chflags(source, original | flag) + measured = fetch_model.copy_with_digest(source, target) + self.assertEqual(measured, (len(b"flagged-source"), sha_of(b"flagged-source"))) + self.assertEqual(target.read_bytes(), b"flagged-source") + target.unlink() # the staged copy must be deletable/replaceable + self.assertFalse(target.exists()) + self.assertEqual(source.read_bytes(), b"flagged-source") + self.assertEqual(os.stat(source).st_flags, original | flag) + finally: + os.chflags(source, original) + + def test_transform_input_mismatch_fails_before_script_launch(self) -> None: + m = {"source": {"transform": {"id": fetch_model.STQ_TRANSFORM, "inputPath": "mt/translator.gguf", + "inputSha256": sha_of(b"audited-upstream-input")}}, + "files": [{"sha256": sha_of(b"derived")}]} + staged = self.repo / "staged.gguf" + staged.write_bytes(b"not-the-audited-input") + with mock.patch.object(fetch_model.subprocess, "run") as command: + with self.assertRaises(fetch_model.IntegrityError): + fetch_model.apply_source_transform(m, staged, allow_derived=True, + staged_digest=sha_of(b"not-the-audited-input")) + command.assert_not_called() + + def test_derived_bundle_shortcut_skips_script_without_a_reread(self) -> None: + m = {"source": {"transform": {"id": fetch_model.STQ_TRANSFORM, "inputPath": "mt/translator.gguf", + "inputSha256": sha_of(b"audited-upstream-input")}}, + "files": [{"sha256": sha_of(b"derived")}]} + staged = self.repo / "staged-derived.gguf" + staged.write_bytes(b"derived") + with mock.patch.object(fetch_model.subprocess, "run") as command: + self.assertIsNone(fetch_model.apply_source_transform(m, staged, allow_derived=True, + staged_digest=sha_of(b"derived"))) + command.assert_not_called() + + def test_transform_output_digest_is_returned_for_the_gate(self) -> None: + m = {"source": {"transform": {"id": fetch_model.STQ_TRANSFORM, "inputPath": "mt/translator.gguf", + "inputSha256": sha_of(b"audited-upstream-input")}}, + "files": [{"sha256": sha_of(b"derived")}]} + staged = self.repo / "staged-input.gguf" + staged.write_bytes(b"audited-upstream-input") + + def fake_run(command, **kwargs): + output = Path(command[3]) + output.write_bytes(b"derived") + return mock.Mock(returncode=0, stderr="") + + with mock.patch.object(fetch_model.subprocess, "run", fake_run): + self.assertEqual(fetch_model.apply_source_transform(m, staged, allow_derived=True, + staged_digest=sha_of(b"audited-upstream-input")), sha_of(b"derived")) + self.assertEqual(staged.read_bytes(), b"derived") + + def test_build_provenance_mismatch_is_integrity_failure(self) -> None: + write_manifest(self.repo, manifest([ + {"path": "asr/a.bin", "sizeBytes": 3, "sha256": sha_of(b"abc")}, + ])) + self.stub_downloader({"a.bin": b"abc"}) + with mock.patch.object(fetch_model, "verify_build_record", + side_effect=fetch_model.ContractError("bad-value", "outputs differ")): + code = self.run_fetch(self.repo / "dest-prov") + self.assertEqual(code, 1, "provenance mismatch is an integrity failure, like validate_models") + self.assertFalse((self.repo / "dest-prov/asr").exists()) + + def test_update_manifest_backfill_failure_fails_request_but_keeps_install(self) -> None: + """--update-manifest is an explicit request: if the backfill write + fails, the request fails (exit 2) while the verified package stays + installed with its own manifest intact.""" + data = b"installed-content" + write_manifest(self.repo, manifest([ + {"path": "asr/a.bin", "sizeBytes": None, "sha256": sha_of(data)}, + ])) + self.stub_downloader({"a.bin": data}) + dest = self.repo / "dest-backfill" + real_write_json = fetch_model.write_json + shared = self.repo / "shared/model-manifests/asr.json" + + def failing_backfill(path, data): + if path == shared: + raise OSError("read-only shared dir") + return real_write_json(path, data) + + with mock.patch.object(fetch_model, "write_json", failing_backfill): + code = fetch_model.main(["--package", "asr", "--dest", str(dest), "--update-manifest"]) + self.assertEqual(code, 2, "a failed explicit update request must not exit 0") + self.assertEqual((dest / "asr/a.bin").read_bytes(), data, "installed package stays") + self.assertTrue((dest / "asr/manifest.json").is_file()) + self.assertIsNone(json.loads(shared.read_text())["files"][0]["sizeBytes"], + "the shared manifest must not claim a backfill that never landed") + + def test_update_manifest_noop_when_sizes_already_pinned(self) -> None: + """With every sizeBytes already pinned, --update-manifest is a no-op: + the shared manifest is not rewritten and the install exits 0.""" + data = b"pinned-content" + write_manifest(self.repo, manifest([ + {"path": "asr/a.bin", "sizeBytes": len(data), "sha256": sha_of(data)}, + ])) + self.stub_downloader({"a.bin": data}) + dest = self.repo / "dest-noop" + real_write_json = fetch_model.write_json + shared = self.repo / "shared/model-manifests/asr.json" + before = shared.read_text() + + def no_shared_write(path, data): + if path == shared: + raise AssertionError("backfill write attempted with nothing pending") + return real_write_json(path, data) + + with mock.patch.object(fetch_model, "write_json", no_shared_write): + code = fetch_model.main(["--package", "asr", "--dest", str(dest), "--update-manifest"]) + self.assertEqual(code, 0, "nothing to backfill: the explicit request is a no-op") + self.assertEqual(shared.read_text(), before) + if __name__ == "__main__": unittest.main() diff --git a/convert/tts_runner.py b/convert/tts_runner.py index 377207d..fcbdd07 100644 --- a/convert/tts_runner.py +++ b/convert/tts_runner.py @@ -42,6 +42,8 @@ from datetime import datetime, timezone from pathlib import Path +from file_integrity import sha256_of as sha256_file + SAMPLE_RATE = 24000 SAMPLES_PER_FRAME = 1920 # 12 Hz codec -> 24 kHz PCM NUM_CODEBOOKS = 16 # RVQ groups sent to the vocoder @@ -779,14 +781,6 @@ def finish_after_eos(self): return np.concatenate(self.audio) -def sha256_file(path: Path) -> str: - h = hashlib.sha256() - with open(path, "rb") as f: - for chunk in iter(lambda: f.read(4 * 1024 * 1024), b""): - h.update(chunk) - return h.hexdigest() - - def compute_model_hashes(model_dir: Path, files=HASHED_FILES) -> dict: """sha256 per graph file; hardlinked external data is hashed once.""" hashes = {} diff --git a/convert/validate_models.py b/convert/validate_models.py index 115529a..a7cb97c 100644 --- a/convert/validate_models.py +++ b/convert/validate_models.py @@ -25,7 +25,6 @@ from __future__ import annotations import argparse -import hashlib import json import sys import time @@ -37,20 +36,12 @@ sys.path.insert(0, str(REPO_ROOT / "convert")) from manifest_contract import ArgParser, ContractError, load_manifest_v2, verify_build_record # noqa: E402 -from fetch_model import package_lease, check_no_symlink_escape -from model_tasks import load_suite, run_task, runner_layout_error +from fetch_model import package_lease, check_no_symlink_escape, sha256_of # noqa: E402 +from model_tasks import load_suite, run_task, runner_layout_error # noqa: E402 EXIT_OK, EXIT_FAIL, EXIT_BLOCKED, EXIT_CONTRACT, EXIT_ARGS = 0, 1, 2, 3, 4 -def sha256_of(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - class Report: def __init__(self) -> None: self.entries: list[dict] = [] diff --git a/docs/brand/README.md b/docs/brand/README.md index b01ac68..08d9c1a 100644 --- a/docs/brand/README.md +++ b/docs/brand/README.md @@ -16,3 +16,5 @@ The mark combines an A with two listening arcs. Pine is the background, mint is [mark.svg](mark.svg) is the canonical geometry. Run `python3 scripts/build_brand_assets` to regenerate the Android foreground/monochrome vectors and iOS icon sizes. The iOS icon stays opaque and square; the operating system supplies its mask. Use system fonts in the applications, and preserve native text scaling. Avoid claims such as “real time,” “production certified,” or “best accuracy” without the corresponding published measurements. The first release is a developer preview. + +The [native motion rules](../specs/2026-09-08-quality/motion.md) and [Remotion showcase](../../showcase/README.md) use this same visual system. diff --git a/docs/specs/2026-09-05-auralis/06-real-runtime-verification.md b/docs/specs/2026-09-05-auralis/06-real-runtime-verification.md index 4922154..baa5b2f 100644 --- a/docs/specs/2026-09-05-auralis/06-real-runtime-verification.md +++ b/docs/specs/2026-09-05-auralis/06-real-runtime-verification.md @@ -4,7 +4,7 @@ ## 可复现来源 -HF 文件使用 source.repoId + 完整 source.revision。GitHub release 模型使用 source.archive（HTTPS URL、归档 SHA-256、精确大小、stripPrefix）；release 标签不是 git commit。归档先验整体哈希，再只复制清单声明的普通文件；路径穿越、链接、重复成员被拒绝。Python/Kotlin/Swift 对这些字段及重复 JSON 键采用一致规则。 +HF 文件使用 source.repoId + 完整 source.revision。GitHub release 模型使用 source.archive（HTTPS URL、归档 SHA-256、精确大小、stripPrefix）；release 标签不是 git commit。归档先验整体哈希，再只复制清单声明的普通文件；路径穿越、链接、清单内文件的重复成员被拒绝；清单外普通文件不安装。Python/Kotlin/Swift 对这些字段及重复 JSON 键采用一致规则。 MT 只允许一个已审计转换 source.transform：hymt-stq42-to43-v1。它记录原始文件名与 93e025… 输入哈希；files 记录 e42935… 的运行时产物、精确大小和 translator 角色，runtimeRevision 固定 1e411d8… 。这不是重新量化；完整张量数据不变。程序不执行来自 manifest 的任意命令。获取工具从原始文件实际转换并验证输出后，与 manifest 一起切换；错误保留旧包。 diff --git a/docs/specs/2026-09-08-quality/README.md b/docs/specs/2026-09-08-quality/README.md new file mode 100644 index 0000000..048b190 --- /dev/null +++ b/docs/specs/2026-09-08-quality/README.md @@ -0,0 +1,20 @@ +# Auralis runtime quality and motion + +Baseline: `v0.1.0-preview.1` / `7e2cb57927d6290c089dce1c33a72bbc9902b6cc`. + +This pass reduces repeated inference copies and acquisition reads, bounds iOS capture memory, and adds restrained native motion. Changes are selected against the baseline while preserving model outputs, sampling, cancellation and package integrity. + +| Area | Selected change | Evidence | +|---|---|---| +| Android TTS | Bounded top-k selection, shared repetition mask, mel/FFT work reduction, prepared-reference ownership and cleanup | [Fixed-seed equivalence and JVM measurements](reports/android-runtime.md) | +| iOS TTS | Direct ORT KV feedback, shared scoped tensor reads, checked shape arithmetic | [Runtime tests and full-model WAV equality](reports/ios-runtime.md) | +| iOS capture | Ten-second utterance bound and eight-chunk callback queue | [PCM continuity and overflow checks](reports/capture.md) | +| Model tools | Hash during acquisition; clone local files when safe; remove repeated hash wrappers | [Failure-path tests and measured I/O ablation](reports/model-tooling.md) | +| Native UI | State, turn, press and live-level motion with reduced-motion support | [Native implementation and verification](reports/native-motion.md) | +| Showcase | One 18-second Remotion composition using the canonical brand | [Render and reduced-motion evidence](reports/showcase.md) | + +Small units are retained where they remove repetition or expose a useful boundary. Extra strategies, ownership frameworks, digest caches and losing benchmark candidates are omitted. The area reports distinguish structural savings, measured host results and limits. + +Run `python3 scripts/verify --mode fast` for contracts, Python and Swift host checks. Android and iOS CI also build their pinned native dependencies and run platform build/test checks. Showcase CI installs the lockfile and type-checks its separate TypeScript source. + +Model weights, precision, pinned revisions and shared-package readiness remain fixed. Host/emulator results do not qualify physical-device latency, memory or energy. The published preview artifacts remain associated with their original tag; this source pass does not silently replace them. diff --git a/docs/specs/2026-09-08-quality/benchmarks/capture-buffer-ablation.swift b/docs/specs/2026-09-08-quality/benchmarks/capture-buffer-ablation.swift new file mode 100644 index 0000000..5bd1497 --- /dev/null +++ b/docs/specs/2026-09-08-quality/benchmarks/capture-buffer-ablation.swift @@ -0,0 +1,38 @@ +import Foundation +func inspect(_ label: String, _ policy: AsyncStream<[Float]>.Continuation.BufferingPolicy) async -> [String: Any] { + let pair = AsyncStream<[Float]>.makeStream(bufferingPolicy: policy) + var drops = 0 + for i in 0..<1000 { + if case .dropped = pair.continuation.yield([Float](repeating: Float(i), count: 3200)) { drops += 1 } + } + pair.continuation.finish() + var count = 0, first = -1, last = -1 + for await chunk in pair.stream { + if count == 0 { first = Int(chunk[0]) } + last = Int(chunk[0]); count += 1 + } + return ["candidate": label, "bufferedChunksObserved": count, "logicalPcmBytes": count * 3200 * 4, + "drops": drops, "firstId": first, "lastId": last] +} + +@main +enum CaptureBufferAblation { + static func main() async throws { + var results: [[String: Any]] = [] + results.append(await inspect("unbounded baseline", .unbounded)) + results.append(await inspect("keep newest eight and continue", .bufferingNewest(8))) + let production = AudioChunkStream() + var firstRejected = -1 + for i in 0..<1000 { + if !production.yield([Float](repeating: Float(i), count: 3200)) { firstRejected = i; break } + } + var ids: [Int] = [] + for await chunk in production.stream { ids.append(Int(chunk[0])) } + results.append(["candidate": "bounded contiguous prefix then terminate", "bufferedChunksObserved": ids.count, + "logicalPcmBytes": ids.count * 3200 * 4, "firstRejectedId": firstRejected, + "firstId": ids.first ?? -1, "lastId": ids.last ?? -1]) + let data = try JSONSerialization.data(withJSONObject: results, options: [.prettyPrinted, .sortedKeys]) + print(String(decoding: data, as: UTF8.self)) + precondition(ids == Array(0..<8) && firstRejected == 8) + } +} diff --git a/docs/specs/2026-09-08-quality/motion.md b/docs/specs/2026-09-08-quality/motion.md new file mode 100644 index 0000000..59470fe --- /dev/null +++ b/docs/specs/2026-09-08-quality/motion.md @@ -0,0 +1,15 @@ +# Native motion + +Auralis uses the existing Pine/Mint/Paper palette and system typography. Motion clarifies state changes and responds to real input. The separate Remotion composition is an interface demonstration, not a model-performance claim. + +| Surface | Motion | Bound | +|---|---|---| +| Session status | Short fade and small vertical change | About160–220ms, no repeated layout work | +| New conversation item | Fade with small displacement and placement spring | Stable turn IDs, preserve reading/focus order | +| Main action | Small press scale and native feedback | Draw/compositing phase; no layout resize | +| Input level | Seven bars from real RMS, smoothed and compressed for quiet speech | One drawing surface, no independent per-bar springs | +| Idle and reduced motion | Static decoration; information remains current | No idle timeline or periodic task | + +Use native Compose/SwiftUI primitives. Keep state logic and inference outputs unchanged. Keep playback distinct from listening; a decorative activity signal does not claim a measured output waveform. Reduced motion must suppress decorative movement and animated scrolling. + +Selection checks: native compile, state readability at larger text sizes, reduced-motion behavior, and idle scheduling. Compare drawing-only level updates with the existing layout-per-bar design. Any remaining device/frame-time claims require measured evidence. diff --git a/docs/specs/2026-09-08-quality/reports/android-runtime.md b/docs/specs/2026-09-08-quality/reports/android-runtime.md new file mode 100644 index 0000000..7b4413c --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/android-runtime.md @@ -0,0 +1,168 @@ +# Android TTS 推理质量/效率报告（2026-09-08） + +Lane：Android 推理三文件（`TtsEngine.kt` / `TtsApi2Runtime.kt` / `SpeakerEmbeddingExtractor.kt`）+ 对应测试。 +基线：`v0.1.0-preview.1` / `7e2cb57927d6290c089dce1c33a72bbc9902b6cc`。 +执行环境：本 quality worktree，macOS aarch64 主机，JDK 23，Gradle wrapper（Groovy 3.0.22）。所有微基准为 **JVM（JIT C2）主机数字**，只刻画算法级工作/分配量变化，**不冒充手机或端到端合成提速**（见 README 验收门：主机结果不满足实体设备资格）。 + +## 候选与基线（开工时清单） + +| # | 候选 | 证据来源 | 判定 | +|---|---|---|---| +| 1 | `Qwen3TtsProtocol.sampleFromLogits` 每次 `probs.copyOf().sortedDescending()`（整排序+拷贝）→ 精确 K-th largest 选择 | 代码审查 + 微基准 | **实施** | +| 2 | `sampleGroup0` 每步 `generated.toSet()`（HashSet+装箱）→ seen-mask | 代码审查 + 微基准 | **实施**（随 #1 同函数） | +| 3 | `logMelSpectrogram` 每 mel 带重复计算 `sqrt(re²+im²+1e-9)`（128×513/帧）→ 每 bin 一次；FFT twiddle 每蝶形重算 sin/cos → 同表达式缓存 | 代码审查（root 提示）+ 微基准 | **实施** | +| 4 | `prepareReference` 的 `.also` 仅成功路径释放 reference_encoder（ICL 失败/取消路径滞留 ~190 MB 会话） | 代码审查 | **实施**（TtsApi2Runtime.kt） | +| 5 | `PreparedReference` 声称不可变但公开 FloatArray 字段可外部修改 | 代码审查 | **实施**（TtsApi2Runtime.kt） | +| 6 | `sampleCodePredictor` 调用点 `cpLogits.copyOfRange(...)` 与函数内尾部 arraycopy 重复拷贝 | 代码审查 | **实施**（两处调用点） | +| 7 | `code flattening is group-major for the vocoder` 测试缺 `@Test`，从未运行 | 测试审查 | **实施**（补注解） | +| 8 | `logMelSpectrogram` 短音频（<256 样本）AIOOBE 崩溃（守卫边界错误） | 新增等价测试暴露 | **实施**（见"意外发现"） | +| 9 | BPE 重复工作（encode/bpe 每合成一次、按文本长度） | 审查 | **不做**：每合成仅 1 次、量级随文本长度，无基准证据表明是热点 | +| 10 | 其它 DSP 重复（window/filterbank 每次 logMel 调用重建） | 审查 | **不做**：每次调用仅 1 次，微基准中不构成可见份额 | +| 11 | 采样 weights DoubleArray 跨步复用（scratch 缓冲） | 构思 | **不做**：需跨两个引擎传 scratch、引入有状态接口；exp() 主导的每次调用成本中分配占比未证明值得（见消融） | +| 12 | API1 `generateFrames` 后不释放 code_predictor 会话（与 decode/vocoder 不对称） | 代码审查 | **不改**：cp 会话小、保留可加速重复合成；无内存受害证据，API2 亦全缓存。记录为观察 | + +基线 JVM 单测：`AURALIS_TTS_MODEL_DIR=$HOME/Library/Caches/Auralis/tts/api2-release ./gradlew :app:testDebugUnitTest --offline --init-script /private/tmp/auralis-existing-cache.init.gradle` → **146 tests 全绿、0 跳过**（含真实 tokenizer 用例，Qwen3TtsProtocolTest 20/20）。 + +## 1. 采样阈值选择（TtsEngine.kt — Qwen3TtsProtocol） + +### 缺陷 +`sampleFromLogits` 在每个采样步执行 `probs.copyOf().sortedDescending()[topK - 1]`：整词表拷贝 + 完整排序 + Kotlin `List` 装箱视图，只为取第 K 大的值。每合成帧调用 16 次（1× group0 vocab 3072 + 15× code predictor vocab 2048）。 + +### 修复 +新增纯数值单元 `Qwen3TtsProtocol.kthLargestFloat(values, k)`（`TtsEngine.kt:475`）：有界堆选择。第 K 大 = 第 (n−K+1) 小，堆取较小一侧（容量 ≤ (n+1)/2），O(n log min(K, n−K+1))；对值只比较、不运算，与排序参照**位等价**。掩码逻辑（`probs[i] < threshold → -inf`，保留并列）、CDF 原顺序扫描、`random.nextDouble()` 恰好一次的消耗、temperature=0/topK=1 短路全部未动。 + +同函数内 `generated.toSet()` → `BooleanArray` seen-mask：逐 index 独立施加惩罚，与 HashSet 迭代顺序无关，逐值等价；require 校验消息不变。 + +### 等价证据（新增测试，Qwen3TtsProtocolTest.kt） +- 冻结基线 oracle：测试内完整复制基线 `sampleGroup0`/`sampleCodePredictor`/`sampleFromLogits`（含 `toSet` 与 `sortedDescending`），标注"故意重复以防两侧同时被改"。 +- `group0 sampler is value-identical to frozen baseline across topK and ties`：4 种分布（uniform / 量化重复 / 稀疏有限值+−inf / 全等）× topK {0,1,2,50,51,vocab−1,vocab} × 150 步，两侧共享同 seed RNG 流——任何消耗次数/顺序偏差都会立即失步。逐值相等。 +- `code predictor sampler ...`：同矩阵 × 200 步，逐值相等。 +- `kth largest matches the sorted multiset reference exactly`：size {1..257 全 K 扫描 + 1024/3072 采样 K 扫描} × 6 类对抗数组（随机、两值、全等、单峰、−inf 混合、±0.0 混合），断言 `kthLargestFloat(a,k)` 与 `a.copyOf().sortedDescending()[k-1]` 按 `floatToRawIntBits` 位精确（混合 ±0.0 时零符号豁免；掩码语义已证等价：`x < -0.0` 与 `x < 0.0` 对所有 x 同值）。 +- `kth largest rejects empty arrays and out-of-range orders`。 + +### 微基准（门控 `AURALIS_PERF_BENCH=1`，`benchmark sampling and mel rework`） +JVM 中位数，warmup=repeats/4+10，400 次（mel 10 次）；两次独立运行（`--rerun`）新实现侧稳定，legacy 侧受 GC/堆状态波动： + +| 项 | legacy | new | 比 | +|---|---|---|---| +| sampleCodePredictor [2048, uniform] | 57.5–62.8 µs | 12.8–13.0 µs | 4.4–4.9× | +| sampleCodePredictor [2048, ties] | 20.3–20.5 µs | 11.4–12.2 µs | 1.7–1.8× | +| sampleGroup0 [3072, gen=512] | 78.6–87.9 µs | 62.7–63.8 µs | 1.25–1.38× | +| 仅阈值 [2048, k=50]：copyOf+sort vs select | 40.3–42.8 µs | 3.3–3.4 µs | ~12× | +| logMelSpectrogram [2s 音频, ~189 帧] | 19.8–30.7 ms | 13.25–13.28 ms | 1.5–2.3× | + +读法（诚实口径）： +- 采样路径剩余成本由 exp() softmax 主导（select 3.4 µs vs 全程 13 µs）；阈值选择本身 ~12× 但只占全路径一部分。 +- 按帧外推：16 次采样/帧，legacy ≈ 0.9–1.0 ms/帧 → new ≈ 0.26–0.28 ms/帧；对每帧 ~100–300 ms 量级的 talker decode + code predictor ONNX 计算是 **<1% 的墙钟改善**——CPU 收益不是本轮主要价值。 +- 主要价值是分配消除：每步不再有 copyOf（8–12 KB）+ sortedDescending 装箱 + HashSet。按 2048 帧预算上限的算术估算：16 次采样/帧 × 2048 帧 ≈ 32k 次调用 × ~11 KB/次 ≈ **350–400 MB 堆垃圾/满预算合成**（比例随实际帧数线性下降；普通短语远低于上限）。这是纯算术推算，**非 ART/设备实测**。 +- legacy 侧 run-to-run 波动（mel 19.8 vs 30.7 ms）表明单次 JVM 微基准对分配重的代码噪声显著；比较以新实现侧稳定性与隔离腿（threshold-only）为准。 + +## 2. logMelSpectrogram 前端（TtsEngine.kt — Qwen3TtsProtocol） + +### 缺陷 +- 幅度：`sqrt(re[k]²+im[k]²+1e-9)` 在 m(128)×k(513) 内循环逐带重算，同一频点每帧重复 128 次。 +- FFT：每帧每蝶形块重算 `cos(angle·k)/sin(angle·k)`——n=1024 时每帧求值 **5120 对**（10 级 × 每级 n/size 个块 × 每块 half 个重复计算），其中独特值仅 **1023 个**（每级 half 个，Σhalf=1023）。 + +### 修复（均位等价） +- 每 bin 每帧算一次 `mag[k]`（**同一表达式**），mel 带循环按原 k 顺序累加 `basis[m·513+k] * mag[k]`——累加顺序与 Float/Double 转换不变，输出位等价。 +- `TwiddleTable`（单一 `@Volatile` 不可变 holder + 双检锁）：每 FFT 尺寸一次构建，值用**与内联完全相同的表达式** `cos(-2.0*PI/size * k)` 计算，位等价；并发读一致。 + +### 位等价证据（新增测试） +- `fft is bit-identical to frozen baseline across sizes`：n ∈ {2,4,8,64,256,1024,4096} 随机复数输入，逐点 delta=0。 +- `log mel is bit-identical to frozen baseline frontend`：全零 / DC / 正弦 / 冲激 / 2s 噪声 / 256 样本（恰好一 hop）/ 1023 样本 / 既有 8192 验证信号，`legacy.data.contentEquals(current.data)` 逐位相等。 +- 微基准见上表（mel 路径合并实测 1.5–2.3×；幅度提升与 twiddle 缓存无独立消融，不做单项墙钟归因）。 + +## 3. 意外发现：短音频 AIOOBE（已修，行为变化=崩溃→干净错误） + +新增等价测试暴露：`logMelSpectrogram` 原守卫 `audio.size < 2` 边界错误。`pad=384`，`padded.size = audio.size+768`；当 `audio.size < 256`（24 kHz 下 ~10.7 ms）时 `padded.size < nFft`，`frames = 1 + (padded.size-nFft)/hop` 整除截断为 1，帧循环读越界 → `ArrayIndexOutOfBoundsException`。基线对 <10.7 ms 的非静音参考音频是**崩溃**，不是设计错误码。 + +修复：守卫改为 `audio.size < hop`（数学下界 `nFft - 2*pad = hop`），异常类型与消息不变（`IllegalArgumentException("reference audio too short for mel frontend")`），`log mel rejects audio shorter than one hop cleanly` 锁定 {2,100,255} 拒绝 + 256 边界通过。iOS 同源实现的对齐项见 §7.3。 + +## 4. TtsApi2Runtime.kt（prepareReference 所有权 / PreparedReference 不可变 / 冗余拷贝） + +### 4.1 prepareReference 失败/取消路径释放（`.also` → try/finally） +缺陷：`withContext(Dispatchers.Default) { ... }.also { releaseRole("reference_encoder") }` 只在 withContext 正常返回时释放。ICL 路径（有 referenceText）中任一失败——tokenizer 缺失、`refTokenIds.size < 6`、runReferenceEncoder 图 I/O 与 codes 校验、`computeVocoderWarmState` 帧数/状态校验——或块完成后的取消（withContext 退出时重抛 CancellationException），都会把 ~190 MB 的 reference_encoder 会话滞留到整个引擎 `release()`。 +修复（TtsApi2Runtime.kt:213-267）：`try { withContext(...) { ... } } finally { modelManager.releaseRole(SUB_DIR, roles, "reference_encoder") }`。块体逐行未改。 +语义核对： +- 成功路径释放的时机/参数/次数与原 `.also` 完全一致（仍在 `mutex.withLock` 内、caller 线程）。 +- xvector-only 路径会话从未加载：`OnnxModelManager.releaseRole`（OnnxModelManager.kt:118-124）→ `release`（:140-142）= `sessions.remove(key)?.close()`，未加载时纯 no-op，无引用计数/重复 close 风险（独立验证与实施代理结论一致）。 +- `releaseRole` 非挂起函数，取消态协程中也能执行。 +- 同文件其余资源路径（input tensor finally close、`Result.use`、`readTalkerState` 校验失败 catch→close→rethrow、`state.close()` 外层 finally）经逐函数核对均完备；全文件 `.also {` 释放模式现为 0 处。 + +### 4.2 PreparedReference 真不可变 +缺陷：KDoc 声称 immutable conditioning package，但 `embedding`/`referenceTokenIds`/`referenceCodes` 是可变数组直通引用；`referenceCodes` 完全无下游校验。 +修复（TtsApi2Runtime.kt:140-177）：构造参数收私有快照（构造期 `copyOf()`），公开属性每次读取 `field?.copyOf()`；`referenceText`（String 不可变）直存；`isIcl` 改为构造期求值的存储属性，语义不变；internal constructor 参数名/顺序/类型逐项保持（既有调用面与 androidTest 零改动）。 +- 有意偏差：实施提示曾建议"构造时不再额外拷贝"，但与"构造入参后续修改不影响实例"的行为规格互斥；按行为规格采用构造期+读取期双重拷贝，代价为每次 prepareReference（秒级 ONNX 推理）多 ≤24 KB memcpy。 +- internal `vocoderWarmState` 别名分析：生产端 `readVocoderState` 从 ORT buffer 拷入新 JVM 数组，快照从不别名 ORT 内存；唯一消费点 `VocoderTurnState.fromSnapshot`（逐数组 copyOf）与 `VocoderWarmState.copy()`；全仓 grep 无其它读取点、无写入点。 +- androidTest（TtsApi2RuntimeDeviceTest.kt）核对：全部用例只读 `embedding.size`/`referenceCodes == null`/`isIcl`/`referenceFrames`/`identity`，无直接构造、无对返回数组的写入依赖——防御性拷贝对其严格增强，零改动。 + +### 4.3 冗余尾部拷贝移除 +`sampleCodePredictor(cpLogits.copyOfRange(cpLogits.size - cfg.cpVocab, cpLogits.size), ...)` → 直接传 `cpLogits`（TtsApi2Runtime.kt:583-589）。等价性：函数内部 `System.arraycopy(logitsLast, logitsLast.size - vocab, probs, 0, vocab)` 自取末尾 vocab 个元素，与预切片逐元素相同；采样序列与 RNG 消耗不变。省去每 codebook 步 2048 float 分配（~30k 次/合成）。TtsEngine.kt 的同型调用点由本 lane 同步修改（同一所有者，改动等价）。 + +## 5. 测试与验证 + +### 命令（本 worktree，`android/` 目录执行） +``` +AURALIS_TTS_MODEL_DIR=$HOME/Library/Caches/Auralis/tts/api2-release \ + ./gradlew :app:testDebugUnitTest --offline \ + --init-script /private/tmp/auralis-existing-cache.init.gradle + +# 微基准（门控）： +AURALIS_PERF_BENCH=1 AURALIS_TTS_MODEL_DIR=$HOME/Library/Caches/Auralis/tts/api2-release \ + ./gradlew :app:testDebugUnitTest --rerun \ + --tests "com.dialect.interpreter.inference.Qwen3TtsProtocolTest" \ + --offline --init-script /private/tmp/auralis-existing-cache.init.gradle +``` +注意：`/private/tmp/auralis-existing-cache.init.gradle` 仅本地离线缓存访问（强制 offline + 镜像 repo 替换），未提交到仓库。 + +### 数字 +| 时点 | tests | skipped | failures | 备注 | +|---|---|---|---|---| +| 基线（未改源） | 146 | 0 | 0 | 含真实 tokenizer 用例 | +| TtsEngine 改动后 | 155 | 1 | 0 | +8 新测试 +1 修复 @Test；skip = 门控基准 | +| + TtsApi2Runtime 改动后 | 162 | 1 | 0 | +7 PreparedReference JVM 契约测试 | +| + 审查修复（最终） | **163** | 1 | 0 | +1 尾部切片钉测试；±0.0 断言精确化；assertSame(warmState) | + +新增测试清单： +- Qwen3TtsProtocolTest：group0 逐值等价（4 分布×7 topK×150 步）、code predictor 逐值等价（同矩阵×200 步）、kthLargestFloat 对排序参照精确匹配（1..257 全 K + 1024/3072 采样 K，6 类对抗数组；位精确、零符号豁免）、padded 尾部切片钉（oracle 对拍 + prefix 不可见性，group0/cp × 2 分布 × 50 步）、kthLargest 拒绝空/越界、FFT 位等价（7 尺寸）、mel 位等价（8 信号）、短音频干净拒绝（{2,100,255} 拒绝 + 256 边界）、门控微基准。 +- TtsApi2PreparedReferenceTest（新文件，JVM）：构造入参改写不渗入 / 取出副本改写不影响实例 / 每次读取独立副本（×3 数组字段）、标量与 engineToken/generation 绑定原样（assertSame 钉住过期绑定语义）、isIcl 不可经副本翻转、xvector-only 形态、VocoderWarmState.copy() 独立性。 + +覆盖边界（如实）：reference_encoder 失败/取消释放没有自动化测试——`OnnxModelManager.sessions` 为 private、无公开可观察面；JVM 侧 OnnxModelManager 需要 Android Context 不可实例化（不可改它），androidTest 侧真实 bundle 全部满足图校验、帧数校验经 JNI 30s 上限不可达（auralis_resample_jni.cpp:39,76），注入故障在真机上不可观测。该路径以 try/finally 重构 + `releaseRole` no-op 语义的源码证据覆盖；不以装饰性用例冒充验证。取消路径的既有覆盖（每帧/每 codebook ensureActive、TtsCancellationTest）未动。 + +androidTest / 真机（TtsApi2RuntimeDeviceTest、TtsCancellationTest、TtsFrameBudgetTest 等）本轮未运行（README 验收门：主机结果不满足实体设备资格）。采样序列逐位不变 + chunking 一致性测试存在，预期设备行为不变。 + +### 对抗性审查轮（3 视角独立审查 + 逐项对抗验证，7 代理） + +4 个原始发现，3 个经对抗验证成立并已修复： + +| 发现 | 严重度 | 处置 | +|---|---|---| +| `kthLargestFloat` 与装箱排序在**混合 ±0.0 输入**下返回的零符号可能不同（堆为 IEEE 比较；文档"bit-identical"措辞过强；测试 delta=0 看不见零符号）。行为零影响：掩码 `x < ±0.0` 对所有 x 同值、exp(±0.0) 相等 | minor | KDoc 改为"逐值相等，唯混合 ±0.0 时零符号可能不同（不改行为）"；oracle 测试改为 `floatToRawIntBits` 位精确断言 + 零符号豁免（`assertSortedThreshold`） | +| **尾部切片语义零覆盖**：生产 CP 图在 g=1 prefill 输出 [1,2,2048]→展平 4096 floats，`sampleCodePredictor` 取尾部 vocab 个；新旧全部测试只传 size==vocab 数组，"头部拷贝"突变可让全套件保持绿色而设备行为改变 | major | 新增 `samplers read the trailing vocab slice of padded logits`：37 元素垃圾前缀 + 尾部分布，oracle 对拍 + padded-vs-tail-only 同 RNG 同 token 双重钉（group0 与 cp 双路径 × 2 分布 × 50 步） | +| `TtsApi2PreparedReferenceTest` 漏掉 `vocoderWarmState` 的直传断言（丢失即每个 ICL turn 硬失败的字段） | major | `scalar and binding fields pass through unchanged` 补 `assertSame(warm, p.vocoderWarmState)` | + +审查中被对抗验证否决的原始发现：1 项（equivalence 视角，与 ±0.0 项重复）。验证阶段有 2 个子代理在安全分类器限流下运行，其结论经主 worker 独立复核（±0.0 场景手工推演 [1f,-0f,0f],k=2：堆 sift 后 [−0.0,1.0]，0f>−0.0 为假不替换，返回 −0.0；掩码语义等价成立）。 + +## 6. 消融与未采纳设计 + +| 设计 | 处置 | 依据 | +|---|---|---| +| 整排序 → 有界堆选择 | **保留** | 隔离腿 ~12×；等价性经排序参照逐值验证 | +| `generated.toSet()` → BooleanArray seen-mask | **保留** | 与阈值选择同函数、合并实测 1.25–1.38×；逐值等价（无独立消融，不归因单项贡献） | +| mel 幅度提升 | **保留** | 每帧冗余 sqrt 65,664→513 次（128 带 × 513 bin 重复），结构计数；合并实测 1.5–2.3× | +| FFT twiddle 缓存 | **保留** | 每帧 5120 对内联求值→0（1023 个独特值缓存）；同表达式缓存保证位等价；与幅度提升合并实测 | +| 采样 weights DoubleArray 跨步 scratch 复用 | **不做** | 需要跨 API1/API2 两个调用方传入 scratch、引入有状态签名；exp() 主导的剩余成本中其占比未证明；收益上限 ~10 µs/帧（<0.1%） | +| BPE 重复工作整理 | **不做** | 每合成 1 次、随文本长度线性；无基准证据为热点 | +| window/filterbank 每次 logMel 调用重建缓存化 | **不做** | 每调用 1 次，微基准不构成可见份额 | +| API1 code_predictor 会话合成后释放 | **不改** | 与 decode/vocoder 不对称但会话小、保留利于重复合成；无内存受害证据（记录为观察，见候选 #12） | +| 失败实验记录 | 短音频守卫初版测试用例（2 样本）暴露的是基线 AIOOBE 而非新代码问题；首个微基准"复跑"因 Gradle up-to-date 未实际执行，经 `--rerun` 修正（教训：up-to-date 跳过的"复现"数字无效） | — | + +## 7. 仍开放的门 / 移交 + +1. **真机/模拟器验证未做**：设备测试套件（含 api2Case 门：main/chunking/lifecycle/stream/binding）与端到端回转写未运行——需 root 排期与设备。预期采样序列逐位不变。 +2. **reference_encoder 释放的设备级观测**：如需自动化，需要 OnnxModelManager 增加测试可观测面（不属于本 lane，需 root 决定）或真机故障注入。 +3. **iOS lane 对齐项**：iOS 若有同源 mel 前端实现，其短音频守卫可能存在同型 AIOOBE（见 §3）；属 iOS lane 文件，未改动。 +4. **微基准口径**：全部为主机 JVM 数字；ART/设备上的收益（尤其按 2048 帧预算上限估算的 ~350–400 MB/满预算合成分配消除，比例随实际帧数线性下降）是算术推算，未实测。 +5. 短音频行为变化（崩溃→干净 IAE，§3）请 root 确认接受；同一输入旧代码崩溃、新代码按既有错误风格拒绝。 +6. 语义保留声明：错误码/异常类型与消息、就绪门（manifest apiContractVersion、角色校验）、取消语义（每帧/每 codebook ensureActive、sink 失败即中止）、API1/API2 行为、采样协议（temperature/topK/penalty/seed）全部未动；模型精度、manifest/schema、品牌与 Apache-2.0 许可未触碰。 +7. **native talker KV 直接复用未回退**：API2 `TalkerState` 仍持有 `OnnxTensor`（past_keys/past_values 留在 ORT 内存，`getFloatBuffer()` 堆拷贝路径未引入）；本轮只把调用点冗余的 logits 预切片去掉，未触碰 KV 传递结构。 diff --git a/docs/specs/2026-09-08-quality/reports/capture-boundary-baseline.json b/docs/specs/2026-09-08-quality/reports/capture-boundary-baseline.json new file mode 100644 index 0000000..32c94dc --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/capture-boundary-baseline.json @@ -0,0 +1,11 @@ +{ + "asrCallsBeforeSilence": 0, + "baseline": "7e2cb57", + "exceeded10sBound": true, + "expectedMaxSamples": 160000, + "inputSpeechSamples": 320000, + "observedUtteranceSamples": [ + 364800 + ], + "scope": "Host session core with synthetic PCM and fake recognition; not ASR accuracy or physical-device performance" +} diff --git a/docs/specs/2026-09-08-quality/reports/capture-buffer-ablation.json b/docs/specs/2026-09-08-quality/reports/capture-buffer-ablation.json new file mode 100644 index 0000000..1361a30 --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/capture-buffer-ablation.json @@ -0,0 +1,26 @@ +[ + { + "bufferedChunksObserved" : 1000, + "candidate" : "unbounded baseline", + "drops" : 0, + "firstId" : 0, + "lastId" : 999, + "logicalPcmBytes" : 12800000 + }, + { + "bufferedChunksObserved" : 8, + "candidate" : "keep newest eight and continue", + "drops" : 992, + "firstId" : 992, + "lastId" : 999, + "logicalPcmBytes" : 102400 + }, + { + "bufferedChunksObserved" : 8, + "candidate" : "bounded contiguous prefix then terminate", + "firstId" : 0, + "firstRejectedId" : 8, + "lastId" : 7, + "logicalPcmBytes" : 102400 + } +] diff --git a/docs/specs/2026-09-08-quality/reports/capture.md b/docs/specs/2026-09-08-quality/reports/capture.md new file mode 100644 index 0000000..1ea1ffb --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/capture.md @@ -0,0 +1,47 @@ +# iOS capture bounds + +The released baseline buffered continuous speech until silence. A host reproduction sent 320,000 voiced samples (20 s at 16 kHz): ASR received nothing before silence, then one 364,800-sample utterance (22.8 s including the VAD tail). The required utterance bound is 160,000 samples. See [baseline evidence](capture-boundary-baseline.json). + +## Change + +`UtteranceSegmenter` emits at most 160,000 samples, including up to 3,200 samples of pre-roll. A single large input can emit several bounded values directly into the existing capacity-two ended-utterance queue. A forced split preserves PCM order and keeps VAD continuity; playback and stop reset partial input. The VAD clock now uses actual sample counts. Non-finite input terminates capture visibly. + +`AudioChunkStream` also bounds the earlier microphone callback queue: eight chunks of at most 3,200 samples, or 102,400 bytes of queued PCM payload. On overflow it closes the stream and stops the matching recording. It preserves the contiguous prefix; it does not splice later audio across a gap. A stream belongs to one recording, so a late callback cannot stop or contaminate the next recording. + +The pipeline's existing unexpected-stream-end path reports capture failure. Partial, unfinished speech is discarded; complete earlier utterances retain their normal queue semantics. No changes were made to inference sampling, model weights or package readiness. + +## Selection experiment + +The experiment sends 1,000 synthetic callback chunks while the consumer is stalled. These are observed queue results and derived PCM payload sizes, not process RSS or device throughput. + +| Candidate | Retained chunks | PCM payload | Result | +|---|---:|---:|---| +| Unbounded baseline | 1,000 | 12,800,000 B | Memory grows with producer duration | +| Keep newest eight and continue | 8 | 102,400 B | Drops 992 chunks and loses the beginning | +| Keep oldest eight, terminate on overflow | 8 | 102,400 B | Preserves IDs 0–7, rejects ID 8, then ends | + +The last candidate is implemented using the native Swift stream. No custom ring-buffer or condition-variable framework is needed. The eight-chunk capacity is an engineering bound, not a value calibrated on physical phones. [Results](capture-buffer-ablation.json). + +Swift's pinned source confirms that `bufferingOldest` rejects new input when full, while `bufferingNewest` discards older buffered input: [Swift 6.1.2 AsyncStream](https://github.com/swiftlang/swift/blob/swift-6.1.2-RELEASE/stdlib/public/Concurrency/AsyncStream.swift#L156-L171). The behavior was also executed in the experiment. + +## Validation + +- Existing 35 host session/playback tests plus 13 new capture regressions: **48 passed, 0 failed**. +- Continuous 20 s speech reaches the actual session-core ASR port as two exact 160,000-sample inputs before silence. This uses a recognition spy, not a real model. +- Tests cover oversized inputs, uneven partitioning, exact PCM order, pre-roll, short pauses, reset, actual-sample VAD timing, invalid PCM, overflow, chunk-length validation, stream lifetime and visible session failure. +- The overflow integration fixture first assumed that two queued slots meant the third send must fail. A waiting iterator legitimately receives one chunk directly. The test now sends four chunks synchronously; production capacity and timing were unchanged. +- Real `AudioRecorder` + `AudioChunkStream` Catalyst typecheck: exit 0, one existing Bluetooth-option deprecation warning. +- Physical microphone/route behavior, whole iOS build and representative ASR quality require subsequent integration/device checks; no such result is claimed here. + +Run the host checks from the repository root: + +```sh +bash ios/DialectInterpreterTests/SessionCoreTests/run_host_tests.sh +``` + +Reproduce the buffering-policy experiment: + +```sh +swiftc -O -parse-as-library ios/DialectInterpreter/Audio/AudioChunkStream.swift docs/specs/2026-09-08-quality/benchmarks/capture-buffer-ablation.swift -o /tmp/auralis-capture-ablation +/tmp/auralis-capture-ablation +``` diff --git a/docs/specs/2026-09-08-quality/reports/ios-runtime-evidence.json b/docs/specs/2026-09-08-quality/reports/ios-runtime-evidence.json new file mode 100644 index 0000000..bb7e890 --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/ios-runtime-evidence.json @@ -0,0 +1,190 @@ +{ + "exitCode": 0, + "binarySha256": "c1b27ecda4e7346c17160052bb106ebfa1de596ca01c9f9fa17c3f412924614e", + "harnessSha256": "aa3bb814b2b520d0c0d1e572fed7de2ccf95b145f746c266fe6145c33cf353df", + "sources": { + "AsrEngine.swift": "239d6c47c971c65f91c0b0ae79ecc691110ec018124bde4514fb7d86b3846604", + "HyMtNativeBridge.swift": "8fac5313df1f370dccc0be92a6933e706650f385369c4b8ae45d8e4eff6a79cc", + "ModelRepository.swift": "b48d523eda0d74f13e247e3f3d7969d0e1756895e331be95c6d42e7d332cd4d8", + "OnnxModelManager.swift": "60fe58b8e535e9df97543c608b54d5d422ed31cb466013d0706b492c0d61018e", + "OrtRuntime.swift": "d647e3e26110bfa53065b5088e4355b3aae1261424cdb36e4acab4529a3cea20", + "PackageVerifier.swift": "83302624fe70cfc958fb50979cdfa8f6a58e8bbbfacf8bd7b8576c7c80f18ceb", + "SharedContracts.swift": "d5a4bf47389d840ba602b58435e32f1892c06a237229b5a7e4309271b1a30ccf", + "TtsEngine.swift": "5af0fc423f86756d937f8eec83619647a909448f33b803aedfa9a5105216a732", + "VoiceProfileRepository.swift": "1fe7bf2ee7db889fff6372db3b4af3d78f357aba74d86fa933466960007f99bc" + }, + "wavs": { + "121-chinese-icl.wav": "840705e933e5fa11c4f3b97a5ca35b4cbd955973d5200a0f00c9265e188b4694", + "121-chinese-xvector.wav": "7227002afbd7a8340f33cd3d54afd4c88cfcc396061e303158507986f0a51b4c", + "121-english-icl.wav": "513aa02c28c71d26677339e26e41190dcb8e4426631d6c34448ec4f068f5f620", + "121-english-xvector.wav": "87c0ad3792cbbd33b3e5b5942f184262d8e7af3066f9c034f5ac47de22232710", + "260-chinese-icl.wav": "de60fec8304e37a140d2a6e7649d72cc1913fcb6755f160a39506715761b45e8", + "260-chinese-xvector.wav": "5ef557363166b0f63d297652ab462f841c02758244f04f23c98687906bbdb23f", + "260-english-icl.wav": "20a33355b3060b2cde94fbcd1eb53755178aaf9ee073000723b0020e358bd547", + "260-english-xvector.wav": "e04f29dc5cdc3c9f00f3fad0e7477f3fc8a9f0c28254e75e9d9c3c89dd1d522a" + }, + "matchesBaseline": true, + "environment": "macOS arm64 host, ONNX Runtime 1.24.2; not physical-device qualification", + "baselineCommit": "7e2cb57927d6290c089dce1c33a72bbc9902b6cc", + "checks": [ + { + "expectedFrames": 106, + "gotFrames": 106, + "hasWarmup": true, + "id": "encoder-121", + "status": "pass", + "warmupPosition": 106, + "xvectorHasWarmup": false + }, + { + "expectedFrames": 88, + "gotFrames": 88, + "hasWarmup": true, + "id": "encoder-260", + "status": "pass", + "warmupPosition": 88, + "xvectorHasWarmup": false + }, + { + "clippingRatio": 0, + "conditioningMode": "icl", + "duration_s": 3.12, + "frames": 39, + "id": "121-english-icl", + "peak": 0.24347658455371857, + "status": "pass", + "wall_s": 10.478690958333573, + "warmupReused": true + }, + { + "clippingRatio": 0, + "conditioningMode": "icl", + "duration_s": 2.72, + "frames": 34, + "id": "121-chinese-icl", + "peak": 0.28037765622138977, + "status": "pass", + "wall_s": 5.167879374999757, + "warmupReused": true + }, + { + "clippingRatio": 0, + "conditioningMode": "icl", + "duration_s": 3.12, + "frames": 39, + "id": "260-english-icl", + "peak": 0.4616669714450836, + "status": "pass", + "wall_s": 5.734880166666699, + "warmupReused": true + }, + { + "clippingRatio": 0, + "conditioningMode": "icl", + "duration_s": 2.96, + "frames": 37, + "id": "260-chinese-icl", + "peak": 0.41179177165031433, + "status": "pass", + "wall_s": 5.315171041666872, + "warmupReused": true + }, + { + "clippingRatio": 0, + "conditioningMode": "xvector", + "duration_s": 3.12, + "frames": 39, + "id": "121-english-xvector", + "peak": 0.38960057497024536, + "status": "pass", + "wall_s": 5.2243536249998215, + "warmupReused": false + }, + { + "clippingRatio": 0, + "conditioningMode": "xvector", + "duration_s": 2.72, + "frames": 34, + "id": "121-chinese-xvector", + "peak": 0.33545371890068054, + "status": "pass", + "wall_s": 4.730405374999464, + "warmupReused": false + }, + { + "clippingRatio": 0, + "conditioningMode": "xvector", + "duration_s": 3.44, + "frames": 43, + "id": "260-english-xvector", + "peak": 0.4602113962173462, + "status": "pass", + "wall_s": 5.682604541666478, + "warmupReused": false + }, + { + "clippingRatio": 0, + "conditioningMode": "xvector", + "duration_s": 2.72, + "frames": 34, + "id": "260-chinese-xvector", + "peak": 0.6433568596839905, + "status": "pass", + "wall_s": 4.5498221249999915, + "warmupReused": false + }, + { + "chunk_vs_oneshot": 2.8759241104125977e-06, + "chunk_vs_synth": 0, + "id": "chunk-vs-oneshot", + "oneshot_vs_synth": 2.8759241104125977e-06, + "samples": 74880, + "status": "pass" + }, + { + "error": "badOutput(\"sink-forced-failure\")", + "id": "sink-throw", + "status": "pass" + }, + { + "frames": 11, + "id": "recover-after-sink", + "status": "pass" + }, + { + "error": "frame budget exhausted before codec EOS; refusing truncated speech", + "id": "noEOS", + "status": "pass" + }, + { + "frames": 11, + "id": "recover-after-noEOS", + "status": "pass" + }, + { + "id": "cancel", + "secondsAfterCancel": 0.08559950000017125, + "status": "pass" + }, + { + "frames": 11, + "id": "recover-after-cancel", + "status": "pass" + }, + { + "error": "ICL requires nonempty reference text and TTS API2", + "id": "empty-icl-text", + "status": "pass" + }, + { + "id": "snapshot-isolation", + "position": 106, + "status": "pass" + }, + { + "error": "notLoaded(\"TTS\")", + "id": "release-invalidates-prepared", + "status": "pass" + } + ] +} diff --git a/docs/specs/2026-09-08-quality/reports/ios-runtime.md b/docs/specs/2026-09-08-quality/reports/ios-runtime.md new file mode 100644 index 0000000..d9ef944 --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/ios-runtime.md @@ -0,0 +1,81 @@ +# iOS TTS runtime review — 2026-09-08 + +Baseline: `v0.1.0-preview.1` (`7e2cb57927d6290c089dce1c33a72bbc9902b6cc`). +Measurements below use macOS arm64 and the official ONNX Runtime 1.24.2 bindings. They do not establish iPhone performance or model readiness. + +## Changes retained + +- Keep talker, code-predictor and streaming-vocoder KV outputs as `OnnxTensor` values and feed them directly into the next graph call. Preserve shape, dtype, finite-value and cancellation checks. API1 still stacks its separate per-layer prefill outputs once, as required by that graph. +- Reject negative or overflowing input shapes before constructing a tensor. Scalar and zero-extent inputs remain supported. +- Share float validation and reading through `withFloatBuffer`; `floatArray` only adds materialization. Retain both the `ORTValue` and Foundation wrapper throughout every float/int64 borrow. +- Compute mel magnitudes once per FFT bin instead of once per mel band. Reuse frame-local buffers while preserving arithmetic order. Require at least one valid frame; the sole production caller already rejects references shorter than 1,024 samples. + +Each KV output is validated before assignment. A later validation failure unwinds the turn; partially replaced local state is never resumed. Prepared-reference warmup state remains an immutable value snapshot. Sampling, precision, model files, public APIs and manifest readiness are unchanged. + +## Copy accounting + +A **set** below contains both keys and values. The old path materialized a set and reconstructed it as input; the new feedback path performs neither operation. Logical transfer counts describe payload movement, not measured memory bandwidth or allocator traffic. + +| State | One set | Removed logical output + input transfer | +|---|---|---| +| Talker `[28,1,8,T,128]` × 2 | `229,376 × T` bytes | `458,752 × T` bytes per step; 88,539,136 bytes at T=193 | +| Code predictor `[5,1,8,T,128]` × 2 | `40,960 × T` bytes | `81,920 × T` bytes per group step | +| Streaming vocoder, conv + KV | 5,193,984 bytes | 10,387,968 bytes per step; normally four audio-code frames per step | + +**CP history resets every audio frame.** Its first group consumes two positions, then the remaining groups add one each: present lengths are 2…16. It does not accumulate `15 × audioFrames` positions. Earlier estimates using T=959 were incorrect; at T=16 the removed logical transfer is 1,310,720 bytes per group step. + +The host probes also distinguish three read paths: + +| Read path | Observed host payload copies | +|---|---| +| Native KV feedback | 0 managed copies; never calls `tensorData()` | +| `withFloatBuffer` | 1 Foundation snapshot | +| Current `floatArray` | Snapshot + Array = 2 | +| Baseline `floatArray` | Snapshot + Data bridge + Array = 3 | + +Although ORT's Objective-C source requests a no-copy `NSMutableData` wrapper, the tested Foundation implementation returned an independent snapshot. A separate Objective-C probe reproduced that behavior without Swift bridging. Other platforms may alias the storage, so both owners are retained during borrowing. Tests assert values and lifetimes, not pointer identity. No private binding or extra ownership framework was added. + +## Validation + +| Check | Result | +|---|---| +| Repository Swift Testing suites | 23 tests / 3 suites pass against real ORT, including the final shared-reader cleanup | +| Synthetic Concat/MatMul graph | 64-step native feedback equals the materialize/rebuild twin bit for bit | +| Synthetic Identity graph | Retained step-zero output remains unchanged through 63 later runs | +| Tensor boundaries | Negative/overflowing dimensions and wrong dtype throw; scalar and empty tensors round-trip | +| Mel regression | LCG, impulse and DC goldens, frame counts and reflection semantics pass | +| Independent mel comparison | 27 input cases, including a 30-second reference and extreme floats, match baseline bit patterns | +| Full-model baseline/candidate comparison | Three serial runs of each binary; all 20 checks per run pass; all eight WAV hashes match across all six runs | +| Full-model final shared-reader check | 20 checks pass; eight WAVs match baseline byte for byte; [source-bound evidence](ios-runtime-evidence.json) | + +The synthetic graphs are explicitly synthetic runtime fixtures. Full-model checks use the real API2 talker, reference encoder, code predictor and streaming vocoder, with references 121/260, Chinese/English targets and ICL/xvector modes. Parameters are fixed at `maxFrames=384`, `seed=2026_0906`. + +All six baseline/candidate runs agree on encoder codes (R=106/88), eight synthesis frame counts (39/34/39/37/39/34/43/34), sink-error recovery, no-EOS rejection, cancellation and release invalidation. Chunk/full-vocoder maximum difference remains `2.8759241104125977e-06`. Cancellation returned in 0.05–0.11 seconds on this host. Every eight-WAV SHA list has digest `eab0d3022a233b78…`. + +The new tests are part of the existing Xcode test target. Host execution does not replace the integrated iOS simulator CI run or physical-device qualification. The scratch host build links macOS 27 native artifacts; its deployment-target linker warnings do not establish macOS 14 compatibility. + +## Measurements and ablation + +Maximum RSS in the three full-model runs was 5.56/4.93/4.95 GB for baseline and 4.25/4.61/4.73 GB for the KV candidate. These process peaks exceed the initial 3–3.5 GB estimate. Per-synthesis wall time varied from 4.1 to 10.6 seconds, with within-binary variation comparable to differences between candidates; no end-to-end speedup is claimed. + +| Candidate | Decision | Evidence or cost | +|---|---|---| +| Native KV feedback | Keep | Full-model WAV equality; eliminates repeated materialization and reconstruction | +| Shared float reader | Keep | Removes duplicate validation/borrow logic; 23 runtime tests pass | +| One magnitude calculation per bin | Keep | Bit-pattern comparison passes; function-level improvement about 1.1–1.2×, less than 1% end to end | +| FFT twiddle cache | Omit | Estimated total benefit too small to justify another retained cache | +| Partial KV conversion | Omit | Keeps two state representations while retaining part of the copy cost | +| Additional owner/retention framework | Omit | Graph lifetime checks pass with ordinary ORTValue ownership and scoped borrowing | +| Private Objective-C/C++ tensor bridge | Omit | The remaining snapshot is used only where values must be inspected; no measured benefit justifies a second binding path | + +## Reproduction and attribution + +Private evidence root: `$AURALIS_CACHE/reports/ios-runtime-20260908/`. Model weights, reference recordings and generated WAVs are excluded from Git. + +- `baseline-src/`: nine files verified against `7e2cb57`; baseline TTS SHA starts `d539cf9b`, ORT wrapper `1ba2ef44`. +- `candidate-src/`: the six-run KV candidate. Binary hashes start `a71ed564` (baseline) and `d46d4202` (candidate). Subsequent first review changed comments only. +- `build-full.sh`: one compilation recipe for both source directories. The frozen harness is `$AURALIS_CACHE/reports/ios-tts-api2/freeze/harness.swift`, SHA starts `aa3bb814`. +- `full/{baseline,candidate}-r{1,2,3}/`: independent results, logs, process peaks and WAV hash lists; `full-run-summary.txt` records six exit-zero results. +- `root-final-src/`, `root-final-source-shas.json`: source snapshot after float-reader consolidation and explicit int64-owner retention. `testrun/root-final-swift-test.log` records 23 passing tests; `full/root-final-r1/` keeps its separate full-model result and binary/source hashes. + +The final source needs integrated CI before merging. Physical-device latency, memory and energy gates remain open; shared packages remain draft. diff --git a/docs/specs/2026-09-08-quality/reports/model-tooling-benchmark.json b/docs/specs/2026-09-08-quality/reports/model-tooling-benchmark.json new file mode 100644 index 0000000..16b2713 --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/model-tooling-benchmark.json @@ -0,0 +1,62 @@ +[ + { + "sizeMiB": 64, + "candidate": "A", + "cache": "warm", + "medianSeconds": 0.040255625000099826, + "minSeconds": 0.038431417000083457, + "stdevSeconds": 0.00115246945525479, + "rounds": 9, + "peakRssBytes": 27181056 + }, + { + "sizeMiB": 64, + "candidate": "B", + "cache": "warm", + "medianSeconds": 0.04220179200001439, + "minSeconds": 0.03461558400022113, + "stdevSeconds": 0.0038223169615079805, + "rounds": 9, + "peakRssBytes": 27181056 + }, + { + "sizeMiB": 64, + "candidate": "C", + "cache": "warm", + "medianSeconds": 0.03676587500012829, + "minSeconds": 0.034586792000027344, + "stdevSeconds": 0.0009893315391121646, + "rounds": 9, + "peakRssBytes": 27033600 + }, + { + "sizeMiB": 256, + "candidate": "A", + "cache": "warm", + "medianSeconds": 0.15810012500014636, + "minSeconds": 0.15446966699983022, + "stdevSeconds": 0.00250657264443712, + "rounds": 9, + "peakRssBytes": 27148288 + }, + { + "sizeMiB": 256, + "candidate": "B", + "cache": "warm", + "medianSeconds": 0.14299645899973257, + "minSeconds": 0.1388483749997249, + "stdevSeconds": 0.0073777146282784185, + "rounds": 9, + "peakRssBytes": 27312128 + }, + { + "sizeMiB": 256, + "candidate": "C", + "cache": "warm", + "medianSeconds": 0.12833366700033366, + "minSeconds": 0.12611904200002755, + "stdevSeconds": 0.001174063602608084, + "rounds": 9, + "peakRssBytes": 27082752 + } +] diff --git a/docs/specs/2026-09-08-quality/reports/model-tooling.md b/docs/specs/2026-09-08-quality/reports/model-tooling.md new file mode 100644 index 0000000..4e693ae --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/model-tooling.md @@ -0,0 +1,64 @@ +# Model acquisition and verification — 2026-09-08 + +Baseline: `v0.1.0-preview.1` (`7e2cb57927d6290c089dce1c33a72bbc9902b6cc`). Model pins, readiness, precision and package layout are unchanged. + +## Implementation + +`file_integrity.py` supplies one SHA-256 reader and one acquisition primitive. `fetch_model`, `validate_models`, `model_tasks`, `mt_runner` and `tts_runner` reuse it. Existing `file_hash` and `sha256_file` names remain import aliases. Python 3.10 uses the chunked fallback when `hashlib.file_digest` is unavailable. The MT runner's default paths now use the current user's cache or `AURALIS_CACHE`, with `HYMT_LIB` / `HYMT_MODEL` overrides retained. + +For local files, APFS `clonefile` followed by one read-back hash avoids copying file contents. Immutable or append-only sources use streaming copy because cloning their flags makes the staged file impossible to replace or delete. The same fallback handles unsupported platforms, filesystems or symbols. Both paths return `(size, digest)` to the existing pinned-hash gate. Digests stay in process and are never persisted as readiness markers. + +Downloads hash incoming bytes and reject overlong or truncated input. Archives are fully authenticated **before** tar parsing; each selected regular member is then hashed while being copied. Local archives retain their required pre-hash pass. Symlink, traversal, duplicate selected-file, missing-file and per-member integrity checks remain intact. + +Staging, leases, rollback, provenance checks and package manifests retain their existing boundaries. `install()` fsyncs every staged file and the affected directories before promotion; no extra source fsync remains in the clone helper. + +| Acquisition path | Baseline full-file passes | Final passes | +|---|---|---| +| Local copy | 2 reads + 1 write | Clone: 1 read; fallback: 1 read + 1 write | +| Downloaded archive | Write + archive hash read + tar read | Hash during write + tar read | +| Local archive | Archive hash read + tar read | Same, to authenticate before parsing | +| Extracted member integrity | Additional read of each staged member | Hash during extraction | +| Transform output gate | Two reads of final output | One verified digest reused by the gate | + +The audited transform script retains its own mandatory input/output verification. These pass counts are structural; they are not multi-GB latency measurements. + +## Failure behavior + +- A build-provenance mismatch is an integrity failure (exit 1), matching `validate_models`. +- `--update-manifest` does no write when all sizes were already pinned. If an explicitly needed metadata update fails after installation, the command exits 2 with `package installed, but the requested manifest update failed`; the verified package and its manifest remain installed. +- Invalid arguments, missing dependencies, contract failures and draft readiness retain their established meanings. + +## Measurements and ablation + +[Recorded warm benchmark](model-tooling-benchmark.json): macOS 27 arm64, Python 3.14.7, APFS, deterministic temporary files; nine rounds, separate process per sample and rotated order. Peak RSS is approximately 26 MiB across candidates, near interpreter baseline. + +| File size | Copy then hash | Fused copy/hash | Clone then hash | +|---|---:|---:|---:| +| 64 MiB | 0.0403 s | 0.0422 s | 0.0368 s | +| 256 MiB | 0.1581 s | 0.1430 s | 0.1283 s | + +At 256 MiB, clone/hash is 19% faster in this host microbenchmark. Fused copying remains the portable fallback, with one fewer read pass. At 64 MiB its timing does not improve over baseline. No model-install or device-wide speedup is inferred. + +An earlier best-effort eviction run measured 0.170/0.157/0.141 seconds at 256 MiB. Cache eviction was not positively verified, and those results precede removal of an extra fsync; they are retained privately as diagnostic data, not a cold-storage guarantee. + +| Candidate | Decision | +|---|---| +| Fused portable copy/hash | Keep: removes a read pass without retaining file-sized buffers | +| APFS clone/hash | Keep: observed benefit; tiny platform-specific path with portable fallback | +| `_HashingReader` archive wrapper | Delete: rehashed already authenticated bytes without saving a read | +| Source fsync inside clone helper | Delete: staged files and directories are already synced before promotion | +| Metadata-repair framework | Omit: route immutable/append-only inputs to ordinary copying | +| New digest/readiness cache | Omit: each acquisition verifies its own bytes | +| `file_digest` as a speed claim | Reject: parity with the loop here; retain it only as the standard-library implementation | + +## Verification + +The implementation pass ran 142 convert tests with six existing optional-dependency skips, 14 scripts tests, compileall and the `file_hash` import probe. Final integration reruns these through `scripts/verify --mode fast`; skipped cases are not counted as passed. + +New regressions cover clone/stream byte equality, immutable and append-only flags on real macOS temporary files, source preservation, replaceable/deletable staging, transform rejection before process launch, provenance classification, explicit-update failure/noop, short/overlong downloads, authenticated archives with bad members, and rejection of bad archives before `tarfile.open` is called. Flag tests restore their temporary source flags in `finally`. + +One additional adversarial finding was confirmed: append-only flags break cleanup just like immutable flags. Both are guarded and tested. The temporary source is assumed stable during acquisition; no general concurrent-mutator framework was added. + +Reproduce lightweight checks with `python3 scripts/verify --mode fast`. Reproduce the warm microbenchmark with `python3 convert/bench_file_integrity.py --rounds 9 --json`. Private raw evidence is under `$AURALIS_CACHE/quality-2026-09-08/model-tooling/`; large temporary benchmark files were removed. + +Real-model integration remains `scripts/verify --scope models`. The separate TTS identity gate remains blocked until its quality criteria are satisfied; this refactor does not make a successful inference imply clone-identity qualification. diff --git a/docs/specs/2026-09-08-quality/reports/native-motion-evidence.json b/docs/specs/2026-09-08-quality/reports/native-motion-evidence.json new file mode 100644 index 0000000..add0969 --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/native-motion-evidence.json @@ -0,0 +1,85 @@ +{ + "unit_tests": { + "tests": 169, + "failures": 0, + "errors": 0, + "skipped": 1 + }, + "build": "testDebugUnitTest assembleDebug: BUILD SUCCESSFUL in 23s", + "lint": "Environment blocked: missing cached lint POM metadata; Google Maven TLS download failure; temporary artifact fallback lacks transitive classes. No dependency changes.", + "apk_sha256": "eb6c555a927bea5ec10a930dc6fa3d1d1be6641ae1361b2687fc78e573bb7bca", + "device": "API 35 arm64 AVD, emulator-5580; separate empty userdata", + "screenshots": [ + { + "name": "large-text-settings.png", + "sha256": "d0650547cd0d3de1d1542728c0f963915a273b193caac7f5b5e527a118a4c826" + }, + { + "name": "reduced-motion-back.png", + "sha256": "b2f6a1850cc2dd004c20ea2e836a62ad31023d2f71d00c08141fb138525fd7d3" + }, + { + "name": "large-text-interpret.png", + "sha256": "1141297b482ca52c6d0e15ca3964b220e1bfd3f96704ff7ec5fbb0862f0b51ee" + }, + { + "name": "normal-settings.png", + "sha256": "34713f42b93e365ab19bae43fed86d5d9d96f6e6e42c2e4317e4a590ed932f55" + }, + { + "name": "reduced-motion-settings.png", + "sha256": "a83d3aa7077e581481e41014b05e9802df6355e3a21e85e214db0678f5fad7c9" + }, + { + "name": "reduced-motion-interpret.png", + "sha256": "e16fe2e5c7277decde43c4fb9d2e2d5f864169151f4e0da4805bfa1eebf61f19" + }, + { + "name": "normal-interpret.png", + "sha256": "90a8dbc519ba1fc07457575206b1afde997900553209f3558cdceab8415e1bfa" + } + ], + "ui_checks": [ + "normal interpretation and Settings navigation", + "200% font interpretation and Settings layout", + "system Settings background/return with animator scale zero", + "reduced-motion forward and back Settings navigation", + "missing-model record action remains disabled" + ], + "limitations": [ + "No microphone/audio used", + "No loaded-model active-meter screenshot", + "Screenshots verify destination/layout, not animation frame timing", + "No physical-device/frame-time qualification" + ], + "settings_restored": true, + "original_userdata_models_profiles": "unchanged", + "disposable_images_removed": [ + "ui-only-userdata.img", + "ui-only-userdata.img.qcow2" + ], + "emulator_stopped": true, + "scope": "Android UI-only AVD evidence; screenshots precede removal of the stale static Settings runtime-version label. No physical-device or active-meter timing qualification.", + "ios": { + "catalystFullSourceTypecheck": { + "exitCode": 0, + "sdk": "MacOSX26.5.sdk / real Mac Catalyst frameworks", + "target": "arm64-apple-ios17.0-macabi", + "remainingWarnings": "two existing allowBluetooth deprecations" + }, + "mappingTests": { + "exitCode": 0, + "framework": "real Swift Testing", + "sourceSha256": "e34dda2a303fd93829f8585f89c53c3f235361be35e5d538fc33ccf894eb4338", + "testSha256": "d389af7469f8d70383366ac756a7e2c6bc883e178ac889771d5feeac9404afca", + "passed": 8 + } + }, + "contrast": { + "background": "#142128", + "oldAccent": "#216B62", + "oldRatio": 2.617, + "headerAccent": "#63B3A4", + "ratio": 6.663 + } +} diff --git a/docs/specs/2026-09-08-quality/reports/native-motion.md b/docs/specs/2026-09-08-quality/reports/native-motion.md new file mode 100644 index 0000000..fd059fe --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/native-motion.md @@ -0,0 +1,46 @@ +# Native motion verification — 2026-09-08 + +Baseline: `v0.1.0-preview.1` (`7e2cb579`). Both apps use native Compose/SwiftUI primitives and the existing Auralis palette. Inference, sampling and model-readiness contracts are unchanged. + +## Selected behavior + +| Surface | Implementation | +|---|---| +| Session status | Short fade and small vertical shift; direct replacement under reduced motion. Compose size animation is disabled. | +| Conversation | Stable turn IDs drive insertion/placement effects; updates within an existing turn do not replay insertion. Automatic scrolling follows the last ID even when the 200-item list is full. | +| Main action | Small press scale and dimming; reduced motion suppresses scale. Playback is labelled as playback and hides the live capture meter. | +| Input meter | One shared scalar drives seven bars on one drawing surface. A logarithmic display window makes small normalized RMS values distinguishable. | +| Idle/reduced motion | No idle TimelineView, floating empty-state loop or independent bar springs. System preference changes are respected after Android resumes and through the SwiftUI environment. | + +The meter uses a display window of 0.005…0.5 normalized RMS, not a calibrated sound-pressure measurement. Android reads its animated scalar in Canvas drawing; SwiftUI interpolates one `animatableData` scalar and draws through Canvas. Reduced motion snaps to the current measured level. The [Apple Animatable contract](https://developer.apple.com/documentation/swiftui/animatable) supplies interpolation without a timer; [Compose ContentTransform](https://developer.android.com/reference/kotlin/androidx/compose/animation/ContentTransform) permits a null size transform while retaining fade/slide. + +The always-dark Android header now uses Mint `#63B3A4` on Pine `#142128` (calculated contrast 6.663:1). Its earlier light-theme accent produced 2.617:1. Light-surface meters keep the light-surface accent. + +## Simplification and corrections + +- Removed the old layout animation for each Android bar and the idle SwiftUI waveform timeline. +- Rejected an initial per-emission EMA: at a low update rate it stepped visibly and identical consecutive measurements could leave it short of the target. Native scalar interpolation replaces that state machine. +- Replaced a Compose lifecycle observer wrapper with one Activity state value refreshed in `onResume`. +- Removed the unused `StatusIndicatorView` after confirming it had no references. The Xcode project uses filesystem-synchronized groups; no project entry remained to remove. +- Replaced one-millisecond reduced-motion transitions with direct content/`EnterTransition.None`/`ExitTransition.None`. +- One flag now arms the three typing dots. A live reduced-motion change disarms the effect; ring and press animations also receive nil when motion is reduced. +- Replaced a test that enumerated more than 100 million adjacent Floats with a bounded grid plus neighboring values at both clamp boundaries. The eight waveform cases use the project's existing Swift Testing framework. +- Removed the stale Settings label `ONNX Runtime 1.22`; the screen now describes the useful fact, local CPU execution, without a duplicated dependency version. + +## Verification + +| Check | Result | +|---|---| +| Android unit/build | 169 tests, zero failures/errors, one explicit benchmark skip; debug APK built | +| Android UI-only API 35 AVD | Normal and 200% text layouts, Settings forward/back navigation, and system Settings return with animations disabled checked; no crash log entries | +| iOS all-source Catalyst typecheck | Exit 0 using the installed real 26.5 SDK, UIKit/SwiftUI and ORT headers; no UIKit shim | +| Swift waveform tests | Eight real Swift Testing cases pass: bounds, quiet-level readability, monotonicity, inactive/non-finite input and bar geometry | +| Integrated platform CI | Required on the final branch head before merge | + +[Small evidence record](native-motion-evidence.json) includes the tested Android APK hash, screenshot hashes, Swift source/test hashes and contrast calculation. Seven screenshots and XML hierarchies remain privately under `$AURALIS_CACHE/quality-2026-09-08/native-motion-root/`; normal and large-text interpretation/settings images were visually inspected. Those screenshots precede only the final removal of the stale static Settings version label. + +The original model-filled AVD lacked installation space. UI checks used a separate empty userdata image with the current debug APK; no model readiness was bypassed, and recording stayed disabled. Test settings were restored, the emulator stopped and disposable images removed. Original model/profile data was preserved. + +Local Android lint could not resolve missing cached metadata because Google Maven failed TLS; project repositories/dependencies were unchanged. Final GitHub CI supplies the actual lint result. Catalyst retained two pre-existing Bluetooth deprecation warnings. The host test recipe required explicit paths to the installed Swift Testing plugin/runtime; no test-library stubs were used. + +Screenshots establish layout and destination state, not animation frame timing. Active microphone-meter motion, iOS visual rendering and physical-device performance remain unqualified. The separate [Remotion showcase](showcase.md) is labelled simulation and does not replace these gates. diff --git a/docs/specs/2026-09-08-quality/reports/showcase-evidence.json b/docs/specs/2026-09-08-quality/reports/showcase-evidence.json new file mode 100644 index 0000000..7e51be6 --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/showcase-evidence.json @@ -0,0 +1,45 @@ +{ + "video": { + "programs": [], + "stream_groups": [], + "streams": [ + { + "codec_name": "h264", + "width": 1600, + "height": 900, + "r_frame_rate": "30/1", + "nb_frames": "540", + "side_data_list": [ + {} + ] + } + ], + "format": { + "duration": "18.000000", + "size": "577138" + }, + "sha256": "80afe5a1f34c5af9b95f1bdac5a6e97d0d1586765c8250fa921e8f368de591bf" + }, + "motionCheck": { + "True": { + "hashes": [ + "23b9d952f45255a19e44b14a628c4aa691d5c3f0866c83360d4b7213c0723f5a", + "23b9d952f45255a19e44b14a628c4aa691d5c3f0866c83360d4b7213c0723f5a" + ], + "identicalWhileListening": true + }, + "False": { + "hashes": [ + "dde407d2c3ff99df0459a7e76355db3d06a3e1d0babf53ba81afb24fdc400b82", + "cdd6a02d85af6af448080ccbcb5d59ebfceea4510dfc8efd66de1551156bcf9a" + ], + "identicalWhileListening": false + } + }, + "scope": "Simulated interface demonstration, not a model inference recording", + "remotionVersion": "4.0.522", + "sampleFrames": [ + 110, + 130 + ] +} diff --git a/docs/specs/2026-09-08-quality/reports/showcase.md b/docs/specs/2026-09-08-quality/reports/showcase.md new file mode 100644 index 0000000..648da42 --- /dev/null +++ b/docs/specs/2026-09-08-quality/reports/showcase.md @@ -0,0 +1,11 @@ +# Motion showcase verification + +The standalone [Remotion source](../../../../showcase/README.md) renders an 18-second, 1600 × 900, 30 fps interface demonstration. It uses the canonical Auralis mark and Pine/Mint/Paper palette. Conversation and meter activity are simulated and labelled; no speech recording, model weights or performance claims are included. + +`npm run check` passed. The final H.264 export contains 540 frames, no audio stream, and is 577,138 bytes. SHA-256: `80afe5a1f34c5af9b95f1bdac5a6e97d0d1586765c8250fa921e8f368de591bf`. + +A poster and six-frame contact sheet were visually reviewed for text clipping, overlap, hierarchy and brand consistency. With `reducedMotion=true`, frames 110 and 130 are byte-identical; the normal variant changes over the same interval. The [small evidence record](showcase-evidence.json) preserves these hashes and video metadata. + +Selection: one composition, existing SVG, system fonts and ordinary React styles. No component framework, web service, downloaded font or native-app JavaScript dependency. Each render uses one worker. Generated media and node_modules remain outside Git; the lockfile and source are checked in CI. [setup-node v4.4.0](https://github.com/actions/setup-node/blob/v4.4.0/README.md) is pinned to its verified official commit. + +Remotion retains its separate license, linked in the source README. The Auralis composition source is Apache-2.0. Native Compose/SwiftUI motion is validated separately. diff --git a/ios/DialectInterpreter/Audio/AudioChunkStream.swift b/ios/DialectInterpreter/Audio/AudioChunkStream.swift new file mode 100644 index 0000000..fc2cbb2 --- /dev/null +++ b/ios/DialectInterpreter/Audio/AudioChunkStream.swift @@ -0,0 +1,39 @@ +import Foundation + +/// One recording's bounded callback-to-async channel. The tap produces +/// 3,200-sample chunks, so eight queued chunks retain at most 100 KiB of PCM. +/// End on the first overflow: preserving the contiguous prefix lets the +/// pipeline report capture failure instead of transcribing across a gap. +struct AudioChunkStream: Sendable { + static let maxChunkSamples = 3200 + let stream: AsyncStream<[Float]> + private let continuation: AsyncStream<[Float]>.Continuation + + init(capacity: Int = 8) { + precondition(capacity > 0) + let pair = AsyncStream<[Float]>.makeStream(bufferingPolicy: .bufferingOldest(capacity)) + stream = pair.stream + continuation = pair.continuation + } + + func yield(_ chunk: [Float]) -> Bool { + guard !chunk.isEmpty, chunk.count <= Self.maxChunkSamples else { + continuation.finish() + return false + } + switch continuation.yield(chunk) { + case .enqueued: + return true + case .dropped: + continuation.finish() + return false + case .terminated: + return false + @unknown default: + continuation.finish() + return false + } + } + + func finish() { continuation.finish() } +} diff --git a/ios/DialectInterpreter/Audio/AudioRecorder.swift b/ios/DialectInterpreter/Audio/AudioRecorder.swift index 1648488..779d610 100644 --- a/ios/DialectInterpreter/Audio/AudioRecorder.swift +++ b/ios/DialectInterpreter/Audio/AudioRecorder.swift @@ -27,19 +27,18 @@ final class AudioRecorder { // Chunk streaming private var recordingID: UUID? - private var chunkStream: AsyncStream<[Float]>? - private var chunkContinuation: AsyncStream<[Float]>.Continuation? - - /// Async stream of Float PCM audio chunks. - var audioChunks: AsyncStream<[Float]> { - if let chunkStream { return chunkStream } - let stream = AsyncStream<[Float]> { continuation in - self.chunkContinuation = continuation - } - chunkStream = stream - return stream + private var chunkPipe: AudioChunkStream? + + private var pipe: AudioChunkStream { + if let chunkPipe { return chunkPipe } + let created = AudioChunkStream() + chunkPipe = created + return created } + /// Async stream of Float PCM audio chunks for this recording. + var audioChunks: AsyncStream<[Float]> { pipe.stream } + /// Check if recording permission is granted. var hasPermission: Bool { AVAudioApplication.shared.recordPermission == .granted @@ -133,10 +132,9 @@ final class AudioRecorder { throw NSError(domain: "AudioRecorder", code: -4, userInfo: [NSLocalizedDescriptionKey: "无法转换麦克风格式"]) } - _ = audioChunks - // Each tap retains only its own stream; a late callback cannot feed - // the next recording or race a mutable continuation on the main actor. - let continuation = chunkContinuation! + // Each tap retains its own bounded stream. Late callbacks cannot + // feed or stop a subsequent recording. + let capturePipe = pipe var accumulationBuffer: [Float] = [] accumulationBuffer.reserveCapacity(Self.chunkSizeSamples * 2) @@ -179,6 +177,14 @@ final class AudioRecorder { let chunk = Array(accumulationBuffer.prefix(Self.chunkSizeSamples)) accumulationBuffer.removeFirst(Self.chunkSizeSamples) + guard capturePipe.yield(chunk) else { + DispatchQueue.main.async { + guard let self, self.recordingID == id else { return } + self.stopRecording() + } + return + } + // Calculate RMS var energy: Double = 0 for sample in chunk { @@ -191,7 +197,6 @@ final class AudioRecorder { self.amplitude = rms } - continuation.yield(chunk) } } @@ -211,9 +216,8 @@ final class AudioRecorder { engine = nil isRecording = false amplitude = 0 - chunkContinuation?.finish() - chunkContinuation = nil - chunkStream = nil + chunkPipe?.finish() + chunkPipe = nil print("[AudioRecorder] Recording stopped") } diff --git a/ios/DialectInterpreter/Audio/UtteranceSegmenter.swift b/ios/DialectInterpreter/Audio/UtteranceSegmenter.swift new file mode 100644 index 0000000..efc68fa --- /dev/null +++ b/ios/DialectInterpreter/Audio/UtteranceSegmenter.swift @@ -0,0 +1,82 @@ +import Foundation + +/// Sample-bounded PCM assembly. At 16 kHz, the defaults retain 200 ms of +/// pre-roll and emit at most 10 s per utterance, including that pre-roll. +/// The callback transfers each completed value without collecting another +/// unbounded list when one input chunk crosses several utterance boundaries. +final class UtteranceSegmenter { + enum InputError: Error { case nonFiniteAudio } + + private let vad = VoiceActivityDetector() + private let sampleRate: Int + private let maxSamples: Int + private let prerollSamples: Int + private var preroll: [Float] = [] + private var speech: [Float] = [] + private var active = false + + init(sampleRate: Int = 16000, maxSamples: Int = 160000, prerollSamples: Int = 3200) { + precondition(sampleRate > 0 && maxSamples > 0) + precondition(prerollSamples >= 0 && prerollSamples < maxSamples) + self.sampleRate = sampleRate + self.maxSamples = maxSamples + self.prerollSamples = prerollSamples + } + + var bufferedSampleCount: Int { speech.count + preroll.count } + + func process(_ chunk: [Float], emit: ([Float]) -> Void) throws { + guard !chunk.isEmpty else { return } + guard chunk.allSatisfy(\.isFinite) else { throw InputError.nonFiniteAudio } + let result = vad.process(audioChunk: chunk, sampleRate: sampleRate) + if result.isSpeech { + if !active { + active = true + append(preroll, emit: emit) + preroll.removeAll(keepingCapacity: true) + } + append(chunk, emit: emit) + } else { + if active { + if result.utteranceComplete && !speech.isEmpty { emitSpeech(emit) } + else { speech.removeAll(keepingCapacity: false) } + active = false + } + retainPreroll(chunk) + } + } + + func reset() { + vad.reset() + preroll.removeAll(keepingCapacity: false) + speech.removeAll(keepingCapacity: false) + active = false + } + + private func append(_ samples: [Float], emit: ([Float]) -> Void) { + var offset = 0 + while offset < samples.count { + let count = min(maxSamples - speech.count, samples.count - offset) + speech.append(contentsOf: samples[offset..<(offset + count)]) + offset += count + if speech.count == maxSamples { emitSpeech(emit) } + } + } + + private func emitSpeech(_ emit: ([Float]) -> Void) { + let completed = speech + speech = [] + emit(completed) + } + + private func retainPreroll(_ samples: [Float]) { + guard prerollSamples > 0 else { return } + if samples.count >= prerollSamples { + preroll = Array(samples.suffix(prerollSamples)) + } else { + let excess = preroll.count + samples.count - prerollSamples + if excess > 0 { preroll.removeFirst(excess) } + preroll.append(contentsOf: samples) + } + } +} diff --git a/ios/DialectInterpreter/Audio/VoiceActivityDetector.swift b/ios/DialectInterpreter/Audio/VoiceActivityDetector.swift index 8a365e9..6eed223 100644 --- a/ios/DialectInterpreter/Audio/VoiceActivityDetector.swift +++ b/ios/DialectInterpreter/Audio/VoiceActivityDetector.swift @@ -20,11 +20,12 @@ final class VoiceActivityDetector { private var state: State = .silence private var speechStartTime: Int64 = 0 private var silenceStartTime: Int64 = 0 - private var audioClockMs: Int64 = 0 + private var audioClockSamples: Int64 = 0 private var energySmoothed: Float = 0 /// Process an audio chunk and return VAD result. - func process(audioChunk: [Float], chunkDurationMs: Int64 = 200) -> VadResult { + func process(audioChunk: [Float], sampleRate: Int = 16000) -> VadResult { + precondition(sampleRate > 0) guard !audioChunk.isEmpty else { return VadResult( isSpeech: state == .speech || state == .trailingSilence, @@ -44,7 +45,7 @@ final class VoiceActivityDetector { energySmoothed = 0.7 * energySmoothed + 0.3 * rms let isSpeech = energySmoothed > Self.energyThreshold - let now = audioClockMs + let now = audioClockSamples * 1000 / Int64(sampleRate) var utteranceComplete = false switch state { @@ -70,7 +71,7 @@ final class VoiceActivityDetector { } } - audioClockMs += max(chunkDurationMs, 1) + audioClockSamples += Int64(audioChunk.count) return VadResult( isSpeech: state == .speech || state == .trailingSilence, @@ -84,7 +85,7 @@ final class VoiceActivityDetector { state = .silence speechStartTime = 0 silenceStartTime = 0 - audioClockMs = 0 + audioClockSamples = 0 energySmoothed = 0 } } diff --git a/ios/DialectInterpreter/Inference/OrtRuntime.swift b/ios/DialectInterpreter/Inference/OrtRuntime.swift index 79884da..5c44905 100644 --- a/ios/DialectInterpreter/Inference/OrtRuntime.swift +++ b/ios/DialectInterpreter/Inference/OrtRuntime.swift @@ -52,10 +52,26 @@ struct OnnxTensor { let shape: [Int] let dtype: OnnxTensorDtype + /// Validate input shapes before allocating a tensor. + private static func elementCount(_ shape: [Int]) throws -> Int { + var count = 1 + for dim in shape { + guard dim >= 0 else { + throw OnnxRuntimeError.inputTensor("shape \(shape) has a negative dimension") + } + let (product, overflow) = count.multipliedReportingOverflow(by: dim) + guard !overflow else { + throw OnnxRuntimeError.inputTensor("shape \(shape) exceeds the addressable element count") + } + count = product + } + return count + } + init(floatData: [Float], shape: [Int]) throws { self.shape = shape self.dtype = .float - let expected = shape.reduce(1, *) + let expected = try Self.elementCount(shape) guard expected == floatData.count else { throw OnnxRuntimeError.inputTensor("element count \(floatData.count) does not match shape \(shape)") } @@ -74,7 +90,7 @@ struct OnnxTensor { init(int64Data: [Int64], shape: [Int]) throws { self.shape = shape self.dtype = .int64 - let expected = shape.reduce(1, *) + let expected = try Self.elementCount(shape) guard expected == int64Data.count else { throw OnnxRuntimeError.inputTensor("element count \(int64Data.count) does not match shape \(shape)") } @@ -108,37 +124,60 @@ struct OnnxTensor { } } - private func rawData() throws -> Data { + /// Keep the Foundation wrapper; bridging to Data may copy its payload. + private func rawMutableData() throws -> NSMutableData { do { - return try value.tensorData() as Data + return try value.tensorData() } catch { throw OnnxRuntimeError.outputTensor("tensorData failed: \(error)") } } func floatArray() throws -> [Float] { - guard dtype == .float else { - throw OnnxRuntimeError.outputTensor("dtype is \(dtype), not float; refusing to reinterpret") - } - let data = try rawData() - guard data.count % MemoryLayout.size == 0 else { - throw OnnxRuntimeError.outputTensor("byte count \(data.count) is not a multiple of Float") - } - return data.withUnsafeBytes { buffer in - Array(buffer.bindMemory(to: Float.self)) - } + try withFloatBuffer { Array($0) } } func int64Array() throws -> [Int64] { guard dtype == .int64 else { throw OnnxRuntimeError.outputTensor("dtype is \(dtype), not int64; refusing to reinterpret") } - let data = try rawData() - guard data.count % MemoryLayout.size == 0 else { - throw OnnxRuntimeError.outputTensor("byte count \(data.count) is not a multiple of Int64") + let data = try rawMutableData() + guard data.length % MemoryLayout.size == 0 else { + throw OnnxRuntimeError.outputTensor("byte count \(data.length) is not a multiple of Int64") } - return data.withUnsafeBytes { buffer in - Array(buffer.bindMemory(to: Int64.self)) + let count = data.length / MemoryLayout.size + return withExtendedLifetime(value) { _ in + withExtendedLifetime(data) { data in + let base = UnsafeRawPointer(data.bytes).assumingMemoryBound(to: Int64.self) + return Array(UnsafeBufferPointer(start: base, count: count)) + } + } + } + + /// Dtype gate with floatArray()'s exact refusal semantics, without + /// reading any tensor data. Used when a large output tensor is validated + /// for reuse as a subsequent input. + func requireFloat() throws { + guard dtype == .float else { + throw OnnxRuntimeError.outputTensor("dtype is \(dtype), not float; refusing to reinterpret") + } + } + + /// Borrow float data while retaining both its runtime and Foundation owners. + /// The pointer must not escape the closure. On the tested macOS runtime, + /// tensorData() creates one snapshot; floatArray() adds an Array copy. + func withFloatBuffer(_ body: (UnsafeBufferPointer) throws -> R) throws -> R { + try requireFloat() + let data = try rawMutableData() + guard data.length % MemoryLayout.size == 0 else { + throw OnnxRuntimeError.outputTensor("byte count \(data.length) is not a multiple of Float") + } + return try withExtendedLifetime(value) { _ in + try withExtendedLifetime(data) { data in + let base = UnsafeRawPointer(data.bytes).assumingMemoryBound(to: Float.self) + return try body(UnsafeBufferPointer(start: base, + count: data.length / MemoryLayout.size)) + } } } } @@ -201,3 +240,12 @@ final class OrtInferenceSession { } } } + +/// Test support: app-hosted test bundles resolve ORT classes from the host +/// binary, so the synthetic-graph tests obtain a real environment through this +/// factory instead of importing the bindings module themselves. +enum OrtInferenceTestSupport { + static func makeEnv() throws -> ORTEnv { + try ORTEnv(loggingLevel: ORTLoggingLevel.warning) + } +} diff --git a/ios/DialectInterpreter/Inference/PipelineOrchestrator.swift b/ios/DialectInterpreter/Inference/PipelineOrchestrator.swift index dc04372..4e91fc4 100644 --- a/ios/DialectInterpreter/Inference/PipelineOrchestrator.swift +++ b/ios/DialectInterpreter/Inference/PipelineOrchestrator.swift @@ -548,8 +548,7 @@ final class PipelineOrchestrator { // down (stop during boot, etc.). guard !Task.isCancelled else { return } - let vad = VoiceActivityDetector() - var utteranceChunks: [[Float]] = [] + let segmenter = UtteranceSegmenter() do { try audioCapture.start() @@ -572,33 +571,27 @@ final class PipelineOrchestrator { // reset VAD state so the post-playback tail is not glued to the // pre-playback utterance. if audioPlayback.isPlaying { - utteranceChunks.removeAll() - vad.reset() + segmenter.reset() continue } - let vadResult = vad.process(audioChunk: chunk) - - if vadResult.isSpeech { - utteranceChunks.append(chunk) - } - - guard vadResult.utteranceComplete, !utteranceChunks.isEmpty else { continue } - - let utterance = utteranceChunks.flatMap { $0 } - utteranceChunks.removeAll() - - // Turn id assigned at capture time, when the job enters the queue. - let job = TurnJob(turnId: Self.nextTurnId(), sessionId: sid, audio: utterance) - let accepted = queue.enqueue(job) - if accepted { - telemetry.utteranceCount += 1 - } else { - // Dropped is a first-class, visible turn status — no silent loss. - telemetry.droppedUtterances = Int64(queue.droppedCount) - emitEvent(.turnUpdated(TurnState( - id: job.turnId, sessionId: sid, status: .dropped, - sourceText: nil, translatedText: nil))) + do { + try segmenter.process(chunk) { utterance in + // Identity is assigned at capture time, including forced + // long-speech splits; the ended queue keeps its own bound. + let job = TurnJob(turnId: Self.nextTurnId(), sessionId: sid, audio: utterance) + if queue.enqueue(job) { + telemetry.utteranceCount += 1 + } else { + telemetry.droppedUtterances = Int64(queue.droppedCount) + emitEvent(.turnUpdated(TurnState( + id: job.turnId, sessionId: sid, status: .dropped, + sourceText: nil, translatedText: nil))) + } + } + } catch { + emitEvent(.error("Captured audio is invalid. Restart recording.")) + return } } } diff --git a/ios/DialectInterpreter/Inference/TtsEngine.swift b/ios/DialectInterpreter/Inference/TtsEngine.swift index 4d7390a..a4eed68 100644 --- a/ios/DialectInterpreter/Inference/TtsEngine.swift +++ b/ios/DialectInterpreter/Inference/TtsEngine.swift @@ -409,14 +409,16 @@ final class TtsEngine { } let codes = try await encodeReferenceCodes(pcm24k: pcm24k, config: cfg) let (_, warmed) = try await streamingVocoderStep( - groupMajor: codes, state: StreamingVocoderState.zero()) + groupMajor: codes, state: try StreamingVocoderState.zero()) try Task.checkCancellation() guard isLoaded, referenceGeneration == generation else { throw OrtInferenceFailure.notLoaded("TTS reference preparation was released") } + // The warm-up snapshot is value-owned: materialize the validated + // output tensors once (one copy) instead of aliasing graph memory. let warmup = PreparedReference.VocoderWarmup( - conv: Array(warmed.conv), keys: Array(warmed.keys), - values: Array(warmed.values), position: warmed.position) + conv: try warmed.conv.floatArray(), keys: try warmed.keys.floatArray(), + values: try warmed.values.floatArray(), position: warmed.position) return PreparedReference( generation: generation, embedding: embedding, pcm24kSamples: pcm24k.count, @@ -446,6 +448,7 @@ final class TtsEngine { private static let streamingVocoderChunkFrames = 4 private static let streamingConvState = 135_232 private static let streamingKvFloats = 8 * 1 * 16 * 71 * 64 + private static let streamingKvShape = [8, 1, 16, 71, 64] private func synthesizeApi2(text: String, language: String, speakerEmbedding: [Float]?, @@ -519,7 +522,7 @@ final class TtsEngine { "past_values": try OnnxTensor(floatData: [], shape: emptyPast), ]) let state = try Qwen3TtsProtocol.PrefillState(api2Outputs: prefillOutputs, config: cfg) - var vocoderState = StreamingVocoderState(warmup: preparedReference?.vocoderWarmup) + var vocoderState = try StreamingVocoderState(warmup: preparedReference?.vocoderWarmup) var pendingFrames: [[Int]] = [] var waveform: [Float] = [] func flushPending() async throws { @@ -618,18 +621,10 @@ final class TtsEngine { guard chunkFrames >= 1, !targetFrames.isEmpty else { throw OrtInferenceFailure.badInput("streaming vocoder requires a positive chunk and target frames") } - var state: StreamingVocoderState - if let warmup { - // Independent working copy; PreparedReference stays immutable. - state = StreamingVocoderState( - conv: Array(warmup.conv), keys: Array(warmup.keys), - values: Array(warmup.values), position: warmup.position) - } else { - state = StreamingVocoderState.zero() - if let reference = referenceCodesGroupMajor { - let (_, next) = try await streamingVocoderStep(groupMajor: reference, state: state) - state = next - } + var state = try StreamingVocoderState(warmup: warmup) + if warmup == nil, let reference = referenceCodesGroupMajor { + let (_, next) = try await streamingVocoderStep(groupMajor: reference, state: state) + state = next } var waveform: [Float] = [] var offset = 0 @@ -649,24 +644,46 @@ final class TtsEngine { return waveform } + /// Turn-local vocoder state. The conv/KV tensors are the vocoder graph's + /// OWN outputs, fed back as the next step's inputs without a managed + /// copy; only the immutable `PreparedReference` snapshot stays value + /// typed. Commit-on-success ordering: the previous tensors stay alive + /// while a run reads them and are released by ARC after the new state is + /// validated and swapped in — no input/output aliasing inside one run. private struct StreamingVocoderState { - var conv: [Float] - var keys: [Float] - var values: [Float] + var conv: OnnxTensor + var keys: OnnxTensor + var values: OnnxTensor var position: Int64 - init(conv: [Float], keys: [Float], values: [Float], position: Int64) { + init(conv: OnnxTensor, keys: OnnxTensor, values: OnnxTensor, position: Int64) { self.conv = conv; self.keys = keys; self.values = values; self.position = position } - init(warmup: PreparedReference.VocoderWarmup?) { + /// Independent working copy from the prepared snapshot; one + /// array->tensor copy per turn, never per step. PreparedReference + /// stays immutable. + init(warmup: PreparedReference.VocoderWarmup?) throws { if let warmup { - self.init(conv: warmup.conv, keys: warmup.keys, values: warmup.values, position: warmup.position) - } else { self = Self.zero() } - } - static func zero() -> StreamingVocoderState { - StreamingVocoderState( - conv: [Float](repeating: 0, count: TtsEngine.streamingConvState), - keys: [Float](repeating: 0, count: TtsEngine.streamingKvFloats), - values: [Float](repeating: 0, count: TtsEngine.streamingKvFloats), + self.init( + conv: try OnnxTensor( + floatData: warmup.conv, shape: [TtsEngine.streamingConvState]), + keys: try OnnxTensor( + floatData: warmup.keys, shape: TtsEngine.streamingKvShape), + values: try OnnxTensor( + floatData: warmup.values, shape: TtsEngine.streamingKvShape), + position: warmup.position) + } else { self = try Self.zero() } + } + static func zero() throws -> StreamingVocoderState { + try StreamingVocoderState( + conv: OnnxTensor( + floatData: [Float](repeating: 0, count: TtsEngine.streamingConvState), + shape: [TtsEngine.streamingConvState]), + keys: OnnxTensor( + floatData: [Float](repeating: 0, count: TtsEngine.streamingKvFloats), + shape: TtsEngine.streamingKvShape), + values: OnnxTensor( + floatData: [Float](repeating: 0, count: TtsEngine.streamingKvFloats), + shape: TtsEngine.streamingKvShape), position: 0) } } @@ -683,19 +700,13 @@ final class TtsEngine { for g in 0..