From 89ba06bd2d08d9a8ca150961fcf5fc65633d756f Mon Sep 17 00:00:00 2001 From: Toby Jackson Date: Wed, 18 Nov 2026 11:26:02 +0000 Subject: [PATCH 1/4] Move VAD into the akiki.audio namespace Refs #11 --- README.md | 37 ++-- docs/releasing.md | 6 +- docs/style.md | 8 +- pyproject.toml | 2 +- src/akiki/__init__.py | 27 ++- src/akiki/__main__.py | 2 +- src/akiki/audio/__init__.py | 5 + src/akiki/audio/vad.py | 357 ++++++++++++++++++++++++++++++ src/akiki/decision.py | 2 +- src/akiki/vad.py | 361 +------------------------------ tests/test_api.py | 14 +- tests/test_decision_inference.py | 3 +- tests/test_library_discovery.py | 2 +- tests/test_minilm.py | 2 +- tests/test_packaging.py | 1 + tests/test_training_api.py | 6 +- tests/test_vad.py | 2 +- verify_dist.py | 24 +- 18 files changed, 448 insertions(+), 413 deletions(-) create mode 100644 src/akiki/audio/__init__.py create mode 100644 src/akiki/audio/vad.py diff --git a/README.md b/README.md index 2104d76..cf2850f 100644 --- a/README.md +++ b/README.md @@ -115,12 +115,12 @@ PyTorch is the second VAD compatibility fallback. ## Python usage -Use `akiki.vad` for VAD and `akiki.decision` for decision models. +Use `akiki.audio.vad` for VAD and `akiki.decision` for decision models. The training and teacher modules support the decision-model workflow: | Module | Main interface | Supporting types | | --- | --- | --- | -| `akiki.vad` | `VAD` for speech detection | `SpeechFrame`, `Engine`, `ModelPrecision`, `FFTPrecision` | +| `akiki.audio.vad` | `VAD` for speech detection | `SpeechFrame`, `Engine`, `ModelPrecision`, `FFTPrecision` | | `akiki.decision` | `DecisionModel` for specialist decisions | `BaseModel`, `Specialist`, `AffineLayer`, `TrainedSpecialist`, `SpecialistEvaluation`, `TrainingAdvice`, `list_base_models`, `Classification`, `ModelPrecision` | | `akiki.training` | Classifier training with a private companion | The installed companion's API, including `LocalModel`, `ModelFamily`, and `Trainer` | | `akiki.teacher` | Experimental review through a local or self-hosted server | `Teacher`, `Verdict`, `DecisionReview`, `TeachingResult` | @@ -128,7 +128,7 @@ The training and teacher modules support the decision-model workflow: Import the primary interfaces from their modules: ```python -from akiki.vad import VAD +from akiki.audio import vad from akiki.decision import DecisionModel, Specialist ``` @@ -148,7 +148,8 @@ modules remain accessible but are omitted from that list. The decision-model module is named `akiki.decision`. Existing imports from `akiki.minilm` remain available for compatibility and use the same classes and precision enum. Named top-level compatibility imports remain available. -Wildcard imports expose only `VAD`, `MiniLM`, `vad`, `decision`, and `__version__`. +Existing imports from `akiki.vad` use the same classes and enums as `akiki.audio.vad`. +Wildcard imports expose only `VAD`, `MiniLM`, `audio`, `decision`, and `__version__`. ## Voice activity detection (VAD) @@ -156,21 +157,21 @@ Akiki accepts mono audio sampled at 16,000 Hz. ```python import soundfile as sf -from akiki import VAD +from akiki.audio import vad audio, sample_rate = sf.read("recording.wav", dtype="float32") -with VAD() as vad: - for frame in vad.push(audio, sample_rate=sample_rate): +with vad.VAD() as detector: + for frame in detector.push(audio, sample_rate=sample_rate): print(frame.start_seconds, frame.end_seconds, frame.probability, frame.speech) ``` -For live audio, call `vad.push()` repeatedly with successive audio buffers. +For live audio, call `detector.push()` repeatedly with successive audio buffers. Buffers can have any length; incomplete frames are retained for the next call. -Call `vad.reset()` before starting a new recording. A final incomplete frame is +Call `detector.reset()` before starting a new recording. A final incomplete frame is not padded or scored. -Use `VAD(threshold=0.6)` to set the speech decision threshold. Every frame still +Use `vad.VAD(threshold=0.6)` to set the speech decision threshold. Every frame still includes its probability, regardless of the threshold. The public API includes docstrings for editor hints and interactive help: @@ -178,15 +179,15 @@ The public API includes docstrings for editor hints and interactive help: ```python import akiki -help(akiki) # Overview of vad, decision, training, and teacher -help(akiki.vad) # Speech detection, configuration, and result types +help(akiki) # Overview of audio, decision, training, and teacher +help(akiki.audio.vad) # Speech detection, configuration, and result types help(akiki.decision) # Decision models, specialists, and result types help(akiki.decision.DecisionModel) # Base-model selection and specialist inference help(akiki.decision.Specialist.train) # Training parameters and saved exports -help(akiki.VAD) # Configuration, attributes, and streaming example -help(akiki.VAD.__init__) # Constructor parameters and defaults -help(akiki.VAD.push) # Audio input, buffering, results, and errors -help(akiki.vad.SpeechFrame) # Result fields and timestamp units +help(akiki.audio.vad.VAD) # Configuration, attributes, and streaming example +help(akiki.audio.vad.VAD.__init__) # Constructor parameters and defaults +help(akiki.audio.vad.VAD.push) # Audio input, buffering, results, and errors +help(akiki.audio.vad.SpeechFrame) # Result fields and timestamp units ``` ### Command line @@ -287,7 +288,7 @@ Choose another precision when creating the detector. For example, to use the original FP32 weights: ```python -from akiki.vad import VAD, Engine, ModelPrecision +from akiki.audio.vad import VAD, Engine, ModelPrecision with VAD(engine=Engine.AKIKI, model_precision=ModelPrecision.FP32) as vad: frames = vad.push(audio, sample_rate=sample_rate) @@ -309,7 +310,7 @@ akiki vad recording.wav --model-precision fp32 | `ModelPrecision.INT8_FP32_INPUT` | INT8 weights; first projection keeps FP32 activations | The Python VAD API requires `ModelPrecision` and `FFTPrecision` enum members, -both exported from `akiki.vad`. Omit `model_precision` or pass `None` to use the +both exported from `akiki.audio.vad`. Omit `model_precision` or pass `None` to use the selected engine's default. CLI precision options retain their lowercase names. The FFT stays FP32: `fft_precision` accepts only `FFTPrecision.FP32`. Recurrent state, diff --git a/docs/releasing.md b/docs/releasing.md index d68deaa..caa0159 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -4,7 +4,7 @@ The public PyPI description comes from `README.md`. Keep it focused on package features and usage. Build and publishing instructions belong here. Akiki provides VAD and decision models. -The primary classes are `akiki.vad.VAD` and `akiki.decision.DecisionModel`. +The primary classes are `akiki.audio.vad.VAD` and `akiki.decision.DecisionModel`. Current native decision inference supports MiniLM/BERT exports. The library filenames below retain the name of that implementation. @@ -195,9 +195,9 @@ target platforms before adding those wheels to `dist/`. Validate the Markdown and metadata with `twine check` for every new artifact. Test a clean install with both portable and platform wheels available and confirm that `VAD().engine is Engine.AKIKI`, loading the installed library -(`Engine` is exported from `akiki`). +(`Engine` is exported from `akiki.audio.vad`). Confirm that `VAD().model_precision is ModelPrecision.INT8` and -`VAD().fft_precision is FFTPrecision.FP32` (both enums are exported from `akiki.vad`), +`VAD().fft_precision is FFTPrecision.FP32` (both enums are exported from `akiki.audio.vad`), and that `akiki.decision._load_library()` loads the installed MiniLM library. When no precision is specified, ONNX and PyTorch compatibility engines use FP32; an explicit non-FP32 request must require the Akiki engine. diff --git a/docs/style.md b/docs/style.md index 7c8856e..7098c72 100644 --- a/docs/style.md +++ b/docs/style.md @@ -19,7 +19,7 @@ ONNX and PyTorch are VAD compatibility fallbacks. Do not use the implementation language as the engine name. Describe the two primary interfaces as VAD and decision models. -Use `akiki.vad.VAD` and `akiki.decision.DecisionModel` in API overviews. +Use `akiki.audio.vad.VAD` and `akiki.decision.DecisionModel` in API overviews. Describe MiniLM/BERT as the family supported by the current decision-model runtime. Use `MiniLM` for direct access to that family. Keep model discovery, training support, and inference support separate. @@ -36,7 +36,7 @@ Use separate enum classes when parameters accept different sets of choices. For example, VAD model precision and FFT precision have different domains: ```python -from akiki.vad import VAD, FFTPrecision, ModelPrecision +from akiki.audio.vad import VAD, FFTPrecision, ModelPrecision with VAD( model_precision=ModelPrecision.INT8, @@ -46,7 +46,7 @@ with VAD( ``` VAD and decision models also have distinct model precision domains. Use -`akiki.vad.ModelPrecision` with `VAD` and `akiki.decision.ModelPrecision` with +`akiki.audio.vad.ModelPrecision` with `VAD` and `akiki.decision.ModelPrecision` with `DecisionModel` or `MiniLM`. Do not share enum types just because some serialized values match. @@ -117,7 +117,7 @@ Check the executable examples in the public inference docstrings: ```bash HF_HUB_OFFLINE=1 python -m pytest -q --doctest-modules \ - src/akiki/__init__.py src/akiki/vad.py src/akiki/decision.py src/akiki/hub.py + src/akiki/__init__.py src/akiki/audio/vad.py src/akiki/decision.py src/akiki/hub.py ``` VAD examples require a working local engine. Examples with placeholder text-model diff --git a/pyproject.toml b/pyproject.toml index 38932f2..629b27a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ include-package-data = false [tool.setuptools.packages.find] where = ["src"] -include = ["akiki"] +include = ["akiki", "akiki.audio"] namespaces = false [tool.setuptools.package-data] diff --git a/src/akiki/__init__.py b/src/akiki/__init__.py index 8ca3a65..2848e40 100644 --- a/src/akiki/__init__.py +++ b/src/akiki/__init__.py @@ -1,10 +1,10 @@ """Local voice activity detection and decision models. -The primary interfaces are ``akiki.vad.VAD`` and ``akiki.decision.DecisionModel``. +The primary interfaces are ``akiki.audio.vad.VAD`` and ``akiki.decision.DecisionModel``. Training and teacher review support the decision-model workflow. The public API is organized into four areas: -``akiki.vad`` +``akiki.audio.vad`` Streaming voice activity detection with ``VAD``. Results are ``SpeechFrame`` objects. Configure inference with this module's ``Engine``, ``ModelPrecision``, and ``FFTPrecision`` enums. The VAD model is bundled. @@ -48,18 +48,20 @@ Find domain-specific configuration and help through the modules:: - >>> from akiki import decision, vad + >>> from akiki import decision + >>> from akiki.audio import vad >>> vad.ModelPrecision.INT8 is decision.ModelPrecision.INT8 False -Use ``help(akiki.vad)`` and ``help(akiki.decision)`` to explore each API, or -``help(akiki.vad.VAD)`` and ``help(akiki.decision.DecisionModel)`` for the +Use ``help(akiki.audio.vad)`` and ``help(akiki.decision)`` to explore each API, or +``help(akiki.audio.vad.VAD)`` and ``help(akiki.decision.DecisionModel)`` for the primary classes. Use ``help(akiki.decision.MiniLM)`` for direct access to the currently supported decision-model family. Explicit model retrieval utilities live in ``akiki.hub``. Compatibility ------------- +``akiki.vad`` re-exports the VAD classes and enums from ``akiki.audio.vad``. ``akiki.minilm`` remains a compatibility module. It re-exports the decision API's MiniLM class, result types, and current precision enum. Existing named top-level imports of ``SpeechFrame``, ``Engine``, ``ModelPrecision``, @@ -74,25 +76,26 @@ Installed Akiki package version. """ -from . import decision, vad +from . import audio, decision from . import minilm as minilm +from . import vad as vad +from .audio.vad import VAD +from .audio.vad import Engine as Engine +from .audio.vad import FFTPrecision as FFTPrecision +from .audio.vad import ModelPrecision as ModelPrecision +from .audio.vad import SpeechFrame as SpeechFrame from .decision import Classification as Classification from .decision import Encoding as Encoding from .decision import MiniLM from .hub import download_model as download_model -from .vad import VAD -from .vad import Engine as Engine -from .vad import FFTPrecision as FFTPrecision -from .vad import ModelPrecision as ModelPrecision -from .vad import SpeechFrame as SpeechFrame __version__ = "0.2.2" __all__ = [ "MiniLM", "VAD", "__version__", + "audio", "decision", - "vad", ] diff --git a/src/akiki/__main__.py b/src/akiki/__main__.py index 1b0f388..191fefd 100644 --- a/src/akiki/__main__.py +++ b/src/akiki/__main__.py @@ -9,8 +9,8 @@ from dataclasses import asdict from . import __version__ +from .audio.vad import Engine, FFTPrecision, ModelPrecision from .decision import ModelPrecision as MiniLMPrecision -from .vad import Engine, FFTPrecision, ModelPrecision def main(): diff --git a/src/akiki/audio/__init__.py b/src/akiki/audio/__init__.py new file mode 100644 index 0000000..092c1f1 --- /dev/null +++ b/src/akiki/audio/__init__.py @@ -0,0 +1,5 @@ +"""Audio inference interfaces.""" + +from . import vad + +__all__ = ["vad"] diff --git a/src/akiki/audio/vad.py b/src/akiki/audio/vad.py new file mode 100644 index 0000000..1eac99f --- /dev/null +++ b/src/akiki/audio/vad.py @@ -0,0 +1,357 @@ +"""Streaming speech detection and its configuration and result types. + +Use ``VAD`` with successive mono, 16 kHz audio buffers. Each complete frame +returns a ``SpeechFrame`` with timestamps, a probability, and a speech decision. +The bundled model processes 256 samples (16 ms) per frame. + +The Akiki engine is the primary engine. ``Engine.AUTO`` tries it first. +The ONNX and PyTorch engines provide compatibility fallbacks. +``ModelPrecision`` selects the VAD model's numeric format; ``FFTPrecision`` configures its FFT. +These enums belong to VAD; MiniLM uses ``akiki.decision.ModelPrecision``. + +Examples +-------- +>>> from akiki.audio import vad +>>> with vad.VAD(model_precision=vad.ModelPrecision.FP32) as detector: +... frames = detector.push([0.0] * 256) +>>> len(frames) +1 +""" + +import json +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from .._engines import Engine +from .._precision import FFTPrecision, ModelPrecision + +__all__ = ["Engine", "FFTPrecision", "ModelPrecision", "SpeechFrame", "VAD"] + +SAMPLE_RATE = 16000 + + +@dataclass(frozen=True) +class SpeechFrame: + """Immutable speech decision for one complete frame returned by ``VAD.push``. + + Timestamps describe the input audio, measured from detector creation or the + last ``VAD.reset()`` call. They do not measure processing or wall-clock time. + The bundled model produces one frame for every 256 samples (16 ms). + + Attributes + ---------- + start_seconds : float + Inclusive start time of the frame, in seconds. + end_seconds : float + Exclusive end time of the frame, in seconds. + probability : float + Model's speech probability, between 0 and 1 inclusive. + speech : bool + Whether ``probability >= detector.threshold`` when the frame was + produced. Decisions are made per frame, without merging speech segments. + + Examples + -------- + >>> frame = SpeechFrame(0.0, 0.016, 0.9, True) + >>> frame.end_seconds - frame.start_seconds + 0.016 + >>> frame.probability, frame.speech + (0.9, True) + """ + + start_seconds: float + end_seconds: float + probability: float + speech: bool + + +class VAD: + """Identify speech in a stream of mono audio sampled at 16,000 Hz. + + Each instance maintains the recurrent state, audio context, and incomplete + frame for one stream. Feed successive buffers to ``push()``; call + ``reset()`` before an independent recording. The bundled model emits a + ``SpeechFrame`` for every 256 samples (16 ms). Custom models use their + supported ``chunk_samples`` value: 256 or 512 samples. + + Models load locally. The Akiki engine is the primary engine. + ``engine=Engine.AUTO`` tries it first, then the ONNX and PyTorch compatibility + fallbacks. The Akiki engine uses INT8 by default. Compatibility engines use FP32. + Explicit non-FP32 precision requires the Akiki engine. Install compatibility + runtimes with ``akiki[onnx]`` or ``akiki[pytorch]`` when needed. + See ``VAD.__init__`` for all constructor options. + + Use the detector as a context manager or call ``close()`` when finished. + Closing does not pad or score a final incomplete frame. + + Attributes + ---------- + engine : Engine + Selected inference engine: ``Engine.AKIKI``, ``Engine.ONNX``, or + ``Engine.PYTORCH``. + engine_errors : dict[Engine, str] + Reasons engines were unavailable before automatic selection succeeded. + model_id : str + Identifier from the loaded model's metadata. + model_precision : ModelPrecision + Active model precision, after resolving the engine's default. + fft_precision : FFTPrecision + FFT precision; always ``FFTPrecision.FP32``. + threshold : float + Minimum probability classified as speech. + chunk_samples : int + Number of input samples needed for one output frame. + context_samples : int + Number of prior audio samples retained as model context. + hidden : int + Number of recurrent hidden units in the loaded model. + completed_samples : int + Number of samples scored since creation or the last reset. + pending : numpy.ndarray + Unscored float32 samples buffered for the next call to ``push``. + context : numpy.ndarray + Float32 audio context with shape ``(context_samples,)``. + state : numpy.ndarray + Float32 recurrent state with shape ``(1, 1, hidden)``. + session : onnxruntime.InferenceSession + Present only when the selected engine is ONNX; ``None`` after closing. + + Notes + ----- + Treat ``pending``, ``context``, and ``state`` as read-only stream state. + Use a separate instance for each independent audio stream. + Calls on one detector are not protected by a lock. Do not call ``push``, + ``reset``, or ``close`` concurrently on the same instance. + + Examples + -------- + Successive buffers share a timeline and incomplete frames are retained:: + + >>> from akiki.audio.vad import VAD + >>> with VAD() as detector: + ... first = detector.push([0.0] * 100) + ... second = detector.push([0.0] * 156) + >>> first + [] + >>> len(second), second[0].end_seconds + (1, 0.016) + """ + + def __init__( + self, + model_path: str | Path | None = None, + threshold: float | None = None, + *, + engine: Engine = Engine.AUTO, + library_path: str | Path | None = None, + model_precision: ModelPrecision | None = None, + fft_precision: FFTPrecision = FFTPrecision.FP32, + cache_dir: str | Path | None = None, + ): + """Load a local model and inference engine, ready for a new stream. + + Parameters + ---------- + model_path : str or pathlib.Path, optional + If omitted, use the bundled model. + For a custom Akiki model, pass its JSON metadata path, such as + ``"models/vad.json"``. The Akiki engine loads the sibling + ``"models/vad-native/"`` directory. It must contain ``config.json``, + ``model.safetensors``, and ``export.json``. + The ONNX and PyTorch compatibility fallbacks use matching ``.onnx`` + and ``.pt`` files beside the metadata file. + Custom models must use the full-waveform GRU format. Supported + ``(chunk_samples, context_samples)`` pairs are ``(256, 320)`` and + ``(512, 64)``. Custom PyTorch metadata also requires ``pytorch_sha256``. + threshold : float, optional + Speech decision threshold between 0 and 1 inclusive. If omitted, + use the threshold in the model's metadata. This changes the boolean + speech decision, not the returned probability. + engine : Engine, default=Engine.AUTO + Automatic selection tries the primary Akiki engine first. + ONNX and PyTorch are compatibility fallbacks, in that order. + An explicitly selected engine must succeed; it does not fall back. + Pass an ``Engine`` member, such as ``Engine.AKIKI``. + library_path : str or pathlib.Path, optional + Specific local Akiki shared library to load. Affects only the Akiki + engine. If omitted, use ``AKIKI_VAD_LIBRARY`` or the bundled library. + Automatic discovery does not search developer directories. + model_precision : ModelPrecision, optional + One of ``ModelPrecision.FP32``, ``FP16``, ``BF16``, ``INT8``, + ``INT4``, or ``INT8_FP32_INPUT``. If omitted, Akiki uses INT8 and + ONNX/PyTorch use FP32. Explicit non-FP32 precision requires Akiki, + including in automatic mode. Reduced-precision Akiki weights are + generated from the verified local FP32 model and reused from the + cache. ``ModelPrecision.INT8_FP32_INPUT`` uses INT8 weights while + keeping the first projection's input in FP32. + fft_precision : FFTPrecision, default=FFTPrecision.FP32 + FFT precision. Only ``FFTPrecision.FP32`` is supported. + cache_dir : str or pathlib.Path, optional + Directory for generated Akiki model variants. If omitted, use + ``AKIKI_CACHE_DIR`` or the platform's user cache directory. Original + model weights are preserved. FP32 inference needs no conversion. + + Raises + ------ + TypeError + ``engine``, ``model_precision``, or ``fft_precision`` has the wrong + enum type. ``model_precision`` also accepts ``None``. + ValueError + A parameter, model specification, or model checksum is invalid. + OSError + The local model metadata cannot be read. + RuntimeError + No compatible engine is available for the requested configuration, + or the directory for a generated model variant is not writable. + """ + from .._engines import load_engine + + if not isinstance(engine, Engine): + raise TypeError("engine must be an Engine member, such as Engine.AUTO or Engine.AKIKI") + if model_precision is not None and not isinstance(model_precision, ModelPrecision): + raise TypeError("model_precision must be a ModelPrecision member or None") + if not isinstance(fft_precision, FFTPrecision): + raise TypeError("fft_precision must be an FFTPrecision member") + if model_precision not in (None, ModelPrecision.FP32) and engine not in ( + Engine.AUTO, + Engine.AKIKI, + ): + raise ValueError("Non-FP32 model_precision requires Engine.AKIKI or Engine.AUTO") + self.fft_precision = fft_precision + path = ( + Path(model_path) + if model_path is not None + else Path(__file__).parents[1] / "models/akiki-vad.onnx" + ) + metadata = json.loads(path.with_suffix(".json").read_text()) + self.model_id = metadata.get("model_id", path.stem) + spec = metadata["spec"] + self.chunk_samples = int(spec["chunk_samples"]) + self.context_samples = int(spec["context_samples"]) + self.hidden = int(spec["hidden"]) + if spec["kind"] != "gru" or (self.chunk_samples, self.context_samples) not in { + (256, 320), + (512, 64), + }: + raise ValueError("Expected an Akiki full-waveform GRU model") + if spec.get("sample_rate", SAMPLE_RATE) != SAMPLE_RATE or self.hidden <= 0: + raise ValueError("Expected a 16 kHz model with a positive hidden size") + self.threshold = float(metadata["threshold"] if threshold is None else threshold) + if not np.isfinite(self.threshold) or not 0 <= self.threshold <= 1: + raise ValueError("Threshold must lie between zero and one") + self._backend, self.engine, self.engine_errors = load_engine( + path, metadata, engine, library_path, model_precision, cache_dir + ) + self.model_precision = model_precision or ( + ModelPrecision.INT8 if self.engine is Engine.AKIKI else ModelPrecision.FP32 + ) + if self.engine is Engine.ONNX: + self.session = self._backend.session + self.reset() + + def reset(self) -> None: + """Start an independent recording with the same model and settings. + + Clear the recurrent state and audio context, discard pending samples, + and reset output timestamps to zero. The model remains loaded. + + Raises + ------ + RuntimeError + The detector has been closed. Create a new instance to process more + audio after closing. + """ + self._backend.reset() + self.context = np.zeros(self.context_samples, dtype=np.float32) + self.state = np.zeros((1, 1, self.hidden), dtype=np.float32) + self.pending = np.empty(0, dtype=np.float32) + self.completed_samples = 0 + + def push(self, samples, sample_rate: int = SAMPLE_RATE) -> list[SpeechFrame]: + """Process the next audio buffer and return decisions for complete frames. + + Parameters + ---------- + samples : array_like + One-dimensional, finite mono waveform samples. Lists and NumPy + arrays are accepted and converted to float32. Samples should + normally be scaled to [-1, 1]; integer PCM is not normalized for + you. Audio is not resampled or mixed down from multiple channels. + sample_rate : int, default=16000 + Input sample rate in Hz. Must be 16000. + + Returns + ------- + list[SpeechFrame] + One result for each complete ``chunk_samples`` input samples, + including any samples buffered by a previous call. Timestamps + continue from earlier calls. Returns an empty list when there is + not enough audio for a complete frame. + + Raises + ------ + TypeError + The input type cannot be converted to numeric audio samples. + ValueError + Audio cannot be converted to float32, is not one-dimensional, + contains nonfinite values, or has the wrong sample rate. These + validation failures leave the stream state unchanged. + RuntimeError + The detector is closed, inference fails, or the model returns an + invalid speech probability. + + Notes + ----- + Buffers may have any length, including zero. A final incomplete frame + stays in ``pending`` until more audio arrives or ``reset()`` discards + it. Closing the detector does not pad or score those samples. + """ + self._backend.check_open() + audio = np.asarray(samples, dtype=np.float32) + if sample_rate != SAMPLE_RATE or audio.ndim != 1 or not np.isfinite(audio).all(): + raise ValueError("Expected finite mono 16 kHz audio samples") + audio = np.concatenate([self.pending, audio]) + complete = len(audio) // self.chunk_samples * self.chunk_samples + frames = [] + for start in range(0, complete, self.chunk_samples): + window = np.concatenate([self.context, audio[start : start + self.chunk_samples]]) + probability, state = self._backend.infer(window, self.state) + probability = float(np.asarray(probability).reshape(-1)[0]) + if not np.isfinite(probability) or not 0 <= probability <= 1: + raise RuntimeError("Model returned an invalid speech probability") + begin = self.completed_samples + self.state = state + self.context = window[-self.context_samples :].copy() + self.completed_samples += self.chunk_samples + frames.append( + SpeechFrame( + begin / SAMPLE_RATE, + self.completed_samples / SAMPLE_RATE, + probability, + probability >= self.threshold, + ) + ) + self.pending = audio[complete:].copy() + return frames + + def close(self) -> None: + """Release this stream's inference resources; safe to call repeatedly. + + Pending samples are not scored. After closing, ``push()``, ``reset()``, + and entering a ``with`` block raise ``RuntimeError``. Previously + returned ``SpeechFrame`` objects remain usable. + """ + self._backend.close() + if self.engine is Engine.ONNX: + self.session = None + + def __enter__(self): + """Return this open detector for use in a ``with`` block.""" + self._backend.check_open() + return self + + def __exit__(self, *_): + """Close the detector on leaving a ``with`` block; propagate exceptions.""" + self.close() diff --git a/src/akiki/decision.py b/src/akiki/decision.py index 966bc89..2134007 100644 --- a/src/akiki/decision.py +++ b/src/akiki/decision.py @@ -765,7 +765,7 @@ def __init__( ``INT8``, ``INT8_FP32_INPUT``, ``INT4``, or ``INT4_G32``. Weights are converted in memory when loading; the files are unchanged. Attention, normalization, and pooling remain FP32. Strings and - members of ``akiki.vad.ModelPrecision`` are rejected. + members of ``akiki.audio.vad.ModelPrecision`` are rejected. chunk_tokens : int, optional Split text into chunks of at most this many tokens, including two special tokens, as Ariadne retrieval models do. Defaults to the diff --git a/src/akiki/vad.py b/src/akiki/vad.py index 9bbc145..7cd6d74 100644 --- a/src/akiki/vad.py +++ b/src/akiki/vad.py @@ -1,357 +1,10 @@ -"""Streaming speech detection and its configuration and result types. +"""Compatibility imports for voice activity detection in ``akiki.audio.vad``.""" -Use ``VAD`` with successive mono, 16 kHz audio buffers. Each complete frame -returns a ``SpeechFrame`` with timestamps, a probability, and a speech decision. -The bundled model processes 256 samples (16 ms) per frame. - -The Akiki engine is the primary engine. ``Engine.AUTO`` tries it first. -The ONNX and PyTorch engines provide compatibility fallbacks. -``ModelPrecision`` selects the VAD model's numeric format; ``FFTPrecision`` configures its FFT. -These enums belong to VAD; MiniLM uses ``akiki.decision.ModelPrecision``. - -Examples --------- ->>> from akiki import vad ->>> with vad.VAD(model_precision=vad.ModelPrecision.FP32) as detector: -... frames = detector.push([0.0] * 256) ->>> len(frames) -1 -""" - -import json -from dataclasses import dataclass -from pathlib import Path - -import numpy as np - -from ._engines import Engine -from ._precision import FFTPrecision, ModelPrecision +from .audio.vad import SAMPLE_RATE as SAMPLE_RATE +from .audio.vad import VAD as VAD +from .audio.vad import Engine as Engine +from .audio.vad import FFTPrecision as FFTPrecision +from .audio.vad import ModelPrecision as ModelPrecision +from .audio.vad import SpeechFrame as SpeechFrame __all__ = ["Engine", "FFTPrecision", "ModelPrecision", "SpeechFrame", "VAD"] - -SAMPLE_RATE = 16000 - - -@dataclass(frozen=True) -class SpeechFrame: - """Immutable speech decision for one complete frame returned by ``VAD.push``. - - Timestamps describe the input audio, measured from detector creation or the - last ``VAD.reset()`` call. They do not measure processing or wall-clock time. - The bundled model produces one frame for every 256 samples (16 ms). - - Attributes - ---------- - start_seconds : float - Inclusive start time of the frame, in seconds. - end_seconds : float - Exclusive end time of the frame, in seconds. - probability : float - Model's speech probability, between 0 and 1 inclusive. - speech : bool - Whether ``probability >= detector.threshold`` when the frame was - produced. Decisions are made per frame, without merging speech segments. - - Examples - -------- - >>> frame = SpeechFrame(0.0, 0.016, 0.9, True) - >>> frame.end_seconds - frame.start_seconds - 0.016 - >>> frame.probability, frame.speech - (0.9, True) - """ - - start_seconds: float - end_seconds: float - probability: float - speech: bool - - -class VAD: - """Identify speech in a stream of mono audio sampled at 16,000 Hz. - - Each instance maintains the recurrent state, audio context, and incomplete - frame for one stream. Feed successive buffers to ``push()``; call - ``reset()`` before an independent recording. The bundled model emits a - ``SpeechFrame`` for every 256 samples (16 ms). Custom models use their - supported ``chunk_samples`` value: 256 or 512 samples. - - Models load locally. The Akiki engine is the primary engine. - ``engine=Engine.AUTO`` tries it first, then the ONNX and PyTorch compatibility - fallbacks. The Akiki engine uses INT8 by default. Compatibility engines use FP32. - Explicit non-FP32 precision requires the Akiki engine. Install compatibility - runtimes with ``akiki[onnx]`` or ``akiki[pytorch]`` when needed. - See ``VAD.__init__`` for all constructor options. - - Use the detector as a context manager or call ``close()`` when finished. - Closing does not pad or score a final incomplete frame. - - Attributes - ---------- - engine : Engine - Selected inference engine: ``Engine.AKIKI``, ``Engine.ONNX``, or - ``Engine.PYTORCH``. - engine_errors : dict[Engine, str] - Reasons engines were unavailable before automatic selection succeeded. - model_id : str - Identifier from the loaded model's metadata. - model_precision : ModelPrecision - Active model precision, after resolving the engine's default. - fft_precision : FFTPrecision - FFT precision; always ``FFTPrecision.FP32``. - threshold : float - Minimum probability classified as speech. - chunk_samples : int - Number of input samples needed for one output frame. - context_samples : int - Number of prior audio samples retained as model context. - hidden : int - Number of recurrent hidden units in the loaded model. - completed_samples : int - Number of samples scored since creation or the last reset. - pending : numpy.ndarray - Unscored float32 samples buffered for the next call to ``push``. - context : numpy.ndarray - Float32 audio context with shape ``(context_samples,)``. - state : numpy.ndarray - Float32 recurrent state with shape ``(1, 1, hidden)``. - session : onnxruntime.InferenceSession - Present only when the selected engine is ONNX; ``None`` after closing. - - Notes - ----- - Treat ``pending``, ``context``, and ``state`` as read-only stream state. - Use a separate instance for each independent audio stream. - Calls on one detector are not protected by a lock. Do not call ``push``, - ``reset``, or ``close`` concurrently on the same instance. - - Examples - -------- - Successive buffers share a timeline and incomplete frames are retained:: - - >>> from akiki import VAD - >>> with VAD() as detector: - ... first = detector.push([0.0] * 100) - ... second = detector.push([0.0] * 156) - >>> first - [] - >>> len(second), second[0].end_seconds - (1, 0.016) - """ - - def __init__( - self, - model_path: str | Path | None = None, - threshold: float | None = None, - *, - engine: Engine = Engine.AUTO, - library_path: str | Path | None = None, - model_precision: ModelPrecision | None = None, - fft_precision: FFTPrecision = FFTPrecision.FP32, - cache_dir: str | Path | None = None, - ): - """Load a local model and inference engine, ready for a new stream. - - Parameters - ---------- - model_path : str or pathlib.Path, optional - If omitted, use the bundled model. - For a custom Akiki model, pass its JSON metadata path, such as - ``"models/vad.json"``. The Akiki engine loads the sibling - ``"models/vad-native/"`` directory. It must contain ``config.json``, - ``model.safetensors``, and ``export.json``. - The ONNX and PyTorch compatibility fallbacks use matching ``.onnx`` - and ``.pt`` files beside the metadata file. - Custom models must use the full-waveform GRU format. Supported - ``(chunk_samples, context_samples)`` pairs are ``(256, 320)`` and - ``(512, 64)``. Custom PyTorch metadata also requires ``pytorch_sha256``. - threshold : float, optional - Speech decision threshold between 0 and 1 inclusive. If omitted, - use the threshold in the model's metadata. This changes the boolean - speech decision, not the returned probability. - engine : Engine, default=Engine.AUTO - Automatic selection tries the primary Akiki engine first. - ONNX and PyTorch are compatibility fallbacks, in that order. - An explicitly selected engine must succeed; it does not fall back. - Pass an ``Engine`` member, such as ``Engine.AKIKI``. - library_path : str or pathlib.Path, optional - Specific local Akiki shared library to load. Affects only the Akiki - engine. If omitted, use ``AKIKI_VAD_LIBRARY`` or the bundled library. - Automatic discovery does not search developer directories. - model_precision : ModelPrecision, optional - One of ``ModelPrecision.FP32``, ``FP16``, ``BF16``, ``INT8``, - ``INT4``, or ``INT8_FP32_INPUT``. If omitted, Akiki uses INT8 and - ONNX/PyTorch use FP32. Explicit non-FP32 precision requires Akiki, - including in automatic mode. Reduced-precision Akiki weights are - generated from the verified local FP32 model and reused from the - cache. ``ModelPrecision.INT8_FP32_INPUT`` uses INT8 weights while - keeping the first projection's input in FP32. - fft_precision : FFTPrecision, default=FFTPrecision.FP32 - FFT precision. Only ``FFTPrecision.FP32`` is supported. - cache_dir : str or pathlib.Path, optional - Directory for generated Akiki model variants. If omitted, use - ``AKIKI_CACHE_DIR`` or the platform's user cache directory. Original - model weights are preserved. FP32 inference needs no conversion. - - Raises - ------ - TypeError - ``engine``, ``model_precision``, or ``fft_precision`` has the wrong - enum type. ``model_precision`` also accepts ``None``. - ValueError - A parameter, model specification, or model checksum is invalid. - OSError - The local model metadata cannot be read. - RuntimeError - No compatible engine is available for the requested configuration, - or the directory for a generated model variant is not writable. - """ - from ._engines import load_engine - - if not isinstance(engine, Engine): - raise TypeError("engine must be an Engine member, such as Engine.AUTO or Engine.AKIKI") - if model_precision is not None and not isinstance(model_precision, ModelPrecision): - raise TypeError("model_precision must be a ModelPrecision member or None") - if not isinstance(fft_precision, FFTPrecision): - raise TypeError("fft_precision must be an FFTPrecision member") - if model_precision not in (None, ModelPrecision.FP32) and engine not in ( - Engine.AUTO, - Engine.AKIKI, - ): - raise ValueError("Non-FP32 model_precision requires Engine.AKIKI or Engine.AUTO") - self.fft_precision = fft_precision - path = ( - Path(model_path) - if model_path is not None - else Path(__file__).parent / "models/akiki-vad.onnx" - ) - metadata = json.loads(path.with_suffix(".json").read_text()) - self.model_id = metadata.get("model_id", path.stem) - spec = metadata["spec"] - self.chunk_samples = int(spec["chunk_samples"]) - self.context_samples = int(spec["context_samples"]) - self.hidden = int(spec["hidden"]) - if spec["kind"] != "gru" or (self.chunk_samples, self.context_samples) not in { - (256, 320), - (512, 64), - }: - raise ValueError("Expected an Akiki full-waveform GRU model") - if spec.get("sample_rate", SAMPLE_RATE) != SAMPLE_RATE or self.hidden <= 0: - raise ValueError("Expected a 16 kHz model with a positive hidden size") - self.threshold = float(metadata["threshold"] if threshold is None else threshold) - if not np.isfinite(self.threshold) or not 0 <= self.threshold <= 1: - raise ValueError("Threshold must lie between zero and one") - self._backend, self.engine, self.engine_errors = load_engine( - path, metadata, engine, library_path, model_precision, cache_dir - ) - self.model_precision = model_precision or ( - ModelPrecision.INT8 if self.engine is Engine.AKIKI else ModelPrecision.FP32 - ) - if self.engine is Engine.ONNX: - self.session = self._backend.session - self.reset() - - def reset(self) -> None: - """Start an independent recording with the same model and settings. - - Clear the recurrent state and audio context, discard pending samples, - and reset output timestamps to zero. The model remains loaded. - - Raises - ------ - RuntimeError - The detector has been closed. Create a new instance to process more - audio after closing. - """ - self._backend.reset() - self.context = np.zeros(self.context_samples, dtype=np.float32) - self.state = np.zeros((1, 1, self.hidden), dtype=np.float32) - self.pending = np.empty(0, dtype=np.float32) - self.completed_samples = 0 - - def push(self, samples, sample_rate: int = SAMPLE_RATE) -> list[SpeechFrame]: - """Process the next audio buffer and return decisions for complete frames. - - Parameters - ---------- - samples : array_like - One-dimensional, finite mono waveform samples. Lists and NumPy - arrays are accepted and converted to float32. Samples should - normally be scaled to [-1, 1]; integer PCM is not normalized for - you. Audio is not resampled or mixed down from multiple channels. - sample_rate : int, default=16000 - Input sample rate in Hz. Must be 16000. - - Returns - ------- - list[SpeechFrame] - One result for each complete ``chunk_samples`` input samples, - including any samples buffered by a previous call. Timestamps - continue from earlier calls. Returns an empty list when there is - not enough audio for a complete frame. - - Raises - ------ - TypeError - The input type cannot be converted to numeric audio samples. - ValueError - Audio cannot be converted to float32, is not one-dimensional, - contains nonfinite values, or has the wrong sample rate. These - validation failures leave the stream state unchanged. - RuntimeError - The detector is closed, inference fails, or the model returns an - invalid speech probability. - - Notes - ----- - Buffers may have any length, including zero. A final incomplete frame - stays in ``pending`` until more audio arrives or ``reset()`` discards - it. Closing the detector does not pad or score those samples. - """ - self._backend.check_open() - audio = np.asarray(samples, dtype=np.float32) - if sample_rate != SAMPLE_RATE or audio.ndim != 1 or not np.isfinite(audio).all(): - raise ValueError("Expected finite mono 16 kHz audio samples") - audio = np.concatenate([self.pending, audio]) - complete = len(audio) // self.chunk_samples * self.chunk_samples - frames = [] - for start in range(0, complete, self.chunk_samples): - window = np.concatenate([self.context, audio[start : start + self.chunk_samples]]) - probability, state = self._backend.infer(window, self.state) - probability = float(np.asarray(probability).reshape(-1)[0]) - if not np.isfinite(probability) or not 0 <= probability <= 1: - raise RuntimeError("Model returned an invalid speech probability") - begin = self.completed_samples - self.state = state - self.context = window[-self.context_samples :].copy() - self.completed_samples += self.chunk_samples - frames.append( - SpeechFrame( - begin / SAMPLE_RATE, - self.completed_samples / SAMPLE_RATE, - probability, - probability >= self.threshold, - ) - ) - self.pending = audio[complete:].copy() - return frames - - def close(self) -> None: - """Release this stream's inference resources; safe to call repeatedly. - - Pending samples are not scored. After closing, ``push()``, ``reset()``, - and entering a ``with`` block raise ``RuntimeError``. Previously - returned ``SpeechFrame`` objects remain usable. - """ - self._backend.close() - if self.engine is Engine.ONNX: - self.session = None - - def __enter__(self): - """Return this open detector for use in a ``with`` block.""" - self._backend.check_open() - return self - - def __exit__(self, *_): - """Close the detector on leaving a ``with`` block; propagate exceptions.""" - self.close() diff --git a/tests/test_api.py b/tests/test_api.py index 2697b5a..35f4e32 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,7 +4,9 @@ import pytest -from akiki import decision, vad +from akiki import audio, decision +from akiki import vad as legacy_vad +from akiki.audio import vad from akiki.hub import download_model @@ -28,6 +30,12 @@ def test_existing_top_level_imports_keep_their_identity(name, expected): assert namespace[name] is expected +@pytest.mark.parametrize("name", vad.__all__) +def test_legacy_vad_imports_keep_their_identity(name): + assert importlib.import_module("akiki.vad") is legacy_vad + assert getattr(legacy_vad, name) is getattr(vad, name) + + @pytest.mark.parametrize("name", importlib.import_module("akiki.minilm").__all__) def test_legacy_minilm_imports_keep_their_identity(name): legacy = importlib.import_module("akiki.minilm") @@ -39,7 +47,9 @@ def test_wildcard_import_exposes_the_main_interfaces_and_modules(): exec("from akiki import *", namespace) assert namespace["VAD"] is vad.VAD assert namespace["MiniLM"] is decision.MiniLM - assert namespace["vad"] is vad + assert namespace["audio"] is audio + assert namespace["audio"].vad is vad + assert "vad" not in namespace assert namespace["decision"] is decision assert "minilm" not in namespace assert "ModelPrecision" not in namespace diff --git a/tests/test_decision_inference.py b/tests/test_decision_inference.py index bbeb8ba..c2dd09c 100644 --- a/tests/test_decision_inference.py +++ b/tests/test_decision_inference.py @@ -13,7 +13,7 @@ import pytest from conftest import write_safetensors -from akiki import decision, vad +from akiki import decision from akiki._affine_cache import _base_identity, _digest from akiki._specialist_training import ( TrainedSpecialist, @@ -22,6 +22,7 @@ _root, _write_json, ) +from akiki.audio import vad from akiki.decision import DecisionModel, MiniLM, ModelPrecision, Specialist diff --git a/tests/test_library_discovery.py b/tests/test_library_discovery.py index 4a757c7..abbab27 100644 --- a/tests/test_library_discovery.py +++ b/tests/test_library_discovery.py @@ -5,7 +5,7 @@ import pytest from akiki import _engines, decision -from akiki.vad import VAD, Engine, ModelPrecision +from akiki.audio.vad import VAD, Engine, ModelPrecision @pytest.fixture(params=["win32", "darwin", "linux"]) diff --git a/tests/test_minilm.py b/tests/test_minilm.py index c0aa782..06bf1ef 100644 --- a/tests/test_minilm.py +++ b/tests/test_minilm.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from akiki import vad +from akiki.audio import vad from akiki.decision import MiniLM, ModelPrecision diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 41675cd..8878d32 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -208,6 +208,7 @@ def test_distributions_exclude_local_work_and_pass_release_verification(packaged directory = project / "dist" with zipfile.ZipFile(next(directory.glob("*.whl"))) as wheel: names = wheel.namelist() + assert "akiki/audio/vad.py" in names assert "akiki/decision.py" in names assert not any(name.startswith("akiki_training/") for name in names) metadata = wheel.read(next(name for name in names if name.endswith(".dist-info/METADATA"))) diff --git a/tests/test_training_api.py b/tests/test_training_api.py index 44fb199..f048307 100644 --- a/tests/test_training_api.py +++ b/tests/test_training_api.py @@ -39,10 +39,10 @@ def find_spec(self, fullname, path=None, target=None): sys.meta_path.insert(0, BlockTraining()) import akiki assert akiki.VAD is not None -assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'decision', 'vad'] +assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'audio', 'decision'] namespace = {} exec('from akiki import *', namespace) -assert namespace['vad'].VAD is akiki.VAD +assert namespace['audio'].vad.VAD is akiki.VAD assert namespace['decision'].MiniLM is akiki.MiniLM assert 'akiki.training' not in sys.modules import importlib @@ -85,7 +85,7 @@ def find_spec(self, fullname, path=None, target=None): from akiki_training import training as legacy assert training is akiki.training -assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'decision', 'training', 'vad'] +assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'audio', 'decision', 'training'] assert dir(training) == sorted(backend.__all__) assert training.__all__ is backend.__all__ for name in backend.__all__: diff --git a/tests/test_vad.py b/tests/test_vad.py index 99f25a3..43eb804 100644 --- a/tests/test_vad.py +++ b/tests/test_vad.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from akiki import VAD, Engine, FFTPrecision, ModelPrecision +from akiki.audio.vad import VAD, Engine, FFTPrecision, ModelPrecision def test_primary_engine_uses_the_akiki_name(): diff --git a/verify_dist.py b/verify_dist.py index 6321e51..1e0db84 100644 --- a/verify_dist.py +++ b/verify_dist.py @@ -43,7 +43,13 @@ else [args.wheel or args.dist_dir / f"akiki-{version}-py3-none-any.whl"] ) source = args.dist_dir / f"akiki-{version}.tar.gz" -model_dir = root / "src/akiki/models" +package_dir = root / "src/akiki" +model_dir = package_dir / "models" +python_modules = { + "akiki/" + path.relative_to(package_dir).as_posix(): path + for package in (package_dir, package_dir / "audio") + for path in package.glob("*.py") +} expected_id = "teacher-only623-v2-s1730" expected_hash = "0fab56bc4ea6725ec881c35973e04813a4963dc5bbf4bf5ccfe8fd7cf468c6f5" checks = {} @@ -85,11 +91,12 @@ def check_no_local_artifacts(names): def check_package_files(names, *, allow_native=False): names = {name for name in names if not name.endswith("/")} - expected = { - "akiki/" + path.relative_to(root / "src/akiki").as_posix() - for path in [*(root / "src/akiki").glob("*.py"), *model_dir.rglob("*")] + expected = set(python_modules) + expected.update( + "akiki/" + path.relative_to(package_dir).as_posix() + for path in model_dir.rglob("*") if path.is_file() - } + ) if allow_native: expected.update( name @@ -228,11 +235,8 @@ def check_models(read, names): f"Wheel licence differs: {name}", ) info["model"] = check_models(archive.read, archive.namelist()) - for path in (root / "src/akiki").glob("*.py"): - require( - archive.read("akiki/" + path.name) == path.read_bytes(), - f"Stale Python module: {path.name}", - ) + for name, path in python_modules.items(): + require(archive.read(name) == path.read_bytes(), f"Stale Python module: {name}") libraries = [ name for name in archive.namelist() From 8a01c603a60ab81031e5a35305bfbdad84cf2646 Mon Sep 17 00:00:00 2001 From: Toby Jackson Date: Wed, 18 Nov 2026 11:45:16 +0000 Subject: [PATCH 2/4] Deprecate legacy VAD imports for removal in 0.3.0 --- .github/workflows/publish.yml | 25 ++++++----- README.md | 5 ++- src/akiki/__init__.py | 34 +++++++++++---- src/akiki/__main__.py | 2 +- src/akiki/vad.py | 28 ++++++++++++- tests/conftest.py | 2 +- tests/test_api.py | 62 ++++++++++++++++++++++++---- tests/test_cli.py | 5 ++- tests/test_hub.py | 3 +- tests/test_quantization.py | 2 +- tests/test_training_api.py | 4 +- tools/benchmark_latency.py | 3 +- tools/build_native.py | 2 +- tools/evaluate_precision_tradeoff.py | 3 +- 14 files changed, 141 insertions(+), 39 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 245a599..09cb272 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -127,15 +127,16 @@ jobs: import importlib.util from pathlib import Path import akiki + from akiki.audio.vad import VAD, Engine, FFTPrecision, ModelPrecision from akiki.decision import _load_library import numpy as np assert importlib.util.find_spec("onnxruntime") is None assert importlib.util.find_spec("torch") is None - with akiki.VAD() as vad: - assert vad.engine is akiki.Engine.AKIKI - assert vad.model_precision is akiki.ModelPrecision.INT8 - assert vad.fft_precision is akiki.FFTPrecision.FP32 + with VAD() as vad: + assert vad.engine is Engine.AKIKI + assert vad.model_precision is ModelPrecision.INT8 + assert vad.fft_precision is FFTPrecision.FP32 assert Path(vad._backend.lib._name).parent == Path(akiki.__file__).parent / "native" assert len(vad.push(np.zeros(256 * 3 + 17, np.float32))) == 3 assert _load_library()[1].parent == Path(akiki.__file__).parent / "native" @@ -151,7 +152,7 @@ jobs: "$RUNNER_TEMP/akiki-onnx/bin/python" -I - <<'PY' import importlib.util import numpy as np - from akiki import Engine, ModelPrecision, VAD + from akiki.audio.vad import VAD, Engine, ModelPrecision assert importlib.util.find_spec("torch") is None with VAD() as vad: @@ -253,6 +254,7 @@ jobs: from pathlib import Path import akiki import numpy as np + from akiki.audio.vad import VAD, Engine, ModelPrecision from akiki.decision import _load_library assert akiki.__version__ == os.environ["AKIKI_EXPECTED_VERSION"] @@ -262,8 +264,8 @@ jobs: assert {p.name for p in native.iterdir()} == { "libakiki_vad_engine.so", "libakiki_minilm_engine.so" } - with akiki.VAD() as vad: - assert vad.engine is akiki.Engine.AKIKI and vad.model_precision is akiki.ModelPrecision.INT8 + with VAD() as vad: + assert vad.engine is Engine.AKIKI and vad.model_precision is ModelPrecision.INT8 assert Path(vad._backend.lib._name).parent == native assert len(vad.push(np.zeros(256 * 3 + 17, np.float32))) == 3 assert _load_library()[1].parent == native @@ -398,15 +400,16 @@ jobs: from pathlib import Path import akiki import numpy as np + from akiki.audio.vad import VAD, Engine, FFTPrecision, ModelPrecision from akiki.decision import _load_library assert importlib.util.find_spec('onnxruntime') is None assert importlib.util.find_spec('torch') is None native = Path(akiki.__file__).parent / 'native' - with akiki.VAD() as vad: - assert vad.engine is akiki.Engine.AKIKI - assert vad.model_precision is akiki.ModelPrecision.INT8 - assert vad.fft_precision is akiki.FFTPrecision.FP32 + with VAD() as vad: + assert vad.engine is Engine.AKIKI + assert vad.model_precision is ModelPrecision.INT8 + assert vad.fft_precision is FFTPrecision.FP32 assert Path(vad._backend.lib._name).parent == native assert len(vad.push(np.zeros(256 * 3 + 17, np.float32))) == 3 assert _load_library()[1].parent == native diff --git a/README.md b/README.md index cf2850f..bef9f6c 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ VAD and decision inference have distinct `ModelPrecision` enums. Their supported formats differ. `MiniLM` is the lower-level interface for direct embeddings and classification with the currently supported MiniLM/BERT family. -The existing `akiki.VAD` and `akiki.MiniLM` shortcuts remain available. +The `akiki.MiniLM` shortcut remains available. The existing top-level result types and VAD enums remain available as compatibility aliases. Import `training` explicitly when its optional companion package is installed; the base package does not load it. @@ -148,7 +148,8 @@ modules remain accessible but are omitted from that list. The decision-model module is named `akiki.decision`. Existing imports from `akiki.minilm` remain available for compatibility and use the same classes and precision enum. Named top-level compatibility imports remain available. -Existing imports from `akiki.vad` use the same classes and enums as `akiki.audio.vad`. +`akiki.vad` and `akiki.VAD` are deprecated and will be removed in version 0.3.0. +Use `from akiki.audio import vad` and call `vad.VAD()`. Wildcard imports expose only `VAD`, `MiniLM`, `audio`, `decision`, and `__version__`. ## Voice activity detection (VAD) diff --git a/src/akiki/__init__.py b/src/akiki/__init__.py index 2848e40..964976b 100644 --- a/src/akiki/__init__.py +++ b/src/akiki/__init__.py @@ -25,7 +25,7 @@ server. Propose supplemental examples for retraining with ``Teacher``. This module is loaded only when explicitly imported. -``VAD`` and ``MiniLM`` are also available directly from ``akiki`` as shortcuts. +``MiniLM`` is also available directly from ``akiki`` as a shortcut. Import supporting result types and configuration enums from their respective modules. VAD and decision precision enums have different supported formats and are not interchangeable. @@ -36,10 +36,10 @@ Examples -------- -Process one frame of silence using the top-level shortcut:: +Process one frame of silence:: - >>> import akiki - >>> with akiki.VAD() as detector: + >>> from akiki.audio import vad + >>> with vad.VAD() as detector: ... frames = detector.push([0.0] * 256) >>> len(frames) 1 @@ -61,7 +61,8 @@ Compatibility ------------- -``akiki.vad`` re-exports the VAD classes and enums from ``akiki.audio.vad``. +``akiki.vad`` and ``akiki.VAD`` will be removed in version 0.3.0. +Use ``from akiki.audio import vad`` and call ``vad.VAD()``. ``akiki.minilm`` remains a compatibility module. It re-exports the decision API's MiniLM class, result types, and current precision enum. Existing named top-level imports of ``SpeechFrame``, ``Engine``, ``ModelPrecision``, @@ -76,10 +77,12 @@ Installed Akiki package version. """ +import warnings +from importlib import import_module +from typing import TYPE_CHECKING + from . import audio, decision from . import minilm as minilm -from . import vad as vad -from .audio.vad import VAD from .audio.vad import Engine as Engine from .audio.vad import FFTPrecision as FFTPrecision from .audio.vad import ModelPrecision as ModelPrecision @@ -89,6 +92,9 @@ from .decision import MiniLM from .hub import download_model as download_model +if TYPE_CHECKING: + from .audio.vad import VAD as VAD + __version__ = "0.2.2" __all__ = [ "MiniLM", @@ -99,6 +105,20 @@ ] +def __getattr__(name: str): + if name == "VAD": + warnings.warn( + "akiki.VAD is deprecated and will be removed in version 0.3.0. " + "Use akiki.audio.vad.VAD instead.", + DeprecationWarning, + stacklevel=2, + ) + return audio.vad.VAD + if name == "vad": + return import_module(".vad", __name__) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + def __dir__() -> list[str]: """List the primary API and explicitly imported training/teacher modules.""" names = __all__.copy() diff --git a/src/akiki/__main__.py b/src/akiki/__main__.py index 191fefd..3c53c3c 100644 --- a/src/akiki/__main__.py +++ b/src/akiki/__main__.py @@ -157,7 +157,7 @@ def _run_vad(args): """Print frame decisions for a local audio file using the requested engine.""" import soundfile as sf - from . import VAD + from .audio.vad import VAD with ( VAD( diff --git a/src/akiki/vad.py b/src/akiki/vad.py index 7cd6d74..d952c77 100644 --- a/src/akiki/vad.py +++ b/src/akiki/vad.py @@ -1,4 +1,7 @@ -"""Compatibility imports for voice activity detection in ``akiki.audio.vad``.""" +"""Deprecated imports for VAD. Use ``akiki.audio.vad``.""" + +import sys +import warnings from .audio.vad import SAMPLE_RATE as SAMPLE_RATE from .audio.vad import VAD as VAD @@ -7,4 +10,27 @@ from .audio.vad import ModelPrecision as ModelPrecision from .audio.vad import SpeechFrame as SpeechFrame + +def _deprecation_stacklevel() -> int: + # The warnings module skips the importlib bootstrap frames itself. + # Also skip the Python import wrapper and our package's lazy import. + level = 2 + frame = sys._getframe(1).f_back + while frame is not None: + module = frame.f_globals.get("__name__", "") + if module in (__package__, "importlib"): + level += 1 + elif not module.startswith("importlib._bootstrap"): + break + frame = frame.f_back + return level + + +warnings.warn( + "akiki.vad is deprecated and will be removed in version 0.3.0. " + "Use 'from akiki.audio import vad' instead.", + DeprecationWarning, + stacklevel=_deprecation_stacklevel(), +) + __all__ = ["Engine", "FFTPrecision", "ModelPrecision", "SpeechFrame", "VAD"] diff --git a/tests/conftest.py b/tests/conftest.py index abc5281..8857ef9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from akiki import VAD, Engine +from akiki.audio.vad import VAD, Engine @pytest.fixture diff --git a/tests/test_api.py b/tests/test_api.py index 35f4e32..1416d57 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,11 +1,15 @@ """Check the grouped public API and its existing import aliases.""" import importlib +import os +import subprocess +import sys +from pathlib import Path import pytest +import akiki from akiki import audio, decision -from akiki import vad as legacy_vad from akiki.audio import vad from akiki.hub import download_model @@ -13,7 +17,6 @@ @pytest.mark.parametrize( "name,expected", [ - ("VAD", vad.VAD), ("MiniLM", decision.MiniLM), ("SpeechFrame", vad.SpeechFrame), ("Engine", vad.Engine), @@ -30,10 +33,54 @@ def test_existing_top_level_imports_keep_their_identity(name, expected): assert namespace[name] is expected -@pytest.mark.parametrize("name", vad.__all__) -def test_legacy_vad_imports_keep_their_identity(name): - assert importlib.import_module("akiki.vad") is legacy_vad - assert getattr(legacy_vad, name) is getattr(vad, name) +@pytest.mark.parametrize("access", ["from akiki import VAD", "VAD = akiki.VAD"]) +def test_legacy_top_level_vad_warns_and_keeps_its_identity(access): + namespace = {"akiki": akiki} + with pytest.warns(DeprecationWarning, match=r"akiki\.VAD .* removed in version 0\.3\.0"): + exec(access, namespace) + assert namespace["VAD"] is vad.VAD + + +@pytest.mark.parametrize( + "access", + [ + "import akiki.vad as legacy", + "from akiki import vad as legacy", + "legacy = akiki.vad", + 'legacy = importlib.import_module("akiki.vad")', + ], +) +def test_legacy_vad_module_warns_and_keeps_its_exports(access): + result = subprocess.run( + [ + sys.executable, + "-c", + f""" +import importlib +import warnings + +with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + import akiki + from akiki.audio import vad + +with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always", DeprecationWarning) + {access} +[warning] = recorded +assert warning.filename == "", warning.filename +assert warning.category is DeprecationWarning +assert "akiki.vad is deprecated and will be removed in version 0.3.0" in str(warning.message) +assert legacy is importlib.import_module("akiki.vad") +for name in vad.__all__: + assert getattr(legacy, name) is getattr(vad, name) +""", + ], + env=os.environ | {"PYTHONPATH": str(Path(akiki.__file__).resolve().parents[1])}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr @pytest.mark.parametrize("name", importlib.import_module("akiki.minilm").__all__) @@ -44,7 +91,8 @@ def test_legacy_minilm_imports_keep_their_identity(name): def test_wildcard_import_exposes_the_main_interfaces_and_modules(): namespace = {} - exec("from akiki import *", namespace) + with pytest.warns(DeprecationWarning, match=r"akiki\.VAD"): + exec("from akiki import *", namespace) assert namespace["VAD"] is vad.VAD assert namespace["MiniLM"] is decision.MiniLM assert namespace["audio"] is audio diff --git a/tests/test_cli.py b/tests/test_cli.py index 90adddc..aaf20b8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,6 +11,7 @@ import akiki from akiki.__main__ import main +from akiki.audio import vad from akiki.decision import ModelPrecision as MiniLMPrecision @@ -144,7 +145,7 @@ def test_help_and_version_do_not_load_an_engine(arguments, expected, monkeypatch def unexpected_engine_load(*args, **kwargs): pytest.fail("Help and version must work without an inference engine") - monkeypatch.setattr(akiki, "VAD", unexpected_engine_load) + monkeypatch.setattr(vad, "VAD", unexpected_engine_load) monkeypatch.setattr(akiki, "MiniLM", unexpected_engine_load) monkeypatch.setitem(sys.modules, "huggingface_hub", None) monkeypatch.setattr(sys, "argv", ["akiki", *arguments]) @@ -193,7 +194,7 @@ def test_vad_engine_errors_include_the_command_prefix(monkeypatch, capsys): def unavailable_engine(*args, **kwargs): raise RuntimeError("No compatible engine") - monkeypatch.setattr(akiki, "VAD", unavailable_engine) + monkeypatch.setattr(vad, "VAD", unavailable_engine) monkeypatch.setattr(sys, "argv", ["akiki", "vad", "recording.wav"]) with pytest.raises(SystemExit) as error: main() diff --git a/tests/test_hub.py b/tests/test_hub.py index c0f08e9..44cbc5d 100644 --- a/tests/test_hub.py +++ b/tests/test_hub.py @@ -14,6 +14,7 @@ import akiki from akiki import MiniLM, download_model from akiki.__main__ import main +from akiki.audio import vad from akiki.decision import ModelPrecision @@ -193,7 +194,7 @@ def unexpected_load(*args, **kwargs): pytest.fail("Downloading a checkpoint must not load an inference engine") monkeypatch.setattr(akiki, "MiniLM", unexpected_load) - monkeypatch.setattr(akiki, "VAD", unexpected_load) + monkeypatch.setattr(vad, "VAD", unexpected_load) destination = tmp_path / "model" monkeypatch.setattr( sys, diff --git a/tests/test_quantization.py b/tests/test_quantization.py index 6414aa8..3097f90 100644 --- a/tests/test_quantization.py +++ b/tests/test_quantization.py @@ -9,8 +9,8 @@ import pytest import akiki -from akiki import VAD, Engine, FFTPrecision, ModelPrecision from akiki import _quantization as quant +from akiki.audio.vad import VAD, Engine, FFTPrecision, ModelPrecision from akiki.decision import ModelPrecision as MiniLMPrecision diff --git a/tests/test_training_api.py b/tests/test_training_api.py index f048307..f20057d 100644 --- a/tests/test_training_api.py +++ b/tests/test_training_api.py @@ -38,11 +38,11 @@ def find_spec(self, fullname, path=None, target=None): sys.meta_path.insert(0, BlockTraining()) import akiki -assert akiki.VAD is not None +assert akiki.audio.vad.VAD is not None assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'audio', 'decision'] namespace = {} exec('from akiki import *', namespace) -assert namespace['audio'].vad.VAD is akiki.VAD +assert namespace['audio'].vad.VAD is akiki.audio.vad.VAD assert namespace['decision'].MiniLM is akiki.MiniLM assert 'akiki.training' not in sys.modules import importlib diff --git a/tools/benchmark_latency.py b/tools/benchmark_latency.py index 8fd930b..d80c8db 100644 --- a/tools/benchmark_latency.py +++ b/tools/benchmark_latency.py @@ -25,7 +25,8 @@ def worker(args): import numpy as np import akiki - from akiki import VAD, Engine, __version__ + from akiki import __version__ + from akiki.audio.vad import VAD, Engine engine = ( Engine.AUTO diff --git a/tools/build_native.py b/tools/build_native.py index 87acd03..bb0fb68 100644 --- a/tools/build_native.py +++ b/tools/build_native.py @@ -7,7 +7,7 @@ import sys from pathlib import Path -# Shared libraries loaded by akiki.VAD and akiki.MiniLM. +# Shared libraries loaded by akiki.audio.vad.VAD and akiki.MiniLM. PACKAGES = ("akiki-vad-engine", "akiki-minilm-engine") diff --git a/tools/evaluate_precision_tradeoff.py b/tools/evaluate_precision_tradeoff.py index f3df381..93dc5d4 100644 --- a/tools/evaluate_precision_tradeoff.py +++ b/tools/evaluate_precision_tradeoff.py @@ -26,7 +26,8 @@ import soundfile as sf from sklearn.metrics import average_precision_score, precision_recall_fscore_support -from akiki import VAD, Engine, FFTPrecision, ModelPrecision, __version__ +from akiki import __version__ +from akiki.audio.vad import VAD, Engine, FFTPrecision, ModelPrecision MODES = tuple(ModelPrecision) From 7f0963eb3e640adfaa03f1385d3e66306f36613d Mon Sep 17 00:00:00 2001 From: Toby Jackson Date: Wed, 18 Nov 2026 12:29:01 +0000 Subject: [PATCH 3/4] Add enum-based TEN-VAD selection from ONNX checkpoints --- README.md | 31 ++++- pyproject.toml | 1 + src/akiki/_atomic.py | 16 +++ src/akiki/_quantization.py | 21 +-- src/akiki/audio/_ten_vad.py | 70 ++++++++++ src/akiki/audio/_ten_vad_model.py | 215 ++++++++++++++++++++++++++++++ src/akiki/audio/vad.py | 122 +++++++++++++---- src/akiki/hub.py | 66 +++++---- src/akiki/vad.py | 3 +- tests/test_ten_vad_model.py | 78 +++++++++++ tests/test_vad.py | 70 +++++++++- verify_dist.py | 4 +- 12 files changed, 623 insertions(+), 74 deletions(-) create mode 100644 src/akiki/audio/_ten_vad.py create mode 100644 src/akiki/audio/_ten_vad_model.py create mode 100644 tests/test_ten_vad_model.py diff --git a/README.md b/README.md index bef9f6c..668067c 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ The training and teacher modules support the decision-model workflow: | Module | Main interface | Supporting types | | --- | --- | --- | -| `akiki.audio.vad` | `VAD` for speech detection | `SpeechFrame`, `Engine`, `ModelPrecision`, `FFTPrecision` | +| `akiki.audio.vad` | `VAD` for speech detection | `SpeechFrame`, `Model`, `Engine`, `ModelPrecision`, `FFTPrecision` | | `akiki.decision` | `DecisionModel` for specialist decisions | `BaseModel`, `Specialist`, `AffineLayer`, `TrainedSpecialist`, `SpecialistEvaluation`, `TrainingAdvice`, `list_base_models`, `Classification`, `ModelPrecision` | | `akiki.training` | Classifier training with a private companion | The installed companion's API, including `LocalModel`, `ModelFamily`, and `Trainer` | | `akiki.teacher` | Experimental review through a local or self-hosted server | `Teacher`, `Verdict`, `DecisionReview`, `TeachingResult` | @@ -191,6 +191,35 @@ help(akiki.audio.vad.VAD.push) # Audio input, buffering, results, and errors help(akiki.audio.vad.SpeechFrame) # Result fields and timestamp units ``` +### Select the VAD model + +`vad.VAD()` and `vad.VAD(model=vad.Model.AKIKI_VAD)` load the bundled Akiki-VAD model. + +To use TEN-VAD, install `akiki[ten-vad]` and set `AKIKI_TEN_VAD_LIBRARY` to +your local Akiki TEN-VAD shared library. You can also pass `library_path`. + +```python +from akiki.audio import vad + +with vad.VAD( + model=vad.Model.TEN_VAD, + model_path="snapshot/src/onnx_model/ten-vad.onnx", +) as detector: + frames = detector.push([0.0] * 256) +``` + +Keep the matching `src/coeff.h` in the snapshot. For a standalone ONNX file, +keep `coeff.h` next to the file. The coefficients supply the audio frontend. +TEN-VAD requires mono 16 kHz samples in `[-1, 1]` and runs in FP32. + +Omit `model_path` to use the supported snapshot already in the Hugging Face +cache. Set `HF_HUB_OFFLINE=1` and configure `HF_HUB_CACHE` if you use a +custom cache directory. [TEN-VAD model on Hugging Face](https://huggingface.co/TEN-framework/ten-vad) + +The first load converts the ONNX weights for the Akiki engine. Later loads +reuse the converted weights in the Akiki cache. The Akiki engine processes +all audio frames. + ### Command line ```bash diff --git a/pyproject.toml b/pyproject.toml index 629b27a..fb90027 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = ["numpy>=1.24", "soundfile>=0.12"] huggingface = ["huggingface-hub>=0.24"] onnx = ["onnxruntime>=1.17"] pytorch = ["torch>=2.1"] +ten-vad = ["huggingface-hub>=0.24", "onnx>=1.16"] training = ["torch>=2.1", "transformers>=4.40,<6", "safetensors>=0.4"] [project.urls] diff --git a/src/akiki/_atomic.py b/src/akiki/_atomic.py index 886f6ae..753f358 100644 --- a/src/akiki/_atomic.py +++ b/src/akiki/_atomic.py @@ -2,6 +2,7 @@ import os import sys +import tempfile import time from pathlib import Path @@ -20,3 +21,18 @@ def replace_file(source: Path, destination: Path) -> None: ): raise time.sleep(0.01 * 2**attempt) + + +def write_bytes(path: Path, data: bytes) -> None: + """Write a complete file before replacing its destination.""" + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=path.parent, prefix=".tmp-", delete=False) as stream: + temporary = Path(stream.name) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + replace_file(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) diff --git a/src/akiki/_quantization.py b/src/akiki/_quantization.py index f3a7337..71ff045 100644 --- a/src/akiki/_quantization.py +++ b/src/akiki/_quantization.py @@ -12,12 +12,11 @@ import os import struct import sys -import tempfile from pathlib import Path import numpy as np -from ._atomic import replace_file +from ._atomic import write_bytes from ._precision import ModelPrecision # AKVAD003 format IDs are fixed independently of enum declaration order. @@ -180,20 +179,6 @@ def default_cache_dir(): return Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "akiki" -def _atomic_write(path, data): - temporary = None - try: - with tempfile.NamedTemporaryFile(dir=path.parent, prefix=".tmp-", delete=False) as stream: - temporary = Path(stream.name) - stream.write(data) - stream.flush() - os.fsync(stream.fileno()) - replace_file(temporary, path) - finally: - if temporary is not None: - temporary.unlink(missing_ok=True) - - def cached_model(folder, receipt, precision: ModelPrecision, cache_dir, validate_source): """Return a checked variant; repair incomplete/corrupt derived cache entries.""" identity = { @@ -239,8 +224,8 @@ def cached_model(folder, receipt, precision: ModelPrecision, cache_dir, validate } try: directory.mkdir(parents=True, exist_ok=True) - _atomic_write(model, payload) - _atomic_write(manifest, (json.dumps(saved, sort_keys=True, indent=2) + "\n").encode()) + write_bytes(model, payload) + write_bytes(manifest, (json.dumps(saved, sort_keys=True, indent=2) + "\n").encode()) except OSError as error: raise RuntimeError( "Cannot write the Akiki model cache; set cache_dir or AKIKI_CACHE_DIR to a writable directory" diff --git a/src/akiki/audio/_ten_vad.py b/src/akiki/audio/_ten_vad.py new file mode 100644 index 0000000..1292592 --- /dev/null +++ b/src/akiki/audio/_ten_vad.py @@ -0,0 +1,70 @@ +"""Local binding for the Akiki TEN-VAD engine.""" + +import ctypes +import weakref +from pathlib import Path + +import numpy as np + +from .._engines import Backend, EngineUnavailable, library_candidates + + +class TenVadBackend(Backend): + chunk_samples = 256 + hidden = 64 + state_shape = (4, hidden) + + @classmethod + def load_library(cls, library_path: str | Path | None): + candidate = next( + library_candidates(library_path, "akiki_ten_vad_engine", "AKIKI_TEN_VAD_LIBRARY") + ).resolve() + try: + lib = ctypes.CDLL(str(candidate)) + cls._bind(lib) + except (OSError, AttributeError) as error: + raise EngineUnavailable( + f"No compatible Akiki TEN-VAD library at {candidate}. " + "Set AKIKI_TEN_VAD_LIBRARY or pass library_path." + ) from error + return lib + + def __init__(self, path: Path, lib): + handle = lib.akiki_ten_vad_create(str(path.resolve()).encode("utf-8")) + if not handle: + raise ValueError(f"The Akiki TEN-VAD engine cannot load the model: {path}") + self.lib, self.handle = lib, handle + self._finalizer = weakref.finalize(self, lib.akiki_ten_vad_free, handle) + + @staticmethod + def _bind(lib): + pointer = ctypes.c_void_p + for name, arguments, result in ( + ("create", [ctypes.c_char_p], pointer), + ("free", [pointer], None), + ("reset", [pointer], None), + ("frame", [pointer, pointer, ctypes.POINTER(ctypes.c_float)], ctypes.c_int), + ("state", [pointer, pointer], ctypes.c_int), + ): + function = getattr(lib, f"akiki_ten_vad_{name}") + function.argtypes, function.restype = arguments, result + + def reset(self): + super().reset() + self.lib.akiki_ten_vad_reset(self.handle) + + def infer(self, window, state): + if window.shape != (self.chunk_samples,): + raise ValueError("TEN-VAD requires exactly 256 samples per frame") + chunk = np.ascontiguousarray(window, dtype=np.float32) + probability = ctypes.c_float() + if self.lib.akiki_ten_vad_frame(self.handle, chunk.ctypes.data, ctypes.byref(probability)): + raise RuntimeError("Akiki TEN-VAD engine could not score an audio frame") + hidden = np.empty(self.state_shape, dtype=np.float32) + if self.lib.akiki_ten_vad_state(self.handle, hidden.ctypes.data): + raise RuntimeError("Akiki TEN-VAD engine could not return its recurrent state") + return probability.value, hidden + + def close(self): + super().close() + self._finalizer() diff --git a/src/akiki/audio/_ten_vad_model.py b/src/akiki/audio/_ten_vad_model.py new file mode 100644 index 0000000..74a0a3c --- /dev/null +++ b/src/akiki/audio/_ten_vad_model.py @@ -0,0 +1,215 @@ +"""Prepare TEN-VAD ONNX weights for the native streaming engine.""" + +import hashlib +import re +from pathlib import Path + +import numpy as np + +from .._atomic import write_bytes +from .._quantization import default_cache_dir +from ..hub import _download_snapshot + +_REPO = "TEN-framework/ten-vad" +_REVISION = "bda8ffc78b1846c5c7cbd38f04e52deff49de707" +_ONNX = "src/onnx_model/ten-vad.onnx" +_COEFFICIENTS = "src/coeff.h" +_HUB_CHECKSUMS = { + _ONNX: "e10b98a0cab1c98e847fbdda14cb3d45a38336d47535a3f63a0fb6c4e0f4cdf4", + _COEFFICIENTS: "0c86dc4a9161e418da6ce03d50dea067dff085749d730e3913b99a8e116397ef", +} +_GRAPH_SHA256 = "bbefc914a8b2a8db673e9b74914cc086b566c09e7a67c22a1f21b26a39edf91a" +_CONVERTER_VERSION = b"ten-vad-onnx-v1" +_NATIVE_MAGIC = b"AKTNV001" +_NATIVE_SIZE = len(_NATIVE_MAGIC) + (75498 + 850) * 4 +_PREFIX = "StatefulPartitionedCall/vad_model/" + + +def _read_sources(model_path: str | Path | None) -> tuple[bytes, bytes]: + if model_path is None: + try: + snapshot = _download_snapshot( + _REPO, + revision=_REVISION, + allow_patterns=tuple(_HUB_CHECKSUMS), + local_files_only=True, + ) + except RuntimeError as error: + raise RuntimeError( + "Cannot load TEN-VAD from the local Hugging Face cache. " + f"Pass model_path to a local TEN-VAD ONNX file. {error}" + ) from error + contents = [] + for filename, checksum in _HUB_CHECKSUMS.items(): + path = snapshot / filename + data = path.read_bytes() + if hashlib.sha256(data).hexdigest() != checksum: + raise ValueError(f"TEN-VAD Hub file has an incorrect checksum: {path}") + contents.append(data) + return contents[0], contents[1] + + weights = Path(model_path).expanduser() + if weights.suffix.lower() != ".onnx": + raise ValueError("TEN-VAD model_path must point to its .onnx weights") + model_data = weights.read_bytes() + source_dir = weights.parent + if source_dir.name == "onnx_model" and not (source_dir / "coeff.h").is_file(): + source_dir = source_dir.parent + coefficients = source_dir / "coeff.h" + if not coefficients.is_file(): + raise FileNotFoundError( + "TEN-VAD requires the matching upstream coeff.h beside the ONNX file " + f"or in the parent of its onnx_model directory: {coefficients}" + ) + return model_data, coefficients.read_bytes() + + +def _network_weights(tensors: dict[str, np.ndarray]) -> list[np.ndarray]: + arrays = [] + used = set() + + def take(name, shape): + value = tensors.get(name) + if value is None or value.shape != shape or not np.isfinite(value).all(): + raise ValueError(f"Invalid or missing TEN-VAD tensor: {name}") + used.add(name) + return value + + for layer, depth, depth_shape, point, point_shape in ( + ( + "separable_conv2d", + "const_fold_opt__178", + (1, 1, 3, 3), + "separable_conv2d/ReadVariableOp_1:0", + (16, 1, 1, 1), + ), + ( + "separable_conv1d", + "const_fold_opt__179", + (16, 1, 1, 3), + "ExpandDims_2:0", + (16, 16, 1, 1), + ), + ( + "separable_conv1d_1", + "const_fold_opt__180", + (16, 1, 1, 3), + "ExpandDims_2:0", + (16, 16, 1, 1), + ), + ): + arrays.extend( + ( + take(depth, depth_shape), + take(f"{_PREFIX}{layer}/{point}", point_shape), + take(f"{_PREFIX}{layer}/BiasAdd/ReadVariableOp:0", (16,)), + ) + ) + + # ONNX uses IOFC gate order. The native engine uses IFCO. + gates = [0, 2, 3, 1] + for input_name, recurrent_name, bias_name, inputs in ( + ("W0__70", "R0__71", "B0__72", 80), + ("W0__99", "R0__100", "B0__101", 64), + ): + weight = take(input_name, (1, 256, inputs)).reshape(4, 64, inputs)[gates] + recurrent = take(recurrent_name, (1, 256, 64)).reshape(4, 64, 64)[gates] + bias = take(bias_name, (1, 512)).reshape(2, 4, 64)[:, gates] + arrays.extend((weight, bias[0], recurrent, bias[1])) + + for layer, inputs, outputs in (("dense_3", 128, 32), ("dense_5", 32, 1)): + arrays.append(take(f"{_PREFIX}{layer}/Tensordot/ReadVariableOp:0", (inputs, outputs)).T) + arrays.append(take(f"{_PREFIX}{layer}/BiasAdd/ReadVariableOp:0", (outputs,))) + if used != tensors.keys(): + raise ValueError("Unsupported TEN-VAD weight set") + return arrays + + +def _frontend_weights(data: bytes) -> list[np.ndarray]: + text = data.decode("utf-8") + arrays = [] + for name, length in ( + ("AUP_AED_FEATURE_MEANS", 41), + ("AUP_AED_FEATURE_STDS", 41), + ("AUP_AED_STFTWindow_Hann768", 768), + ): + match = re.search(rf"\b{name}\[[^\]]+\]\s*=\s*\{{([^}}]+)\}}", text) + if match is None: + raise ValueError(f"Missing TEN-VAD frontend coefficient array: {name}") + values = np.array( + [float(value.strip().rstrip("f")) for value in match[1].split(",") if value.strip()], + dtype=" 1)).any(): + raise ValueError("Invalid TEN-VAD frontend normalization or window") + return arrays + + +def _convert(model_data: bytes, coefficients: bytes) -> bytes: + try: + import onnx + from onnx import numpy_helper + except ImportError as error: + raise RuntimeError("TEN-VAD ONNX import requires pip install 'akiki[ten-vad]'") from error + + try: + graph = onnx.load_model_from_string(model_data).graph + except Exception as error: + raise ValueError("Cannot parse the TEN-VAD ONNX checkpoint") from error + + topology = onnx.GraphProto() + topology.CopyFrom(graph) + weights = {} + for initializer, tensor in zip(graph.initializer, topology.initializer): + if initializer.external_data or initializer.data_location == onnx.TensorProto.EXTERNAL: + raise ValueError("TEN-VAD ONNX must contain its weights in a single file") + if initializer.data_type == onnx.TensorProto.FLOAT: + if initializer.name in weights: + raise ValueError(f"Duplicate TEN-VAD tensor: {initializer.name}") + weights[initializer.name] = numpy_helper.to_array(initializer) + for field in ("raw_data", "float_data", "double_data"): + tensor.ClearField(field) + if hashlib.sha256(topology.SerializeToString()).hexdigest() != _GRAPH_SHA256: + raise ValueError("Unsupported ONNX graph. Use the TEN-framework TEN-VAD 1.0 model.") + + arrays = _network_weights(weights) + _frontend_weights(coefficients) + return _NATIVE_MAGIC + b"".join(array.astype(" Path: + """Return a checked native cache entry for the selected TEN-VAD ONNX files.""" + weights, coefficients = _read_sources(model_path) + key = hashlib.sha256( + _CONVERTER_VERSION + + hashlib.sha256(weights).digest() + + hashlib.sha256(coefficients).digest() + ).hexdigest() + root = Path(cache_dir).expanduser() if cache_dir is not None else default_cache_dir() + directory = root / "ten-vad" / key + model = directory / "model.bin" + checksum = directory / "sha256" + try: + cached = model.read_bytes() + if ( + len(cached) == _NATIVE_SIZE + and cached.startswith(_NATIVE_MAGIC) + and hashlib.sha256(cached).hexdigest().encode() == checksum.read_bytes() + ): + return model + except OSError: + pass + + payload = _convert(weights, coefficients) + try: + directory.mkdir(parents=True, exist_ok=True) + write_bytes(model, payload) + write_bytes(checksum, hashlib.sha256(payload).hexdigest().encode()) + except OSError as error: + raise RuntimeError( + "Cannot write the TEN-VAD model cache. Set cache_dir or AKIKI_CACHE_DIR." + ) from error + return model diff --git a/src/akiki/audio/vad.py b/src/akiki/audio/vad.py index 1eac99f..70eff85 100644 --- a/src/akiki/audio/vad.py +++ b/src/akiki/audio/vad.py @@ -20,6 +20,7 @@ import json from dataclasses import dataclass +from enum import Enum from pathlib import Path import numpy as np @@ -27,11 +28,18 @@ from .._engines import Engine from .._precision import FFTPrecision, ModelPrecision -__all__ = ["Engine", "FFTPrecision", "ModelPrecision", "SpeechFrame", "VAD"] +__all__ = ["Engine", "FFTPrecision", "Model", "ModelPrecision", "SpeechFrame", "VAD"] SAMPLE_RATE = 16000 +class Model(Enum): + """Model family to use for voice activity detection.""" + + AKIKI_VAD = "akiki-vad" + TEN_VAD = "ten-vad" + + @dataclass(frozen=True) class SpeechFrame: """Immutable speech decision for one complete frame returned by ``VAD.push``. @@ -76,11 +84,10 @@ class VAD: ``SpeechFrame`` for every 256 samples (16 ms). Custom models use their supported ``chunk_samples`` value: 256 or 512 samples. - Models load locally. The Akiki engine is the primary engine. - ``engine=Engine.AUTO`` tries it first, then the ONNX and PyTorch compatibility - fallbacks. The Akiki engine uses INT8 by default. Compatibility engines use FP32. - Explicit non-FP32 precision requires the Akiki engine. Install compatibility - runtimes with ``akiki[onnx]`` or ``akiki[pytorch]`` when needed. + The Akiki engine is the primary engine. For Akiki-VAD, ``Engine.AUTO`` + tries it first, then ONNX, then PyTorch. The Akiki engine uses INT8 by + default for Akiki-VAD. The compatibility engines use FP32. + TEN-VAD uses the Akiki engine with FP32. See ``VAD.__init__`` for all constructor options. Use the detector as a context manager or call ``close()`` when finished. @@ -94,7 +101,7 @@ class VAD: engine_errors : dict[Engine, str] Reasons engines were unavailable before automatic selection succeeded. model_id : str - Identifier from the loaded model's metadata. + Identifier from the Akiki-VAD metadata, or ``"ten-vad"``. model_precision : ModelPrecision Active model precision, after resolving the engine's default. fft_precision : FFTPrecision @@ -113,8 +120,10 @@ class VAD: Unscored float32 samples buffered for the next call to ``push``. context : numpy.ndarray Float32 audio context with shape ``(context_samples,)``. + TEN-VAD keeps its audio context in the engine. This array is empty for TEN-VAD. state : numpy.ndarray - Float32 recurrent state with shape ``(1, 1, hidden)``. + Float32 recurrent state. Akiki-VAD uses shape ``(1, 1, hidden)``. + TEN-VAD uses shape ``(4, 64)`` with hidden and cell state for each LSTM. session : onnxruntime.InferenceSession Present only when the selected engine is ONNX; ``None`` after closing. @@ -144,18 +153,22 @@ def __init__( model_path: str | Path | None = None, threshold: float | None = None, *, + model: Model = Model.AKIKI_VAD, engine: Engine = Engine.AUTO, library_path: str | Path | None = None, model_precision: ModelPrecision | None = None, fft_precision: FFTPrecision = FFTPrecision.FP32, cache_dir: str | Path | None = None, ): - """Load a local model and inference engine, ready for a new stream. + """Load a local model and inference engine for a new stream. Parameters ---------- model_path : str or pathlib.Path, optional - If omitted, use the bundled model. + For TEN-VAD, pass the path to its official ``.onnx`` file. + Keep the matching ``coeff.h`` beside the weights or use the + official snapshot layout. If omitted for TEN-VAD, use the pinned + Hugging Face snapshot from the local cache. For a custom Akiki model, pass its JSON metadata path, such as ``"models/vad.json"``. The Akiki engine loads the sibling ``"models/vad-native/"`` directory. It must contain ``config.json``, @@ -167,8 +180,11 @@ def __init__( ``(512, 64)``. Custom PyTorch metadata also requires ``pytorch_sha256``. threshold : float, optional Speech decision threshold between 0 and 1 inclusive. If omitted, - use the threshold in the model's metadata. This changes the boolean - speech decision, not the returned probability. + use the threshold in the model's metadata, or 0.5 for TEN-VAD. + The threshold applies to the speech decision. + model : Model, default=Model.AKIKI_VAD + Select the VAD model family. Use ``Model.TEN_VAD`` for TEN-VAD. + With ``Model.AKIKI_VAD`` and no ``model_path``, use the bundled model. engine : Engine, default=Engine.AUTO Automatic selection tries the primary Akiki engine first. ONNX and PyTorch are compatibility fallbacks, in that order. @@ -176,8 +192,9 @@ def __init__( Pass an ``Engine`` member, such as ``Engine.AKIKI``. library_path : str or pathlib.Path, optional Specific local Akiki shared library to load. Affects only the Akiki - engine. If omitted, use ``AKIKI_VAD_LIBRARY`` or the bundled library. - Automatic discovery does not search developer directories. + engine. If omitted, use ``AKIKI_VAD_LIBRARY`` or the bundled + Akiki-VAD library. For TEN-VAD, use ``AKIKI_TEN_VAD_LIBRARY`` + or pass its library path here. model_precision : ModelPrecision, optional One of ``ModelPrecision.FP32``, ``FP16``, ``BF16``, ``INT8``, ``INT4``, or ``INT8_FP32_INPUT``. If omitted, Akiki uses INT8 and @@ -185,19 +202,20 @@ def __init__( including in automatic mode. Reduced-precision Akiki weights are generated from the verified local FP32 model and reused from the cache. ``ModelPrecision.INT8_FP32_INPUT`` uses INT8 weights while - keeping the first projection's input in FP32. + keeping the first projection's input in FP32. TEN-VAD uses FP32. fft_precision : FFTPrecision, default=FFTPrecision.FP32 FFT precision. Only ``FFTPrecision.FP32`` is supported. cache_dir : str or pathlib.Path, optional Directory for generated Akiki model variants. If omitted, use ``AKIKI_CACHE_DIR`` or the platform's user cache directory. Original - model weights are preserved. FP32 inference needs no conversion. + model weights are preserved. TEN-VAD prepares an engine cache + entry from its ONNX weights once per checkpoint. Raises ------ TypeError - ``engine``, ``model_precision``, or ``fft_precision`` has the wrong - enum type. ``model_precision`` also accepts ``None``. + ``model``, ``engine``, ``model_precision``, or ``fft_precision`` + has the wrong enum type. ``model_precision`` also accepts ``None``. ValueError A parameter, model specification, or model checksum is invalid. OSError @@ -206,8 +224,10 @@ def __init__( No compatible engine is available for the requested configuration, or the directory for a generated model variant is not writable. """ - from .._engines import load_engine - + if not isinstance(model, Model): + raise TypeError( + "model must be a Model member, such as Model.AKIKI_VAD or Model.TEN_VAD" + ) if not isinstance(engine, Engine): raise TypeError("engine must be an Engine member, such as Engine.AUTO or Engine.AKIKI") if model_precision is not None and not isinstance(model_precision, ModelPrecision): @@ -220,6 +240,22 @@ def __init__( ): raise ValueError("Non-FP32 model_precision requires Engine.AKIKI or Engine.AUTO") self.fft_precision = fft_precision + self._model = model + if model is Model.TEN_VAD: + self._load_ten_vad( + model_path, threshold, engine, library_path, model_precision, cache_dir + ) + else: + self._load_akiki_vad( + model_path, threshold, engine, library_path, model_precision, cache_dir + ) + self.reset() + + def _load_akiki_vad( + self, model_path, threshold, engine, library_path, model_precision, cache_dir + ): + from .._engines import load_engine + path = ( Path(model_path) if model_path is not None @@ -238,9 +274,10 @@ def __init__( raise ValueError("Expected an Akiki full-waveform GRU model") if spec.get("sample_rate", SAMPLE_RATE) != SAMPLE_RATE or self.hidden <= 0: raise ValueError("Expected a 16 kHz model with a positive hidden size") - self.threshold = float(metadata["threshold"] if threshold is None else threshold) - if not np.isfinite(self.threshold) or not 0 <= self.threshold <= 1: - raise ValueError("Threshold must lie between zero and one") + self._state_shape = (1, 1, self.hidden) + self.threshold = self._validate_threshold( + metadata["threshold"] if threshold is None else threshold + ) self._backend, self.engine, self.engine_errors = load_engine( path, metadata, engine, library_path, model_precision, cache_dir ) @@ -249,7 +286,35 @@ def __init__( ) if self.engine is Engine.ONNX: self.session = self._backend.session - self.reset() + + def _load_ten_vad( + self, model_path, threshold, engine, library_path, model_precision, cache_dir + ): + from ._ten_vad import TenVadBackend + from ._ten_vad_model import prepare_model + + if engine not in (Engine.AUTO, Engine.AKIKI): + raise ValueError("TEN-VAD requires the Akiki engine") + if model_precision not in (None, ModelPrecision.FP32): + raise ValueError("TEN-VAD requires ModelPrecision.FP32") + self.threshold = self._validate_threshold(0.5 if threshold is None else threshold) + library = TenVadBackend.load_library(library_path) + native_path = prepare_model(model_path, cache_dir) + self._backend = TenVadBackend(native_path, library) + self.engine, self.engine_errors = Engine.AKIKI, {} + self.model_id = Model.TEN_VAD.value + self.model_precision = ModelPrecision.FP32 + self.chunk_samples = self._backend.chunk_samples + self.context_samples = 0 + self.hidden = self._backend.hidden + self._state_shape = self._backend.state_shape + + @staticmethod + def _validate_threshold(threshold) -> float: + value = float(threshold) + if not np.isfinite(value) or not 0 <= value <= 1: + raise ValueError("Threshold must lie between zero and one") + return value def reset(self) -> None: """Start an independent recording with the same model and settings. @@ -265,7 +330,7 @@ def reset(self) -> None: """ self._backend.reset() self.context = np.zeros(self.context_samples, dtype=np.float32) - self.state = np.zeros((1, 1, self.hidden), dtype=np.float32) + self.state = np.zeros(self._state_shape, dtype=np.float32) self.pending = np.empty(0, dtype=np.float32) self.completed_samples = 0 @@ -278,7 +343,8 @@ def push(self, samples, sample_rate: int = SAMPLE_RATE) -> list[SpeechFrame]: One-dimensional, finite mono waveform samples. Lists and NumPy arrays are accepted and converted to float32. Samples should normally be scaled to [-1, 1]; integer PCM is not normalized for - you. Audio is not resampled or mixed down from multiple channels. + you. TEN-VAD requires samples in [-1, 1]. + Audio is not resampled or mixed down from multiple channels. sample_rate : int, default=16000 Input sample rate in Hz. Must be 16000. @@ -312,6 +378,8 @@ def push(self, samples, sample_rate: int = SAMPLE_RATE) -> list[SpeechFrame]: audio = np.asarray(samples, dtype=np.float32) if sample_rate != SAMPLE_RATE or audio.ndim != 1 or not np.isfinite(audio).all(): raise ValueError("Expected finite mono 16 kHz audio samples") + if self._model is Model.TEN_VAD and (np.abs(audio) > 1).any(): + raise ValueError("TEN-VAD requires audio samples in [-1, 1]") audio = np.concatenate([self.pending, audio]) complete = len(audio) // self.chunk_samples * self.chunk_samples frames = [] @@ -323,7 +391,7 @@ def push(self, samples, sample_rate: int = SAMPLE_RATE) -> list[SpeechFrame]: raise RuntimeError("Model returned an invalid speech probability") begin = self.completed_samples self.state = state - self.context = window[-self.context_samples :].copy() + self.context = window[len(window) - self.context_samples :].copy() self.completed_samples += self.chunk_samples frames.append( SpeechFrame( diff --git a/src/akiki/hub.py b/src/akiki/hub.py index 6f47b0d..1be2285 100644 --- a/src/akiki/hub.py +++ b/src/akiki/hub.py @@ -1,8 +1,6 @@ -"""Optional checkpoint retrieval for the current MiniLM/BERT decision runtime. +"""Optional checkpoint retrieval from the Hugging Face Hub. -This helper supports the current engine's file layout. -Retrieving another model family does not add decision-model inference support. -Ordinary VAD and decision inference use local files. +The public download_model helper supports MiniLM/BERT decision models. """ import os @@ -66,6 +64,44 @@ def download_model( ... "sentence-transformers/all-MiniLM-L6-v2", local_files_only=True ... ) """ + root = _download_snapshot( + repo_id, + allow_patterns=_MODEL_FILES, + revision=revision, + cache_dir=cache_dir, + local_dir=local_dir, + local_files_only=local_files_only, + token=token, + ) + if (root / "base_model").is_dir() and all( + (root / name).is_file() for name in _CHECKPOINT_FILES + ): + raise ValueError( + "Hugging Face snapshot contains both a root checkpoint and base_model/; " + "use a clean local_dir to avoid mixing models" + ) + checkpoint = root / "base_model" if (root / "base_model").is_dir() else root + required = [checkpoint / name for name in _CHECKPOINT_FILES] + if checkpoint != root: + required.append(root / "adapter.safetensors") + missing = [str(path) for path in required if not path.is_file()] + if missing: + raise FileNotFoundError( + "Hugging Face snapshot is missing required MiniLM files: " + ", ".join(missing) + ) + return root + + +def _download_snapshot( + repo_id: str, + *, + allow_patterns: tuple[str, ...], + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + local_files_only: bool = False, + token: str | bool | None = None, +) -> Path: try: from huggingface_hub import snapshot_download except ImportError as error: @@ -83,27 +119,9 @@ def download_model( local_dir=Path(local_dir).expanduser() if local_dir is not None else None, local_files_only=local_files_only or offline, token=token, - allow_patterns=list(_MODEL_FILES), + allow_patterns=list(allow_patterns), ) except Exception as error: # Hub HTTP exception types differ across supported dependency versions. raise RuntimeError(f"Could not resolve Hugging Face model {repo_id!r}: {error}") from error - - root = Path(snapshot) - if (root / "base_model").is_dir() and all( - (root / name).is_file() for name in _CHECKPOINT_FILES - ): - raise ValueError( - "Hugging Face snapshot contains both a root checkpoint and base_model/; " - "use a clean local_dir to avoid mixing models" - ) - checkpoint = root / "base_model" if (root / "base_model").is_dir() else root - required = [checkpoint / name for name in _CHECKPOINT_FILES] - if checkpoint != root: - required.append(root / "adapter.safetensors") - missing = [str(path) for path in required if not path.is_file()] - if missing: - raise FileNotFoundError( - "Hugging Face snapshot is missing required MiniLM files: " + ", ".join(missing) - ) - return root + return Path(snapshot) diff --git a/src/akiki/vad.py b/src/akiki/vad.py index d952c77..97093a0 100644 --- a/src/akiki/vad.py +++ b/src/akiki/vad.py @@ -7,6 +7,7 @@ from .audio.vad import VAD as VAD from .audio.vad import Engine as Engine from .audio.vad import FFTPrecision as FFTPrecision +from .audio.vad import Model as Model from .audio.vad import ModelPrecision as ModelPrecision from .audio.vad import SpeechFrame as SpeechFrame @@ -33,4 +34,4 @@ def _deprecation_stacklevel() -> int: stacklevel=_deprecation_stacklevel(), ) -__all__ = ["Engine", "FFTPrecision", "ModelPrecision", "SpeechFrame", "VAD"] +__all__ = ["Engine", "FFTPrecision", "Model", "ModelPrecision", "SpeechFrame", "VAD"] diff --git a/tests/test_ten_vad_model.py b/tests/test_ten_vad_model.py new file mode 100644 index 0000000..7dbc64b --- /dev/null +++ b/tests/test_ten_vad_model.py @@ -0,0 +1,78 @@ +"""Test TEN-VAD source selection and native cache handling offline.""" + +import hashlib +import sys +from types import SimpleNamespace + +import pytest + +from akiki.audio import _ten_vad_model + + +def test_pinned_snapshot_is_resolved_without_network(tmp_path, monkeypatch): + calls = [] + + def snapshot_download(**kwargs): + calls.append(kwargs) + return tmp_path + + sources = { + _ten_vad_model._ONNX: b"ONNX model", + _ten_vad_model._COEFFICIENTS: b"frontend coefficients", + } + for name, data in sources.items(): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + monkeypatch.delenv("HF_HUB_OFFLINE", raising=False) + monkeypatch.setitem( + sys.modules, "huggingface_hub", SimpleNamespace(snapshot_download=snapshot_download) + ) + monkeypatch.setattr( + _ten_vad_model, + "_HUB_CHECKSUMS", + {name: hashlib.sha256(data).hexdigest() for name, data in sources.items()}, + ) + + assert _ten_vad_model._read_sources(None) == tuple(sources.values()) + assert calls[0]["repo_id"] == "TEN-framework/ten-vad" + assert calls[0]["revision"] == _ten_vad_model._REVISION + assert calls[0]["local_files_only"] is True + assert calls[0]["allow_patterns"] == list(sources) + + (tmp_path / _ten_vad_model._ONNX).write_bytes(b"changed model") + with pytest.raises(ValueError, match="incorrect checksum"): + _ten_vad_model._read_sources(None) + + +def test_local_onnx_reuses_and_repairs_native_cache(tmp_path, monkeypatch): + model = tmp_path / "src/onnx_model/ten-vad.onnx" + model.parent.mkdir(parents=True) + model.write_bytes(b"local weights") + (model.parent.parent / "coeff.h").write_bytes(b"local coefficients") + calls = [] + payload = _ten_vad_model._NATIVE_MAGIC + bytes(_ten_vad_model._NATIVE_SIZE - 8) + + def convert(weights, coefficients): + calls.append((weights, coefficients)) + return payload + + monkeypatch.setitem(sys.modules, "huggingface_hub", None) + monkeypatch.setattr(_ten_vad_model, "_convert", convert) + + cache = tmp_path / "cache" + cached = _ten_vad_model.prepare_model(model, cache) + assert cached.read_bytes() == payload + assert _ten_vad_model.prepare_model(model, cache) == cached + assert calls == [(b"local weights", b"local coefficients")] + + cached.write_bytes(payload[:-1] + b"\x01") + assert _ten_vad_model.prepare_model(model, cache) == cached + assert cached.read_bytes() == payload + assert len(calls) == 2 + + +def test_ten_vad_rejects_native_files_as_public_input(tmp_path): + with pytest.raises(ValueError, match="must point to its .onnx weights"): + _ten_vad_model.prepare_model(tmp_path / "weights.tnv", tmp_path / "cache") diff --git a/tests/test_vad.py b/tests/test_vad.py index 43eb804..70076a0 100644 --- a/tests/test_vad.py +++ b/tests/test_vad.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from akiki.audio.vad import VAD, Engine, FFTPrecision, ModelPrecision +from akiki.audio.vad import VAD, Engine, FFTPrecision, Model, ModelPrecision def test_primary_engine_uses_the_akiki_name(): @@ -110,7 +110,9 @@ def test_default_is_selected_release_checkpoint(): model_dir = Path(akiki.__file__).parent / "models" metadata = json.loads((model_dir / "akiki-vad.json").read_text()) assert akiki.__version__ == "0.2.2" - assert VAD().model_id == "teacher-only623-v2-s1730" + with VAD() as default, VAD(model=Model.AKIKI_VAD) as named: + assert default.model_id == named.model_id == "teacher-only623-v2-s1730" + assert default.push([0.0] * 256) == named.push([0.0] * 256) assert metadata["origin"]["seed"] == 1730 assert metadata["release_selection"]["checkpoint_window"] == 31 assert ( @@ -119,6 +121,64 @@ def test_default_is_selected_release_checkpoint(): ) +def test_named_ten_vad_streams_with_its_own_state(monkeypatch, tmp_path): + from akiki._engines import Backend + from akiki.audio import _ten_vad, _ten_vad_model + + model_path = tmp_path / "weights.onnx" + native_path = tmp_path / "model.bin" + library_path = tmp_path / "engine.so" + + class FakeTenVad(Backend): + chunk_samples = 256 + hidden = 64 + state_shape = (4, hidden) + + @staticmethod + def load_library(path): + assert path == library_path + return library_path + + def __init__(self, path, library): + assert path == native_path + assert library == library_path + + def infer(self, window, state): + assert window.shape == (self.chunk_samples,) + return 0.25, state + 1 + + def prepare(path, cache): + assert path == model_path + assert cache is None + return native_path + + monkeypatch.setattr(_ten_vad_model, "prepare_model", prepare) + monkeypatch.setattr(_ten_vad, "TenVadBackend", FakeTenVad) + with VAD(model=Model.TEN_VAD, model_path=model_path, library_path=library_path) as detector: + assert detector.model_id == "ten-vad" + assert detector.model_precision is ModelPrecision.FP32 + assert detector.threshold == 0.5 + assert detector.push([0.0] * 128) == [] + with pytest.raises(ValueError, match="requires audio samples"): + detector.push([0.0] * 256 + [1.5]) + assert detector.completed_samples == 0 + assert len(detector.pending) == 128 + frames = detector.push([0.0] * 384) + assert [(frame.start_seconds, frame.end_seconds) for frame in frames] == [ + (0.0, 0.016), + (0.016, 0.032), + ] + assert all(not frame.speech for frame in frames) + assert detector.context.size == 0 + np.testing.assert_array_equal(detector.state, np.full((4, 64), 2)) + detector.reset() + np.testing.assert_array_equal(detector.state, np.zeros((4, 64))) + assert detector.push([0.0] * 512) == frames + + with pytest.raises(RuntimeError, match="closed"): + detector.push([]) + + def test_engines_agree(engine): pytest.importorskip("onnxruntime") audio = np.random.default_rng(42).normal(0, 0.05, 16000).astype(np.float32) @@ -229,6 +289,12 @@ def test_corrupt_native_model_is_not_hidden_by_fallback(tmp_path, precision): ) +@pytest.mark.parametrize("model", ["TEN-VAD", "akiki-vad", "unknown", Engine.AKIKI, None]) +def test_non_enum_model_is_rejected(model): + with pytest.raises(TypeError, match="model must be a Model member"): + VAD(model=model) + + @pytest.mark.parametrize("engine", ["auto", "akiki", "onnx", "pytorch", "unknown", None, 1, []]) def test_non_enum_engine_is_rejected(engine): with pytest.raises(TypeError, match="engine must be an Engine member"): diff --git a/verify_dist.py b/verify_dist.py index 1e0db84..4b74122 100644 --- a/verify_dist.py +++ b/verify_dist.py @@ -131,7 +131,7 @@ def check_metadata(data): ) require( set(metadata.get_all("Provides-Extra") or []) - == {"huggingface", "onnx", "pytorch", "training"}, + == {"huggingface", "onnx", "pytorch", "ten-vad", "training"}, "Missing optional feature extras", ) require( @@ -142,6 +142,8 @@ def check_metadata(data): 'huggingface-hub>=0.24; extra == "huggingface"', 'onnxruntime>=1.17; extra == "onnx"', 'torch>=2.1; extra == "pytorch"', + 'huggingface-hub>=0.24; extra == "ten-vad"', + 'onnx>=1.16; extra == "ten-vad"', 'torch>=2.1; extra == "training"', 'transformers<6,>=4.40; extra == "training"', 'safetensors>=0.4; extra == "training"', From 31bb63370de86e372a110e427a5c9bba13c28c2c Mon Sep 17 00:00:00 2001 From: Toby Jackson Date: Wed, 18 Nov 2026 12:46:25 +0000 Subject: [PATCH 4/4] Test both VAD models on the Akiki and ONNX runtimes --- .github/workflows/publish.yml | 16 +++ README.md | 10 +- docs/releasing.md | 4 + src/akiki/_engines.py | 35 +++-- src/akiki/audio/_ten_vad.py | 57 +++++++- src/akiki/audio/_ten_vad_model.py | 10 +- src/akiki/audio/vad.py | 30 +++-- tests/test_ten_vad_model.py | 7 +- tests/test_vad.py | 2 +- tests/test_vad_runtimes.py | 211 ++++++++++++++++++++++++++++++ 10 files changed, 345 insertions(+), 37 deletions(-) create mode 100644 tests/test_vad_runtimes.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 09cb272..c892aa5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -92,6 +92,12 @@ jobs: - name: Build native engines offline run: python tools/build_native.py --inference-dir .cache/akiki-inference + - name: Build the local TEN-VAD library for runtime tests + run: | + cargo build --offline --locked --release --lib -p akiki-ten-vad-engine \ + --manifest-path .cache/akiki-inference/Cargo.toml \ + --target-dir .cache/native-target + - name: Build portable wheel and source archive run: python -m build --no-isolation @@ -143,6 +149,16 @@ jobs: print("Native wheel: bundled Akiki engines, INT8, base dependencies only") PY + - name: Test both VAD models on both CPU runtimes from the installed wheel + env: + AKIKI_TEST_VAD_RUNTIMES: "1" + run: | + python="$RUNNER_TEMP/akiki-native/bin/python" + wheels=(dist/akiki-*-manylinux_2_34_x86_64.whl) + "$python" -m pip install "${wheels[0]}[onnx,ten-vad]" pytest + AKIKI_TEN_VAD_LIBRARY="$PWD/.cache/native-target/release/libakiki_ten_vad_engine.so" \ + "$python" -m pytest -q -o pythonpath= tests/test_vad_runtimes.py + - name: Test portable wheel with the ONNX extra run: | version="$(python verify_dist.py --version)" diff --git a/README.md b/README.md index 668067c..b9a04dd 100644 --- a/README.md +++ b/README.md @@ -216,9 +216,13 @@ Omit `model_path` to use the supported snapshot already in the Hugging Face cache. Set `HF_HUB_OFFLINE=1` and configure `HF_HUB_CACHE` if you use a custom cache directory. [TEN-VAD model on Hugging Face](https://huggingface.co/TEN-framework/ten-vad) -The first load converts the ONNX weights for the Akiki engine. Later loads -reuse the converted weights in the Akiki cache. The Akiki engine processes -all audio frames. +The first load prepares the model for the local Akiki TEN-VAD library. Later +loads reuse the prepared data in the Akiki cache. The Akiki engine runs the +network by default. + +To run the network in ONNX Runtime, install `akiki[onnx,ten-vad]` and pass +`engine=vad.Engine.ONNX`. TEN-VAD uses the same local Akiki library to +extract audio features for both engines. ### Command line diff --git a/docs/releasing.md b/docs/releasing.md index caa0159..08744e7 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -23,6 +23,10 @@ Its tests use public dependencies and synthetic local checkpoints. Private engine and training checks run on `main` and trusted manual builds. Fork PR jobs do not receive private checkout credentials. +The trusted Linux x86-64 build tests Akiki-VAD and TEN-VAD on the Akiki engine +and ONNX Runtime from an installed wheel. It builds the TEN-VAD library from +the pinned engine checkout. The tests generate the TEN-VAD ONNX fixture locally. + Use version tags to record releases and release candidates. For example, use `v0.2.1` for version `0.2.1` and `v0.2.2rc1` for `0.2.2rc1`. Do not create an RC branch or release branch for each version. diff --git a/src/akiki/_engines.py b/src/akiki/_engines.py index 2be8aef..be0da78 100644 --- a/src/akiki/_engines.py +++ b/src/akiki/_engines.py @@ -57,24 +57,31 @@ def close(self): self.closed = True +def _require_onnx_runtime(): + try: + import onnxruntime as ort + except (ImportError, OSError) as error: + raise EngineUnavailable( + "Install the ONNX engine with: pip install 'akiki[onnx]'" + ) from error + return ort + + +def _create_onnx_session(ort, model: str | bytes): + options = ort.SessionOptions() + options.intra_op_num_threads = options.inter_op_num_threads = 1 + try: + return ort.InferenceSession(model, sess_options=options, providers=["CPUExecutionProvider"]) + except Exception as error: + raise EngineUnavailable(f"ONNX Runtime could not load the model: {error}") from error + + class OnnxBackend(Backend): def __init__(self, path, metadata): - try: - import onnxruntime as ort - except (ImportError, OSError) as error: - raise EngineUnavailable( - "Install the ONNX engine with: pip install 'akiki[onnx]'" - ) from error + ort = _require_onnx_runtime() path = path.with_suffix(".onnx") verify_file(path, metadata["onnx_sha256"]) - options = ort.SessionOptions() - options.intra_op_num_threads = options.inter_op_num_threads = 1 - try: - self.session = ort.InferenceSession( - str(path), sess_options=options, providers=["CPUExecutionProvider"] - ) - except Exception as error: - raise EngineUnavailable(f"ONNX Runtime could not load the model: {error}") from error + self.session = _create_onnx_session(ort, str(path)) if {v.name for v in self.session.get_inputs()} != {"audio_with_context", "h"}: raise ValueError("Unexpected ONNX inputs") diff --git a/src/akiki/audio/_ten_vad.py b/src/akiki/audio/_ten_vad.py index 1292592..b82b87a 100644 --- a/src/akiki/audio/_ten_vad.py +++ b/src/akiki/audio/_ten_vad.py @@ -1,4 +1,4 @@ -"""Local binding for the Akiki TEN-VAD engine.""" +"""TEN-VAD inference with the Akiki audio frontend.""" import ctypes import weakref @@ -6,7 +6,13 @@ import numpy as np -from .._engines import Backend, EngineUnavailable, library_candidates +from .._engines import ( + Backend, + EngineUnavailable, + _create_onnx_session, + _require_onnx_runtime, + library_candidates, +) class TenVadBackend(Backend): @@ -68,3 +74,50 @@ def infer(self, window, state): def close(self): super().close() self._finalizer() + + +class TenVadOnnxBackend(TenVadBackend): + # The upstream streaming wrapper resets recurrent state every 30 seconds. + _reset_frames = 1875 + + def __init__(self, path: Path, lib, model_data: bytes): + session = _create_onnx_session(_require_onnx_runtime(), model_data) + self._input_names = tuple(value.name for value in session.get_inputs()) + if len(self._input_names) != 5 or len(session.get_outputs()) != 5: + raise ValueError("Expected TEN-VAD features, probability, and four recurrent states") + super().__init__(path, lib) + self.session = session + self._features = np.empty((1, 3, 41), dtype=np.float32) + self._frames = 0 + + @staticmethod + def _bind(lib): + TenVadBackend._bind(lib) + pointer = ctypes.c_void_p + lib.akiki_ten_vad_features.argtypes = [pointer, pointer, pointer] + lib.akiki_ten_vad_features.restype = ctypes.c_int + + def reset(self): + super().reset() + self._frames = 0 + + def infer(self, window, state): + if window.shape != (self.chunk_samples,): + raise ValueError("TEN-VAD requires exactly 256 samples per frame") + chunk = np.ascontiguousarray(window, dtype=np.float32) + if self.lib.akiki_ten_vad_features( + self.handle, chunk.ctypes.data, self._features.ctypes.data + ): + raise RuntimeError("Akiki TEN-VAD frontend could not extract audio features") + inputs = dict(zip(self._input_names, (self._features, *state[:, None, :]), strict=True)) + probability, *states = self.session.run(None, inputs) + state = np.stack(states).reshape(self.state_shape) + self._frames += 1 + if self._frames == self._reset_frames: + state.fill(0) + self._frames = 0 + return probability, state + + def close(self): + super().close() + self.session = None diff --git a/src/akiki/audio/_ten_vad_model.py b/src/akiki/audio/_ten_vad_model.py index 74a0a3c..a3aede2 100644 --- a/src/akiki/audio/_ten_vad_model.py +++ b/src/akiki/audio/_ten_vad_model.py @@ -180,8 +180,10 @@ def _convert(model_data: bytes, coefficients: bytes) -> bytes: return _NATIVE_MAGIC + b"".join(array.astype(" Path: - """Return a checked native cache entry for the selected TEN-VAD ONNX files.""" +def prepare_model( + model_path: str | Path | None, cache_dir: str | Path | None +) -> tuple[Path, bytes]: + """Return the checked native cache entry and its source ONNX bytes.""" weights, coefficients = _read_sources(model_path) key = hashlib.sha256( _CONVERTER_VERSION @@ -199,7 +201,7 @@ def prepare_model(model_path: str | Path | None, cache_dir: str | Path | None) - and cached.startswith(_NATIVE_MAGIC) and hashlib.sha256(cached).hexdigest().encode() == checksum.read_bytes() ): - return model + return model, weights except OSError: pass @@ -212,4 +214,4 @@ def prepare_model(model_path: str | Path | None, cache_dir: str | Path | None) - raise RuntimeError( "Cannot write the TEN-VAD model cache. Set cache_dir or AKIKI_CACHE_DIR." ) from error - return model + return model, weights diff --git a/src/akiki/audio/vad.py b/src/akiki/audio/vad.py index 70eff85..46aea13 100644 --- a/src/akiki/audio/vad.py +++ b/src/akiki/audio/vad.py @@ -87,7 +87,8 @@ class VAD: The Akiki engine is the primary engine. For Akiki-VAD, ``Engine.AUTO`` tries it first, then ONNX, then PyTorch. The Akiki engine uses INT8 by default for Akiki-VAD. The compatibility engines use FP32. - TEN-VAD uses the Akiki engine with FP32. + TEN-VAD uses FP32 and selects the Akiki engine by default. + With ``Engine.ONNX``, the Akiki frontend supplies features to ONNX Runtime. See ``VAD.__init__`` for all constructor options. Use the detector as a context manager or call ``close()`` when finished. @@ -190,9 +191,11 @@ def __init__( ONNX and PyTorch are compatibility fallbacks, in that order. An explicitly selected engine must succeed; it does not fall back. Pass an ``Engine`` member, such as ``Engine.AKIKI``. + For TEN-VAD, select ``Engine.ONNX`` to run the network in ONNX Runtime. + Both TEN-VAD engines require the local Akiki TEN-VAD library. library_path : str or pathlib.Path, optional - Specific local Akiki shared library to load. Affects only the Akiki - engine. If omitted, use ``AKIKI_VAD_LIBRARY`` or the bundled + Specific local Akiki shared library to load. For Akiki-VAD, this + applies to the Akiki engine. If omitted, use ``AKIKI_VAD_LIBRARY`` or the bundled Akiki-VAD library. For TEN-VAD, use ``AKIKI_TEN_VAD_LIBRARY`` or pass its library path here. model_precision : ModelPrecision, optional @@ -290,18 +293,25 @@ def _load_akiki_vad( def _load_ten_vad( self, model_path, threshold, engine, library_path, model_precision, cache_dir ): - from ._ten_vad import TenVadBackend + from ._ten_vad import TenVadBackend, TenVadOnnxBackend from ._ten_vad_model import prepare_model - if engine not in (Engine.AUTO, Engine.AKIKI): - raise ValueError("TEN-VAD requires the Akiki engine") + if engine is Engine.PYTORCH: + raise ValueError("TEN-VAD supports the Akiki and ONNX engines") if model_precision not in (None, ModelPrecision.FP32): raise ValueError("TEN-VAD requires ModelPrecision.FP32") self.threshold = self._validate_threshold(0.5 if threshold is None else threshold) - library = TenVadBackend.load_library(library_path) - native_path = prepare_model(model_path, cache_dir) - self._backend = TenVadBackend(native_path, library) - self.engine, self.engine_errors = Engine.AKIKI, {} + backend = TenVadOnnxBackend if engine is Engine.ONNX else TenVadBackend + library = backend.load_library(library_path) + native_path, model_data = prepare_model(model_path, cache_dir) + if engine is Engine.ONNX: + self._backend = backend(native_path, library, model_data) + self.engine = Engine.ONNX + self.session = self._backend.session + else: + self._backend = backend(native_path, library) + self.engine = Engine.AKIKI + self.engine_errors = {} self.model_id = Model.TEN_VAD.value self.model_precision = ModelPrecision.FP32 self.chunk_samples = self._backend.chunk_samples diff --git a/tests/test_ten_vad_model.py b/tests/test_ten_vad_model.py index 7dbc64b..fa8d740 100644 --- a/tests/test_ten_vad_model.py +++ b/tests/test_ten_vad_model.py @@ -62,13 +62,14 @@ def convert(weights, coefficients): monkeypatch.setattr(_ten_vad_model, "_convert", convert) cache = tmp_path / "cache" - cached = _ten_vad_model.prepare_model(model, cache) + cached, source = _ten_vad_model.prepare_model(model, cache) + assert source == b"local weights" assert cached.read_bytes() == payload - assert _ten_vad_model.prepare_model(model, cache) == cached + assert _ten_vad_model.prepare_model(model, cache) == (cached, source) assert calls == [(b"local weights", b"local coefficients")] cached.write_bytes(payload[:-1] + b"\x01") - assert _ten_vad_model.prepare_model(model, cache) == cached + assert _ten_vad_model.prepare_model(model, cache) == (cached, source) assert cached.read_bytes() == payload assert len(calls) == 2 diff --git a/tests/test_vad.py b/tests/test_vad.py index 70076a0..ec8b45f 100644 --- a/tests/test_vad.py +++ b/tests/test_vad.py @@ -150,7 +150,7 @@ def infer(self, window, state): def prepare(path, cache): assert path == model_path assert cache is None - return native_path + return native_path, b"fake ONNX data" monkeypatch.setattr(_ten_vad_model, "prepare_model", prepare) monkeypatch.setattr(_ten_vad, "TenVadBackend", FakeTenVad) diff --git a/tests/test_vad_runtimes.py b/tests/test_vad_runtimes.py new file mode 100644 index 0000000..e7a02f6 --- /dev/null +++ b/tests/test_vad_runtimes.py @@ -0,0 +1,211 @@ +"""Run VAD inference with the real CPU engines and local test data.""" + +import hashlib +import os + +import numpy as np +import pytest + +from akiki.audio import _ten_vad_model, vad + +pytestmark = pytest.mark.skipif( + os.environ.get("AKIKI_TEST_VAD_RUNTIMES") != "1", + reason="Run the native and ONNX Runtime matrix with AKIKI_TEST_VAD_RUNTIMES=1", +) + + +@pytest.fixture +def ten_vad_onnx(tmp_path, monkeypatch): + """Generate a TEN-shaped ONNX graph with synthetic weights.""" + import onnx + from onnx import TensorProto, helper, numpy_helper + + rng = np.random.default_rng(27) + nodes, initializers = [], [] + + def initializer(name, values): + initializers.append(numpy_helper.from_array(np.asarray(values), name=name)) + return name + + def weight(name, shape): + return initializer(name, rng.uniform(-0.15, 0.15, shape).astype(np.float32)) + + def node(kind, inputs, output, **attributes): + nodes.append(helper.make_node(kind, inputs, [output], **attributes)) + return output + + prefix = "StatefulPartitionedCall/vad_model/" + axis_zero = initializer("axis_zero", np.array([0], np.int64)) + axis_one = initializer("axis_one", np.array([1], np.int64)) + features = node("Unsqueeze", ["features", axis_one], "features_nchw") + for layer, index, depth_shape, point, point_shape, group, pads in ( + ( + "separable_conv2d", + 178, + (1, 1, 3, 3), + "separable_conv2d/ReadVariableOp_1:0", + (16, 1, 1, 1), + 1, + [0, 0, 0, 0], + ), + ( + "separable_conv1d", + 179, + (16, 1, 1, 3), + "ExpandDims_2:0", + (16, 16, 1, 1), + 16, + [0, 1, 0, 1], + ), + ( + "separable_conv1d_1", + 180, + (16, 1, 1, 3), + "ExpandDims_2:0", + (16, 16, 1, 1), + 16, + [0, 0, 0, 1], + ), + ): + depth = weight(f"const_fold_opt__{index}", depth_shape) + depth_out = node( + "Conv", + [features, depth], + f"depth_{index}", + group=group, + pads=pads, + strides=[1, 1 if index == 178 else 2], + ) + projection = weight(f"{prefix}{layer}/{point}", point_shape) + bias = weight(f"{prefix}{layer}/BiasAdd/ReadVariableOp:0", (16,)) + conv = node("Conv", [depth_out, projection, bias], f"point_{index}") + features = node("Relu", [conv], f"relu_{index}") + if index == 178: + features = node("MaxPool", [features], "pooled", kernel_shape=[1, 3], strides=[1, 2]) + + ordered = node("Transpose", [features], "frequency_major", perm=[0, 2, 3, 1]) + lstm_shape = initializer("lstm_shape", np.array([1, 1, 80], np.int64)) + features = node("Reshape", [ordered, lstm_shape], "lstm_input") + state_names = ("hidden1", "cell1", "hidden2", "cell2") + hidden_outputs = [] + for index, (weight_name, recurrent_name, bias_name, inputs) in enumerate( + (("W0__70", "R0__71", "B0__72", 80), ("W0__99", "R0__100", "B0__101", 64)) + ): + hidden_name, cell_name = state_names[2 * index : 2 * index + 2] + hidden = node("Unsqueeze", [hidden_name, axis_zero], f"{hidden_name}_3d") + cell = node("Unsqueeze", [cell_name, axis_zero], f"{cell_name}_3d") + hidden_out, cell_out = f"{hidden_name}_out_3d", f"{cell_name}_out_3d" + nodes.append( + helper.make_node( + "LSTM", + [ + features, + weight(weight_name, (1, 256, inputs)), + weight(recurrent_name, (1, 256, 64)), + weight(bias_name, (1, 512)), + "", + hidden, + cell, + ], + [f"lstm_sequence{index}", hidden_out, cell_out], + hidden_size=64, + ) + ) + node("Squeeze", [hidden_out, axis_zero], f"{hidden_name}_out") + node("Squeeze", [cell_out, axis_zero], f"{cell_name}_out") + hidden_outputs.append(hidden_out) + features = hidden_out + + features = node("Concat", hidden_outputs[::-1], "joined", axis=2) + for layer, inputs, outputs in (("dense_3", 128, 32), ("dense_5", 32, 1)): + projection = weight(f"{prefix}{layer}/Tensordot/ReadVariableOp:0", (inputs, outputs)) + bias = weight(f"{prefix}{layer}/BiasAdd/ReadVariableOp:0", (outputs,)) + features = node("MatMul", [features, projection], f"{layer}_projection") + features = node("Add", [features, bias], f"{layer}_output") + if layer == "dense_3": + features = node("Relu", [features], "dense_relu") + node("Sigmoid", [features], "probability") + + def value(name, shape): + return helper.make_tensor_value_info(name, TensorProto.FLOAT, shape) + + graph = helper.make_graph( + nodes, + "synthetic-ten-vad", + [value("features", [1, 3, 41]), *(value(name, [1, 64]) for name in state_names)], + [value("probability", [1, 1, 1]), *(value(f"{name}_out", [1, 64]) for name in state_names)], + initializer=initializers, + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)], ir_version=8) + onnx.checker.check_model(model) + path = tmp_path / "ten-vad.onnx" + onnx.save(model, path) + + # The production converter pins the original graph. Permit only this test graph here. + for tensor in graph.initializer: + if tensor.data_type == TensorProto.FLOAT: + tensor.ClearField("raw_data") + monkeypatch.setattr( + _ten_vad_model, "_GRAPH_SHA256", hashlib.sha256(graph.SerializeToString()).hexdigest() + ) + + window = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(768) / 768) + arrays = ( + ("AUP_AED_FEATURE_MEANS", np.zeros(41)), + ("AUP_AED_FEATURE_STDS", np.ones(41)), + ("AUP_AED_STFTWindow_Hann768", window), + ) + (tmp_path / "coeff.h").write_text( + "".join( + f"float {name}[{len(values)}] = {{{','.join(map(str, values))}}};\n" + for name, values in arrays + ) + ) + return path + + +@pytest.mark.parametrize("model", list(vad.Model)) +def test_vad_scores_and_resets_on_both_cpu_runtimes(model, request, tmp_path): + is_ten = model is vad.Model.TEN_VAD + model_path = request.getfixturevalue("ten_vad_onnx") if is_ten else None + options = dict( + model=model, + model_path=model_path, + model_precision=vad.ModelPrecision.FP32, + cache_dir=tmp_path / "cache", + ) + boundary = (1875 if is_ten else 9) * 256 + audio = np.random.default_rng(28).uniform(-0.3, 0.3, boundary + 256 + 17).astype(np.float32) + with ( + vad.VAD(engine=vad.Engine.AKIKI, **options) as native, + vad.VAD(engine=vad.Engine.ONNX, **options) as onnx, + ): + assert native.engine is vad.Engine.AKIKI + assert onnx.engine is vad.Engine.ONNX + assert onnx.session.get_providers() == ["CPUExecutionProvider"] + results = [] + for detector in (native, onnx): + assert detector.push(audio[:37]) == [] + frames = detector.push(audio[37 : boundary - 256]) + assert np.any(detector.state) + frames += detector.push(audio[boundary - 256 : boundary]) + if is_ten: + np.testing.assert_array_equal(detector.state, 0) + frames += detector.push(audio[boundary:]) + assert len(detector.pending) == 17 + assert np.any(detector.state) + assert len(frames) == boundary // 256 + 1 + assert frames[0].start_seconds == 0 + assert frames[-1].end_seconds == (boundary + 256) / 16000 + detector.reset() + assert detector.push(audio[:256]) == frames[:1] + results.append(frames) + + native_frames, onnx_frames = results + np.testing.assert_allclose( + [frame.probability for frame in native_frames], + [frame.probability for frame in onnx_frames], + atol=2e-5, + rtol=1e-5, + ) + assert [frame.speech for frame in native_frames] == [frame.speech for frame in onnx_frames]