diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 245a599..c892aa5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -92,6 +92,12 @@ jobs: - name: Build native engines offline run: python tools/build_native.py --inference-dir .cache/akiki-inference + - name: Build the local TEN-VAD library for runtime tests + run: | + cargo build --offline --locked --release --lib -p akiki-ten-vad-engine \ + --manifest-path .cache/akiki-inference/Cargo.toml \ + --target-dir .cache/native-target + - name: Build portable wheel and source archive run: python -m build --no-isolation @@ -127,21 +133,32 @@ jobs: import importlib.util from pathlib import Path import akiki + from akiki.audio.vad import VAD, Engine, FFTPrecision, ModelPrecision from akiki.decision import _load_library import numpy as np assert importlib.util.find_spec("onnxruntime") is None assert importlib.util.find_spec("torch") is None - with akiki.VAD() as vad: - assert vad.engine is akiki.Engine.AKIKI - assert vad.model_precision is akiki.ModelPrecision.INT8 - assert vad.fft_precision is akiki.FFTPrecision.FP32 + with VAD() as vad: + assert vad.engine is Engine.AKIKI + assert vad.model_precision is ModelPrecision.INT8 + assert vad.fft_precision is FFTPrecision.FP32 assert Path(vad._backend.lib._name).parent == Path(akiki.__file__).parent / "native" assert len(vad.push(np.zeros(256 * 3 + 17, np.float32))) == 3 assert _load_library()[1].parent == Path(akiki.__file__).parent / "native" print("Native wheel: bundled Akiki engines, INT8, base dependencies only") PY + - name: Test both VAD models on both CPU runtimes from the installed wheel + env: + AKIKI_TEST_VAD_RUNTIMES: "1" + run: | + python="$RUNNER_TEMP/akiki-native/bin/python" + wheels=(dist/akiki-*-manylinux_2_34_x86_64.whl) + "$python" -m pip install "${wheels[0]}[onnx,ten-vad]" pytest + AKIKI_TEN_VAD_LIBRARY="$PWD/.cache/native-target/release/libakiki_ten_vad_engine.so" \ + "$python" -m pytest -q -o pythonpath= tests/test_vad_runtimes.py + - name: Test portable wheel with the ONNX extra run: | version="$(python verify_dist.py --version)" @@ -151,7 +168,7 @@ jobs: "$RUNNER_TEMP/akiki-onnx/bin/python" -I - <<'PY' import importlib.util import numpy as np - from akiki import Engine, ModelPrecision, VAD + from akiki.audio.vad import VAD, Engine, ModelPrecision assert importlib.util.find_spec("torch") is None with VAD() as vad: @@ -253,6 +270,7 @@ jobs: from pathlib import Path import akiki import numpy as np + from akiki.audio.vad import VAD, Engine, ModelPrecision from akiki.decision import _load_library assert akiki.__version__ == os.environ["AKIKI_EXPECTED_VERSION"] @@ -262,8 +280,8 @@ jobs: assert {p.name for p in native.iterdir()} == { "libakiki_vad_engine.so", "libakiki_minilm_engine.so" } - with akiki.VAD() as vad: - assert vad.engine is akiki.Engine.AKIKI and vad.model_precision is akiki.ModelPrecision.INT8 + with VAD() as vad: + assert vad.engine is Engine.AKIKI and vad.model_precision is ModelPrecision.INT8 assert Path(vad._backend.lib._name).parent == native assert len(vad.push(np.zeros(256 * 3 + 17, np.float32))) == 3 assert _load_library()[1].parent == native @@ -398,15 +416,16 @@ jobs: from pathlib import Path import akiki import numpy as np + from akiki.audio.vad import VAD, Engine, FFTPrecision, ModelPrecision from akiki.decision import _load_library assert importlib.util.find_spec('onnxruntime') is None assert importlib.util.find_spec('torch') is None native = Path(akiki.__file__).parent / 'native' - with akiki.VAD() as vad: - assert vad.engine is akiki.Engine.AKIKI - assert vad.model_precision is akiki.ModelPrecision.INT8 - assert vad.fft_precision is akiki.FFTPrecision.FP32 + with VAD() as vad: + assert vad.engine is Engine.AKIKI + assert vad.model_precision is ModelPrecision.INT8 + assert vad.fft_precision is FFTPrecision.FP32 assert Path(vad._backend.lib._name).parent == native assert len(vad.push(np.zeros(256 * 3 + 17, np.float32))) == 3 assert _load_library()[1].parent == native diff --git a/README.md b/README.md index fee1705..1483f11 100644 --- a/README.md +++ b/README.md @@ -115,13 +115,13 @@ PyTorch is the second VAD compatibility fallback. ## Python usage -Use `akiki.vad` for VAD and `akiki.decision` for decision models. +Use `akiki.audio.vad` for VAD and `akiki.decision` for decision models. Use `akiki.vision.object_detection` for object detection. The training and teacher modules support the decision-model workflow: | Module | Main interface | Supporting types | | --- | --- | --- | -| `akiki.vad` | `VAD` for speech detection | `SpeechFrame`, `Engine`, `ModelPrecision`, `FFTPrecision` | +| `akiki.audio.vad` | `VAD` for speech detection | `SpeechFrame`, `Model`, `Engine`, `ModelPrecision`, `FFTPrecision` | | `akiki.vision.object_detection` | `ObjectDetector` for object detection | `Detection` | | `akiki.decision` | `DecisionModel` for specialist decisions | `BaseModel`, `Specialist`, `AffineLayer`, `TrainedSpecialist`, `SpecialistEvaluation`, `TrainingAdvice`, `list_base_models`, `Classification`, `ModelPrecision` | | `akiki.training` | Classifier training with a private companion | The installed companion's API, including `LocalModel`, `ModelFamily`, and `Trainer` | @@ -130,8 +130,9 @@ The training and teacher modules support the decision-model workflow: Import the primary interfaces from their modules: ```python -from akiki.vad import VAD +from akiki.audio import vad from akiki.decision import DecisionModel, Specialist +from akiki.vision import object_detection ``` Import configuration enums and result types from the relevant module. @@ -139,7 +140,7 @@ VAD and decision inference have distinct `ModelPrecision` enums. Their supported formats differ. `MiniLM` is the lower-level interface for direct embeddings and classification with the currently supported MiniLM/BERT family. -The existing `akiki.VAD` and `akiki.MiniLM` shortcuts remain available. +The `akiki.MiniLM` shortcut remains available. The existing top-level result types and VAD enums remain available as compatibility aliases. Import `training` explicitly when its optional companion package is installed; the base package does not load it. @@ -150,7 +151,9 @@ modules remain accessible but are omitted from that list. The decision-model module is named `akiki.decision`. Existing imports from `akiki.minilm` remain available for compatibility and use the same classes and precision enum. Named top-level compatibility imports remain available. -Wildcard imports expose only `VAD`, `MiniLM`, `vad`, `decision`, `vision`, and `__version__`. +`akiki.vad` and `akiki.VAD` are deprecated and will be removed in version 0.3.0. +Use `from akiki.audio import vad` and call `vad.VAD()`. +Wildcard imports expose only `VAD`, `MiniLM`, `audio`, `decision`, `vision`, and `__version__`. ## Object detection @@ -189,21 +192,21 @@ Akiki accepts mono audio sampled at 16,000 Hz. ```python import soundfile as sf -from akiki import VAD +from akiki.audio import vad audio, sample_rate = sf.read("recording.wav", dtype="float32") -with VAD() as vad: - for frame in vad.push(audio, sample_rate=sample_rate): +with vad.VAD() as detector: + for frame in detector.push(audio, sample_rate=sample_rate): print(frame.start_seconds, frame.end_seconds, frame.probability, frame.speech) ``` -For live audio, call `vad.push()` repeatedly with successive audio buffers. +For live audio, call `detector.push()` repeatedly with successive audio buffers. Buffers can have any length; incomplete frames are retained for the next call. -Call `vad.reset()` before starting a new recording. A final incomplete frame is +Call `detector.reset()` before starting a new recording. A final incomplete frame is not padded or scored. -Use `VAD(threshold=0.6)` to set the speech decision threshold. Every frame still +Use `vad.VAD(threshold=0.6)` to set the speech decision threshold. Every frame still includes its probability, regardless of the threshold. The public API includes docstrings for editor hints and interactive help: @@ -211,17 +214,50 @@ The public API includes docstrings for editor hints and interactive help: ```python import akiki -help(akiki) # Overview of vad, decision, training, and teacher -help(akiki.vad) # Speech detection, configuration, and result types +help(akiki) # Overview of audio, decision, training, and teacher +help(akiki.audio.vad) # Speech detection, configuration, and result types help(akiki.decision) # Decision models, specialists, and result types help(akiki.decision.DecisionModel) # Base-model selection and specialist inference help(akiki.decision.Specialist.train) # Training parameters and saved exports -help(akiki.VAD) # Configuration, attributes, and streaming example -help(akiki.VAD.__init__) # Constructor parameters and defaults -help(akiki.VAD.push) # Audio input, buffering, results, and errors -help(akiki.vad.SpeechFrame) # Result fields and timestamp units +help(akiki.audio.vad.VAD) # Configuration, attributes, and streaming example +help(akiki.audio.vad.VAD.__init__) # Constructor parameters and defaults +help(akiki.audio.vad.VAD.push) # Audio input, buffering, results, and errors +help(akiki.audio.vad.SpeechFrame) # Result fields and timestamp units +``` + +### Select the VAD model + +`vad.VAD()` and `vad.VAD(model=vad.Model.AKIKI_VAD)` load the bundled Akiki-VAD model. + +To use TEN-VAD, install `akiki[ten-vad]` and set `AKIKI_TEN_VAD_LIBRARY` to +your local Akiki TEN-VAD shared library. You can also pass `library_path`. + +```python +from akiki.audio import vad + +with vad.VAD( + model=vad.Model.TEN_VAD, + model_path="snapshot/src/onnx_model/ten-vad.onnx", +) as detector: + frames = detector.push([0.0] * 256) ``` +Keep the matching `src/coeff.h` in the snapshot. For a standalone ONNX file, +keep `coeff.h` next to the file. The coefficients supply the audio frontend. +TEN-VAD requires mono 16 kHz samples in `[-1, 1]` and runs in FP32. + +Omit `model_path` to use the supported snapshot already in the Hugging Face +cache. Set `HF_HUB_OFFLINE=1` and configure `HF_HUB_CACHE` if you use a +custom cache directory. [TEN-VAD model on Hugging Face](https://huggingface.co/TEN-framework/ten-vad) + +The first load prepares the model for the local Akiki TEN-VAD library. Later +loads reuse the prepared data in the Akiki cache. The Akiki engine runs the +network by default. + +To run the network in ONNX Runtime, install `akiki[onnx,ten-vad]` and pass +`engine=vad.Engine.ONNX`. TEN-VAD uses the same local Akiki library to +extract audio features for both engines. + ### Command line ```bash @@ -320,7 +356,7 @@ Choose another precision when creating the detector. For example, to use the original FP32 weights: ```python -from akiki.vad import VAD, Engine, ModelPrecision +from akiki.audio.vad import VAD, Engine, ModelPrecision with VAD(engine=Engine.AKIKI, model_precision=ModelPrecision.FP32) as vad: frames = vad.push(audio, sample_rate=sample_rate) @@ -342,7 +378,7 @@ akiki vad recording.wav --model-precision fp32 | `ModelPrecision.INT8_FP32_INPUT` | INT8 weights; first projection keeps FP32 activations | The Python VAD API requires `ModelPrecision` and `FFTPrecision` enum members, -both exported from `akiki.vad`. Omit `model_precision` or pass `None` to use the +both exported from `akiki.audio.vad`. Omit `model_precision` or pass `None` to use the selected engine's default. CLI precision options retain their lowercase names. The FFT stays FP32: `fft_precision` accepts only `FFTPrecision.FP32`. Recurrent state, diff --git a/docs/releasing.md b/docs/releasing.md index d68deaa..08744e7 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -4,7 +4,7 @@ The public PyPI description comes from `README.md`. Keep it focused on package features and usage. Build and publishing instructions belong here. Akiki provides VAD and decision models. -The primary classes are `akiki.vad.VAD` and `akiki.decision.DecisionModel`. +The primary classes are `akiki.audio.vad.VAD` and `akiki.decision.DecisionModel`. Current native decision inference supports MiniLM/BERT exports. The library filenames below retain the name of that implementation. @@ -23,6 +23,10 @@ Its tests use public dependencies and synthetic local checkpoints. Private engine and training checks run on `main` and trusted manual builds. Fork PR jobs do not receive private checkout credentials. +The trusted Linux x86-64 build tests Akiki-VAD and TEN-VAD on the Akiki engine +and ONNX Runtime from an installed wheel. It builds the TEN-VAD library from +the pinned engine checkout. The tests generate the TEN-VAD ONNX fixture locally. + Use version tags to record releases and release candidates. For example, use `v0.2.1` for version `0.2.1` and `v0.2.2rc1` for `0.2.2rc1`. Do not create an RC branch or release branch for each version. @@ -195,9 +199,9 @@ target platforms before adding those wheels to `dist/`. Validate the Markdown and metadata with `twine check` for every new artifact. Test a clean install with both portable and platform wheels available and confirm that `VAD().engine is Engine.AKIKI`, loading the installed library -(`Engine` is exported from `akiki`). +(`Engine` is exported from `akiki.audio.vad`). Confirm that `VAD().model_precision is ModelPrecision.INT8` and -`VAD().fft_precision is FFTPrecision.FP32` (both enums are exported from `akiki.vad`), +`VAD().fft_precision is FFTPrecision.FP32` (both enums are exported from `akiki.audio.vad`), and that `akiki.decision._load_library()` loads the installed MiniLM library. When no precision is specified, ONNX and PyTorch compatibility engines use FP32; an explicit non-FP32 request must require the Akiki engine. diff --git a/docs/style.md b/docs/style.md index 7c8856e..7098c72 100644 --- a/docs/style.md +++ b/docs/style.md @@ -19,7 +19,7 @@ ONNX and PyTorch are VAD compatibility fallbacks. Do not use the implementation language as the engine name. Describe the two primary interfaces as VAD and decision models. -Use `akiki.vad.VAD` and `akiki.decision.DecisionModel` in API overviews. +Use `akiki.audio.vad.VAD` and `akiki.decision.DecisionModel` in API overviews. Describe MiniLM/BERT as the family supported by the current decision-model runtime. Use `MiniLM` for direct access to that family. Keep model discovery, training support, and inference support separate. @@ -36,7 +36,7 @@ Use separate enum classes when parameters accept different sets of choices. For example, VAD model precision and FFT precision have different domains: ```python -from akiki.vad import VAD, FFTPrecision, ModelPrecision +from akiki.audio.vad import VAD, FFTPrecision, ModelPrecision with VAD( model_precision=ModelPrecision.INT8, @@ -46,7 +46,7 @@ with VAD( ``` VAD and decision models also have distinct model precision domains. Use -`akiki.vad.ModelPrecision` with `VAD` and `akiki.decision.ModelPrecision` with +`akiki.audio.vad.ModelPrecision` with `VAD` and `akiki.decision.ModelPrecision` with `DecisionModel` or `MiniLM`. Do not share enum types just because some serialized values match. @@ -117,7 +117,7 @@ Check the executable examples in the public inference docstrings: ```bash HF_HUB_OFFLINE=1 python -m pytest -q --doctest-modules \ - src/akiki/__init__.py src/akiki/vad.py src/akiki/decision.py src/akiki/hub.py + src/akiki/__init__.py src/akiki/audio/vad.py src/akiki/decision.py src/akiki/hub.py ``` VAD examples require a working local engine. Examples with placeholder text-model diff --git a/pyproject.toml b/pyproject.toml index bd9f768..0c5b0e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = ["numpy>=1.24", "soundfile>=0.12"] huggingface = ["huggingface-hub>=0.24"] onnx = ["onnxruntime>=1.17"] pytorch = ["torch>=2.1"] +ten-vad = ["huggingface-hub>=0.24", "onnx>=1.16"] training = ["torch>=2.1", "transformers>=4.40,<6", "safetensors>=0.4"] [project.urls] @@ -45,7 +46,7 @@ include-package-data = false [tool.setuptools.packages.find] where = ["src"] -include = ["akiki", "akiki.vision"] +include = ["akiki", "akiki.audio", "akiki.vision"] namespaces = false [tool.setuptools.package-data] diff --git a/src/akiki/__init__.py b/src/akiki/__init__.py index 9b1aeed..df2cb56 100644 --- a/src/akiki/__init__.py +++ b/src/akiki/__init__.py @@ -1,13 +1,15 @@ """Local voice activity detection, object detection, and decision models. -The primary interfaces are ``akiki.vad.VAD`` and ``akiki.decision.DecisionModel``. +The primary interfaces are ``akiki.audio.vad.VAD``, +``akiki.vision.object_detection.ObjectDetector``, and ``akiki.decision.DecisionModel``. Training and teacher review support the decision-model workflow. The public API has five areas. -``akiki.vad`` +``akiki.audio.vad`` Streaming voice activity detection with ``VAD``. Results are - ``SpeechFrame`` objects. Configure inference with this module's ``Engine``, - ``ModelPrecision``, and ``FFTPrecision`` enums. The VAD model is bundled. + ``SpeechFrame`` objects. Select the model with ``Model``. Configure inference + with ``Engine``, ``ModelPrecision``, and ``FFTPrecision``. + The default Akiki-VAD model is bundled. ``akiki.vision.object_detection`` Use ``ObjectDetector`` to find objects in RGB images. Each ``Detection`` gives a family, score, and box. @@ -29,7 +31,7 @@ server. Propose supplemental examples for retraining with ``Teacher``. This module is loaded only when explicitly imported. -``VAD`` and ``MiniLM`` are also available directly from ``akiki`` as shortcuts. +``MiniLM`` is also available directly from ``akiki`` as a shortcut. Import supporting result types and configuration enums from their respective modules. VAD and decision precision enums have different supported formats and are not interchangeable. @@ -40,10 +42,10 @@ Examples -------- -Process one frame of silence using the top-level shortcut:: +Process one frame of silence:: - >>> import akiki - >>> with akiki.VAD() as detector: + >>> from akiki.audio import vad + >>> with vad.VAD() as detector: ... frames = detector.push([0.0] * 256) >>> len(frames) 1 @@ -52,18 +54,21 @@ Find domain-specific configuration and help through the modules:: - >>> from akiki import decision, vad + >>> from akiki import decision + >>> from akiki.audio import vad >>> vad.ModelPrecision.INT8 is decision.ModelPrecision.INT8 False -Use ``help(akiki.vad)`` and ``help(akiki.decision)`` to explore each API, or -``help(akiki.vad.VAD)`` and ``help(akiki.decision.DecisionModel)`` for the +Use ``help(akiki.audio.vad)`` and ``help(akiki.decision)`` to explore each API, or +``help(akiki.audio.vad.VAD)`` and ``help(akiki.decision.DecisionModel)`` for the primary classes. Use ``help(akiki.decision.MiniLM)`` for direct access to the currently supported decision-model family. Explicit model retrieval utilities live in ``akiki.hub``. Compatibility ------------- +``akiki.vad`` and ``akiki.VAD`` will be removed in version 0.3.0. +Use ``from akiki.audio import vad`` and call ``vad.VAD()``. ``akiki.minilm`` remains a compatibility module. It re-exports the decision API's MiniLM class, result types, and current precision enum. Existing named top-level imports of ``SpeechFrame``, ``Engine``, ``ModelPrecision``, @@ -78,29 +83,49 @@ Installed Akiki package version. """ -from . import decision, vad, vision +import warnings +from importlib import import_module +from typing import TYPE_CHECKING + +from . import audio, decision, vision from . import minilm as minilm +from .audio.vad import Engine as Engine +from .audio.vad import FFTPrecision as FFTPrecision +from .audio.vad import ModelPrecision as ModelPrecision +from .audio.vad import SpeechFrame as SpeechFrame from .decision import Classification as Classification from .decision import Encoding as Encoding from .decision import MiniLM from .hub import download_model as download_model -from .vad import VAD -from .vad import Engine as Engine -from .vad import FFTPrecision as FFTPrecision -from .vad import ModelPrecision as ModelPrecision -from .vad import SpeechFrame as SpeechFrame + +if TYPE_CHECKING: + from .audio.vad import VAD as VAD __version__ = "0.2.2" __all__ = [ "MiniLM", "VAD", "__version__", + "audio", "decision", - "vad", "vision", ] +def __getattr__(name: str): + if name == "VAD": + warnings.warn( + "akiki.VAD is deprecated and will be removed in version 0.3.0. " + "Use akiki.audio.vad.VAD instead.", + DeprecationWarning, + stacklevel=2, + ) + return audio.vad.VAD + if name == "vad": + return import_module(".vad", __name__) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + def __dir__() -> list[str]: """List the primary API and explicitly imported training/teacher modules.""" names = __all__.copy() diff --git a/src/akiki/__main__.py b/src/akiki/__main__.py index 1b0f388..3c53c3c 100644 --- a/src/akiki/__main__.py +++ b/src/akiki/__main__.py @@ -9,8 +9,8 @@ from dataclasses import asdict from . import __version__ +from .audio.vad import Engine, FFTPrecision, ModelPrecision from .decision import ModelPrecision as MiniLMPrecision -from .vad import Engine, FFTPrecision, ModelPrecision def main(): @@ -157,7 +157,7 @@ def _run_vad(args): """Print frame decisions for a local audio file using the requested engine.""" import soundfile as sf - from . import VAD + from .audio.vad import VAD with ( VAD( diff --git a/src/akiki/_atomic.py b/src/akiki/_atomic.py index 886f6ae..753f358 100644 --- a/src/akiki/_atomic.py +++ b/src/akiki/_atomic.py @@ -2,6 +2,7 @@ import os import sys +import tempfile import time from pathlib import Path @@ -20,3 +21,18 @@ def replace_file(source: Path, destination: Path) -> None: ): raise time.sleep(0.01 * 2**attempt) + + +def write_bytes(path: Path, data: bytes) -> None: + """Write a complete file before replacing its destination.""" + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=path.parent, prefix=".tmp-", delete=False) as stream: + temporary = Path(stream.name) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + replace_file(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) diff --git a/src/akiki/_engines.py b/src/akiki/_engines.py index 2be8aef..be0da78 100644 --- a/src/akiki/_engines.py +++ b/src/akiki/_engines.py @@ -57,24 +57,31 @@ def close(self): self.closed = True +def _require_onnx_runtime(): + try: + import onnxruntime as ort + except (ImportError, OSError) as error: + raise EngineUnavailable( + "Install the ONNX engine with: pip install 'akiki[onnx]'" + ) from error + return ort + + +def _create_onnx_session(ort, model: str | bytes): + options = ort.SessionOptions() + options.intra_op_num_threads = options.inter_op_num_threads = 1 + try: + return ort.InferenceSession(model, sess_options=options, providers=["CPUExecutionProvider"]) + except Exception as error: + raise EngineUnavailable(f"ONNX Runtime could not load the model: {error}") from error + + class OnnxBackend(Backend): def __init__(self, path, metadata): - try: - import onnxruntime as ort - except (ImportError, OSError) as error: - raise EngineUnavailable( - "Install the ONNX engine with: pip install 'akiki[onnx]'" - ) from error + ort = _require_onnx_runtime() path = path.with_suffix(".onnx") verify_file(path, metadata["onnx_sha256"]) - options = ort.SessionOptions() - options.intra_op_num_threads = options.inter_op_num_threads = 1 - try: - self.session = ort.InferenceSession( - str(path), sess_options=options, providers=["CPUExecutionProvider"] - ) - except Exception as error: - raise EngineUnavailable(f"ONNX Runtime could not load the model: {error}") from error + self.session = _create_onnx_session(ort, str(path)) if {v.name for v in self.session.get_inputs()} != {"audio_with_context", "h"}: raise ValueError("Unexpected ONNX inputs") diff --git a/src/akiki/_quantization.py b/src/akiki/_quantization.py index f3a7337..71ff045 100644 --- a/src/akiki/_quantization.py +++ b/src/akiki/_quantization.py @@ -12,12 +12,11 @@ import os import struct import sys -import tempfile from pathlib import Path import numpy as np -from ._atomic import replace_file +from ._atomic import write_bytes from ._precision import ModelPrecision # AKVAD003 format IDs are fixed independently of enum declaration order. @@ -180,20 +179,6 @@ def default_cache_dir(): return Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "akiki" -def _atomic_write(path, data): - temporary = None - try: - with tempfile.NamedTemporaryFile(dir=path.parent, prefix=".tmp-", delete=False) as stream: - temporary = Path(stream.name) - stream.write(data) - stream.flush() - os.fsync(stream.fileno()) - replace_file(temporary, path) - finally: - if temporary is not None: - temporary.unlink(missing_ok=True) - - def cached_model(folder, receipt, precision: ModelPrecision, cache_dir, validate_source): """Return a checked variant; repair incomplete/corrupt derived cache entries.""" identity = { @@ -239,8 +224,8 @@ def cached_model(folder, receipt, precision: ModelPrecision, cache_dir, validate } try: directory.mkdir(parents=True, exist_ok=True) - _atomic_write(model, payload) - _atomic_write(manifest, (json.dumps(saved, sort_keys=True, indent=2) + "\n").encode()) + write_bytes(model, payload) + write_bytes(manifest, (json.dumps(saved, sort_keys=True, indent=2) + "\n").encode()) except OSError as error: raise RuntimeError( "Cannot write the Akiki model cache; set cache_dir or AKIKI_CACHE_DIR to a writable directory" diff --git a/src/akiki/audio/__init__.py b/src/akiki/audio/__init__.py new file mode 100644 index 0000000..092c1f1 --- /dev/null +++ b/src/akiki/audio/__init__.py @@ -0,0 +1,5 @@ +"""Audio inference interfaces.""" + +from . import vad + +__all__ = ["vad"] diff --git a/src/akiki/audio/_ten_vad.py b/src/akiki/audio/_ten_vad.py new file mode 100644 index 0000000..b82b87a --- /dev/null +++ b/src/akiki/audio/_ten_vad.py @@ -0,0 +1,123 @@ +"""TEN-VAD inference with the Akiki audio frontend.""" + +import ctypes +import weakref +from pathlib import Path + +import numpy as np + +from .._engines import ( + Backend, + EngineUnavailable, + _create_onnx_session, + _require_onnx_runtime, + library_candidates, +) + + +class TenVadBackend(Backend): + chunk_samples = 256 + hidden = 64 + state_shape = (4, hidden) + + @classmethod + def load_library(cls, library_path: str | Path | None): + candidate = next( + library_candidates(library_path, "akiki_ten_vad_engine", "AKIKI_TEN_VAD_LIBRARY") + ).resolve() + try: + lib = ctypes.CDLL(str(candidate)) + cls._bind(lib) + except (OSError, AttributeError) as error: + raise EngineUnavailable( + f"No compatible Akiki TEN-VAD library at {candidate}. " + "Set AKIKI_TEN_VAD_LIBRARY or pass library_path." + ) from error + return lib + + def __init__(self, path: Path, lib): + handle = lib.akiki_ten_vad_create(str(path.resolve()).encode("utf-8")) + if not handle: + raise ValueError(f"The Akiki TEN-VAD engine cannot load the model: {path}") + self.lib, self.handle = lib, handle + self._finalizer = weakref.finalize(self, lib.akiki_ten_vad_free, handle) + + @staticmethod + def _bind(lib): + pointer = ctypes.c_void_p + for name, arguments, result in ( + ("create", [ctypes.c_char_p], pointer), + ("free", [pointer], None), + ("reset", [pointer], None), + ("frame", [pointer, pointer, ctypes.POINTER(ctypes.c_float)], ctypes.c_int), + ("state", [pointer, pointer], ctypes.c_int), + ): + function = getattr(lib, f"akiki_ten_vad_{name}") + function.argtypes, function.restype = arguments, result + + def reset(self): + super().reset() + self.lib.akiki_ten_vad_reset(self.handle) + + def infer(self, window, state): + if window.shape != (self.chunk_samples,): + raise ValueError("TEN-VAD requires exactly 256 samples per frame") + chunk = np.ascontiguousarray(window, dtype=np.float32) + probability = ctypes.c_float() + if self.lib.akiki_ten_vad_frame(self.handle, chunk.ctypes.data, ctypes.byref(probability)): + raise RuntimeError("Akiki TEN-VAD engine could not score an audio frame") + hidden = np.empty(self.state_shape, dtype=np.float32) + if self.lib.akiki_ten_vad_state(self.handle, hidden.ctypes.data): + raise RuntimeError("Akiki TEN-VAD engine could not return its recurrent state") + return probability.value, hidden + + def close(self): + super().close() + self._finalizer() + + +class TenVadOnnxBackend(TenVadBackend): + # The upstream streaming wrapper resets recurrent state every 30 seconds. + _reset_frames = 1875 + + def __init__(self, path: Path, lib, model_data: bytes): + session = _create_onnx_session(_require_onnx_runtime(), model_data) + self._input_names = tuple(value.name for value in session.get_inputs()) + if len(self._input_names) != 5 or len(session.get_outputs()) != 5: + raise ValueError("Expected TEN-VAD features, probability, and four recurrent states") + super().__init__(path, lib) + self.session = session + self._features = np.empty((1, 3, 41), dtype=np.float32) + self._frames = 0 + + @staticmethod + def _bind(lib): + TenVadBackend._bind(lib) + pointer = ctypes.c_void_p + lib.akiki_ten_vad_features.argtypes = [pointer, pointer, pointer] + lib.akiki_ten_vad_features.restype = ctypes.c_int + + def reset(self): + super().reset() + self._frames = 0 + + def infer(self, window, state): + if window.shape != (self.chunk_samples,): + raise ValueError("TEN-VAD requires exactly 256 samples per frame") + chunk = np.ascontiguousarray(window, dtype=np.float32) + if self.lib.akiki_ten_vad_features( + self.handle, chunk.ctypes.data, self._features.ctypes.data + ): + raise RuntimeError("Akiki TEN-VAD frontend could not extract audio features") + inputs = dict(zip(self._input_names, (self._features, *state[:, None, :]), strict=True)) + probability, *states = self.session.run(None, inputs) + state = np.stack(states).reshape(self.state_shape) + self._frames += 1 + if self._frames == self._reset_frames: + state.fill(0) + self._frames = 0 + return probability, state + + def close(self): + super().close() + self.session = None diff --git a/src/akiki/audio/_ten_vad_model.py b/src/akiki/audio/_ten_vad_model.py new file mode 100644 index 0000000..a3aede2 --- /dev/null +++ b/src/akiki/audio/_ten_vad_model.py @@ -0,0 +1,217 @@ +"""Prepare TEN-VAD ONNX weights for the native streaming engine.""" + +import hashlib +import re +from pathlib import Path + +import numpy as np + +from .._atomic import write_bytes +from .._quantization import default_cache_dir +from ..hub import _download_snapshot + +_REPO = "TEN-framework/ten-vad" +_REVISION = "bda8ffc78b1846c5c7cbd38f04e52deff49de707" +_ONNX = "src/onnx_model/ten-vad.onnx" +_COEFFICIENTS = "src/coeff.h" +_HUB_CHECKSUMS = { + _ONNX: "e10b98a0cab1c98e847fbdda14cb3d45a38336d47535a3f63a0fb6c4e0f4cdf4", + _COEFFICIENTS: "0c86dc4a9161e418da6ce03d50dea067dff085749d730e3913b99a8e116397ef", +} +_GRAPH_SHA256 = "bbefc914a8b2a8db673e9b74914cc086b566c09e7a67c22a1f21b26a39edf91a" +_CONVERTER_VERSION = b"ten-vad-onnx-v1" +_NATIVE_MAGIC = b"AKTNV001" +_NATIVE_SIZE = len(_NATIVE_MAGIC) + (75498 + 850) * 4 +_PREFIX = "StatefulPartitionedCall/vad_model/" + + +def _read_sources(model_path: str | Path | None) -> tuple[bytes, bytes]: + if model_path is None: + try: + snapshot = _download_snapshot( + _REPO, + revision=_REVISION, + allow_patterns=tuple(_HUB_CHECKSUMS), + local_files_only=True, + ) + except RuntimeError as error: + raise RuntimeError( + "Cannot load TEN-VAD from the local Hugging Face cache. " + f"Pass model_path to a local TEN-VAD ONNX file. {error}" + ) from error + contents = [] + for filename, checksum in _HUB_CHECKSUMS.items(): + path = snapshot / filename + data = path.read_bytes() + if hashlib.sha256(data).hexdigest() != checksum: + raise ValueError(f"TEN-VAD Hub file has an incorrect checksum: {path}") + contents.append(data) + return contents[0], contents[1] + + weights = Path(model_path).expanduser() + if weights.suffix.lower() != ".onnx": + raise ValueError("TEN-VAD model_path must point to its .onnx weights") + model_data = weights.read_bytes() + source_dir = weights.parent + if source_dir.name == "onnx_model" and not (source_dir / "coeff.h").is_file(): + source_dir = source_dir.parent + coefficients = source_dir / "coeff.h" + if not coefficients.is_file(): + raise FileNotFoundError( + "TEN-VAD requires the matching upstream coeff.h beside the ONNX file " + f"or in the parent of its onnx_model directory: {coefficients}" + ) + return model_data, coefficients.read_bytes() + + +def _network_weights(tensors: dict[str, np.ndarray]) -> list[np.ndarray]: + arrays = [] + used = set() + + def take(name, shape): + value = tensors.get(name) + if value is None or value.shape != shape or not np.isfinite(value).all(): + raise ValueError(f"Invalid or missing TEN-VAD tensor: {name}") + used.add(name) + return value + + for layer, depth, depth_shape, point, point_shape in ( + ( + "separable_conv2d", + "const_fold_opt__178", + (1, 1, 3, 3), + "separable_conv2d/ReadVariableOp_1:0", + (16, 1, 1, 1), + ), + ( + "separable_conv1d", + "const_fold_opt__179", + (16, 1, 1, 3), + "ExpandDims_2:0", + (16, 16, 1, 1), + ), + ( + "separable_conv1d_1", + "const_fold_opt__180", + (16, 1, 1, 3), + "ExpandDims_2:0", + (16, 16, 1, 1), + ), + ): + arrays.extend( + ( + take(depth, depth_shape), + take(f"{_PREFIX}{layer}/{point}", point_shape), + take(f"{_PREFIX}{layer}/BiasAdd/ReadVariableOp:0", (16,)), + ) + ) + + # ONNX uses IOFC gate order. The native engine uses IFCO. + gates = [0, 2, 3, 1] + for input_name, recurrent_name, bias_name, inputs in ( + ("W0__70", "R0__71", "B0__72", 80), + ("W0__99", "R0__100", "B0__101", 64), + ): + weight = take(input_name, (1, 256, inputs)).reshape(4, 64, inputs)[gates] + recurrent = take(recurrent_name, (1, 256, 64)).reshape(4, 64, 64)[gates] + bias = take(bias_name, (1, 512)).reshape(2, 4, 64)[:, gates] + arrays.extend((weight, bias[0], recurrent, bias[1])) + + for layer, inputs, outputs in (("dense_3", 128, 32), ("dense_5", 32, 1)): + arrays.append(take(f"{_PREFIX}{layer}/Tensordot/ReadVariableOp:0", (inputs, outputs)).T) + arrays.append(take(f"{_PREFIX}{layer}/BiasAdd/ReadVariableOp:0", (outputs,))) + if used != tensors.keys(): + raise ValueError("Unsupported TEN-VAD weight set") + return arrays + + +def _frontend_weights(data: bytes) -> list[np.ndarray]: + text = data.decode("utf-8") + arrays = [] + for name, length in ( + ("AUP_AED_FEATURE_MEANS", 41), + ("AUP_AED_FEATURE_STDS", 41), + ("AUP_AED_STFTWindow_Hann768", 768), + ): + match = re.search(rf"\b{name}\[[^\]]+\]\s*=\s*\{{([^}}]+)\}}", text) + if match is None: + raise ValueError(f"Missing TEN-VAD frontend coefficient array: {name}") + values = np.array( + [float(value.strip().rstrip("f")) for value in match[1].split(",") if value.strip()], + dtype=" 1)).any(): + raise ValueError("Invalid TEN-VAD frontend normalization or window") + return arrays + + +def _convert(model_data: bytes, coefficients: bytes) -> bytes: + try: + import onnx + from onnx import numpy_helper + except ImportError as error: + raise RuntimeError("TEN-VAD ONNX import requires pip install 'akiki[ten-vad]'") from error + + try: + graph = onnx.load_model_from_string(model_data).graph + except Exception as error: + raise ValueError("Cannot parse the TEN-VAD ONNX checkpoint") from error + + topology = onnx.GraphProto() + topology.CopyFrom(graph) + weights = {} + for initializer, tensor in zip(graph.initializer, topology.initializer): + if initializer.external_data or initializer.data_location == onnx.TensorProto.EXTERNAL: + raise ValueError("TEN-VAD ONNX must contain its weights in a single file") + if initializer.data_type == onnx.TensorProto.FLOAT: + if initializer.name in weights: + raise ValueError(f"Duplicate TEN-VAD tensor: {initializer.name}") + weights[initializer.name] = numpy_helper.to_array(initializer) + for field in ("raw_data", "float_data", "double_data"): + tensor.ClearField(field) + if hashlib.sha256(topology.SerializeToString()).hexdigest() != _GRAPH_SHA256: + raise ValueError("Unsupported ONNX graph. Use the TEN-framework TEN-VAD 1.0 model.") + + arrays = _network_weights(weights) + _frontend_weights(coefficients) + return _NATIVE_MAGIC + b"".join(array.astype(" tuple[Path, bytes]: + """Return the checked native cache entry and its source ONNX bytes.""" + weights, coefficients = _read_sources(model_path) + key = hashlib.sha256( + _CONVERTER_VERSION + + hashlib.sha256(weights).digest() + + hashlib.sha256(coefficients).digest() + ).hexdigest() + root = Path(cache_dir).expanduser() if cache_dir is not None else default_cache_dir() + directory = root / "ten-vad" / key + model = directory / "model.bin" + checksum = directory / "sha256" + try: + cached = model.read_bytes() + if ( + len(cached) == _NATIVE_SIZE + and cached.startswith(_NATIVE_MAGIC) + and hashlib.sha256(cached).hexdigest().encode() == checksum.read_bytes() + ): + return model, weights + except OSError: + pass + + payload = _convert(weights, coefficients) + try: + directory.mkdir(parents=True, exist_ok=True) + write_bytes(model, payload) + write_bytes(checksum, hashlib.sha256(payload).hexdigest().encode()) + except OSError as error: + raise RuntimeError( + "Cannot write the TEN-VAD model cache. Set cache_dir or AKIKI_CACHE_DIR." + ) from error + return model, weights diff --git a/src/akiki/audio/vad.py b/src/akiki/audio/vad.py new file mode 100644 index 0000000..46aea13 --- /dev/null +++ b/src/akiki/audio/vad.py @@ -0,0 +1,435 @@ +"""Streaming speech detection and its configuration and result types. + +Use ``VAD`` with successive mono, 16 kHz audio buffers. Each complete frame +returns a ``SpeechFrame`` with timestamps, a probability, and a speech decision. +The bundled model processes 256 samples (16 ms) per frame. + +The Akiki engine is the primary engine. ``Engine.AUTO`` tries it first. +The ONNX and PyTorch engines provide compatibility fallbacks. +``ModelPrecision`` selects the VAD model's numeric format; ``FFTPrecision`` configures its FFT. +These enums belong to VAD; MiniLM uses ``akiki.decision.ModelPrecision``. + +Examples +-------- +>>> from akiki.audio import vad +>>> with vad.VAD(model_precision=vad.ModelPrecision.FP32) as detector: +... frames = detector.push([0.0] * 256) +>>> len(frames) +1 +""" + +import json +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + +import numpy as np + +from .._engines import Engine +from .._precision import FFTPrecision, ModelPrecision + +__all__ = ["Engine", "FFTPrecision", "Model", "ModelPrecision", "SpeechFrame", "VAD"] + +SAMPLE_RATE = 16000 + + +class Model(Enum): + """Model family to use for voice activity detection.""" + + AKIKI_VAD = "akiki-vad" + TEN_VAD = "ten-vad" + + +@dataclass(frozen=True) +class SpeechFrame: + """Immutable speech decision for one complete frame returned by ``VAD.push``. + + Timestamps describe the input audio, measured from detector creation or the + last ``VAD.reset()`` call. They do not measure processing or wall-clock time. + The bundled model produces one frame for every 256 samples (16 ms). + + Attributes + ---------- + start_seconds : float + Inclusive start time of the frame, in seconds. + end_seconds : float + Exclusive end time of the frame, in seconds. + probability : float + Model's speech probability, between 0 and 1 inclusive. + speech : bool + Whether ``probability >= detector.threshold`` when the frame was + produced. Decisions are made per frame, without merging speech segments. + + Examples + -------- + >>> frame = SpeechFrame(0.0, 0.016, 0.9, True) + >>> frame.end_seconds - frame.start_seconds + 0.016 + >>> frame.probability, frame.speech + (0.9, True) + """ + + start_seconds: float + end_seconds: float + probability: float + speech: bool + + +class VAD: + """Identify speech in a stream of mono audio sampled at 16,000 Hz. + + Each instance maintains the recurrent state, audio context, and incomplete + frame for one stream. Feed successive buffers to ``push()``; call + ``reset()`` before an independent recording. The bundled model emits a + ``SpeechFrame`` for every 256 samples (16 ms). Custom models use their + supported ``chunk_samples`` value: 256 or 512 samples. + + The Akiki engine is the primary engine. For Akiki-VAD, ``Engine.AUTO`` + tries it first, then ONNX, then PyTorch. The Akiki engine uses INT8 by + default for Akiki-VAD. The compatibility engines use FP32. + TEN-VAD uses FP32 and selects the Akiki engine by default. + With ``Engine.ONNX``, the Akiki frontend supplies features to ONNX Runtime. + See ``VAD.__init__`` for all constructor options. + + Use the detector as a context manager or call ``close()`` when finished. + Closing does not pad or score a final incomplete frame. + + Attributes + ---------- + engine : Engine + Selected inference engine: ``Engine.AKIKI``, ``Engine.ONNX``, or + ``Engine.PYTORCH``. + engine_errors : dict[Engine, str] + Reasons engines were unavailable before automatic selection succeeded. + model_id : str + Identifier from the Akiki-VAD metadata, or ``"ten-vad"``. + model_precision : ModelPrecision + Active model precision, after resolving the engine's default. + fft_precision : FFTPrecision + FFT precision; always ``FFTPrecision.FP32``. + threshold : float + Minimum probability classified as speech. + chunk_samples : int + Number of input samples needed for one output frame. + context_samples : int + Number of prior audio samples retained as model context. + hidden : int + Number of recurrent hidden units in the loaded model. + completed_samples : int + Number of samples scored since creation or the last reset. + pending : numpy.ndarray + Unscored float32 samples buffered for the next call to ``push``. + context : numpy.ndarray + Float32 audio context with shape ``(context_samples,)``. + TEN-VAD keeps its audio context in the engine. This array is empty for TEN-VAD. + state : numpy.ndarray + Float32 recurrent state. Akiki-VAD uses shape ``(1, 1, hidden)``. + TEN-VAD uses shape ``(4, 64)`` with hidden and cell state for each LSTM. + session : onnxruntime.InferenceSession + Present only when the selected engine is ONNX; ``None`` after closing. + + Notes + ----- + Treat ``pending``, ``context``, and ``state`` as read-only stream state. + Use a separate instance for each independent audio stream. + Calls on one detector are not protected by a lock. Do not call ``push``, + ``reset``, or ``close`` concurrently on the same instance. + + Examples + -------- + Successive buffers share a timeline and incomplete frames are retained:: + + >>> from akiki.audio.vad import VAD + >>> with VAD() as detector: + ... first = detector.push([0.0] * 100) + ... second = detector.push([0.0] * 156) + >>> first + [] + >>> len(second), second[0].end_seconds + (1, 0.016) + """ + + def __init__( + self, + model_path: str | Path | None = None, + threshold: float | None = None, + *, + model: Model = Model.AKIKI_VAD, + engine: Engine = Engine.AUTO, + library_path: str | Path | None = None, + model_precision: ModelPrecision | None = None, + fft_precision: FFTPrecision = FFTPrecision.FP32, + cache_dir: str | Path | None = None, + ): + """Load a local model and inference engine for a new stream. + + Parameters + ---------- + model_path : str or pathlib.Path, optional + For TEN-VAD, pass the path to its official ``.onnx`` file. + Keep the matching ``coeff.h`` beside the weights or use the + official snapshot layout. If omitted for TEN-VAD, use the pinned + Hugging Face snapshot from the local cache. + For a custom Akiki model, pass its JSON metadata path, such as + ``"models/vad.json"``. The Akiki engine loads the sibling + ``"models/vad-native/"`` directory. It must contain ``config.json``, + ``model.safetensors``, and ``export.json``. + The ONNX and PyTorch compatibility fallbacks use matching ``.onnx`` + and ``.pt`` files beside the metadata file. + Custom models must use the full-waveform GRU format. Supported + ``(chunk_samples, context_samples)`` pairs are ``(256, 320)`` and + ``(512, 64)``. Custom PyTorch metadata also requires ``pytorch_sha256``. + threshold : float, optional + Speech decision threshold between 0 and 1 inclusive. If omitted, + use the threshold in the model's metadata, or 0.5 for TEN-VAD. + The threshold applies to the speech decision. + model : Model, default=Model.AKIKI_VAD + Select the VAD model family. Use ``Model.TEN_VAD`` for TEN-VAD. + With ``Model.AKIKI_VAD`` and no ``model_path``, use the bundled model. + engine : Engine, default=Engine.AUTO + Automatic selection tries the primary Akiki engine first. + ONNX and PyTorch are compatibility fallbacks, in that order. + An explicitly selected engine must succeed; it does not fall back. + Pass an ``Engine`` member, such as ``Engine.AKIKI``. + For TEN-VAD, select ``Engine.ONNX`` to run the network in ONNX Runtime. + Both TEN-VAD engines require the local Akiki TEN-VAD library. + library_path : str or pathlib.Path, optional + Specific local Akiki shared library to load. For Akiki-VAD, this + applies to the Akiki engine. If omitted, use ``AKIKI_VAD_LIBRARY`` or the bundled + Akiki-VAD library. For TEN-VAD, use ``AKIKI_TEN_VAD_LIBRARY`` + or pass its library path here. + model_precision : ModelPrecision, optional + One of ``ModelPrecision.FP32``, ``FP16``, ``BF16``, ``INT8``, + ``INT4``, or ``INT8_FP32_INPUT``. If omitted, Akiki uses INT8 and + ONNX/PyTorch use FP32. Explicit non-FP32 precision requires Akiki, + including in automatic mode. Reduced-precision Akiki weights are + generated from the verified local FP32 model and reused from the + cache. ``ModelPrecision.INT8_FP32_INPUT`` uses INT8 weights while + keeping the first projection's input in FP32. TEN-VAD uses FP32. + fft_precision : FFTPrecision, default=FFTPrecision.FP32 + FFT precision. Only ``FFTPrecision.FP32`` is supported. + cache_dir : str or pathlib.Path, optional + Directory for generated Akiki model variants. If omitted, use + ``AKIKI_CACHE_DIR`` or the platform's user cache directory. Original + model weights are preserved. TEN-VAD prepares an engine cache + entry from its ONNX weights once per checkpoint. + + Raises + ------ + TypeError + ``model``, ``engine``, ``model_precision``, or ``fft_precision`` + has the wrong enum type. ``model_precision`` also accepts ``None``. + ValueError + A parameter, model specification, or model checksum is invalid. + OSError + The local model metadata cannot be read. + RuntimeError + No compatible engine is available for the requested configuration, + or the directory for a generated model variant is not writable. + """ + if not isinstance(model, Model): + raise TypeError( + "model must be a Model member, such as Model.AKIKI_VAD or Model.TEN_VAD" + ) + if not isinstance(engine, Engine): + raise TypeError("engine must be an Engine member, such as Engine.AUTO or Engine.AKIKI") + if model_precision is not None and not isinstance(model_precision, ModelPrecision): + raise TypeError("model_precision must be a ModelPrecision member or None") + if not isinstance(fft_precision, FFTPrecision): + raise TypeError("fft_precision must be an FFTPrecision member") + if model_precision not in (None, ModelPrecision.FP32) and engine not in ( + Engine.AUTO, + Engine.AKIKI, + ): + raise ValueError("Non-FP32 model_precision requires Engine.AKIKI or Engine.AUTO") + self.fft_precision = fft_precision + self._model = model + if model is Model.TEN_VAD: + self._load_ten_vad( + model_path, threshold, engine, library_path, model_precision, cache_dir + ) + else: + self._load_akiki_vad( + model_path, threshold, engine, library_path, model_precision, cache_dir + ) + self.reset() + + def _load_akiki_vad( + self, model_path, threshold, engine, library_path, model_precision, cache_dir + ): + from .._engines import load_engine + + path = ( + Path(model_path) + if model_path is not None + else Path(__file__).parents[1] / "models/akiki-vad.onnx" + ) + metadata = json.loads(path.with_suffix(".json").read_text()) + self.model_id = metadata.get("model_id", path.stem) + spec = metadata["spec"] + self.chunk_samples = int(spec["chunk_samples"]) + self.context_samples = int(spec["context_samples"]) + self.hidden = int(spec["hidden"]) + if spec["kind"] != "gru" or (self.chunk_samples, self.context_samples) not in { + (256, 320), + (512, 64), + }: + raise ValueError("Expected an Akiki full-waveform GRU model") + if spec.get("sample_rate", SAMPLE_RATE) != SAMPLE_RATE or self.hidden <= 0: + raise ValueError("Expected a 16 kHz model with a positive hidden size") + self._state_shape = (1, 1, self.hidden) + self.threshold = self._validate_threshold( + metadata["threshold"] if threshold is None else threshold + ) + self._backend, self.engine, self.engine_errors = load_engine( + path, metadata, engine, library_path, model_precision, cache_dir + ) + self.model_precision = model_precision or ( + ModelPrecision.INT8 if self.engine is Engine.AKIKI else ModelPrecision.FP32 + ) + if self.engine is Engine.ONNX: + self.session = self._backend.session + + def _load_ten_vad( + self, model_path, threshold, engine, library_path, model_precision, cache_dir + ): + from ._ten_vad import TenVadBackend, TenVadOnnxBackend + from ._ten_vad_model import prepare_model + + if engine is Engine.PYTORCH: + raise ValueError("TEN-VAD supports the Akiki and ONNX engines") + if model_precision not in (None, ModelPrecision.FP32): + raise ValueError("TEN-VAD requires ModelPrecision.FP32") + self.threshold = self._validate_threshold(0.5 if threshold is None else threshold) + backend = TenVadOnnxBackend if engine is Engine.ONNX else TenVadBackend + library = backend.load_library(library_path) + native_path, model_data = prepare_model(model_path, cache_dir) + if engine is Engine.ONNX: + self._backend = backend(native_path, library, model_data) + self.engine = Engine.ONNX + self.session = self._backend.session + else: + self._backend = backend(native_path, library) + self.engine = Engine.AKIKI + self.engine_errors = {} + self.model_id = Model.TEN_VAD.value + self.model_precision = ModelPrecision.FP32 + self.chunk_samples = self._backend.chunk_samples + self.context_samples = 0 + self.hidden = self._backend.hidden + self._state_shape = self._backend.state_shape + + @staticmethod + def _validate_threshold(threshold) -> float: + value = float(threshold) + if not np.isfinite(value) or not 0 <= value <= 1: + raise ValueError("Threshold must lie between zero and one") + return value + + def reset(self) -> None: + """Start an independent recording with the same model and settings. + + Clear the recurrent state and audio context, discard pending samples, + and reset output timestamps to zero. The model remains loaded. + + Raises + ------ + RuntimeError + The detector has been closed. Create a new instance to process more + audio after closing. + """ + self._backend.reset() + self.context = np.zeros(self.context_samples, dtype=np.float32) + self.state = np.zeros(self._state_shape, dtype=np.float32) + self.pending = np.empty(0, dtype=np.float32) + self.completed_samples = 0 + + def push(self, samples, sample_rate: int = SAMPLE_RATE) -> list[SpeechFrame]: + """Process the next audio buffer and return decisions for complete frames. + + Parameters + ---------- + samples : array_like + One-dimensional, finite mono waveform samples. Lists and NumPy + arrays are accepted and converted to float32. Samples should + normally be scaled to [-1, 1]; integer PCM is not normalized for + you. TEN-VAD requires samples in [-1, 1]. + Audio is not resampled or mixed down from multiple channels. + sample_rate : int, default=16000 + Input sample rate in Hz. Must be 16000. + + Returns + ------- + list[SpeechFrame] + One result for each complete ``chunk_samples`` input samples, + including any samples buffered by a previous call. Timestamps + continue from earlier calls. Returns an empty list when there is + not enough audio for a complete frame. + + Raises + ------ + TypeError + The input type cannot be converted to numeric audio samples. + ValueError + Audio cannot be converted to float32, is not one-dimensional, + contains nonfinite values, or has the wrong sample rate. These + validation failures leave the stream state unchanged. + RuntimeError + The detector is closed, inference fails, or the model returns an + invalid speech probability. + + Notes + ----- + Buffers may have any length, including zero. A final incomplete frame + stays in ``pending`` until more audio arrives or ``reset()`` discards + it. Closing the detector does not pad or score those samples. + """ + self._backend.check_open() + audio = np.asarray(samples, dtype=np.float32) + if sample_rate != SAMPLE_RATE or audio.ndim != 1 or not np.isfinite(audio).all(): + raise ValueError("Expected finite mono 16 kHz audio samples") + if self._model is Model.TEN_VAD and (np.abs(audio) > 1).any(): + raise ValueError("TEN-VAD requires audio samples in [-1, 1]") + audio = np.concatenate([self.pending, audio]) + complete = len(audio) // self.chunk_samples * self.chunk_samples + frames = [] + for start in range(0, complete, self.chunk_samples): + window = np.concatenate([self.context, audio[start : start + self.chunk_samples]]) + probability, state = self._backend.infer(window, self.state) + probability = float(np.asarray(probability).reshape(-1)[0]) + if not np.isfinite(probability) or not 0 <= probability <= 1: + raise RuntimeError("Model returned an invalid speech probability") + begin = self.completed_samples + self.state = state + self.context = window[len(window) - self.context_samples :].copy() + self.completed_samples += self.chunk_samples + frames.append( + SpeechFrame( + begin / SAMPLE_RATE, + self.completed_samples / SAMPLE_RATE, + probability, + probability >= self.threshold, + ) + ) + self.pending = audio[complete:].copy() + return frames + + def close(self) -> None: + """Release this stream's inference resources; safe to call repeatedly. + + Pending samples are not scored. After closing, ``push()``, ``reset()``, + and entering a ``with`` block raise ``RuntimeError``. Previously + returned ``SpeechFrame`` objects remain usable. + """ + self._backend.close() + if self.engine is Engine.ONNX: + self.session = None + + def __enter__(self): + """Return this open detector for use in a ``with`` block.""" + self._backend.check_open() + return self + + def __exit__(self, *_): + """Close the detector on leaving a ``with`` block; propagate exceptions.""" + self.close() diff --git a/src/akiki/decision.py b/src/akiki/decision.py index 966bc89..2134007 100644 --- a/src/akiki/decision.py +++ b/src/akiki/decision.py @@ -765,7 +765,7 @@ def __init__( ``INT8``, ``INT8_FP32_INPUT``, ``INT4``, or ``INT4_G32``. Weights are converted in memory when loading; the files are unchanged. Attention, normalization, and pooling remain FP32. Strings and - members of ``akiki.vad.ModelPrecision`` are rejected. + members of ``akiki.audio.vad.ModelPrecision`` are rejected. chunk_tokens : int, optional Split text into chunks of at most this many tokens, including two special tokens, as Ariadne retrieval models do. Defaults to the diff --git a/src/akiki/hub.py b/src/akiki/hub.py index 6f47b0d..1be2285 100644 --- a/src/akiki/hub.py +++ b/src/akiki/hub.py @@ -1,8 +1,6 @@ -"""Optional checkpoint retrieval for the current MiniLM/BERT decision runtime. +"""Optional checkpoint retrieval from the Hugging Face Hub. -This helper supports the current engine's file layout. -Retrieving another model family does not add decision-model inference support. -Ordinary VAD and decision inference use local files. +The public download_model helper supports MiniLM/BERT decision models. """ import os @@ -66,6 +64,44 @@ def download_model( ... "sentence-transformers/all-MiniLM-L6-v2", local_files_only=True ... ) """ + root = _download_snapshot( + repo_id, + allow_patterns=_MODEL_FILES, + revision=revision, + cache_dir=cache_dir, + local_dir=local_dir, + local_files_only=local_files_only, + token=token, + ) + if (root / "base_model").is_dir() and all( + (root / name).is_file() for name in _CHECKPOINT_FILES + ): + raise ValueError( + "Hugging Face snapshot contains both a root checkpoint and base_model/; " + "use a clean local_dir to avoid mixing models" + ) + checkpoint = root / "base_model" if (root / "base_model").is_dir() else root + required = [checkpoint / name for name in _CHECKPOINT_FILES] + if checkpoint != root: + required.append(root / "adapter.safetensors") + missing = [str(path) for path in required if not path.is_file()] + if missing: + raise FileNotFoundError( + "Hugging Face snapshot is missing required MiniLM files: " + ", ".join(missing) + ) + return root + + +def _download_snapshot( + repo_id: str, + *, + allow_patterns: tuple[str, ...], + revision: str | None = None, + cache_dir: str | Path | None = None, + local_dir: str | Path | None = None, + local_files_only: bool = False, + token: str | bool | None = None, +) -> Path: try: from huggingface_hub import snapshot_download except ImportError as error: @@ -83,27 +119,9 @@ def download_model( local_dir=Path(local_dir).expanduser() if local_dir is not None else None, local_files_only=local_files_only or offline, token=token, - allow_patterns=list(_MODEL_FILES), + allow_patterns=list(allow_patterns), ) except Exception as error: # Hub HTTP exception types differ across supported dependency versions. raise RuntimeError(f"Could not resolve Hugging Face model {repo_id!r}: {error}") from error - - root = Path(snapshot) - if (root / "base_model").is_dir() and all( - (root / name).is_file() for name in _CHECKPOINT_FILES - ): - raise ValueError( - "Hugging Face snapshot contains both a root checkpoint and base_model/; " - "use a clean local_dir to avoid mixing models" - ) - checkpoint = root / "base_model" if (root / "base_model").is_dir() else root - required = [checkpoint / name for name in _CHECKPOINT_FILES] - if checkpoint != root: - required.append(root / "adapter.safetensors") - missing = [str(path) for path in required if not path.is_file()] - if missing: - raise FileNotFoundError( - "Hugging Face snapshot is missing required MiniLM files: " + ", ".join(missing) - ) - return root + return Path(snapshot) diff --git a/src/akiki/vad.py b/src/akiki/vad.py index 9bbc145..97093a0 100644 --- a/src/akiki/vad.py +++ b/src/akiki/vad.py @@ -1,357 +1,37 @@ -"""Streaming speech detection and its configuration and result types. - -Use ``VAD`` with successive mono, 16 kHz audio buffers. Each complete frame -returns a ``SpeechFrame`` with timestamps, a probability, and a speech decision. -The bundled model processes 256 samples (16 ms) per frame. - -The Akiki engine is the primary engine. ``Engine.AUTO`` tries it first. -The ONNX and PyTorch engines provide compatibility fallbacks. -``ModelPrecision`` selects the VAD model's numeric format; ``FFTPrecision`` configures its FFT. -These enums belong to VAD; MiniLM uses ``akiki.decision.ModelPrecision``. - -Examples --------- ->>> from akiki import vad ->>> with vad.VAD(model_precision=vad.ModelPrecision.FP32) as detector: -... frames = detector.push([0.0] * 256) ->>> len(frames) -1 -""" - -import json -from dataclasses import dataclass -from pathlib import Path - -import numpy as np - -from ._engines import Engine -from ._precision import FFTPrecision, ModelPrecision - -__all__ = ["Engine", "FFTPrecision", "ModelPrecision", "SpeechFrame", "VAD"] - -SAMPLE_RATE = 16000 - - -@dataclass(frozen=True) -class SpeechFrame: - """Immutable speech decision for one complete frame returned by ``VAD.push``. - - Timestamps describe the input audio, measured from detector creation or the - last ``VAD.reset()`` call. They do not measure processing or wall-clock time. - The bundled model produces one frame for every 256 samples (16 ms). - - Attributes - ---------- - start_seconds : float - Inclusive start time of the frame, in seconds. - end_seconds : float - Exclusive end time of the frame, in seconds. - probability : float - Model's speech probability, between 0 and 1 inclusive. - speech : bool - Whether ``probability >= detector.threshold`` when the frame was - produced. Decisions are made per frame, without merging speech segments. - - Examples - -------- - >>> frame = SpeechFrame(0.0, 0.016, 0.9, True) - >>> frame.end_seconds - frame.start_seconds - 0.016 - >>> frame.probability, frame.speech - (0.9, True) - """ - - start_seconds: float - end_seconds: float - probability: float - speech: bool - - -class VAD: - """Identify speech in a stream of mono audio sampled at 16,000 Hz. - - Each instance maintains the recurrent state, audio context, and incomplete - frame for one stream. Feed successive buffers to ``push()``; call - ``reset()`` before an independent recording. The bundled model emits a - ``SpeechFrame`` for every 256 samples (16 ms). Custom models use their - supported ``chunk_samples`` value: 256 or 512 samples. - - Models load locally. The Akiki engine is the primary engine. - ``engine=Engine.AUTO`` tries it first, then the ONNX and PyTorch compatibility - fallbacks. The Akiki engine uses INT8 by default. Compatibility engines use FP32. - Explicit non-FP32 precision requires the Akiki engine. Install compatibility - runtimes with ``akiki[onnx]`` or ``akiki[pytorch]`` when needed. - See ``VAD.__init__`` for all constructor options. - - Use the detector as a context manager or call ``close()`` when finished. - Closing does not pad or score a final incomplete frame. - - Attributes - ---------- - engine : Engine - Selected inference engine: ``Engine.AKIKI``, ``Engine.ONNX``, or - ``Engine.PYTORCH``. - engine_errors : dict[Engine, str] - Reasons engines were unavailable before automatic selection succeeded. - model_id : str - Identifier from the loaded model's metadata. - model_precision : ModelPrecision - Active model precision, after resolving the engine's default. - fft_precision : FFTPrecision - FFT precision; always ``FFTPrecision.FP32``. - threshold : float - Minimum probability classified as speech. - chunk_samples : int - Number of input samples needed for one output frame. - context_samples : int - Number of prior audio samples retained as model context. - hidden : int - Number of recurrent hidden units in the loaded model. - completed_samples : int - Number of samples scored since creation or the last reset. - pending : numpy.ndarray - Unscored float32 samples buffered for the next call to ``push``. - context : numpy.ndarray - Float32 audio context with shape ``(context_samples,)``. - state : numpy.ndarray - Float32 recurrent state with shape ``(1, 1, hidden)``. - session : onnxruntime.InferenceSession - Present only when the selected engine is ONNX; ``None`` after closing. - - Notes - ----- - Treat ``pending``, ``context``, and ``state`` as read-only stream state. - Use a separate instance for each independent audio stream. - Calls on one detector are not protected by a lock. Do not call ``push``, - ``reset``, or ``close`` concurrently on the same instance. - - Examples - -------- - Successive buffers share a timeline and incomplete frames are retained:: - - >>> from akiki import VAD - >>> with VAD() as detector: - ... first = detector.push([0.0] * 100) - ... second = detector.push([0.0] * 156) - >>> first - [] - >>> len(second), second[0].end_seconds - (1, 0.016) - """ - - def __init__( - self, - model_path: str | Path | None = None, - threshold: float | None = None, - *, - engine: Engine = Engine.AUTO, - library_path: str | Path | None = None, - model_precision: ModelPrecision | None = None, - fft_precision: FFTPrecision = FFTPrecision.FP32, - cache_dir: str | Path | None = None, - ): - """Load a local model and inference engine, ready for a new stream. - - Parameters - ---------- - model_path : str or pathlib.Path, optional - If omitted, use the bundled model. - For a custom Akiki model, pass its JSON metadata path, such as - ``"models/vad.json"``. The Akiki engine loads the sibling - ``"models/vad-native/"`` directory. It must contain ``config.json``, - ``model.safetensors``, and ``export.json``. - The ONNX and PyTorch compatibility fallbacks use matching ``.onnx`` - and ``.pt`` files beside the metadata file. - Custom models must use the full-waveform GRU format. Supported - ``(chunk_samples, context_samples)`` pairs are ``(256, 320)`` and - ``(512, 64)``. Custom PyTorch metadata also requires ``pytorch_sha256``. - threshold : float, optional - Speech decision threshold between 0 and 1 inclusive. If omitted, - use the threshold in the model's metadata. This changes the boolean - speech decision, not the returned probability. - engine : Engine, default=Engine.AUTO - Automatic selection tries the primary Akiki engine first. - ONNX and PyTorch are compatibility fallbacks, in that order. - An explicitly selected engine must succeed; it does not fall back. - Pass an ``Engine`` member, such as ``Engine.AKIKI``. - library_path : str or pathlib.Path, optional - Specific local Akiki shared library to load. Affects only the Akiki - engine. If omitted, use ``AKIKI_VAD_LIBRARY`` or the bundled library. - Automatic discovery does not search developer directories. - model_precision : ModelPrecision, optional - One of ``ModelPrecision.FP32``, ``FP16``, ``BF16``, ``INT8``, - ``INT4``, or ``INT8_FP32_INPUT``. If omitted, Akiki uses INT8 and - ONNX/PyTorch use FP32. Explicit non-FP32 precision requires Akiki, - including in automatic mode. Reduced-precision Akiki weights are - generated from the verified local FP32 model and reused from the - cache. ``ModelPrecision.INT8_FP32_INPUT`` uses INT8 weights while - keeping the first projection's input in FP32. - fft_precision : FFTPrecision, default=FFTPrecision.FP32 - FFT precision. Only ``FFTPrecision.FP32`` is supported. - cache_dir : str or pathlib.Path, optional - Directory for generated Akiki model variants. If omitted, use - ``AKIKI_CACHE_DIR`` or the platform's user cache directory. Original - model weights are preserved. FP32 inference needs no conversion. - - Raises - ------ - TypeError - ``engine``, ``model_precision``, or ``fft_precision`` has the wrong - enum type. ``model_precision`` also accepts ``None``. - ValueError - A parameter, model specification, or model checksum is invalid. - OSError - The local model metadata cannot be read. - RuntimeError - No compatible engine is available for the requested configuration, - or the directory for a generated model variant is not writable. - """ - from ._engines import load_engine - - if not isinstance(engine, Engine): - raise TypeError("engine must be an Engine member, such as Engine.AUTO or Engine.AKIKI") - if model_precision is not None and not isinstance(model_precision, ModelPrecision): - raise TypeError("model_precision must be a ModelPrecision member or None") - if not isinstance(fft_precision, FFTPrecision): - raise TypeError("fft_precision must be an FFTPrecision member") - if model_precision not in (None, ModelPrecision.FP32) and engine not in ( - Engine.AUTO, - Engine.AKIKI, - ): - raise ValueError("Non-FP32 model_precision requires Engine.AKIKI or Engine.AUTO") - self.fft_precision = fft_precision - path = ( - Path(model_path) - if model_path is not None - else Path(__file__).parent / "models/akiki-vad.onnx" - ) - metadata = json.loads(path.with_suffix(".json").read_text()) - self.model_id = metadata.get("model_id", path.stem) - spec = metadata["spec"] - self.chunk_samples = int(spec["chunk_samples"]) - self.context_samples = int(spec["context_samples"]) - self.hidden = int(spec["hidden"]) - if spec["kind"] != "gru" or (self.chunk_samples, self.context_samples) not in { - (256, 320), - (512, 64), - }: - raise ValueError("Expected an Akiki full-waveform GRU model") - if spec.get("sample_rate", SAMPLE_RATE) != SAMPLE_RATE or self.hidden <= 0: - raise ValueError("Expected a 16 kHz model with a positive hidden size") - self.threshold = float(metadata["threshold"] if threshold is None else threshold) - if not np.isfinite(self.threshold) or not 0 <= self.threshold <= 1: - raise ValueError("Threshold must lie between zero and one") - self._backend, self.engine, self.engine_errors = load_engine( - path, metadata, engine, library_path, model_precision, cache_dir - ) - self.model_precision = model_precision or ( - ModelPrecision.INT8 if self.engine is Engine.AKIKI else ModelPrecision.FP32 - ) - if self.engine is Engine.ONNX: - self.session = self._backend.session - self.reset() - - def reset(self) -> None: - """Start an independent recording with the same model and settings. - - Clear the recurrent state and audio context, discard pending samples, - and reset output timestamps to zero. The model remains loaded. - - Raises - ------ - RuntimeError - The detector has been closed. Create a new instance to process more - audio after closing. - """ - self._backend.reset() - self.context = np.zeros(self.context_samples, dtype=np.float32) - self.state = np.zeros((1, 1, self.hidden), dtype=np.float32) - self.pending = np.empty(0, dtype=np.float32) - self.completed_samples = 0 - - def push(self, samples, sample_rate: int = SAMPLE_RATE) -> list[SpeechFrame]: - """Process the next audio buffer and return decisions for complete frames. - - Parameters - ---------- - samples : array_like - One-dimensional, finite mono waveform samples. Lists and NumPy - arrays are accepted and converted to float32. Samples should - normally be scaled to [-1, 1]; integer PCM is not normalized for - you. Audio is not resampled or mixed down from multiple channels. - sample_rate : int, default=16000 - Input sample rate in Hz. Must be 16000. - - Returns - ------- - list[SpeechFrame] - One result for each complete ``chunk_samples`` input samples, - including any samples buffered by a previous call. Timestamps - continue from earlier calls. Returns an empty list when there is - not enough audio for a complete frame. - - Raises - ------ - TypeError - The input type cannot be converted to numeric audio samples. - ValueError - Audio cannot be converted to float32, is not one-dimensional, - contains nonfinite values, or has the wrong sample rate. These - validation failures leave the stream state unchanged. - RuntimeError - The detector is closed, inference fails, or the model returns an - invalid speech probability. - - Notes - ----- - Buffers may have any length, including zero. A final incomplete frame - stays in ``pending`` until more audio arrives or ``reset()`` discards - it. Closing the detector does not pad or score those samples. - """ - self._backend.check_open() - audio = np.asarray(samples, dtype=np.float32) - if sample_rate != SAMPLE_RATE or audio.ndim != 1 or not np.isfinite(audio).all(): - raise ValueError("Expected finite mono 16 kHz audio samples") - audio = np.concatenate([self.pending, audio]) - complete = len(audio) // self.chunk_samples * self.chunk_samples - frames = [] - for start in range(0, complete, self.chunk_samples): - window = np.concatenate([self.context, audio[start : start + self.chunk_samples]]) - probability, state = self._backend.infer(window, self.state) - probability = float(np.asarray(probability).reshape(-1)[0]) - if not np.isfinite(probability) or not 0 <= probability <= 1: - raise RuntimeError("Model returned an invalid speech probability") - begin = self.completed_samples - self.state = state - self.context = window[-self.context_samples :].copy() - self.completed_samples += self.chunk_samples - frames.append( - SpeechFrame( - begin / SAMPLE_RATE, - self.completed_samples / SAMPLE_RATE, - probability, - probability >= self.threshold, - ) - ) - self.pending = audio[complete:].copy() - return frames - - def close(self) -> None: - """Release this stream's inference resources; safe to call repeatedly. - - Pending samples are not scored. After closing, ``push()``, ``reset()``, - and entering a ``with`` block raise ``RuntimeError``. Previously - returned ``SpeechFrame`` objects remain usable. - """ - self._backend.close() - if self.engine is Engine.ONNX: - self.session = None - - def __enter__(self): - """Return this open detector for use in a ``with`` block.""" - self._backend.check_open() - return self - - def __exit__(self, *_): - """Close the detector on leaving a ``with`` block; propagate exceptions.""" - self.close() +"""Deprecated imports for VAD. Use ``akiki.audio.vad``.""" + +import sys +import warnings + +from .audio.vad import SAMPLE_RATE as SAMPLE_RATE +from .audio.vad import VAD as VAD +from .audio.vad import Engine as Engine +from .audio.vad import FFTPrecision as FFTPrecision +from .audio.vad import Model as Model +from .audio.vad import ModelPrecision as ModelPrecision +from .audio.vad import SpeechFrame as SpeechFrame + + +def _deprecation_stacklevel() -> int: + # The warnings module skips the importlib bootstrap frames itself. + # Also skip the Python import wrapper and our package's lazy import. + level = 2 + frame = sys._getframe(1).f_back + while frame is not None: + module = frame.f_globals.get("__name__", "") + if module in (__package__, "importlib"): + level += 1 + elif not module.startswith("importlib._bootstrap"): + break + frame = frame.f_back + return level + + +warnings.warn( + "akiki.vad is deprecated and will be removed in version 0.3.0. " + "Use 'from akiki.audio import vad' instead.", + DeprecationWarning, + stacklevel=_deprecation_stacklevel(), +) + +__all__ = ["Engine", "FFTPrecision", "Model", "ModelPrecision", "SpeechFrame", "VAD"] diff --git a/tests/conftest.py b/tests/conftest.py index abc5281..8857ef9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from akiki import VAD, Engine +from akiki.audio.vad import VAD, Engine @pytest.fixture diff --git a/tests/test_api.py b/tests/test_api.py index 1c32cc4..ad8281d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,17 +1,22 @@ """Check the grouped public API and its existing import aliases.""" import importlib +import os +import subprocess +import sys +from pathlib import Path import pytest -from akiki import decision, vad, vision +import akiki +from akiki import audio, decision, vision +from akiki.audio import vad from akiki.hub import download_model @pytest.mark.parametrize( "name,expected", [ - ("VAD", vad.VAD), ("MiniLM", decision.MiniLM), ("SpeechFrame", vad.SpeechFrame), ("Engine", vad.Engine), @@ -28,6 +33,56 @@ def test_existing_top_level_imports_keep_their_identity(name, expected): assert namespace[name] is expected +@pytest.mark.parametrize("access", ["from akiki import VAD", "VAD = akiki.VAD"]) +def test_legacy_top_level_vad_warns_and_keeps_its_identity(access): + namespace = {"akiki": akiki} + with pytest.warns(DeprecationWarning, match=r"akiki\.VAD .* removed in version 0\.3\.0"): + exec(access, namespace) + assert namespace["VAD"] is vad.VAD + + +@pytest.mark.parametrize( + "access", + [ + "import akiki.vad as legacy", + "from akiki import vad as legacy", + "legacy = akiki.vad", + 'legacy = importlib.import_module("akiki.vad")', + ], +) +def test_legacy_vad_module_warns_and_keeps_its_exports(access): + result = subprocess.run( + [ + sys.executable, + "-c", + f""" +import importlib +import warnings + +with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + import akiki + from akiki.audio import vad + +with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always", DeprecationWarning) + {access} +[warning] = recorded +assert warning.filename == "", warning.filename +assert warning.category is DeprecationWarning +assert "akiki.vad is deprecated and will be removed in version 0.3.0" in str(warning.message) +assert legacy is importlib.import_module("akiki.vad") +for name in vad.__all__: + assert getattr(legacy, name) is getattr(vad, name) +""", + ], + env=os.environ | {"PYTHONPATH": str(Path(akiki.__file__).resolve().parents[1])}, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + @pytest.mark.parametrize("name", importlib.import_module("akiki.minilm").__all__) def test_legacy_minilm_imports_keep_their_identity(name): legacy = importlib.import_module("akiki.minilm") @@ -36,10 +91,13 @@ def test_legacy_minilm_imports_keep_their_identity(name): def test_wildcard_import_exposes_the_main_interfaces_and_modules(): namespace = {} - exec("from akiki import *", namespace) + with pytest.warns(DeprecationWarning, match=r"akiki\.VAD"): + exec("from akiki import *", namespace) assert namespace["VAD"] is vad.VAD assert namespace["MiniLM"] is decision.MiniLM - assert namespace["vad"] is vad + assert namespace["audio"] is audio + assert namespace["audio"].vad is vad + assert "vad" not in namespace assert namespace["decision"] is decision assert namespace["vision"] is vision assert "minilm" not in namespace diff --git a/tests/test_cli.py b/tests/test_cli.py index 90adddc..aaf20b8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,6 +11,7 @@ import akiki from akiki.__main__ import main +from akiki.audio import vad from akiki.decision import ModelPrecision as MiniLMPrecision @@ -144,7 +145,7 @@ def test_help_and_version_do_not_load_an_engine(arguments, expected, monkeypatch def unexpected_engine_load(*args, **kwargs): pytest.fail("Help and version must work without an inference engine") - monkeypatch.setattr(akiki, "VAD", unexpected_engine_load) + monkeypatch.setattr(vad, "VAD", unexpected_engine_load) monkeypatch.setattr(akiki, "MiniLM", unexpected_engine_load) monkeypatch.setitem(sys.modules, "huggingface_hub", None) monkeypatch.setattr(sys, "argv", ["akiki", *arguments]) @@ -193,7 +194,7 @@ def test_vad_engine_errors_include_the_command_prefix(monkeypatch, capsys): def unavailable_engine(*args, **kwargs): raise RuntimeError("No compatible engine") - monkeypatch.setattr(akiki, "VAD", unavailable_engine) + monkeypatch.setattr(vad, "VAD", unavailable_engine) monkeypatch.setattr(sys, "argv", ["akiki", "vad", "recording.wav"]) with pytest.raises(SystemExit) as error: main() diff --git a/tests/test_decision_inference.py b/tests/test_decision_inference.py index bbeb8ba..c2dd09c 100644 --- a/tests/test_decision_inference.py +++ b/tests/test_decision_inference.py @@ -13,7 +13,7 @@ import pytest from conftest import write_safetensors -from akiki import decision, vad +from akiki import decision from akiki._affine_cache import _base_identity, _digest from akiki._specialist_training import ( TrainedSpecialist, @@ -22,6 +22,7 @@ _root, _write_json, ) +from akiki.audio import vad from akiki.decision import DecisionModel, MiniLM, ModelPrecision, Specialist diff --git a/tests/test_hub.py b/tests/test_hub.py index c0f08e9..44cbc5d 100644 --- a/tests/test_hub.py +++ b/tests/test_hub.py @@ -14,6 +14,7 @@ import akiki from akiki import MiniLM, download_model from akiki.__main__ import main +from akiki.audio import vad from akiki.decision import ModelPrecision @@ -193,7 +194,7 @@ def unexpected_load(*args, **kwargs): pytest.fail("Downloading a checkpoint must not load an inference engine") monkeypatch.setattr(akiki, "MiniLM", unexpected_load) - monkeypatch.setattr(akiki, "VAD", unexpected_load) + monkeypatch.setattr(vad, "VAD", unexpected_load) destination = tmp_path / "model" monkeypatch.setattr( sys, diff --git a/tests/test_library_discovery.py b/tests/test_library_discovery.py index 4a757c7..abbab27 100644 --- a/tests/test_library_discovery.py +++ b/tests/test_library_discovery.py @@ -5,7 +5,7 @@ import pytest from akiki import _engines, decision -from akiki.vad import VAD, Engine, ModelPrecision +from akiki.audio.vad import VAD, Engine, ModelPrecision @pytest.fixture(params=["win32", "darwin", "linux"]) diff --git a/tests/test_minilm.py b/tests/test_minilm.py index c0aa782..06bf1ef 100644 --- a/tests/test_minilm.py +++ b/tests/test_minilm.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from akiki import vad +from akiki.audio import vad from akiki.decision import MiniLM, ModelPrecision diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 40f3afc..6c8dd2d 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -208,6 +208,7 @@ def test_distributions_exclude_local_work_and_pass_release_verification(packaged directory = project / "dist" with zipfile.ZipFile(next(directory.glob("*.whl"))) as wheel: names = wheel.namelist() + assert "akiki/audio/vad.py" in names assert "akiki/decision.py" in names assert "akiki/vision/__init__.py" in names assert "akiki/vision/object_detection.py" in names diff --git a/tests/test_quantization.py b/tests/test_quantization.py index 6414aa8..3097f90 100644 --- a/tests/test_quantization.py +++ b/tests/test_quantization.py @@ -9,8 +9,8 @@ import pytest import akiki -from akiki import VAD, Engine, FFTPrecision, ModelPrecision from akiki import _quantization as quant +from akiki.audio.vad import VAD, Engine, FFTPrecision, ModelPrecision from akiki.decision import ModelPrecision as MiniLMPrecision diff --git a/tests/test_ten_vad_model.py b/tests/test_ten_vad_model.py new file mode 100644 index 0000000..fa8d740 --- /dev/null +++ b/tests/test_ten_vad_model.py @@ -0,0 +1,79 @@ +"""Test TEN-VAD source selection and native cache handling offline.""" + +import hashlib +import sys +from types import SimpleNamespace + +import pytest + +from akiki.audio import _ten_vad_model + + +def test_pinned_snapshot_is_resolved_without_network(tmp_path, monkeypatch): + calls = [] + + def snapshot_download(**kwargs): + calls.append(kwargs) + return tmp_path + + sources = { + _ten_vad_model._ONNX: b"ONNX model", + _ten_vad_model._COEFFICIENTS: b"frontend coefficients", + } + for name, data in sources.items(): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + monkeypatch.delenv("HF_HUB_OFFLINE", raising=False) + monkeypatch.setitem( + sys.modules, "huggingface_hub", SimpleNamespace(snapshot_download=snapshot_download) + ) + monkeypatch.setattr( + _ten_vad_model, + "_HUB_CHECKSUMS", + {name: hashlib.sha256(data).hexdigest() for name, data in sources.items()}, + ) + + assert _ten_vad_model._read_sources(None) == tuple(sources.values()) + assert calls[0]["repo_id"] == "TEN-framework/ten-vad" + assert calls[0]["revision"] == _ten_vad_model._REVISION + assert calls[0]["local_files_only"] is True + assert calls[0]["allow_patterns"] == list(sources) + + (tmp_path / _ten_vad_model._ONNX).write_bytes(b"changed model") + with pytest.raises(ValueError, match="incorrect checksum"): + _ten_vad_model._read_sources(None) + + +def test_local_onnx_reuses_and_repairs_native_cache(tmp_path, monkeypatch): + model = tmp_path / "src/onnx_model/ten-vad.onnx" + model.parent.mkdir(parents=True) + model.write_bytes(b"local weights") + (model.parent.parent / "coeff.h").write_bytes(b"local coefficients") + calls = [] + payload = _ten_vad_model._NATIVE_MAGIC + bytes(_ten_vad_model._NATIVE_SIZE - 8) + + def convert(weights, coefficients): + calls.append((weights, coefficients)) + return payload + + monkeypatch.setitem(sys.modules, "huggingface_hub", None) + monkeypatch.setattr(_ten_vad_model, "_convert", convert) + + cache = tmp_path / "cache" + cached, source = _ten_vad_model.prepare_model(model, cache) + assert source == b"local weights" + assert cached.read_bytes() == payload + assert _ten_vad_model.prepare_model(model, cache) == (cached, source) + assert calls == [(b"local weights", b"local coefficients")] + + cached.write_bytes(payload[:-1] + b"\x01") + assert _ten_vad_model.prepare_model(model, cache) == (cached, source) + assert cached.read_bytes() == payload + assert len(calls) == 2 + + +def test_ten_vad_rejects_native_files_as_public_input(tmp_path): + with pytest.raises(ValueError, match="must point to its .onnx weights"): + _ten_vad_model.prepare_model(tmp_path / "weights.tnv", tmp_path / "cache") diff --git a/tests/test_training_api.py b/tests/test_training_api.py index 68eeb01..12aed04 100644 --- a/tests/test_training_api.py +++ b/tests/test_training_api.py @@ -38,11 +38,11 @@ def find_spec(self, fullname, path=None, target=None): sys.meta_path.insert(0, BlockTraining()) import akiki -assert akiki.VAD is not None -assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'decision', 'vad', 'vision'] +assert akiki.audio.vad.VAD is not None +assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'audio', 'decision', 'vision'] namespace = {} exec('from akiki import *', namespace) -assert namespace['vad'].VAD is akiki.VAD +assert namespace['audio'].vad.VAD is akiki.audio.vad.VAD assert namespace['decision'].MiniLM is akiki.MiniLM assert 'akiki.training' not in sys.modules import importlib @@ -85,7 +85,7 @@ def find_spec(self, fullname, path=None, target=None): from akiki_training import training as legacy assert training is akiki.training -assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'decision', 'training', 'vad', 'vision'] +assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'audio', 'decision', 'training', 'vision'] assert dir(training) == sorted(backend.__all__) assert training.__all__ is backend.__all__ for name in backend.__all__: diff --git a/tests/test_vad.py b/tests/test_vad.py index 99f25a3..ec8b45f 100644 --- a/tests/test_vad.py +++ b/tests/test_vad.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from akiki import VAD, Engine, FFTPrecision, ModelPrecision +from akiki.audio.vad import VAD, Engine, FFTPrecision, Model, ModelPrecision def test_primary_engine_uses_the_akiki_name(): @@ -110,7 +110,9 @@ def test_default_is_selected_release_checkpoint(): model_dir = Path(akiki.__file__).parent / "models" metadata = json.loads((model_dir / "akiki-vad.json").read_text()) assert akiki.__version__ == "0.2.2" - assert VAD().model_id == "teacher-only623-v2-s1730" + with VAD() as default, VAD(model=Model.AKIKI_VAD) as named: + assert default.model_id == named.model_id == "teacher-only623-v2-s1730" + assert default.push([0.0] * 256) == named.push([0.0] * 256) assert metadata["origin"]["seed"] == 1730 assert metadata["release_selection"]["checkpoint_window"] == 31 assert ( @@ -119,6 +121,64 @@ def test_default_is_selected_release_checkpoint(): ) +def test_named_ten_vad_streams_with_its_own_state(monkeypatch, tmp_path): + from akiki._engines import Backend + from akiki.audio import _ten_vad, _ten_vad_model + + model_path = tmp_path / "weights.onnx" + native_path = tmp_path / "model.bin" + library_path = tmp_path / "engine.so" + + class FakeTenVad(Backend): + chunk_samples = 256 + hidden = 64 + state_shape = (4, hidden) + + @staticmethod + def load_library(path): + assert path == library_path + return library_path + + def __init__(self, path, library): + assert path == native_path + assert library == library_path + + def infer(self, window, state): + assert window.shape == (self.chunk_samples,) + return 0.25, state + 1 + + def prepare(path, cache): + assert path == model_path + assert cache is None + return native_path, b"fake ONNX data" + + monkeypatch.setattr(_ten_vad_model, "prepare_model", prepare) + monkeypatch.setattr(_ten_vad, "TenVadBackend", FakeTenVad) + with VAD(model=Model.TEN_VAD, model_path=model_path, library_path=library_path) as detector: + assert detector.model_id == "ten-vad" + assert detector.model_precision is ModelPrecision.FP32 + assert detector.threshold == 0.5 + assert detector.push([0.0] * 128) == [] + with pytest.raises(ValueError, match="requires audio samples"): + detector.push([0.0] * 256 + [1.5]) + assert detector.completed_samples == 0 + assert len(detector.pending) == 128 + frames = detector.push([0.0] * 384) + assert [(frame.start_seconds, frame.end_seconds) for frame in frames] == [ + (0.0, 0.016), + (0.016, 0.032), + ] + assert all(not frame.speech for frame in frames) + assert detector.context.size == 0 + np.testing.assert_array_equal(detector.state, np.full((4, 64), 2)) + detector.reset() + np.testing.assert_array_equal(detector.state, np.zeros((4, 64))) + assert detector.push([0.0] * 512) == frames + + with pytest.raises(RuntimeError, match="closed"): + detector.push([]) + + def test_engines_agree(engine): pytest.importorskip("onnxruntime") audio = np.random.default_rng(42).normal(0, 0.05, 16000).astype(np.float32) @@ -229,6 +289,12 @@ def test_corrupt_native_model_is_not_hidden_by_fallback(tmp_path, precision): ) +@pytest.mark.parametrize("model", ["TEN-VAD", "akiki-vad", "unknown", Engine.AKIKI, None]) +def test_non_enum_model_is_rejected(model): + with pytest.raises(TypeError, match="model must be a Model member"): + VAD(model=model) + + @pytest.mark.parametrize("engine", ["auto", "akiki", "onnx", "pytorch", "unknown", None, 1, []]) def test_non_enum_engine_is_rejected(engine): with pytest.raises(TypeError, match="engine must be an Engine member"): diff --git a/tests/test_vad_runtimes.py b/tests/test_vad_runtimes.py new file mode 100644 index 0000000..e7a02f6 --- /dev/null +++ b/tests/test_vad_runtimes.py @@ -0,0 +1,211 @@ +"""Run VAD inference with the real CPU engines and local test data.""" + +import hashlib +import os + +import numpy as np +import pytest + +from akiki.audio import _ten_vad_model, vad + +pytestmark = pytest.mark.skipif( + os.environ.get("AKIKI_TEST_VAD_RUNTIMES") != "1", + reason="Run the native and ONNX Runtime matrix with AKIKI_TEST_VAD_RUNTIMES=1", +) + + +@pytest.fixture +def ten_vad_onnx(tmp_path, monkeypatch): + """Generate a TEN-shaped ONNX graph with synthetic weights.""" + import onnx + from onnx import TensorProto, helper, numpy_helper + + rng = np.random.default_rng(27) + nodes, initializers = [], [] + + def initializer(name, values): + initializers.append(numpy_helper.from_array(np.asarray(values), name=name)) + return name + + def weight(name, shape): + return initializer(name, rng.uniform(-0.15, 0.15, shape).astype(np.float32)) + + def node(kind, inputs, output, **attributes): + nodes.append(helper.make_node(kind, inputs, [output], **attributes)) + return output + + prefix = "StatefulPartitionedCall/vad_model/" + axis_zero = initializer("axis_zero", np.array([0], np.int64)) + axis_one = initializer("axis_one", np.array([1], np.int64)) + features = node("Unsqueeze", ["features", axis_one], "features_nchw") + for layer, index, depth_shape, point, point_shape, group, pads in ( + ( + "separable_conv2d", + 178, + (1, 1, 3, 3), + "separable_conv2d/ReadVariableOp_1:0", + (16, 1, 1, 1), + 1, + [0, 0, 0, 0], + ), + ( + "separable_conv1d", + 179, + (16, 1, 1, 3), + "ExpandDims_2:0", + (16, 16, 1, 1), + 16, + [0, 1, 0, 1], + ), + ( + "separable_conv1d_1", + 180, + (16, 1, 1, 3), + "ExpandDims_2:0", + (16, 16, 1, 1), + 16, + [0, 0, 0, 1], + ), + ): + depth = weight(f"const_fold_opt__{index}", depth_shape) + depth_out = node( + "Conv", + [features, depth], + f"depth_{index}", + group=group, + pads=pads, + strides=[1, 1 if index == 178 else 2], + ) + projection = weight(f"{prefix}{layer}/{point}", point_shape) + bias = weight(f"{prefix}{layer}/BiasAdd/ReadVariableOp:0", (16,)) + conv = node("Conv", [depth_out, projection, bias], f"point_{index}") + features = node("Relu", [conv], f"relu_{index}") + if index == 178: + features = node("MaxPool", [features], "pooled", kernel_shape=[1, 3], strides=[1, 2]) + + ordered = node("Transpose", [features], "frequency_major", perm=[0, 2, 3, 1]) + lstm_shape = initializer("lstm_shape", np.array([1, 1, 80], np.int64)) + features = node("Reshape", [ordered, lstm_shape], "lstm_input") + state_names = ("hidden1", "cell1", "hidden2", "cell2") + hidden_outputs = [] + for index, (weight_name, recurrent_name, bias_name, inputs) in enumerate( + (("W0__70", "R0__71", "B0__72", 80), ("W0__99", "R0__100", "B0__101", 64)) + ): + hidden_name, cell_name = state_names[2 * index : 2 * index + 2] + hidden = node("Unsqueeze", [hidden_name, axis_zero], f"{hidden_name}_3d") + cell = node("Unsqueeze", [cell_name, axis_zero], f"{cell_name}_3d") + hidden_out, cell_out = f"{hidden_name}_out_3d", f"{cell_name}_out_3d" + nodes.append( + helper.make_node( + "LSTM", + [ + features, + weight(weight_name, (1, 256, inputs)), + weight(recurrent_name, (1, 256, 64)), + weight(bias_name, (1, 512)), + "", + hidden, + cell, + ], + [f"lstm_sequence{index}", hidden_out, cell_out], + hidden_size=64, + ) + ) + node("Squeeze", [hidden_out, axis_zero], f"{hidden_name}_out") + node("Squeeze", [cell_out, axis_zero], f"{cell_name}_out") + hidden_outputs.append(hidden_out) + features = hidden_out + + features = node("Concat", hidden_outputs[::-1], "joined", axis=2) + for layer, inputs, outputs in (("dense_3", 128, 32), ("dense_5", 32, 1)): + projection = weight(f"{prefix}{layer}/Tensordot/ReadVariableOp:0", (inputs, outputs)) + bias = weight(f"{prefix}{layer}/BiasAdd/ReadVariableOp:0", (outputs,)) + features = node("MatMul", [features, projection], f"{layer}_projection") + features = node("Add", [features, bias], f"{layer}_output") + if layer == "dense_3": + features = node("Relu", [features], "dense_relu") + node("Sigmoid", [features], "probability") + + def value(name, shape): + return helper.make_tensor_value_info(name, TensorProto.FLOAT, shape) + + graph = helper.make_graph( + nodes, + "synthetic-ten-vad", + [value("features", [1, 3, 41]), *(value(name, [1, 64]) for name in state_names)], + [value("probability", [1, 1, 1]), *(value(f"{name}_out", [1, 64]) for name in state_names)], + initializer=initializers, + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)], ir_version=8) + onnx.checker.check_model(model) + path = tmp_path / "ten-vad.onnx" + onnx.save(model, path) + + # The production converter pins the original graph. Permit only this test graph here. + for tensor in graph.initializer: + if tensor.data_type == TensorProto.FLOAT: + tensor.ClearField("raw_data") + monkeypatch.setattr( + _ten_vad_model, "_GRAPH_SHA256", hashlib.sha256(graph.SerializeToString()).hexdigest() + ) + + window = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(768) / 768) + arrays = ( + ("AUP_AED_FEATURE_MEANS", np.zeros(41)), + ("AUP_AED_FEATURE_STDS", np.ones(41)), + ("AUP_AED_STFTWindow_Hann768", window), + ) + (tmp_path / "coeff.h").write_text( + "".join( + f"float {name}[{len(values)}] = {{{','.join(map(str, values))}}};\n" + for name, values in arrays + ) + ) + return path + + +@pytest.mark.parametrize("model", list(vad.Model)) +def test_vad_scores_and_resets_on_both_cpu_runtimes(model, request, tmp_path): + is_ten = model is vad.Model.TEN_VAD + model_path = request.getfixturevalue("ten_vad_onnx") if is_ten else None + options = dict( + model=model, + model_path=model_path, + model_precision=vad.ModelPrecision.FP32, + cache_dir=tmp_path / "cache", + ) + boundary = (1875 if is_ten else 9) * 256 + audio = np.random.default_rng(28).uniform(-0.3, 0.3, boundary + 256 + 17).astype(np.float32) + with ( + vad.VAD(engine=vad.Engine.AKIKI, **options) as native, + vad.VAD(engine=vad.Engine.ONNX, **options) as onnx, + ): + assert native.engine is vad.Engine.AKIKI + assert onnx.engine is vad.Engine.ONNX + assert onnx.session.get_providers() == ["CPUExecutionProvider"] + results = [] + for detector in (native, onnx): + assert detector.push(audio[:37]) == [] + frames = detector.push(audio[37 : boundary - 256]) + assert np.any(detector.state) + frames += detector.push(audio[boundary - 256 : boundary]) + if is_ten: + np.testing.assert_array_equal(detector.state, 0) + frames += detector.push(audio[boundary:]) + assert len(detector.pending) == 17 + assert np.any(detector.state) + assert len(frames) == boundary // 256 + 1 + assert frames[0].start_seconds == 0 + assert frames[-1].end_seconds == (boundary + 256) / 16000 + detector.reset() + assert detector.push(audio[:256]) == frames[:1] + results.append(frames) + + native_frames, onnx_frames = results + np.testing.assert_allclose( + [frame.probability for frame in native_frames], + [frame.probability for frame in onnx_frames], + atol=2e-5, + rtol=1e-5, + ) + assert [frame.speech for frame in native_frames] == [frame.speech for frame in onnx_frames] diff --git a/tools/benchmark_latency.py b/tools/benchmark_latency.py index 8fd930b..d80c8db 100644 --- a/tools/benchmark_latency.py +++ b/tools/benchmark_latency.py @@ -25,7 +25,8 @@ def worker(args): import numpy as np import akiki - from akiki import VAD, Engine, __version__ + from akiki import __version__ + from akiki.audio.vad import VAD, Engine engine = ( Engine.AUTO diff --git a/tools/build_native.py b/tools/build_native.py index 87acd03..bb0fb68 100644 --- a/tools/build_native.py +++ b/tools/build_native.py @@ -7,7 +7,7 @@ import sys from pathlib import Path -# Shared libraries loaded by akiki.VAD and akiki.MiniLM. +# Shared libraries loaded by akiki.audio.vad.VAD and akiki.MiniLM. PACKAGES = ("akiki-vad-engine", "akiki-minilm-engine") diff --git a/tools/evaluate_precision_tradeoff.py b/tools/evaluate_precision_tradeoff.py index f3df381..93dc5d4 100644 --- a/tools/evaluate_precision_tradeoff.py +++ b/tools/evaluate_precision_tradeoff.py @@ -26,7 +26,8 @@ import soundfile as sf from sklearn.metrics import average_precision_score, precision_recall_fscore_support -from akiki import VAD, Engine, FFTPrecision, ModelPrecision, __version__ +from akiki import __version__ +from akiki.audio.vad import VAD, Engine, FFTPrecision, ModelPrecision MODES = tuple(ModelPrecision) diff --git a/verify_dist.py b/verify_dist.py index 369dd79..c448dbc 100644 --- a/verify_dist.py +++ b/verify_dist.py @@ -47,7 +47,7 @@ model_dir = package_dir / "models" python_modules = { "akiki/" + path.relative_to(package_dir).as_posix(): path - for package in (package_dir, package_dir / "vision") + for package in (package_dir, package_dir / "audio", package_dir / "vision") for path in package.glob("*.py") } expected_id = "teacher-only623-v2-s1730" @@ -131,7 +131,7 @@ def check_metadata(data): ) require( set(metadata.get_all("Provides-Extra") or []) - == {"huggingface", "onnx", "pytorch", "training"}, + == {"huggingface", "onnx", "pytorch", "ten-vad", "training"}, "Missing optional feature extras", ) require( @@ -142,6 +142,8 @@ def check_metadata(data): 'huggingface-hub>=0.24; extra == "huggingface"', 'onnxruntime>=1.17; extra == "onnx"', 'torch>=2.1; extra == "pytorch"', + 'huggingface-hub>=0.24; extra == "ten-vad"', + 'onnx>=1.16; extra == "ten-vad"', 'torch>=2.1; extra == "training"', 'transformers<6,>=4.40; extra == "training"', 'safetensors>=0.4; extra == "training"', @@ -236,10 +238,7 @@ def check_models(read, names): ) info["model"] = check_models(archive.read, archive.namelist()) for name, path in python_modules.items(): - require( - archive.read(name) == path.read_bytes(), - f"Stale Python module: {name}", - ) + require(archive.read(name) == path.read_bytes(), f"Stale Python module: {name}") libraries = [ name for name in archive.namelist()