From 97072e0c0c04f5feda2430f2ea5533cb039c66ea Mon Sep 17 00:00:00 2001 From: Toby Jackson Date: Wed, 18 Nov 2026 12:41:05 +0000 Subject: [PATCH] Add cached Seen object detection API --- README.md | 35 +++- pyproject.toml | 4 +- src/akiki/__init__.py | 11 +- src/akiki/vision/__init__.py | 5 + src/akiki/vision/object_detection.py | 252 +++++++++++++++++++++++++++ tests/test_api.py | 3 +- tests/test_object_detection.py | 140 +++++++++++++++ tests/test_packaging.py | 2 + tests/test_training_api.py | 4 +- verify_dist.py | 23 ++- 10 files changed, 462 insertions(+), 17 deletions(-) create mode 100644 src/akiki/vision/__init__.py create mode 100644 src/akiki/vision/object_detection.py create mode 100644 tests/test_object_detection.py diff --git a/README.md b/README.md index 2104d76..fee1705 100644 --- a/README.md +++ b/README.md @@ -116,11 +116,13 @@ PyTorch is the second VAD compatibility fallback. ## Python usage Use `akiki.vad` for VAD and `akiki.decision` for decision models. +Use `akiki.vision.object_detection` for object detection. The training and teacher modules support the decision-model workflow: | Module | Main interface | Supporting types | | --- | --- | --- | | `akiki.vad` | `VAD` for speech detection | `SpeechFrame`, `Engine`, `ModelPrecision`, `FFTPrecision` | +| `akiki.vision.object_detection` | `ObjectDetector` for object detection | `Detection` | | `akiki.decision` | `DecisionModel` for specialist decisions | `BaseModel`, `Specialist`, `AffineLayer`, `TrainedSpecialist`, `SpecialistEvaluation`, `TrainingAdvice`, `list_base_models`, `Classification`, `ModelPrecision` | | `akiki.training` | Classifier training with a private companion | The installed companion's API, including `LocalModel`, `ModelFamily`, and `Trainer` | | `akiki.teacher` | Experimental review through a local or self-hosted server | `Teacher`, `Verdict`, `DecisionReview`, `TeachingResult` | @@ -148,7 +150,38 @@ modules remain accessible but are omitted from that list. The decision-model module is named `akiki.decision`. Existing imports from `akiki.minilm` remain available for compatibility and use the same classes and precision enum. Named top-level compatibility imports remain available. -Wildcard imports expose only `VAD`, `MiniLM`, `vad`, `decision`, and `__version__`. +Wildcard imports expose only `VAD`, `MiniLM`, `vad`, `decision`, `vision`, and `__version__`. + +## Object detection + +Use `akiki.vision.object_detection` to find objects in an RGB image. +This interface supports `Akiki-AI/akiki-cam-seen`. +It loads the Seen r4c files from the local Hugging Face cache. + +Set `HF_HUB_OFFLINE=1` when you use the Hugging Face cache. +Use the Hugging Face extra to locate the cached snapshot. +You can also pass an existing snapshot folder as `model_path`. +Set `AKIKI_SEEN_LIBRARY` to the path of your local Akiki Seen library, or +pass `library_path` when you create the detector. + +This example uses Pillow to read an image file. + +```python +import numpy as np +from PIL import Image + +from akiki.vision import object_detection + +with Image.open("frame.jpg") as frame: + image = np.asarray(frame.convert("RGB")) + +with object_detection.ObjectDetector() as detector: + for result in detector.detect(image): + print(result.word, result.score, (result.x0, result.y0, result.x1, result.y1)) +``` + +Use `uint8` images with shape `(height, width, 3)` and RGB channel order. +The box coordinates are in pixels of the source image. ## Voice activity detection (VAD) diff --git a/pyproject.toml b/pyproject.toml index 38932f2..bd9f768 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "akiki" version = "0.2.2" -description = "Local voice activity detection and decision models" +description = "Local voice activity detection, object detection, and decision models" readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" @@ -45,7 +45,7 @@ include-package-data = false [tool.setuptools.packages.find] where = ["src"] -include = ["akiki"] +include = ["akiki", "akiki.vision"] namespaces = false [tool.setuptools.package-data] diff --git a/src/akiki/__init__.py b/src/akiki/__init__.py index 8ca3a65..9b1aeed 100644 --- a/src/akiki/__init__.py +++ b/src/akiki/__init__.py @@ -1,13 +1,17 @@ -"""Local voice activity detection and decision models. +"""Local voice activity detection, object detection, and decision models. The primary interfaces are ``akiki.vad.VAD`` and ``akiki.decision.DecisionModel``. Training and teacher review support the decision-model workflow. -The public API is organized into four areas: +The public API has five areas. ``akiki.vad`` Streaming voice activity detection with ``VAD``. Results are ``SpeechFrame`` objects. Configure inference with this module's ``Engine``, ``ModelPrecision``, and ``FFTPrecision`` enums. The VAD model is bundled. +``akiki.vision.object_detection`` + Use ``ObjectDetector`` to find objects in RGB images. + Each ``Detection`` gives a family, score, and box. + Load Akiki Cam Seen from its local Hugging Face snapshot. ``akiki.decision`` Select a local base checkpoint with ``DecisionModel`` and ``list_base_models``. Classify saved specialists with ``DecisionModel`` through the Akiki engine. @@ -74,7 +78,7 @@ Installed Akiki package version. """ -from . import decision, vad +from . import decision, vad, vision from . import minilm as minilm from .decision import Classification as Classification from .decision import Encoding as Encoding @@ -93,6 +97,7 @@ "__version__", "decision", "vad", + "vision", ] diff --git a/src/akiki/vision/__init__.py b/src/akiki/vision/__init__.py new file mode 100644 index 0000000..c2d1689 --- /dev/null +++ b/src/akiki/vision/__init__.py @@ -0,0 +1,5 @@ +"""Vision inference interfaces.""" + +from . import object_detection + +__all__ = ["object_detection"] diff --git a/src/akiki/vision/object_detection.py b/src/akiki/vision/object_detection.py new file mode 100644 index 0000000..e31dee2 --- /dev/null +++ b/src/akiki/vision/object_detection.py @@ -0,0 +1,252 @@ +"""Object detection in RGB images with the local Akiki engine. + +Use ObjectDetector with the cached Akiki-AI/akiki-cam-seen snapshot. +Each Detection gives an object family, a score, and a box in image pixels. +""" + +import ctypes +import math +import os +import threading +import weakref +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from .._engines import library_candidates + +__all__ = ["Detection", "ObjectDetector"] + +_MODEL_ID = "Akiki-AI/akiki-cam-seen" +_FILES = ( + "seen-r4c-56508271-model.json", + "seen-r4c-56508271-weights.json", + "seen-r4c-56508271-weights.bin", +) +_MAX_DETECTIONS = 100 + + +@dataclass(frozen=True) +class Detection: + """One object family and its bounding box. + + The box gives the left, top, right, and bottom edges in source image pixels. + """ + + family: str + word: str + score: float + x0: float + y0: float + x1: float + y1: float + + +class _NativeDetection(ctypes.Structure): + _fields_ = [ + ("family", ctypes.c_uint32), + ("score", ctypes.c_float), + ("x0", ctypes.c_float), + ("y0", ctypes.c_float), + ("x1", ctypes.c_float), + ("y1", ctypes.c_float), + ] + + +def _cached_snapshot(cache_dir: str | Path | None) -> Path: + try: + from huggingface_hub import snapshot_download + from huggingface_hub.utils import LocalEntryNotFoundError + except ImportError as error: + raise RuntimeError( + "Install akiki[huggingface] to find the cached Seen model, " + "or pass model_path for a local snapshot" + ) from error + try: + return Path( + snapshot_download( + repo_id=_MODEL_ID, + cache_dir=cache_dir, + local_files_only=True, + ) + ) + except LocalEntryNotFoundError as error: + raise FileNotFoundError( + f"The Hugging Face cache has no snapshot for {_MODEL_ID}. " + "Pass model_path for an existing local snapshot" + ) from error + + +def _load_library(library_path: str | Path | None) -> tuple[ctypes.CDLL, Path]: + candidate = next(library_candidates(library_path, "akiki_seen_engine", "AKIKI_SEEN_LIBRARY")) + path = candidate.resolve() + pointer, size = ctypes.c_void_p, ctypes.c_size_t + bindings = { + "load": ([ctypes.c_char_p, ctypes.POINTER(size), size], pointer), + "free": ([pointer], None), + "family_count": ([pointer], size), + "family_id": ([pointer, size], ctypes.c_char_p), + "family_word": ([pointer, size], ctypes.c_char_p), + "last_error": ([], ctypes.c_char_p), + "detect": ( + [ + pointer, + pointer, + ctypes.c_uint32, + ctypes.c_uint32, + size, + ctypes.c_uint32, + ctypes.c_float, + ctypes.c_int32, + ctypes.POINTER(_NativeDetection), + size, + ], + ctypes.c_int64, + ), + } + try: + library = ctypes.CDLL(str(path)) + for name, (args, result) in bindings.items(): + function = getattr(library, f"akiki_seen_{name}") + function.argtypes = args + function.restype = result + except (OSError, AttributeError) as error: + raise RuntimeError( + f"No compatible Akiki Seen library at {path}: {error}. " + "Set AKIKI_SEEN_LIBRARY or pass library_path for a local library" + ) from error + return library, path + + +def _error_text(library: ctypes.CDLL) -> str: + return (library.akiki_seen_last_error() or b"unknown engine error").decode("utf-8", "replace") + + +class ObjectDetector: + """Find the object families supported by Akiki Cam Seen. + + Pass a local Hugging Face snapshot folder as model_path. If you omit it, + the detector looks in the configured Hugging Face cache. Use cache_dir to + select a different cache folder. The cache lookup is local only. + + Pass library_path or set AKIKI_SEEN_LIBRARY to select a local Akiki engine + library. Pass cpus to pin each engine worker to one CPU. If you omit cpus, + the engine selects the CPU placement. + + Use a with block or call close() to release the engine. + """ + + model_id = _MODEL_ID + + def __init__( + self, + model_path: str | Path | None = None, + *, + cache_dir: str | Path | None = None, + library_path: str | Path | None = None, + cpus: Sequence[int] | None = None, + ) -> None: + cpu_ids = tuple(cpus) if cpus is not None else () + max_cpu = 2 ** (8 * ctypes.sizeof(ctypes.c_size_t)) - 1 + if any(type(cpu) is not int or not 0 <= cpu <= max_cpu for cpu in cpu_ids): + raise ValueError("cpus must contain nonnegative integer CPU IDs") + if len(set(cpu_ids)) != len(cpu_ids): + raise ValueError("cpus must contain distinct CPU IDs") + + root = Path(model_path) if model_path is not None else _cached_snapshot(cache_dir) + self.model_path = root.expanduser().resolve() + weights_dir = self.model_path / "weights" + missing = [name for name in _FILES if not (weights_dir / name).is_file()] + if missing: + raise FileNotFoundError( + f"Missing Seen model files in {weights_dir}: {', '.join(missing)}" + ) + + library, self.library_path = _load_library(library_path) + native_cpus = (ctypes.c_size_t * len(cpu_ids))(*cpu_ids) if cpu_ids else None + handle = library.akiki_seen_load(os.fsencode(weights_dir), native_cpus, len(cpu_ids)) + if not handle: + raise ValueError(f"Akiki engine cannot load {self.model_path}: {_error_text(library)}") + + self._library = library + self._handle = handle + self._lock = threading.Lock() + self._finalizer = weakref.finalize(self, library.akiki_seen_free, handle) + try: + self._families = tuple( + ( + library.akiki_seen_family_id(handle, index).decode("utf-8"), + library.akiki_seen_family_word(handle, index).decode("utf-8"), + ) + for index in range(library.akiki_seen_family_count(handle)) + ) + except Exception: + self.close() + raise + + def detect( + self, image: np.ndarray, *, threshold: float | None = None, policy: bool = True + ) -> list[Detection]: + """Find objects in an image with shape (height, width, 3) and dtype uint8. + + Use RGB channel order. Omit threshold to use the model's score threshold, + or give a value greater than zero and at most one. Set policy to False + to skip the model's per-family score limits. Results are in score order. + """ + with self._lock: + self._check_open() + image = np.asarray(image) + if image.dtype != np.uint8 or image.ndim != 3 or image.shape[2] != 3: + raise ValueError("Expected an RGB uint8 image with shape (height, width, 3)") + height, width = image.shape[:2] + if not (0 < height < 2**32 and 0 < width < 2**32): + raise ValueError("Image height and width must be positive 32-bit dimensions") + confidence = 0.0 if threshold is None else float(threshold) + if threshold is not None and not (math.isfinite(confidence) and 0 < confidence <= 1): + raise ValueError("threshold must be greater than zero and at most one") + if not isinstance(policy, bool): + raise TypeError("policy must be a bool") + + pixels = np.ascontiguousarray(image) + results = (_NativeDetection * _MAX_DETECTIONS)() + count = self._library.akiki_seen_detect( + self._handle, + pixels.ctypes.data, + width, + height, + 0, + 0, + confidence, + int(policy), + results, + len(results), + ) + if count < 0: + raise RuntimeError(f"Seen detection failed: {_error_text(self._library)}") + return [ + Detection( + *self._families[item.family], item.score, item.x0, item.y0, item.x1, item.y1 + ) + for item in results[:count] + ] + + def _check_open(self) -> None: + if not self._finalizer.alive: + raise RuntimeError("The object detector is closed") + + def close(self) -> None: + """Release the engine. You can call this method more than once.""" + with self._lock: + self._finalizer() + + def __enter__(self): + """Return this open detector for use in a with block.""" + with self._lock: + self._check_open() + return self + + def __exit__(self, *exc) -> None: + """Release the engine on block exit.""" + self.close() diff --git a/tests/test_api.py b/tests/test_api.py index 2697b5a..1c32cc4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,7 +4,7 @@ import pytest -from akiki import decision, vad +from akiki import decision, vad, vision from akiki.hub import download_model @@ -41,6 +41,7 @@ def test_wildcard_import_exposes_the_main_interfaces_and_modules(): assert namespace["MiniLM"] is decision.MiniLM assert namespace["vad"] is vad assert namespace["decision"] is decision + assert namespace["vision"] is vision assert "minilm" not in namespace assert "ModelPrecision" not in namespace assert "SpeechFrame" not in namespace diff --git a/tests/test_object_detection.py b/tests/test_object_detection.py new file mode 100644 index 0000000..4a3e0b8 --- /dev/null +++ b/tests/test_object_detection.py @@ -0,0 +1,140 @@ +"""Check the cached Seen model path and the native detector interface.""" + +import ctypes +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +from akiki.vision import object_detection + +FILES = ( + "seen-r4c-56508271-model.json", + "seen-r4c-56508271-weights.json", + "seen-r4c-56508271-weights.bin", +) + + +@pytest.fixture +def snapshot(tmp_path): + folder = tmp_path / "snapshot" + weights = folder / "weights" + weights.mkdir(parents=True) + for name in FILES: + (weights / name).touch() + return folder + + +@pytest.fixture +def cached_hub(monkeypatch): + class CacheMiss(Exception): + pass + + hub = SimpleNamespace() + monkeypatch.setitem(sys.modules, "huggingface_hub", hub) + monkeypatch.setitem( + sys.modules, "huggingface_hub.utils", SimpleNamespace(LocalEntryNotFoundError=CacheMiss) + ) + return hub, CacheMiss + + +@pytest.fixture +def native(monkeypatch, tmp_path): + library_path = tmp_path / "libseen" + calls = SimpleNamespace(load=[], free=[], detect=[]) + + def load(path, cpus, count): + calls.load.append((Path(os.fsdecode(path)), tuple(cpus[i] for i in range(count)))) + return 123 + + def detect(handle, pixels, width, height, stride, format_id, threshold, policy, out, size): + data = (ctypes.c_uint8 * (width * height * 3)).from_address(pixels) + calls.detect.append( + (handle, width, height, stride, format_id, threshold, policy, bytes(data)) + ) + assert size >= 1 + out[0].family = 0 + out[0].score = 0.75 + out[0].x0, out[0].y0, out[0].x1, out[0].y1 = 1.0, 2.0, 3.0, 4.0 + return 1 + + library = SimpleNamespace( + akiki_seen_load=load, + akiki_seen_free=calls.free.append, + akiki_seen_family_count=lambda handle: 1, + akiki_seen_family_id=lambda handle, index: b"teddy-bear", + akiki_seen_family_word=lambda handle, index: b"teddy bear", + akiki_seen_detect=detect, + ) + monkeypatch.setattr(object_detection, "_load_library", lambda path: (library, library_path)) + return calls + + +def test_cached_seen_model_uses_local_files_only(snapshot, cached_hub, native, tmp_path): + hub, _ = cached_hub + cache_dir = tmp_path / "hub" + requests = [] + + def lookup(**kwargs): + requests.append(kwargs) + return str(snapshot) + + hub.snapshot_download = lookup + with object_detection.ObjectDetector(cache_dir=cache_dir) as detector: + assert detector.model_id == "Akiki-AI/akiki-cam-seen" + assert detector.model_path == snapshot + assert requests == [ + { + "repo_id": "Akiki-AI/akiki-cam-seen", + "cache_dir": cache_dir, + "local_files_only": True, + } + ] + assert native.load == [(snapshot / "weights", ())] + assert native.free == [123] + + +def test_cache_miss_requests_existing_snapshot(cached_hub): + hub, cache_miss = cached_hub + + def lookup(**kwargs): + raise cache_miss + + hub.snapshot_download = lookup + with pytest.raises(FileNotFoundError, match="model_path for an existing local snapshot"): + object_detection.ObjectDetector() + + +def test_partial_snapshot_is_rejected_before_loading_an_engine(snapshot, native): + (snapshot / "weights" / FILES[2]).unlink() + with pytest.raises(FileNotFoundError, match=FILES[2]): + object_detection.ObjectDetector(snapshot) + assert native.load == [] + + +def test_detection_copies_noncontiguous_rgb_and_frees_handle_once(snapshot, native): + image = np.arange(90, dtype=np.uint8).reshape(5, 6, 3)[:, ::2] + with object_detection.ObjectDetector(snapshot, cpus=(0, 1)) as detector: + detection = detector.detect(image, threshold=0.5, policy=False) + assert detection == [object_detection.Detection("teddy-bear", "teddy bear", 0.75, 1, 2, 3, 4)] + assert native.load == [(snapshot / "weights", (0, 1))] + assert native.detect == [(123, 3, 5, 0, 0, 0.5, 0, image.tobytes())] + detector.close() + assert native.free == [123] + with pytest.raises(RuntimeError, match="closed"): + detector.detect(image) + + +def test_invalid_rgb_and_threshold_are_rejected_before_ffi(snapshot, native): + image = np.zeros((4, 5, 3), dtype=np.uint8) + with object_detection.ObjectDetector(snapshot) as detector: + with pytest.raises(ValueError, match="RGB uint8"): + detector.detect(image.astype(np.float32)) + with pytest.raises(ValueError, match="positive 32-bit"): + detector.detect(image[:0]) + with pytest.raises(ValueError, match="threshold"): + detector.detect(image, threshold=float("nan")) + assert native.detect == [] diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 41675cd..40f3afc 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -209,6 +209,8 @@ def test_distributions_exclude_local_work_and_pass_release_verification(packaged with zipfile.ZipFile(next(directory.glob("*.whl"))) as wheel: names = wheel.namelist() assert "akiki/decision.py" in names + assert "akiki/vision/__init__.py" in names + assert "akiki/vision/object_detection.py" in names assert not any(name.startswith("akiki_training/") for name in names) metadata = wheel.read(next(name for name in names if name.endswith(".dist-info/METADATA"))) assert b"Requires-Dist: akiki-training" not in metadata diff --git a/tests/test_training_api.py b/tests/test_training_api.py index 44fb199..68eeb01 100644 --- a/tests/test_training_api.py +++ b/tests/test_training_api.py @@ -39,7 +39,7 @@ def find_spec(self, fullname, path=None, target=None): sys.meta_path.insert(0, BlockTraining()) import akiki assert akiki.VAD is not None -assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'decision', 'vad'] +assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'decision', 'vad', 'vision'] namespace = {} exec('from akiki import *', namespace) assert namespace['vad'].VAD is akiki.VAD @@ -85,7 +85,7 @@ def find_spec(self, fullname, path=None, target=None): from akiki_training import training as legacy assert training is akiki.training -assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'decision', 'training', 'vad'] +assert dir(akiki) == ['MiniLM', 'VAD', '__version__', 'decision', 'training', 'vad', 'vision'] assert dir(training) == sorted(backend.__all__) assert training.__all__ is backend.__all__ for name in backend.__all__: diff --git a/verify_dist.py b/verify_dist.py index 6321e51..369dd79 100644 --- a/verify_dist.py +++ b/verify_dist.py @@ -43,7 +43,13 @@ else [args.wheel or args.dist_dir / f"akiki-{version}-py3-none-any.whl"] ) source = args.dist_dir / f"akiki-{version}.tar.gz" -model_dir = root / "src/akiki/models" +package_dir = root / "src/akiki" +model_dir = package_dir / "models" +python_modules = { + "akiki/" + path.relative_to(package_dir).as_posix(): path + for package in (package_dir, package_dir / "vision") + for path in package.glob("*.py") +} expected_id = "teacher-only623-v2-s1730" expected_hash = "0fab56bc4ea6725ec881c35973e04813a4963dc5bbf4bf5ccfe8fd7cf468c6f5" checks = {} @@ -85,11 +91,12 @@ def check_no_local_artifacts(names): def check_package_files(names, *, allow_native=False): names = {name for name in names if not name.endswith("/")} - expected = { - "akiki/" + path.relative_to(root / "src/akiki").as_posix() - for path in [*(root / "src/akiki").glob("*.py"), *model_dir.rglob("*")] + expected = set(python_modules) + expected.update( + "akiki/" + path.relative_to(package_dir).as_posix() + for path in model_dir.rglob("*") if path.is_file() - } + ) if allow_native: expected.update( name @@ -228,10 +235,10 @@ def check_models(read, names): f"Wheel licence differs: {name}", ) info["model"] = check_models(archive.read, archive.namelist()) - for path in (root / "src/akiki").glob("*.py"): + for name, path in python_modules.items(): require( - archive.read("akiki/" + path.name) == path.read_bytes(), - f"Stale Python module: {path.name}", + archive.read(name) == path.read_bytes(), + f"Stale Python module: {name}", ) libraries = [ name