From db146492d463a000a4cbbba189be67e39a3e4f3e Mon Sep 17 00:00:00 2001 From: Cloud Lab Works <325058966+cloudlabworks@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:01:29 -0800 Subject: [PATCH] feat: privacy startup event, MCP access log, model hash check, confidence stats Evidence for the CloudLab Works AI-CAIQ rows CCC-07.1, LOG-03.1, LOG-03.2, LOG-04.1, LOG-09.1, MDS-08.1, MDS-08.2 and MDS-10.1. - run: a `privacy` event before DepthAI loads (private, telemetry_opt_in, deviations); camera_health returns the latest one. - run --model-sha256 HEX: refuse to start on a mismatch, before the camera is opened; the archive SHA-256 is recorded in every camera `up` event. - run --confidence-stats: per-zone detection-confidence statistics in alive events (off by default). - mcp --access-log FILE: one JSON line per tool call with argument names and value sizes, result count and outcome; never values, results or image content (off by default; must be outside the events directory). - Tests (26 new), README, CHANGELOG, alerting example, fuzz checks. Drafted with an AI assistant; for maintainer review before merge. --- CHANGELOG.md | 17 ++ README.md | 31 ++- examples/webhook.md | 27 +++ fuzz/corpus/events-privacy | Bin 0 -> 582 bytes fuzz/events.dict | 3 + fuzz/fuzz_events.py | 7 +- src/oak_events/camera.py | 56 ++++- src/oak_events/cli.py | 24 ++- src/oak_events/events.py | 61 +++++- src/oak_events/mcp_server.py | 125 +++++++++-- src/oak_events/privacy.py | 16 ++ tests/test_logging_integrity.py | 370 ++++++++++++++++++++++++++++++++ 12 files changed, 706 insertions(+), 31 deletions(-) create mode 100644 fuzz/corpus/events-privacy create mode 100644 tests/test_logging_integrity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index faab6d1..6ebcf1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,23 @@ All notable changes to this project are documented here. The format follows - Configurable `oak-events mcp --stale-after SECONDS` threshold for camera health. - Atheris fuzz harness `fuzz/fuzz_events.py` for zones files and event JSONL files, run for 60 s on every push and pull request (`fuzz` workflow). +- `oak-events run` writes a `privacy` event before DepthAI loads, recording + whether the DepthAI privacy defaults hold (`private`), whether telemetry + was opted in (`telemetry_opt_in`) and which variables differ + (`deviations`). `camera_health` returns the latest one and `private`. +- `oak-events run --model-sha256 HEX`: refuse to start (exit code 3, camera + not opened) when the model archive's SHA-256 differs. The archive's + SHA-256 is now recorded in every `camera` `up` event. +- `oak-events run --confidence-stats`: per-zone detection-confidence + statistics (`n`, `mean`, `min`, `max`, `below`) in `alive` events. Off by + default. +- `oak-events mcp --access-log FILE`: optional audit log, one JSON line per + tool call (time, tool, argument names and value sizes, result count, + outcome, MCP client process id). Never argument values, results or image + content. Off by default. +- `examples/webhook.md`: alerting on privacy and model-integrity events. +- Fuzz harness also checks the access log's result count and the + `camera_health` privacy fields. ### Changed diff --git a/README.md b/README.md index 18441bb..8a4810f 100644 --- a/README.md +++ b/README.md @@ -114,9 +114,24 @@ the colour camera only approximately, so leave some margin. | Command | What it does | | --- | --- | -| `oak-events run --model PATH [--device-id ID] [--out-dir DIR] [--zones FILE] [--fps 10]` | Run the camera; print events, optionally append to `DIR/YYYY-MM-DD.jsonl` | +| `oak-events run --model PATH [--device-id ID] [--out-dir DIR] [--zones FILE] [--fps 10] [--model-sha256 HEX] [--confidence-stats]` | Run the camera; print events, optionally append to `DIR/YYYY-MM-DD.jsonl` | | `oak-events doctor [--json]` | depthai version, visible devices, bootloader advice, privacy env state | -| `oak-events mcp --events-dir DIR [--stale-after SECONDS]` | MCP stdio server with `recent_events`, `zone_state`, `camera_health`; `camera_health` defaults to stale after 900 seconds | +| `oak-events mcp --events-dir DIR [--stale-after SECONDS] [--access-log FILE]` | MCP stdio server with `recent_events`, `zone_state`, `camera_health`; `camera_health` defaults to stale after 900 seconds | + +Optional checks, all off unless you set them: + +- `--model-sha256 HEX`: compare the model archive's SHA-256 with `HEX` at + start and refuse to run on a mismatch (`camera` event `error`, exit code 3, + camera not opened). The hash is computed and recorded in the `camera` `up` + event on every start, with or without this option. The reference hash is in + [`docs/MODEL_CARD.md`](docs/MODEL_CARD.md). +- `--confidence-stats`: add per-zone detection-confidence statistics to each + `alive` event (see the spot-check procedure in the model card). +- `--access-log FILE`: append one JSON line per MCP tool call to `FILE` (mode + 0600): time, tool name, argument names with the size of each value, result + count, success, and the process id of the MCP client. Argument values and + results are not written, and there is no image content to write. `FILE` + must be outside the events directory. ## Event schema @@ -129,8 +144,14 @@ Every line is a JSON object with `ts` (ISO-8601 local time with offset), | `person_exit` | `zone`, `dwell_s` | nobody seen for `gone_seconds` (10); dwell = first to last sighting | | `depth_baseline` | `zone`, `cells_judged`, `cells_noisy`, `grid` | baseline built; noisy cells (stereo noise, low texture) are ignored | | `depth_change` | `zone`, `cells` (e.g. `["r0c3"]`), `delta_mm` | a cell's depth moved > 15 % for `--hold-seconds` (60) with nobody present | -| `camera` | `state` (`up`, `not_found`, `error`, `reconnect`, `pipeline_stopped`, `stopped`), plus `device`, `device_model`, `bootloader`, `model`, `detail` as relevant | camera lifecycle | -| `alive` | `det_fps`, `person`, `occupied` | heartbeat every `--alive-every` seconds (300) | +| `camera` | `state` (`up`, `not_found`, `error`, `reconnect`, `pipeline_stopped`, `stopped`), plus `device`, `device_model`, `bootloader`, `model`, `model_sha256`, `model_sha256_checked`, `expected_sha256`, `detail` as relevant | camera lifecycle | +| `alive` | `det_fps`, `person`, `occupied`; with `--confidence-stats` also `confidence`: per zone `n`, `mean`, `min`, `max`, `below` | heartbeat every `--alive-every` seconds (300) | +| `privacy` | `private`, `telemetry_opt_in`, `deviations` | written once per run, before DepthAI loads: `private` is `false` if any DepthAI privacy variable differs from the private default, `deviations` names them | + +In `confidence`, `n` counts per-frame person detections in the zone at or +above `--person-confidence` and `below` counts those under it; `mean`, `min` +and `max` describe the accepted ones (`null` when `n` is 0). Each `alive` +event covers the time since the previous one. Example: @@ -148,7 +169,7 @@ Python SDK). Tools, all returning JSON text: - `recent_events(limit=50, kind=None)` - latest events, oldest first - `zone_state()` - per zone: person present or not, latest person and depth events -- `camera_health()` - latest `camera` and `alive` events with their age and a status (`ok`, `stale`, `no_data`, or the last camera state) +- `camera_health()` - latest `camera` and `alive` events with their age and a status (`ok`, `stale`, `no_data`, or the last camera state), plus the latest `privacy` event and `private` (`true`, `false`, or `null` if no run has written one) See `examples/claude-desktop-mcp.json` for a client config, and `examples/openclaw-stream.md` / `examples/webhook.md` for wiring the event diff --git a/examples/webhook.md b/examples/webhook.md index 9b790bd..7ca5d5c 100644 --- a/examples/webhook.md +++ b/examples/webhook.md @@ -26,3 +26,30 @@ tail -F ~/.local/share/oak-events/events/"$(date +%F)".jsonl | ... Note that `tail -F` on a dated file does not roll over at midnight; restart it daily or read stdout instead. + +## Alerting on security events + +Two events mean something security-relevant changed. Forward them to a place +a person reads: + +- `privacy` with `"private": false`: DepthAI telemetry, crash-dump upload or + auto-calibration is not in its private default for this run (`deviations` + says which). +- `camera` with `"state": "error"` and a `detail` starting `model archive + SHA-256 mismatch`: the model file is not the one you pinned with + `--model-sha256`. + +```bash +oak-events run --model ~/models/yolov6-nano.rvc2.tar.xz --model-sha256 "$MODEL_SHA256" \ + | jq -c --unbuffered 'select((.kind == "privacy" and .private != true) + or (.kind == "camera" and .state == "error"))' \ + | while IFS= read -r line; do + curl -sS -X POST -H 'Content-Type: application/json' \ + --data-binary "$line" "$ALERT_WEBHOOK_URL" > /dev/null + done +``` + +An agent can check the same thing through MCP: `camera_health` returns +`"private": false` after such a run. If you run `oak-events mcp --access-log +FILE`, `FILE` is a record of which tool calls the agent made; keep it with +your other logs. diff --git a/fuzz/corpus/events-privacy b/fuzz/corpus/events-privacy new file mode 100644 index 0000000000000000000000000000000000000000..b9ecd0ee10daa03bf032f5570d33924a5db375c4 GIT binary patch literal 582 zcmbtR%WlIU5bQZ$5pivcuq&r^zN5D&LR-96P%t3CPMpa9UY0{uwaK9>TA`hN%`!Y( zS&D4CX0@m{&!WoqF5b36WP2<(ds?%jwjCAupeIl>g^HUnM^hLv2ND__FojtTuGKw6nyt+b^s8q}6Xy1p@{k&X}Muve6S9T!)6%8hC=h&zG;Dj0dQj4dui? z&(E1s9Z(&S ``recent_events``, ``zone_state`` and ``camera_health``. Malformed lines are documented as skipped, so no file content may raise, and every result must still be - JSON-serialisable (the MCP tools return ``json.dumps`` of it). + JSON-serialisable (the MCP tools return ``json.dumps`` of it). The access + log's ``result_count`` must accept every result, and ``camera_health``'s + ``private`` must be a boolean or ``None`` whatever the ``privacy`` event says. No camera and no depthai: only the pure modules are imported. """ @@ -80,6 +82,9 @@ def check_events(data: bytes) -> None: ] for result in results: json.dumps(result) + for tool, result in zip(("recent_events", "recent_events", "zone_state", "camera_health"), results): + assert mcp_server.result_count(tool, result) >= 0 # what the access log records + assert results[3]["private"] in (True, False, None), results[3]["private"] assert results[3]["status"] in ("ok", "stale", "no_data") or results[3]["last_camera"] is not None diff --git a/src/oak_events/camera.py b/src/oak_events/camera.py index 595a5d7..067d88e 100644 --- a/src/oak_events/camera.py +++ b/src/oak_events/camera.py @@ -7,16 +7,18 @@ feed the on-chip network and are never linked to a host queue. * Each depth map is reduced to a coarse grid of cell medians and dropped. Nothing but JSON lines is ever written. -* Telemetry, crash dumps and auto-calibration are off (see ``privacy``). +* Telemetry, crash dumps and auto-calibration are off (see ``privacy``), and + every run records that state in a ``privacy`` event before DepthAI loads. """ -from . import privacy # noqa: F401 (must stay first: sets DepthAI env before depthai loads) +from . import privacy # must stay first: sets DepthAI env before depthai loads +import hashlib import time from pathlib import Path from typing import Any, Callable -from .events import DepthChangeDetector, PersonTracker, Zone +from .events import ConfidenceStats, DepthChangeDetector, PersonTracker, Zone, make_event from .sink import JsonlSink INSTALL_HINT = "depthai is not installed. Install the camera extra: pip install 'oak-events[camera]'" @@ -54,6 +56,15 @@ def discover(dai: Any, device_id: str | None = None, timeout_s: float = DISCOVER sleep(poll_s) +def file_sha256(path: str | Path, chunk_size: int = 1 << 20) -> str: + """Lower-case hex SHA-256 of a file, read in chunks.""" + digest = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + def _label_name(labels: list[str], idx: int) -> str: return labels[idx] if 0 <= idx < len(labels) else str(idx) @@ -97,10 +108,19 @@ def run( hold_seconds: float = 60.0, noisy_frac: float = 0.10, discovery_timeout_s: float = DISCOVERY_TIMEOUT_S, + model_sha256: str | None = None, + confidence_stats: bool = False, ) -> int: """Run until the pipeline stops. Returns a process exit code. - Exit codes: 1 pipeline stopped, 2 no device found, 3 bad model path. + Exit codes: 1 pipeline stopped, 2 no device found, 3 bad model path or + model hash mismatch. + Just before DepthAI loads, a ``privacy`` event records the privacy state. + The model archive's SHA-256 is + computed at every start and recorded in the ``camera`` ``up`` event; with + ``model_sha256`` set, a different hash stops the run before the camera is + opened. With ``confidence_stats``, ``alive`` events carry per-zone + detection-confidence statistics. Exceptions propagate; the CLI turns them into a ``camera`` ``error`` event. Run under a supervisor that restarts on exit. """ @@ -109,7 +129,13 @@ def run( if not model_path.is_file(): sink.emit("camera", state="error", detail=f"model archive not found: {model_path.name}") return 3 + model_digest = file_sha256(model_path) + if model_sha256 is not None and model_digest != model_sha256.strip().lower(): + sink.emit("camera", state="error", detail=f"model archive SHA-256 mismatch: {model_path.name}", + model=model_path.name, model_sha256=model_digest, expected_sha256=model_sha256.strip().lower()) + return 3 + sink.emit("privacy", **privacy.event_fields()) dai = import_depthai() info = discover(dai, device_id, timeout_s=discovery_timeout_s) if info is None: @@ -120,7 +146,8 @@ def run( device.setMaxReconnectionAttempts( MAX_RECONNECT_ATTEMPTS, lambda status: sink.emit("camera", state="reconnect", detail=str(status)) ) - trackers = [PersonTracker(confirm_frames, gone_seconds, person_confidence, zone, person_label) for zone in zones] + trackers = [PersonTracker(confirm_frames, gone_seconds, person_confidence, zone, person_label, + stats=ConfidenceStats() if confidence_stats else None) for zone in zones] detectors = [DepthChangeDetector(grid, baseline_frames, noisy_frac, change_frac, hold_seconds, zone) for zone in zones] @@ -138,7 +165,8 @@ def run( q_depth = stereo.depth.createOutputQueue(maxSize=2, blocking=False) p.start() sink.emit("camera", state="up", device=info.getDeviceId(), device_model=_device_model(device), - bootloader=_bootloader(device), model=model_path.name, zones=[z.name for z in zones]) + bootloader=_bootloader(device), model=model_path.name, model_sha256=model_digest, + model_sha256_checked=model_sha256 is not None, zones=[z.name for z in zones]) last_alive, frames = time.monotonic(), 0 while p.isRunning(): @@ -163,15 +191,25 @@ def run( sink.write(ev) del frame # reduced to cell medians above; nothing is kept if now - last_alive >= alive_every_s: - sink.emit("alive", det_fps=round(frames / (now - last_alive), 1), - person=any(t.present for t in trackers), - occupied=[t.zone.name for t in trackers if t.present]) + sink.write(alive_event(frames / (now - last_alive), trackers)) last_alive, frames = now, 0 time.sleep(0.01) sink.emit("camera", state="pipeline_stopped") return 1 +def alive_event(det_fps: float, trackers: list[PersonTracker]) -> dict[str, Any]: + """The heartbeat event; adds per-zone ``confidence`` statistics when the trackers collect them.""" + fields: dict[str, Any] = { + "det_fps": round(det_fps, 1), + "person": any(t.present for t in trackers), + "occupied": [t.zone.name for t in trackers if t.present], + } + if any(t.stats is not None for t in trackers): + fields["confidence"] = {t.zone.name: t.stats.snapshot() for t in trackers if t.stats is not None} + return make_event("alive", **fields) + + def list_devices() -> list[dict[str, Any]]: """Describe visible devices for ``doctor`` (raises CameraUnavailable without depthai).""" dai = import_depthai() diff --git a/src/oak_events/cli.py b/src/oak_events/cli.py index de68f6d..b99e2da 100644 --- a/src/oak_events/cli.py +++ b/src/oak_events/cli.py @@ -48,6 +48,13 @@ def _positive_seconds(value: str) -> float: return seconds +def _sha256_hex(value: str) -> str: + digest = value.strip().lower() + if len(digest) != 64 or any(ch not in "0123456789abcdef" for ch in digest): + raise argparse.ArgumentTypeError("must be 64 hexadecimal characters (a SHA-256 digest)") + return digest + + def cmd_run(args: argparse.Namespace) -> int: from . import camera # lazy: imports privacy first, depthai only when running @@ -68,6 +75,8 @@ def cmd_run(args: argparse.Namespace) -> int: person_label=args.person_label, person_confidence=args.person_confidence, hold_seconds=args.hold_seconds, + model_sha256=args.model_sha256, + confidence_stats=args.confidence_stats, ) except camera.CameraUnavailable as exc: print(f"oak-events: {exc}", file=sys.stderr) @@ -138,8 +147,15 @@ def _print_doctor(report: dict) -> None: def cmd_mcp(args: argparse.Namespace) -> int: from . import mcp_server + extra: dict[str, object] = {} + if args.access_log is not None: + try: + extra["access_log"] = mcp_server.check_access_log_path(args.events_dir, args.access_log) + except ValueError as exc: + print(f"oak-events: bad --access-log: {exc}", file=sys.stderr) + return 2 try: - mcp_server.serve(args.events_dir, stale_after_s=args.stale_after) + mcp_server.serve(args.events_dir, stale_after_s=args.stale_after, **extra) except RuntimeError as exc: print(f"oak-events: {exc}", file=sys.stderr) return 4 @@ -167,6 +183,10 @@ def build_parser() -> argparse.ArgumentParser: run.add_argument("--person-confidence", type=float, default=0.5, help="minimum confidence (default 0.5)") run.add_argument("--hold-seconds", type=float, default=60.0, help="how long a depth change must persist before it is reported (default 60)") + run.add_argument("--model-sha256", type=_sha256_hex, metavar="HEX", + help="refuse to start unless the model archive has this SHA-256 (see docs/MODEL_CARD.md)") + run.add_argument("--confidence-stats", action="store_true", + help="add per-zone detection-confidence statistics to alive events") run.add_argument("--quiet", action="store_true", help="do not echo events to stdout") run.set_defaults(func=cmd_run) @@ -178,6 +198,8 @@ def build_parser() -> argparse.ArgumentParser: mcp.add_argument("--events-dir", required=True, help="directory written by 'run --out-dir'") mcp.add_argument("--stale-after", type=_positive_seconds, default=900.0, metavar="SECONDS", help="mark camera health stale after this many seconds (default 900)") + mcp.add_argument("--access-log", metavar="FILE", + help="append one JSON line per tool call to FILE (off by default; outside the events directory)") mcp.set_defaults(func=cmd_mcp) return parser diff --git a/src/oak_events/events.py b/src/oak_events/events.py index 3609bd2..237fa4c 100644 --- a/src/oak_events/events.py +++ b/src/oak_events/events.py @@ -8,7 +8,7 @@ * ``ts`` - ISO-8601 local time with UTC offset, second precision * ``kind`` - event kind (``person_enter``, ``person_exit``, ``depth_baseline``, - ``depth_change``, ``camera``, ``alive``) + ``depth_change``, ``camera``, ``alive``, ``privacy``) * kind-specific fields; person and depth events carry ``zone``. """ @@ -169,6 +169,50 @@ def load_zones(path: str | Path | None) -> list[Zone]: Detection = Sequence[Any] +class ConfidenceStats: + """Detection-confidence statistics for one zone, reset at each snapshot. + + Counts per-frame person detections inside the zone: ``n`` at or above the + tracker's threshold (with their ``mean``, ``min`` and ``max`` confidence) + and ``below`` for those the threshold rejected. On a scene that does not + change, a falling mean or a rising ``below`` count means the model, the + camera or the light changed. Only numbers are kept, never a picture. + """ + + def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + self.n = 0 + self.below = 0 + self._sum = 0.0 + self._min: float | None = None + self._max: float | None = None + + def add(self, confidence: float, accepted: bool) -> None: + if not math.isfinite(confidence): + return # not a usable number; NaN would also break the JSON line + if not accepted: + self.below += 1 + return + self.n += 1 + self._sum += confidence + self._min = confidence if self._min is None else min(self._min, confidence) + self._max = confidence if self._max is None else max(self._max, confidence) + + def snapshot(self) -> dict[str, Any]: + """The statistics since the last snapshot, then reset.""" + out: dict[str, Any] = { + "n": self.n, + "mean": round(self._sum / self.n, 3) if self.n else None, + "min": round(self._min, 3) if self._min is not None else None, + "max": round(self._max, 3) if self._max is not None else None, + "below": self.below, + } + self.reset() + return out + + class PersonTracker: """Turn per-frame detections into debounced ``person_enter`` / ``person_exit`` events. @@ -177,6 +221,8 @@ class PersonTracker: * ``person_exit`` fires once nobody has been seen for ``gone_seconds``; it carries ``dwell_s`` (first confirmed sighting to last sighting). * ``now`` is a monotonic clock in seconds supplied by the caller. + * With ``stats``, every person detection in the zone is also counted in + that :class:`ConfidenceStats` (``oak-events run --confidence-stats``). """ def __init__( @@ -186,6 +232,7 @@ def __init__( min_confidence: float = 0.5, zone: Zone | None = None, label: str = "person", + stats: ConfidenceStats | None = None, ) -> None: if confirm_frames < 1: raise ValueError("confirm_frames must be >= 1") @@ -194,6 +241,7 @@ def __init__( self.min_confidence = min_confidence self.zone = zone or Zone() self.label = label + self.stats = stats self._streak = 0 self._since: float | None = None self._last_seen = 0.0 @@ -206,10 +254,17 @@ def _people(self, detections: Iterable[Detection]) -> list[Detection]: out = [] for det in detections: label, conf, _x, _y, z = det[:5] - if str(label) != self.label or conf < self.min_confidence: + if str(label) != self.label: + continue + accepted = not conf < self.min_confidence + if not accepted and self.stats is None: continue cx, cy = (det[5], det[6]) if len(det) >= 7 else (None, None) - if self.zone.contains(z, cx, cy): + if not self.zone.contains(z, cx, cy): + continue + if self.stats is not None: + self.stats.add(float(conf), accepted) + if accepted: out.append(det) return out diff --git a/src/oak_events/mcp_server.py b/src/oak_events/mcp_server.py index d6b5e8e..3c93327 100644 --- a/src/oak_events/mcp_server.py +++ b/src/oak_events/mcp_server.py @@ -8,14 +8,20 @@ ``camera_health``) so it can be used and tested without the ``mcp`` package; ``build_server`` wraps them with FastMCP from the official MCP Python SDK (``MCPServer`` in mcp 2.x, ``FastMCP`` in mcp 1.x). + +With ``--access-log FILE`` every tool call is recorded in ``FILE`` (see +``AccessLog``). It is off by default. """ from __future__ import annotations import datetime as _dt import json +import os from pathlib import Path -from typing import Any, Iterator +from typing import Any, Callable, Iterator + +from .events import make_event INSTALL_HINT = "the mcp package is not installed. Install the extra: pip install 'oak-events[mcp]'" MAX_LIMIT = 500 @@ -106,16 +112,24 @@ def _age_s(ts: str | None, now: _dt.datetime) -> float | None: def camera_health(events_dir: str | Path, stale_after_s: float = 900.0, now: _dt.datetime | None = None) -> dict[str, Any]: - """Latest ``camera`` and ``alive`` events, their age, and a one-word status.""" + """Latest ``camera`` and ``alive`` events, their age, and a one-word status. + + Also the latest ``privacy`` event and ``private``: ``True`` or ``False`` + from that event, ``None`` when there is none (older versions do not write it). + """ now = now or _dt.datetime.now().astimezone() - last_camera = last_alive = None + last_camera = last_alive = last_privacy = None for ev in iter_events_newest_first(events_dir, max_files=SCAN_DAYS): - if last_camera is None and ev.get("kind") == "camera": + kind = ev.get("kind") + if last_camera is None and kind == "camera": last_camera = ev - elif last_alive is None and ev.get("kind") == "alive": + elif last_alive is None and kind == "alive": last_alive = ev - if last_camera and last_alive: + elif last_privacy is None and kind == "privacy": + last_privacy = ev + if last_camera and last_alive and last_privacy: break + private = last_privacy.get("private") if last_privacy else None camera_age = _age_s(last_camera.get("ts") if last_camera else None, now) alive_age = _age_s(last_alive.get("ts") if last_alive else None, now) ages = [a for a in (camera_age, alive_age) if a is not None] @@ -137,9 +151,94 @@ def camera_health(events_dir: str | Path, stale_after_s: float = 900.0, "last_alive": last_alive, "last_alive_age_s": alive_age, "stale_after_s": stale_after_s, + "last_privacy": last_privacy, + "private": private if isinstance(private, bool) else None, } +# --------------------------------------------------------------------------- access log + + +def _value_size(value: Any) -> int: + """Size of an argument value: characters for text, JSON length otherwise, 0 for None.""" + if value is None: + return 0 + if isinstance(value, str): + return len(value) + return len(json.dumps(value, default=str)) + + +def result_count(tool: str, result: Any) -> int: + """How many items a tool returned: events, zones, or (for camera_health) events found.""" + if tool == "recent_events" and isinstance(result, list): + return len(result) + if tool == "zone_state" and isinstance(result, dict): + return len(result.get("zones") or {}) + if tool == "camera_health" and isinstance(result, dict): + return sum(result.get(k) is not None for k in ("last_camera", "last_alive", "last_privacy")) + return 0 + + +def check_access_log_path(events_dir: str | Path, path: str | Path) -> Path: + """The access log must not live in the events directory, or the tools would serve it as events.""" + log = Path(path).expanduser().resolve() + if log.parent == Path(events_dir).expanduser().resolve(): + raise ValueError("the access log must be outside the events directory") + if log.is_dir(): + raise ValueError(f"access log path is a directory: {log.name}") + return log + + +class AccessLog: + """Audit record of MCP tool calls, one JSON line per call, file mode 0600. + + Each line has ``ts``, ``kind`` ``mcp_access``, ``tool``, ``args`` (each + argument name with the size of its value), ``result_count``, ``ok``, + ``error`` (exception type) when a call failed, and ``client_pid`` (the + process that started this stdio server, normally the MCP client). + Argument values and results are never written: the log shows that an + agent read events, not what it read. The server has no image content. + """ + + def __init__(self, path: str | Path) -> None: + self.path = Path(path).expanduser() + self.path.parent.mkdir(parents=True, exist_ok=True) + + def record(self, tool: str, args: dict[str, Any], count: int | None, error: str | None = None) -> dict[str, Any]: + fields: dict[str, Any] = { + "tool": tool, + "args": {key: _value_size(value) for key, value in sorted(args.items())}, + "result_count": count, + "ok": error is None, + "client_pid": os.getppid(), + } + if error is not None: + fields["error"] = error + entry = make_event("mcp_access", **fields) + data = (json.dumps(entry, ensure_ascii=False) + "\n").encode("utf-8") + fd = os.open(self.path, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600) + try: + written = os.write(fd, data) + while written < len(data): + written += os.write(fd, data[written:]) + finally: + os.close(fd) + return entry + + +def call_tool(tool: str, fn: Callable[[], Any], args: dict[str, Any], access_log: AccessLog | None = None) -> str: + """Run one tool, record it in the access log if there is one, and return its JSON text.""" + try: + result = fn() + except Exception as exc: + if access_log is not None: + access_log.record(tool, args, None, error=type(exc).__name__) + raise + if access_log is not None: + access_log.record(tool, args, result_count(tool, result)) + return json.dumps(result) + + def _server_class() -> Any: """FastMCP from the official SDK: ``MCPServer`` in mcp 2.x, ``FastMCP`` in mcp 1.x (same API here).""" try: @@ -154,10 +253,11 @@ def _server_class() -> Any: raise RuntimeError(INSTALL_HINT) from exc -def build_server(events_dir: str | Path, stale_after_s: float = 900.0) -> Any: +def build_server(events_dir: str | Path, stale_after_s: float = 900.0, access_log: str | Path | None = None) -> Any: """Create the MCP server with the three text-only tools.""" server_cls = _server_class() events_dir = Path(events_dir).expanduser() + log = AccessLog(check_access_log_path(events_dir, access_log)) if access_log is not None else None server = server_cls( "oak-events", instructions=( @@ -169,20 +269,21 @@ def build_server(events_dir: str | Path, stale_after_s: float = 900.0) -> Any: @server.tool(name="recent_events") def recent_events_tool(limit: int = 50, kind: str | None = None) -> str: """Most recent camera events as JSON (oldest first). Optional kind filter, e.g. person_enter.""" - return json.dumps(recent_events(events_dir, limit=limit, kind=kind)) + return call_tool("recent_events", lambda: recent_events(events_dir, limit=limit, kind=kind), + {"limit": limit, "kind": kind}, log) @server.tool(name="zone_state") def zone_state_tool() -> str: """Who/what is present per zone, from the latest events, as JSON.""" - return json.dumps(zone_state(events_dir)) + return call_tool("zone_state", lambda: zone_state(events_dir), {}, log) @server.tool(name="camera_health") def camera_health_tool() -> str: """Last camera status event and heartbeat, with their age in seconds, as JSON.""" - return json.dumps(camera_health(events_dir, stale_after_s=stale_after_s)) + return call_tool("camera_health", lambda: camera_health(events_dir, stale_after_s=stale_after_s), {}, log) return server -def serve(events_dir: str | Path, stale_after_s: float = 900.0) -> None: - build_server(events_dir, stale_after_s=stale_after_s).run() # stdio transport +def serve(events_dir: str | Path, stale_after_s: float = 900.0, access_log: str | Path | None = None) -> None: + build_server(events_dir, stale_after_s=stale_after_s, access_log=access_log).run() # stdio transport diff --git a/src/oak_events/privacy.py b/src/oak_events/privacy.py index 7a5d734..a457466 100644 --- a/src/oak_events/privacy.py +++ b/src/oak_events/privacy.py @@ -53,4 +53,20 @@ def state(environ: MutableMapping[str, str] | None = None) -> dict[str, object]: } +def event_fields(environ: MutableMapping[str, str] | None = None) -> dict[str, object]: + """Flat fields for the ``privacy`` event that ``oak-events run`` emits at startup. + + ``deviations`` names the variables that differ from the private defaults, + so a deployer can alert on ``private: false`` without parsing the values. + """ + current = state(environ) + env = current["env"] + assert isinstance(env, dict) + return { + "private": current["private"], + "telemetry_opt_in": current["opted_in_to_telemetry"], + "deviations": sorted(k for k, v in PRIVACY_ENV.items() if env.get(k) != v), + } + + APPLIED = apply() diff --git a/tests/test_logging_integrity.py b/tests/test_logging_integrity.py new file mode 100644 index 0000000..7c8fd70 --- /dev/null +++ b/tests/test_logging_integrity.py @@ -0,0 +1,370 @@ +"""Startup privacy event, MCP access log, model hash check, confidence statistics.""" + +import asyncio +import hashlib +import io +import json +import math +import stat +import sys +import types + +import pytest + +from oak_events import camera, cli, mcp_server as m, privacy +from oak_events.events import ConfidenceStats, PersonTracker, Zone +from oak_events.sink import JsonlSink + + +def _events(stream: io.StringIO) -> list[dict]: + return [json.loads(line) for line in stream.getvalue().splitlines()] + + +@pytest.fixture +def model(tmp_path): + path = tmp_path / "model.tar.xz" + path.write_bytes(b"not really a model archive") + return path + + +@pytest.fixture +def private_env(monkeypatch): + monkeypatch.delenv(privacy.OPT_IN_ENV, raising=False) + for key, value in privacy.PRIVACY_ENV.items(): + monkeypatch.setenv(key, value) + + +# --------------------------------------------------------------------------- privacy event + + +def test_privacy_event_fields_private(private_env): + assert privacy.event_fields() == {"private": True, "telemetry_opt_in": False, "deviations": []} + + +def test_privacy_event_fields_opted_in(monkeypatch): + env = {privacy.OPT_IN_ENV: "1", "DEPTHAI_TELEMETRY": "1"} + fields = privacy.event_fields(env) + assert fields["private"] is False + assert fields["telemetry_opt_in"] is True + assert fields["deviations"] == sorted(privacy.PRIVACY_ENV) + + +def test_run_emits_privacy_event_before_depthai_loads(monkeypatch, model, private_env): + def no_depthai(): + raise camera.CameraUnavailable("no depthai") + + monkeypatch.setattr(camera, "import_depthai", no_depthai) + out = io.StringIO() + with pytest.raises(camera.CameraUnavailable): + camera.run(JsonlSink(stream=out), model=model) + (ev,) = _events(out) + assert ev["kind"] == "privacy" + assert ev["private"] is True and ev["telemetry_opt_in"] is False + + +def test_run_privacy_event_shows_opt_in(monkeypatch, model): + monkeypatch.setenv(privacy.OPT_IN_ENV, "1") + monkeypatch.setenv("DEPTHAI_TELEMETRY", "1") + monkeypatch.setattr(camera, "import_depthai", lambda: (_ for _ in ()).throw(camera.CameraUnavailable("x"))) + out = io.StringIO() + with pytest.raises(camera.CameraUnavailable): + camera.run(JsonlSink(stream=out), model=model) + (ev,) = _events(out) + assert ev["private"] is False and ev["telemetry_opt_in"] is True + assert "DEPTHAI_TELEMETRY" in ev["deviations"] + + +def test_camera_health_reports_privacy_state(events_dir): + from conftest import write_jsonl + + assert m.camera_health(events_dir)["private"] is None # older event files have no privacy event + write_jsonl(events_dir / "2026-01-03.jsonl", [ + {"ts": "2026-01-03T08:00:00+00:00", "kind": "privacy", "private": False, "telemetry_opt_in": True, + "deviations": ["DEPTHAI_TELEMETRY"]}, + ]) + health = m.camera_health(events_dir) + assert health["private"] is False + assert health["last_privacy"]["telemetry_opt_in"] is True + write_jsonl(events_dir / "2026-01-04.jsonl", [{"ts": "2026-01-04T08:00:00+00:00", "kind": "privacy", + "private": "yes"}]) + assert m.camera_health(events_dir)["private"] is None # not a boolean: unknown, not private + + +# --------------------------------------------------------------------------- model hash + + +def test_file_sha256(model): + assert camera.file_sha256(model, chunk_size=4) == hashlib.sha256(model.read_bytes()).hexdigest() + + +def test_model_sha256_mismatch_refuses_to_start(monkeypatch, model): + monkeypatch.setattr(camera, "import_depthai", lambda: pytest.fail("depthai must not load on a hash mismatch")) + out = io.StringIO() + assert camera.run(JsonlSink(stream=out), model=model, model_sha256="0" * 64) == 3 + (ev,) = _events(out) + assert ev["kind"] == "camera" and ev["state"] == "error" + assert "SHA-256 mismatch" in ev["detail"] + assert ev["model_sha256"] == hashlib.sha256(model.read_bytes()).hexdigest() + assert ev["expected_sha256"] == "0" * 64 + assert str(model.parent) not in json.dumps(ev) # file name only, no host paths in events + + +def test_model_sha256_match_continues(monkeypatch, model): + loaded = [] + + def no_depthai(): + loaded.append(True) + raise camera.CameraUnavailable("no depthai") + + monkeypatch.setattr(camera, "import_depthai", no_depthai) + digest = hashlib.sha256(model.read_bytes()).hexdigest().upper() # case does not matter + with pytest.raises(camera.CameraUnavailable): + camera.run(JsonlSink(stdout=False), model=model, model_sha256=digest) + assert loaded == [True] + + +@pytest.mark.parametrize("value", ["abc", "g" * 64, "0" * 63, "0" * 65]) +def test_cli_rejects_malformed_model_sha256(value, model): + with pytest.raises(SystemExit) as exc: + cli.build_parser().parse_args(["run", "--model", str(model), "--model-sha256", value]) + assert exc.value.code == 2 + + +def test_cli_mismatch_exit_code(model, capsys): + assert cli.main(["run", "--model", str(model), "--model-sha256", "f" * 64]) == 3 + ev = json.loads(capsys.readouterr().out.strip()) + assert ev["state"] == "error" and "mismatch" in ev["detail"] + + +# --------------------------------------------------------------------------- fake camera run + + +def _fake_depthai(frames: list[list[tuple]], labels=("person",)): + """Just enough of the DepthAI v3 API for camera.run: one device, detections, no depth frames.""" + remaining = list(frames) + + class Info: + def getDeviceId(self): + return "FAKE-DEVICE" + + class Device: + @staticmethod + def getAllAvailableDevices(): + return [Info()] + + def __init__(self, info): + pass + + def setMaxReconnectionAttempts(self, n, cb): + pass + + def getProductName(self): + return "OAK-FAKE" + + def getBootloaderVersion(self): + return None + + class Queue: + def __init__(self, items): + self.items = items + + def tryGet(self): + return self.items.pop(0) if self.items else None + + class Node: + def __init__(self): + self.left = self.right = None + self.out = self + self.depth = types.SimpleNamespace(createOutputQueue=lambda **kw: Queue([])) + + def build(self, *args, **kwargs): + return self + + def requestOutput(self, size): + return types.SimpleNamespace(link=lambda target: None) + + def getClasses(self): + return list(labels) + + def createOutputQueue(self, **kwargs): + return Queue([_det(rows) for rows in remaining]) + + def _det(rows): + dets = [] + for label, conf, z, cx in rows: + dets.append(types.SimpleNamespace( + label=label, confidence=conf, spatialCoordinates=types.SimpleNamespace(x=0.0, y=0.0, z=z), + xmin=cx - 0.05, xmax=cx + 0.05, ymin=0.45, ymax=0.55)) + return types.SimpleNamespace(detections=dets) + + class Pipeline: + def __init__(self, device): + self.loops = len(frames) + 1 + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def create(self, cls): + return Node() + + def start(self): + pass + + def isRunning(self): + self.loops -= 1 + return self.loops >= 0 + + node = types.SimpleNamespace(Camera=object, StereoDepth=object, SpatialDetectionNetwork=object) + sockets = types.SimpleNamespace(CAM_A="A", CAM_B="B", CAM_C="C") + return types.SimpleNamespace(Device=Device, Pipeline=Pipeline, node=node, CameraBoardSocket=sockets, + NNArchive=lambda path: path) + + +def test_camera_up_event_records_model_hash(monkeypatch, model, private_env): + monkeypatch.setattr(camera, "import_depthai", lambda: _fake_depthai([])) + out = io.StringIO() + digest = hashlib.sha256(model.read_bytes()).hexdigest() + assert camera.run(JsonlSink(stream=out), model=model, model_sha256=digest, discovery_timeout_s=0) == 1 + kinds = [e["kind"] for e in _events(out)] + assert kinds == ["privacy", "camera", "camera"] + up = _events(out)[1] + assert up["state"] == "up" and up["model_sha256"] == digest and up["model_sha256_checked"] is True + + +def test_alive_events_carry_confidence_stats_only_when_enabled(monkeypatch, model): + frames = [[(0, 0.9, 1500.0, 0.2), (0, 0.3, 1500.0, 0.2), (0, 0.7, 1500.0, 0.8)]] * 2 + zones = [Zone("left", (0.0, 0.0, 0.5, 1.0)), Zone("right", (0.5, 0.0, 1.0, 1.0))] + for enabled in (False, True): + monkeypatch.setattr(camera, "import_depthai", lambda: _fake_depthai(frames)) + out = io.StringIO() + camera.run(JsonlSink(stream=out), model=model, zones=zones, alive_every_s=0.005, discovery_timeout_s=0, + confidence_stats=enabled) + alive = [e for e in _events(out) if e["kind"] == "alive"] + assert alive + if not enabled: + assert all("confidence" not in e for e in alive) + continue + total_left = sum(e["confidence"]["left"]["n"] for e in alive) + below_left = sum(e["confidence"]["left"]["below"] for e in alive) + assert (total_left, below_left) == (2, 2) + assert sum(e["confidence"]["right"]["n"] for e in alive) == 2 + assert max(e["confidence"]["left"]["max"] or 0 for e in alive) == 0.9 + + +# --------------------------------------------------------------------------- confidence statistics + + +def test_confidence_stats_snapshot_and_reset(): + s = ConfidenceStats() + for conf, ok in ((0.6, True), (0.8, True), (0.2, False), (math.nan, True)): + s.add(conf, ok) + assert s.snapshot() == {"n": 2, "mean": 0.7, "min": 0.6, "max": 0.8, "below": 1} + assert s.snapshot() == {"n": 0, "mean": None, "min": None, "max": None, "below": 0} + + +def test_tracker_stats_count_zone_and_label_only(): + stats = ConfidenceStats() + t = PersonTracker(confirm_frames=1, zone=Zone("near", z_max_mm=2000), stats=stats) + t.update([("person", 0.9, 0, 0, 1000), ("person", 0.4, 0, 0, 1000), ("person", 0.9, 0, 0, 3000), + ("chair", 0.9, 0, 0, 1000)], now=0.0) + assert stats.snapshot() == {"n": 1, "mean": 0.9, "min": 0.9, "max": 0.9, "below": 1} + + +def test_tracker_events_unchanged_by_stats(): + frames = [[("person", 0.9, 0, 0, 1000)], [("person", 0.4, 0, 0, 1000)], [("person", 0.8, 0, 0, 1000)]] * 3 + times = [i * 0.5 for i in range(len(frames))] + frames.append([]) + times.append(30.0) + plain, counted = PersonTracker(confirm_frames=2), PersonTracker(confirm_frames=2, stats=ConfidenceStats()) + for rows, now in zip(frames, times): + a = [(e["kind"], e.get("dwell_s")) for e in plain.update(rows, now)] + b = [(e["kind"], e.get("dwell_s")) for e in counted.update(rows, now)] + assert a == b + + +# --------------------------------------------------------------------------- MCP access log + + +def test_access_log_records_keys_and_sizes_not_values(tmp_path): + log = m.AccessLog(tmp_path / "audit" / "mcp-access.jsonl") + entry = log.record("recent_events", {"limit": 50, "kind": "person_enter"}, 7) + assert entry["kind"] == "mcp_access" and entry["tool"] == "recent_events" + assert entry["args"] == {"kind": 12, "limit": 2} + assert entry["result_count"] == 7 and entry["ok"] is True + assert isinstance(entry["client_pid"], int) + text = log.path.read_text(encoding="utf-8") + assert "person_enter" not in text # argument values are never written + assert stat.S_IMODE(log.path.stat().st_mode) == 0o600 + + +def test_call_tool_logs_counts_and_never_results(events_dir, tmp_path): + log = m.AccessLog(tmp_path / "access.jsonl") + text = m.call_tool("recent_events", lambda: m.recent_events(events_dir, limit=3), {"limit": 3, "kind": None}, log) + assert len(json.loads(text)) == 3 + m.call_tool("zone_state", lambda: m.zone_state(events_dir), {}, log) + m.call_tool("camera_health", lambda: m.camera_health(events_dir), {}, log) + lines = [json.loads(line) for line in log.path.read_text(encoding="utf-8").splitlines()] + assert [(e["tool"], e["result_count"]) for e in lines] == [("recent_events", 3), ("zone_state", 3), + ("camera_health", 2)] + assert lines[0]["args"] == {"kind": 0, "limit": 1} + assert "desk" not in log.path.read_text(encoding="utf-8") # no event content in the audit log + + +def test_call_tool_logs_failures(tmp_path): + log = m.AccessLog(tmp_path / "access.jsonl") + + def boom(): + raise OSError("disk gone") + + with pytest.raises(OSError): + m.call_tool("zone_state", boom, {}, log) + (entry,) = [json.loads(line) for line in log.path.read_text(encoding="utf-8").splitlines()] + assert entry["ok"] is False and entry["error"] == "OSError" and entry["result_count"] is None + + +def test_call_tool_without_log_writes_nothing(events_dir, tmp_path): + before = sorted(p.name for p in tmp_path.rglob("*")) + m.call_tool("zone_state", lambda: m.zone_state(events_dir), {}, None) + assert sorted(p.name for p in tmp_path.rglob("*")) == before + + +def test_access_log_must_be_outside_events_dir(events_dir, tmp_path): + with pytest.raises(ValueError): + m.check_access_log_path(events_dir, events_dir / "access.jsonl") + with pytest.raises(ValueError): + m.check_access_log_path(events_dir, tmp_path) # a directory + assert m.check_access_log_path(events_dir, tmp_path / "access.jsonl").name == "access.jsonl" + + +def test_cli_mcp_access_log(monkeypatch, events_dir, tmp_path, capsys): + called = {} + monkeypatch.setattr(m, "serve", lambda events_dir, **kw: called.update(kw)) + assert cli.main(["mcp", "--events-dir", str(events_dir), "--access-log", str(events_dir / "a.jsonl")]) == 2 + assert "outside the events directory" in capsys.readouterr().err + assert cli.main(["mcp", "--events-dir", str(events_dir), "--access-log", str(tmp_path / "a.jsonl")]) == 0 + assert called["access_log"] == (tmp_path / "a.jsonl").resolve() + called.clear() + assert cli.main(["mcp", "--events-dir", str(events_dir)]) == 0 + assert "access_log" not in called # off by default + + +def test_mcp_server_tools_write_access_log(events_dir, tmp_path): + pytest.importorskip("mcp") + log_path = tmp_path / "access.jsonl" + server = m.build_server(events_dir, access_log=log_path) + asyncio.run(server.call_tool("recent_events", {"limit": 2})) + asyncio.run(server.call_tool("camera_health", {})) + lines = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines()] + assert [e["tool"] for e in lines] == ["recent_events", "camera_health"] + assert lines[0]["result_count"] == 2 + + +def test_mcp_server_without_access_log_writes_no_file(events_dir, tmp_path): + pytest.importorskip("mcp") + server = m.build_server(events_dir) + asyncio.run(server.call_tool("zone_state", {})) + assert not list(tmp_path.glob("*.jsonl")) + assert sys.modules["oak_events.mcp_server"] is m