From 82ee284a16888689fd1ef842133b23e141796b30 Mon Sep 17 00:00:00 2001 From: Yukkta Seelam Date: Sat, 31 Oct 2026 13:43:29 -0700 Subject: [PATCH 1/3] fix(patchcore): stamp Insight video frames with the source frame identity The annotated frame was pushed to Insight as a bare tensor, so the video got its own timestamp while the verdict metadata carried the source frame's. Insight pairs the two by timestamp, so verdicts could land on the wrong frame when inference dropped or delayed frames. Push each annotated frame as a sample stamped with the source pts and frame ID, and send the verdict with the same values. Video-file mode stamps both with the same computed pts. Same change in C++ and Python. Co-Authored-By: Claude Opus 5.5 --- .../src/cpp/main.cpp | 41 ++++++++++++++----- .../src/python/main.py | 32 +++++++++++---- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/examples/anomaly-detection/patchcore-anomaly-detector/src/cpp/main.cpp b/examples/anomaly-detection/patchcore-anomaly-detector/src/cpp/main.cpp index 1b88a237..7284ad11 100644 --- a/examples/anomaly-detection/patchcore-anomaly-detector/src/cpp/main.cpp +++ b/examples/anomaly-detection/patchcore-anomaly-detector/src/cpp/main.cpp @@ -600,7 +600,7 @@ std::unique_ptr make_metadata_sender(const Config& /// Insight draws `top_classes` as a label on the video; `anomaly` carries the raw values /// for other consumers. void send_verdict(simaai::neat::MetadataSender& sender, float score, float threshold, - bool anomalous, int64_t timestamp_ms, int frame_id) { + bool anomalous, int64_t timestamp_ms, int64_t frame_id) { const std::string verdict = anomalous ? "ANOMALOUS" : "normal"; char label[96]; std::snprintf(label, sizeof(label), "%s score=%.1f thr=%.1f", verdict.c_str(), score, threshold); @@ -618,8 +618,23 @@ void send_verdict(simaai::neat::MetadataSender& sender, float score, float thres } } -void stream_frame(simaai::neat::Run& run, const cv::Mat& frame_bgr) { - if (!run.push(simaai::neat::TensorList{rgb_tensor(frame_bgr)})) { +/// Stamps the annotated frame with its source frame's timestamp and ID, so Insight pairs it +/// with the verdict sent for that frame. +simaai::neat::Sample annotated_frame_sample(const cv::Mat& frame_bgr, int64_t pts_ns, + int64_t frame_id) { + simaai::neat::Sample sample = simaai::neat::make_tensor_sample("", rgb_tensor(frame_bgr)); + sample.payload_type = simaai::neat::PayloadType::Image; + sample.media_type = "video/x-raw"; + sample.format = "RGB"; + sample.payload_tag = sample.format; + sample.pts_ns = pts_ns; + sample.frame_id = frame_id; + return sample; +} + +void stream_frame(simaai::neat::Run& run, const cv::Mat& frame_bgr, int64_t pts_ns, + int64_t frame_id) { + if (!run.push(annotated_frame_sample(frame_bgr, pts_ns, frame_id))) { throw std::runtime_error("Insight video push failed"); } } @@ -674,9 +689,10 @@ int cmd_score_video_file(const Config& cfg, const patchcore::MemoryBank& bank, f << " threshold=" << threshold << " verdict=" << (anomalous ? "ANOMALOUS" : "normal") << " (mla=" << mla_ms << "ms host=" << host_ms << "ms)\n"; - stream_frame(video_sender.run, overlay); - send_verdict(*metadata_sender, scored.image_score, threshold, anomalous, - static_cast(std::lround((processed - 1) * 1000.0 / fps)), processed); + const auto pts_ns = static_cast(std::llround((processed - 1) * 1e9 / fps)); + stream_frame(video_sender.run, overlay, pts_ns, processed); + send_verdict(*metadata_sender, scored.image_score, threshold, anomalous, pts_ns / 1'000'000, + processed); if (cfg.save_every > 0 && processed % cfg.save_every == 0) { const fs::path snapshot_path = cfg.output_dir / ("frame_" + std::to_string(processed) + ".jpg"); if (!cv::imwrite(snapshot_path.string(), overlay)) { @@ -895,7 +911,8 @@ int cmd_score_rtsp(const Config& cfg, const patchcore::MemoryBank& bank, float t cv::Mat bgr; patchcore::PatchEmbeddings embedding; double mla_ms = 0.0; - int64_t timestamp_ms = -1; + int64_t pts_ns = -1; + int64_t source_frame_id = -1; { simaai::neat::Sample sample; simaai::neat::PullError pull_error; @@ -913,7 +930,8 @@ int cmd_score_rtsp(const Config& cfg, const patchcore::MemoryBank& bank, float t // Copy both fields out; the sample (and its pooled buffers) is released at scope end. embedding = embedding_from_sample(sample); bgr = frame_bgr_from_sample(sample); - timestamp_ms = sample.pts_ns >= 0 ? sample.pts_ns / 1'000'000 : -1; + pts_ns = sample.pts_ns; + source_frame_id = sample.frame_id; } const double host_start = time_ms(); @@ -927,9 +945,12 @@ int cmd_score_rtsp(const Config& cfg, const patchcore::MemoryBank& bank, float t << " threshold=" << threshold << " verdict=" << (anomalous ? "ANOMALOUS" : "normal") << " (mla=" << mla_ms << "ms host=" << (time_ms() - host_start) << "ms)\n"; - stream_frame(video_sender.run, overlay); + // The video frame and its verdict carry the same source identity. + const int64_t frame_id = source_frame_id >= 0 ? source_frame_id : processed; + stream_frame(video_sender.run, overlay, pts_ns, frame_id); + const int64_t timestamp_ms = pts_ns >= 0 ? pts_ns / 1'000'000 : -1; send_verdict(*metadata_sender, scored.image_score, threshold, anomalous, timestamp_ms, - processed); + frame_id); if (save_frames && processed % cfg.save_every == 0) { const fs::path snapshot_path = cfg.output_dir / ("frame_" + std::to_string(processed) + ".jpg"); if (!cv::imwrite(snapshot_path.string(), overlay)) { diff --git a/examples/anomaly-detection/patchcore-anomaly-detector/src/python/main.py b/examples/anomaly-detection/patchcore-anomaly-detector/src/python/main.py index ca96e124..770560d5 100644 --- a/examples/anomaly-detection/patchcore-anomaly-detector/src/python/main.py +++ b/examples/anomaly-detection/patchcore-anomaly-detector/src/python/main.py @@ -535,8 +535,21 @@ def send_verdict(sender, score: float, threshold: float, verdict: str, timestamp str(frame_id)) -def stream_frame(run, frame_bgr) -> None: - if not run.push([rgb_tensor(frame_bgr)]): +def annotated_frame_sample(frame_bgr, pts_ns: int, frame_id: int) -> "pyneat.Sample": + """Stamps the annotated frame with its source frame's timestamp and ID, so Insight + pairs it with the verdict sent for that frame.""" + sample = pyneat.make_tensor_sample("", rgb_tensor(frame_bgr)) + sample.payload_type = pyneat.PayloadType.Image + sample.media_type = "video/x-raw" + sample.format = "RGB" + sample.payload_tag = sample.format + sample.pts_ns = pts_ns + sample.frame_id = frame_id + return sample + + +def stream_frame(run, frame_bgr, pts_ns: int, frame_id: int) -> None: + if not run.push([annotated_frame_sample(frame_bgr, pts_ns, frame_id)]): raise RuntimeError("Insight video push failed") @@ -589,9 +602,10 @@ def cmd_score_video_file(cfg: AppConfig, bank: MemoryBank, threshold: float, num f"verdict={verdict} (mla={mla_ms:.1f}ms host={host_ms:.1f}ms)" ) - stream_frame(video_run, overlay) + pts_ns = round((processed - 1) * 1_000_000_000 / fps) + stream_frame(video_run, overlay, pts_ns, processed) send_verdict(metadata_sender, scored.image_score, threshold, verdict, - int((processed - 1) * 1000 / fps), processed) + pts_ns // 1_000_000, processed) if cfg.output.save_every > 0 and processed % cfg.output.save_every == 0: snapshot_path = save_dir / f"frame_{processed}.jpg" if not cv2.imwrite(str(snapshot_path), overlay): @@ -883,7 +897,8 @@ def cmd_score_rtsp(cfg: AppConfig, bank: MemoryBank, threshold: float, num_neigh # Copy both fields out, then drop the sample so its buffers return to the pools. embedding = embedding_from_sample(sample) bgr = frame_bgr_from_sample(sample) - timestamp_ms = int(sample.pts_ns // 1_000_000) if sample.pts_ns >= 0 else -1 + pts_ns = int(sample.pts_ns) + source_frame_id = int(sample.frame_id) del sample host_start = time_ms() @@ -897,8 +912,11 @@ def cmd_score_rtsp(cfg: AppConfig, bank: MemoryBank, threshold: float, num_neigh f"verdict={verdict} (mla={mla_ms:.1f}ms host={time_ms() - host_start:.1f}ms)" ) - stream_frame(video_run, overlay) - send_verdict(metadata_sender, scored.image_score, threshold, verdict, timestamp_ms, processed) + # The video frame and its verdict carry the same source identity. + frame_id = source_frame_id if source_frame_id >= 0 else processed + stream_frame(video_run, overlay, pts_ns, frame_id) + timestamp_ms = pts_ns // 1_000_000 if pts_ns >= 0 else -1 + send_verdict(metadata_sender, scored.image_score, threshold, verdict, timestamp_ms, frame_id) if save_frames and processed % cfg.output.save_every == 0: snapshot_path = save_dir / f"frame_{processed}.jpg" if not cv2.imwrite(str(snapshot_path), overlay): From 9f4ef10dc1a4294bf618a76b4843f7c49ee96bc5 Mon Sep 17 00:00:00 2001 From: Yukkta Seelam Date: Sat, 31 Oct 2026 14:39:29 -0700 Subject: [PATCH 2/3] fix(patchcore): keep pushed frame timestamps in the Insight video sender The video sender's input left appsrc do-timestamp on, so it restamped each pushed frame with its push time and dropped the source timestamp the previous commit set. On a DevKit only 2 of 600 RTSP verdicts landed on a video frame. With do_timestamp off, all 600 RTSP and 450 video-file verdicts match their frame within 1 ms. Co-Authored-By: Claude Opus 5.5 --- .../patchcore-anomaly-detector/src/cpp/main.cpp | 2 ++ .../patchcore-anomaly-detector/src/python/main.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/examples/anomaly-detection/patchcore-anomaly-detector/src/cpp/main.cpp b/examples/anomaly-detection/patchcore-anomaly-detector/src/cpp/main.cpp index 7284ad11..76bcd330 100644 --- a/examples/anomaly-detection/patchcore-anomaly-detector/src/cpp/main.cpp +++ b/examples/anomaly-detection/patchcore-anomaly-detector/src/cpp/main.cpp @@ -566,6 +566,8 @@ VideoSender build_video_sender(const Config& cfg, double fps, int width, int hei input_options.fps_n = output_fps; input_options.fps_d = 1; input_options.memory_policy = simaai::neat::InputMemoryPolicy::Ev74; + // Keep each pushed frame's source timestamp; Insight pairs it with the verdict by timestamp. + input_options.do_timestamp = false; auto sender_options = simaai::neat::nodes::groups::VideoSenderOptions::FromRaw({.fps = output_fps}); diff --git a/examples/anomaly-detection/patchcore-anomaly-detector/src/python/main.py b/examples/anomaly-detection/patchcore-anomaly-detector/src/python/main.py index 770560d5..b12b6f8e 100644 --- a/examples/anomaly-detection/patchcore-anomaly-detector/src/python/main.py +++ b/examples/anomaly-detection/patchcore-anomaly-detector/src/python/main.py @@ -494,6 +494,8 @@ def build_video_sender(cfg: AppConfig, fps: float, width: int, height: int): input_options.fps_n = output_fps input_options.fps_d = 1 input_options.memory_policy = pyneat.InputMemoryPolicy.Ev74 + # Keep each pushed frame's source timestamp; Insight pairs it with the verdict by timestamp. + input_options.do_timestamp = False encode = pyneat.SimaEncodeOptions() encode.fps = output_fps From dba4fbfbde6619a14fab3f3105c992feacd27b0b Mon Sep 17 00:00:00 2001 From: Yukkta Seelam Date: Mon, 16 Nov 2026 12:21:39 -0800 Subject: [PATCH 3/3] test(patchcore): check that each verdict pairs with its video frame Insight draws a verdict on the video frame whose RTP timestamp is within 90 ticks (1 ms) of the verdict's timestamp_ms x 90. The live e2e tests checked the verdict metadata but not this pairing, so dropping the frame's source timestamp would bring back the original bug while every test still passed. Both languages' video-file and RTSP live runs now collect the RTP timestamp of every packet on the video port in a reader thread and require at least 95% of verdicts to pair with a frame. The RTSP runs also check that the verdict metadata arrives. On the DevKit (Core 0.6.1+develop.695a41a6a3d8): this branch passes, Python 2/2 live tests and C++ 8/8 checks. With develop's entrypoints the same tests fail: Python 107 of 120 and 57 of 60 verdicts unpaired, C++ 113 of 120 and 60 of 60. --- .../tests/cpp/test_e2e.cpp | 150 +++++++++++++++++- .../tests/python/test_e2e.py | 76 ++++++++- 2 files changed, 216 insertions(+), 10 deletions(-) diff --git a/examples/anomaly-detection/patchcore-anomaly-detector/tests/cpp/test_e2e.cpp b/examples/anomaly-detection/patchcore-anomaly-detector/tests/cpp/test_e2e.cpp index b92865f0..47a4134b 100644 --- a/examples/anomaly-detection/patchcore-anomaly-detector/tests/cpp/test_e2e.cpp +++ b/examples/anomaly-detection/patchcore-anomaly-detector/tests/cpp/test_e2e.cpp @@ -6,20 +6,31 @@ #include "support/testing/test_config.h" #include "support/testing/test_process.h" +#include +#include #include #include #include #include +#include +#include +#include #include +#include +#include +#include #include +#include #include #include #include #include #include #include +#include #include #include +#include #include #include @@ -638,6 +649,102 @@ int metadata_port() { return env_int_or_default("SIMANEAT_APPS_TEST_INSIGHT_METADATA_PORT", 9100); } +int video_port() { + return env_int_or_default("SIMANEAT_APPS_TEST_INSIGHT_VIDEO_PORT", 9000); +} + +// Collects the RTP timestamp of every packet the app sends to Insight's video port. A +// thread reads while the app runs, so a burst of video packets cannot overflow the socket +// buffer before the test looks at them. +class VideoRtpTimestamps { +public: + explicit VideoRtpTimestamps(int port) { + fd_ = ::socket(AF_INET, SOCK_DGRAM, 0); + if (fd_ < 0) { + error_ = std::string("socket failed: ") + std::strerror(errno); + return; + } + int reuse = 1; + (void)::setsockopt(fd_, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + timeval poll_interval{0, 200000}; + (void)::setsockopt(fd_, SOL_SOCKET, SO_RCVTIMEO, &poll_interval, sizeof(poll_interval)); + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_port = htons(static_cast(port)); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + if (::bind(fd_, reinterpret_cast(&addr), sizeof(addr)) < 0) { + error_ = "bind failed for video port " + std::to_string(port) + ": " + std::strerror(errno); + return; + } + reader_ = std::thread([this] { read(); }); + } + ~VideoRtpTimestamps() { + stop(); + if (fd_ >= 0) { + ::close(fd_); + } + } + bool ok() const { return error_.empty(); } + const std::string& error() const { return error_; } + // Stops reading and returns every RTP timestamp received. + const std::set& stop() { + stopping_ = true; + if (reader_.joinable()) { + reader_.join(); + } + return timestamps_; + } + +private: + void read() { + std::vector packet(65536); + while (!stopping_) { + const ssize_t n = ::recv(fd_, packet.data(), packet.size(), 0); + if (n >= 12 && (packet[0] >> 6) == 2) { + timestamps_.insert((static_cast(packet[4]) << 24) | + (static_cast(packet[5]) << 16) | + (static_cast(packet[6]) << 8) | packet[7]); + } + } + } + + int fd_ = -1; + std::string error_; + std::atomic stopping_{false}; + std::set timestamps_; + std::thread reader_; +}; + +// Empty when Insight can draw the verdicts on their video frames: it pairs a verdict with +// the frame whose RTP timestamp is within 90 ticks (1 ms) of timestamp_ms x 90. Allows 5% +// unpaired for a lost UDP packet. +std::string pairing_problem(const std::vector& messages, + const std::set& frames) { + if (frames.empty()) { + return "no video RTP packets reached the video port"; + } + if (messages.empty()) { + return "no verdict metadata to pair with the video"; + } + std::vector unpaired; + for (const auto& message : messages) { + const auto target = static_cast(message.timestamp_ms * 90); + uint32_t nearest = UINT32_MAX; + for (const uint32_t frame : frames) { + nearest = std::min({nearest, frame - target, target - frame}); + } + if (nearest > 90) { + unpaired.push_back(message.frame_id); + } + } + if (unpaired.size() * 20 > messages.size()) { + return std::to_string(unpaired.size()) + " of " + std::to_string(messages.size()) + + " verdicts have no video frame within 1 ms (first frame id " + unpaired.front() + + "), so Insight cannot draw them on their frames"; + } + return ""; +} + // Runs --calibrate on the bundled nominal set, then one live run with the given source // settings. Returns the live run's stdout, or nullopt after printing a failure. std::optional @@ -651,7 +758,6 @@ calibrate_and_run_live(const std::string& binary, const std::string& model_path, } const fs::path run_dir = fs::path(out_dir).parent_path(); const fs::path config_path = run_dir / "config.yaml"; - const char* port = env_or_null("SIMANEAT_APPS_TEST_INSIGHT_VIDEO_PORT"); ConfigScalars values = { {"model.path", model_path}, {"calibration.nominal_images_dir", "assets/datasets/patchcore/nominal"}, @@ -661,7 +767,7 @@ calibrate_and_run_live(const std::string& binary, const std::string& model_path, {"output.dir", out_dir}, {"output.save_every", std::to_string(save_every)}, {"output.insight.host", "127.0.0.1"}, - {"output.insight.video_port", port && *port ? port : "9000"}, + {"output.insight.video_port", std::to_string(video_port())}, {"output.insight.metadata_port", std::to_string(metadata_port())}, {"runtime.frames", std::to_string(frames)}, }; @@ -740,6 +846,11 @@ int run_video_file_live_run(const std::string& binary, const std::string& model_ std::cerr << "[FAIL] video_file live run: metadata listener: " << listener.error() << "\n"; return 1; } + VideoRtpTimestamps video_frames(video_port()); + if (!video_frames.ok()) { + std::cerr << "[FAIL] video_file live run: video listener: " << video_frames.error() << "\n"; + return 1; + } std::string out_dir; const auto stdout_text = calibrate_and_run_live( binary, model_path, "test_video_file_live_run", @@ -785,6 +896,9 @@ int run_video_file_live_run(const std::string& binary, const std::string& model_ problem = "expected verdict metadata for ~120 frames, got " + std::to_string(matched) + " " + metadata.error; } + if (problem.empty()) { + problem = pairing_problem(metadata.messages, video_frames.stop()); + } } remove_dir(out_dir); if (!problem.empty()) { @@ -792,17 +906,31 @@ int run_video_file_live_run(const std::string& binary, const std::string& model_ return 1; } std::cout << "[OK] video_file: plain frames normal, defect frames ANOMALOUS, verdict metadata " - "sent, 3 snapshots\n"; + "paired with its video frames, 3 snapshots\n"; return 0; } -// rtsp runs the model inside the graph, scores every joined frame, streams to Insight, -// and saves snapshots at the stream's size. +// rtsp runs the model inside the graph, scores every joined frame, streams to Insight with +// each verdict paired to its video frame, and saves snapshots at the stream's size. int run_rtsp_live_run(const std::string& binary, const std::string& model_path) { const auto urls = rtsp_h264_urls_from_env(); if (urls.empty()) { return skip_or_fail("SIMANEAT_TEST_RTSP_H264_URL is not set"); } + sima_examples::testing::MetadataJsonListenerOptions listener_options; + listener_options.base_port = metadata_port(); + listener_options.metadata_type = "classification"; + listener_options.data_array_key = "top_classes"; + sima_examples::testing::MetadataJsonListener listener(listener_options); + if (!listener.ok()) { + std::cerr << "[FAIL] rtsp live run: metadata listener: " << listener.error() << "\n"; + return 1; + } + VideoRtpTimestamps video_frames(video_port()); + if (!video_frames.ok()) { + std::cerr << "[FAIL] rtsp live run: video listener: " << video_frames.error() << "\n"; + return 1; + } std::string out_dir; const auto stdout_text = calibrate_and_run_live( binary, model_path, "test_rtsp_live_run", @@ -818,12 +946,22 @@ int run_rtsp_live_run(const std::string& binary, const std::string& model_path) if (problem.empty()) { problem = snapshots_problem(out_dir, 3, cv::Size()); } + if (problem.empty()) { + const auto metadata = listener.drain_pending(); + if (metadata.messages.size() < 50) { + problem = "expected verdict metadata for ~60 frames, got " + + std::to_string(metadata.messages.size()) + " " + metadata.error; + } else { + problem = pairing_problem(metadata.messages, video_frames.stop()); + } + } remove_dir(out_dir); if (!problem.empty()) { std::cerr << "[FAIL] rtsp live run: " << problem << "\n"; return 1; } - std::cout << "[OK] rtsp: 60 frames scored in the graph, 3 snapshots\n"; + std::cout << "[OK] rtsp: 60 frames scored in the graph, verdict metadata paired with its " + "video frames, 3 snapshots\n"; return 0; } diff --git a/examples/anomaly-detection/patchcore-anomaly-detector/tests/python/test_e2e.py b/examples/anomaly-detection/patchcore-anomaly-detector/tests/python/test_e2e.py index 0c36fe4e..97f354d2 100644 --- a/examples/anomaly-detection/patchcore-anomaly-detector/tests/python/test_e2e.py +++ b/examples/anomaly-detection/patchcore-anomaly-detector/tests/python/test_e2e.py @@ -5,8 +5,11 @@ import json import os import re +import socket +import struct import subprocess import sys +import threading from pathlib import Path import numpy as np @@ -24,6 +27,7 @@ DEMO_VIDEO = APPS_ROOT / "assets" / "datasets" / "patchcore" / "video" / "test.mp4" E2E_INSIGHT_HOST = "127.0.0.1" METADATA_PORT = int(os.environ.get("SIMANEAT_APPS_TEST_INSIGHT_METADATA_PORT", "") or 9100) +VIDEO_PORT = int(os.environ.get("SIMANEAT_APPS_TEST_INSIGHT_VIDEO_PORT", "") or 9000) FRAME_RE = re.compile(r"^frame=(?P\d+): score=(?P[-\d.]+) threshold=(?P[-\d.]+) " r"verdict=(?P\w+)", re.MULTILINE) NOMINAL_IMAGE = "plain_0.png" @@ -81,7 +85,7 @@ def _live_config(e2e_config_writer, tmp_output_dir, source: dict, frames: int, s "save_every": save_every, "insight": { "host": E2E_INSIGHT_HOST, - "video_port": int(os.environ.get("SIMANEAT_APPS_TEST_INSIGHT_VIDEO_PORT", "") or 9000), + "video_port": VIDEO_PORT, "metadata_port": METADATA_PORT, }, }, @@ -90,6 +94,59 @@ def _live_config(e2e_config_writer, tmp_output_dir, source: dict, frames: int, s ) +class _VideoRtpTimestamps: + """Collects the RTP timestamp of every packet the app sends to Insight's video port. + + A thread reads while the app runs, so a burst of video packets cannot overflow the + socket buffer before the test looks at them. + """ + + def __init__(self, port: int) -> None: + self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._sock.bind((E2E_INSIGHT_HOST, port)) + self._sock.settimeout(0.2) + self._stop = threading.Event() + self._thread = threading.Thread(target=self._read, daemon=True) + self.timestamps: set[int] = set() + + def __enter__(self) -> "_VideoRtpTimestamps": + self._thread.start() + return self + + def __exit__(self, *exc) -> None: + self._stop.set() + self._thread.join() + self._sock.close() + + def _read(self) -> None: + while not self._stop.is_set(): + try: + packet = self._sock.recv(65536) + except socket.timeout: + continue + if len(packet) >= 12 and packet[0] >> 6 == 2: + self.timestamps.add(struct.unpack("!I", packet[4:8])[0]) + + +def _assert_verdicts_pair_with_video(messages, rtp_timestamps: set[int]) -> None: + """Insight draws a verdict on the video frame whose RTP timestamp is within 90 ticks + (1 ms) of the verdict's timestamp_ms x 90. Allow 5% unpaired for a lost UDP packet.""" + frames = sorted(rtp_timestamps) + assert frames, "no video RTP packets reached the video port" + assert messages, "no verdict metadata to pair with the video" + + def nearest(timestamp_ms: int) -> int: + target = (timestamp_ms * 90) & 0xFFFFFFFF + return min(min((f - target) & 0xFFFFFFFF, (target - f) & 0xFFFFFFFF) for f in frames) + + unpaired = [m.frame_id for m in messages if nearest(m.timestamp_ms) > 90] + assert len(unpaired) * 20 <= len(messages), ( + f"{len(unpaired)} of {len(messages)} verdicts have no video frame within 1 ms " + f"(frame ids {unpaired[:10]}), so Insight cannot draw them on their frames" + ) + + def _run_main(args: list[str], timeout_s: float) -> subprocess.CompletedProcess: result = subprocess.run( [sys.executable, str(MAIN_PY), *args], @@ -457,7 +514,8 @@ def test_video_file_live_run( timeout_s = test_timeout_ms / 1000 _run_main(["--calibrate", "--config", str(config_path)], timeout_s) with MetadataJsonListener(E2E_INSIGHT_HOST, METADATA_PORT, num_ports=1, - metadata_type="classification", data_array_key="top_classes") as listener: + metadata_type="classification", data_array_key="top_classes") as listener, \ + _VideoRtpTimestamps(VIDEO_PORT) as video: result = _run_main(["--config", str(config_path)], timeout_s) metadata = listener.drain_pending() @@ -468,6 +526,7 @@ def test_video_file_live_run( assert len(sent) >= 100, f"expected verdict metadata for ~120 frames, got {len(sent)}: {metadata.error}" mismatched = [f for f, v in sent.items() if v != verdicts.get(f)] assert not mismatched, f"metadata verdicts differ from printed verdicts for frames {mismatched[:10]}" + _assert_verdicts_pair_with_video(metadata.messages, video.timestamps) normal_frames = [f for f in range(1, 51) if verdicts[f] != "normal"] defect_frames = [f for f in range(71, 121) if verdicts[f] != "ANOMALOUS"] assert not normal_frames, f"plain frames flagged ANOMALOUS: {normal_frames}" @@ -483,16 +542,25 @@ def test_rtsp_live_run( e2e_config_writer, ): """rtsp runs the model inside the graph, scores every joined frame, streams to - Insight, and saves snapshots at the stream's size.""" + Insight with each verdict paired to its video frame, and saves snapshots at the + stream's size.""" config_path = _live_config(e2e_config_writer, tmp_output_dir, {"type": "rtsp", "rtsp": {"url": rtsp_h264_url}}, frames=60, save_every=20) timeout_s = test_timeout_ms / 1000 _run_main(["--calibrate", "--config", str(config_path)], timeout_s) - result = _run_main(["--config", str(config_path)], timeout_s) + with MetadataJsonListener(E2E_INSIGHT_HOST, METADATA_PORT, num_ports=1, + metadata_type="classification", data_array_key="top_classes") as listener, \ + _VideoRtpTimestamps(VIDEO_PORT) as video: + result = _run_main(["--config", str(config_path)], timeout_s) + metadata = listener.drain_pending() frames = [int(m["frame"]) for m in FRAME_RE.finditer(result.stdout)] assert frames == list(range(1, 61)), f"expected frames 1-60 with a verdict, got {frames[:5]}..." + assert len(metadata.messages) >= 50, ( + f"expected verdict metadata for ~60 frames, got {len(metadata.messages)}: {metadata.error}" + ) + _assert_verdicts_pair_with_video(metadata.messages, video.timestamps) import cv2 snapshots = sorted(tmp_output_dir.glob("frame_*.jpg"))