Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,8 @@ VideoSender build_video_sender(const Config& cfg, double fps, int width, int hei
input_options.depth = nv12 ? 1 : 3;
input_options.fps_n = output_fps;
input_options.fps_d = 1;
// Keep each pushed frame's source timestamp; Insight pairs it with the verdict by timestamp.
input_options.do_timestamp = false;

auto sender_options =
simaai::neat::nodes::groups::VideoSenderOptions::FromRaw({.fps = output_fps});
Expand Down Expand Up @@ -578,7 +580,7 @@ std::unique_ptr<simaai::neat::MetadataSender> make_metadata_sender(const Config&
/// Insight draws `top_classes` as a label on the video; `anomaly` carries the raw values
/// for other consumers.
void send_verdict(simaai::neat::MetadataSender& sender, float score, float threshold,
bool anomalous, int64_t timestamp_ms, int frame_id) {
bool anomalous, int64_t timestamp_ms, int64_t frame_id) {
const std::string verdict = anomalous ? "ANOMALOUS" : "normal";
char label[96];
std::snprintf(label, sizeof(label), "%s score=%.1f thr=%.1f", verdict.c_str(), score, threshold);
Expand All @@ -596,12 +598,22 @@ void send_verdict(simaai::neat::MetadataSender& sender, float score, float thres
}
}

void stream_frame(simaai::neat::Run& run, const cv::Mat& frame, bool nv12 = false) {
/// Stamps the annotated frame with its source frame's timestamp and ID, so Insight pairs it
/// with the verdict sent for that frame.
void stream_frame(simaai::neat::Run& run, const cv::Mat& frame, int64_t pts_ns, int64_t frame_id,
bool nv12 = false) {
const auto tensor =
nv12 ? nv12_tensor(frame)
: simaai::neat::Tensor::from_cv_mat(frame, simaai::neat::ImageSpec::PixelFormat::BGR,
simaai::neat::TensorMemory::CPU);
if (!run.push(simaai::neat::TensorList{tensor})) {
simaai::neat::Sample sample = simaai::neat::make_tensor_sample("", tensor);
sample.payload_type = simaai::neat::PayloadType::Image;
sample.media_type = "video/x-raw";
sample.format = nv12 ? "NV12" : "BGR";
sample.payload_tag = sample.format;
sample.pts_ns = pts_ns;
sample.frame_id = frame_id;
if (!run.push(sample)) {
throw std::runtime_error("Insight video push failed");
}
}
Expand Down Expand Up @@ -655,9 +667,10 @@ int cmd_score_video_file(const Config& cfg, const patchcore::MemoryBank& bank, f
<< " threshold=" << threshold << " verdict=" << (anomalous ? "ANOMALOUS" : "normal")
<< " (mla=" << mla_ms << "ms host=" << host_ms << "ms)\n";

stream_frame(video_sender.run, overlay);
send_verdict(*metadata_sender, scored.image_score, threshold, anomalous,
static_cast<int64_t>(std::lround((processed - 1) * 1000.0 / fps)), processed);
const auto pts_ns = static_cast<int64_t>(std::llround((processed - 1) * 1e9 / fps));
stream_frame(video_sender.run, overlay, pts_ns, processed);
send_verdict(*metadata_sender, scored.image_score, threshold, anomalous, pts_ns / 1'000'000,
processed);
if (cfg.save_every > 0 && processed % cfg.save_every == 0) {
const fs::path snapshot_path = cfg.output_dir / ("frame_" + std::to_string(processed) + ".jpg");
if (!cv::imwrite(snapshot_path.string(), overlay)) {
Expand Down Expand Up @@ -886,7 +899,8 @@ int cmd_score_rtsp(const Config& cfg, const patchcore::MemoryBank& bank, float t
cv::Mat frame;
ScoredEmbedding scored_embedding;
double mla_ms = 0.0, score_ms = 0.0;
int64_t timestamp_ms = -1;
int64_t pts_ns = -1;
int64_t source_frame_id = -1;
{
simaai::neat::Sample sample;
simaai::neat::PullError pull_error;
Expand All @@ -906,7 +920,8 @@ int cmd_score_rtsp(const Config& cfg, const patchcore::MemoryBank& bank, float t
scored_embedding = score_embedding_from_sample(sample, bank, num_neighbors);
score_ms = time_ms() - score_start;
frame = frame_nv12_from_sample(sample);
timestamp_ms = sample.pts_ns >= 0 ? sample.pts_ns / 1'000'000 : -1;
pts_ns = sample.pts_ns;
source_frame_id = sample.frame_id;
}

const double host_start = time_ms();
Expand All @@ -920,9 +935,12 @@ int cmd_score_rtsp(const Config& cfg, const patchcore::MemoryBank& bank, float t
<< " threshold=" << threshold << " verdict=" << (anomalous ? "ANOMALOUS" : "normal")
<< " (mla=" << mla_ms << "ms host=" << (score_ms + time_ms() - host_start) << "ms)\n";

stream_frame(video_sender.run, frame, true);
// The video frame and its verdict carry the same source identity.
const int64_t frame_id = source_frame_id >= 0 ? source_frame_id : processed;
stream_frame(video_sender.run, frame, pts_ns, frame_id, true);
const int64_t timestamp_ms = pts_ns >= 0 ? pts_ns / 1'000'000 : -1;
send_verdict(*metadata_sender, scored.image_score, threshold, anomalous, timestamp_ms,
processed);
frame_id);
if (save_frames && processed % cfg.save_every == 0) {
const fs::path snapshot_path =
cfg.output_dir / ("frame_" + std::to_string(processed) + ".jpg");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,8 @@ def build_video_sender(cfg: AppConfig, fps: float, width: int, height: int, nv12
input_options.depth = 1 if nv12 else 3
input_options.fps_n = output_fps
input_options.fps_d = 1
# Keep each pushed frame's source timestamp; Insight pairs it with the verdict by timestamp.
input_options.do_timestamp = False
Comment thread
Yukkta-Seelam marked this conversation as resolved.
Resolved

encode = pyneat.SimaEncodeOptions()
encode.fps = output_fps
Expand Down Expand Up @@ -519,11 +521,20 @@ def send_verdict(sender, score: float, threshold: float, verdict: str, timestamp
str(frame_id))


def stream_frame(run, frame, nv12=False) -> None:
def stream_frame(run, frame, pts_ns: int, frame_id: int, nv12=False) -> None:
"""Stamps the annotated frame with its source frame's timestamp and ID, so Insight
pairs it with the verdict sent for that frame."""
tensor = nv12_tensor(frame) if nv12 else pyneat.Tensor.from_numpy(
frame, copy=True, image_format=pyneat.PixelFormat.BGR, memory=pyneat.TensorMemory.CPU
)
if not run.push([tensor]):
sample = pyneat.make_tensor_sample("", tensor)
sample.payload_type = pyneat.PayloadType.Image
sample.media_type = "video/x-raw"
sample.format = "NV12" if nv12 else "BGR"
sample.payload_tag = sample.format
sample.pts_ns = pts_ns
sample.frame_id = frame_id
if not run.push([sample]):
raise RuntimeError("Insight video push failed")


Expand Down Expand Up @@ -576,9 +587,10 @@ def cmd_score_video_file(cfg: AppConfig, bank: MemoryBank, threshold: float, num
f"verdict={verdict} (mla={mla_ms:.1f}ms host={host_ms:.1f}ms)"
)

stream_frame(video_run, overlay)
pts_ns = round((processed - 1) * 1_000_000_000 / fps)
stream_frame(video_run, overlay, pts_ns, processed)
send_verdict(metadata_sender, scored.image_score, threshold, verdict,
int((processed - 1) * 1000 / fps), processed)
pts_ns // 1_000_000, processed)
if cfg.output.save_every > 0 and processed % cfg.output.save_every == 0:
snapshot_path = save_dir / f"frame_{processed}.jpg"
if not cv2.imwrite(str(snapshot_path), overlay):
Expand Down Expand Up @@ -865,7 +877,8 @@ def cmd_score_rtsp(cfg: AppConfig, bank: MemoryBank, threshold: float, num_neigh
# Own the frame; release the embedding view immediately after scoring.
embedding = embedding_from_sample(sample)
frame = frame_nv12_from_sample(sample)
timestamp_ms = int(sample.pts_ns // 1_000_000) if sample.pts_ns >= 0 else -1
pts_ns = int(sample.pts_ns)
source_frame_id = int(sample.frame_id)
del sample

host_start = time_ms()
Expand All @@ -880,8 +893,11 @@ def cmd_score_rtsp(cfg: AppConfig, bank: MemoryBank, threshold: float, num_neigh
f"verdict={verdict} (mla={mla_ms:.1f}ms host={time_ms() - host_start:.1f}ms)"
)

stream_frame(video_run, frame, nv12=True)
send_verdict(metadata_sender, scored.image_score, threshold, verdict, timestamp_ms, processed)
# The video frame and its verdict carry the same source identity.
frame_id = source_frame_id if source_frame_id >= 0 else processed
stream_frame(video_run, frame, pts_ns, frame_id, nv12=True)
timestamp_ms = pts_ns // 1_000_000 if pts_ns >= 0 else -1
send_verdict(metadata_sender, scored.image_score, threshold, verdict, timestamp_ms, frame_id)
if save_frames and processed % cfg.output.save_every == 0:
snapshot_path = save_dir / f"frame_{processed}.jpg"
if not cv2.imwrite(str(snapshot_path), cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_NV12)):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,31 @@
#include "support/testing/test_config.h"
#include "support/testing/test_process.h"

#include <arpa/inet.h>
#include <netinet/in.h>
#include <nlohmann/json.hpp>
#include <opencv2/core.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
#include <sys/socket.h>
#include <sys/time.h>
#include <unistd.h>
#include <algorithm>
#include <atomic>
#include <cerrno>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <map>
#include <optional>
#include <regex>
#include <set>
#include <sstream>
#include <string>
#include <thread>
#include <utility>
#include <vector>

Expand Down Expand Up @@ -638,6 +649,102 @@ int metadata_port() {
return env_int_or_default("SIMANEAT_APPS_TEST_INSIGHT_METADATA_PORT", 9100);
}

int video_port() {
return env_int_or_default("SIMANEAT_APPS_TEST_INSIGHT_VIDEO_PORT", 9000);
}

// Collects the RTP timestamp of every packet the app sends to Insight's video port. A
// thread reads while the app runs, so a burst of video packets cannot overflow the socket
// buffer before the test looks at them.
class VideoRtpTimestamps {
public:
explicit VideoRtpTimestamps(int port) {
fd_ = ::socket(AF_INET, SOCK_DGRAM, 0);
if (fd_ < 0) {
error_ = std::string("socket failed: ") + std::strerror(errno);
return;
}
int reuse = 1;
(void)::setsockopt(fd_, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));
timeval poll_interval{0, 200000};
(void)::setsockopt(fd_, SOL_SOCKET, SO_RCVTIMEO, &poll_interval, sizeof(poll_interval));
sockaddr_in addr{};
addr.sin_family = AF_INET;
addr.sin_port = htons(static_cast<uint16_t>(port));
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
if (::bind(fd_, reinterpret_cast<const sockaddr*>(&addr), sizeof(addr)) < 0) {
error_ = "bind failed for video port " + std::to_string(port) + ": " + std::strerror(errno);
return;
}
reader_ = std::thread([this] { read(); });
}
~VideoRtpTimestamps() {
stop();
if (fd_ >= 0) {
::close(fd_);
}
}
bool ok() const { return error_.empty(); }
const std::string& error() const { return error_; }
// Stops reading and returns every RTP timestamp received.
const std::set<uint32_t>& stop() {
stopping_ = true;
if (reader_.joinable()) {
reader_.join();
}
return timestamps_;
}

private:
void read() {
std::vector<uint8_t> packet(65536);
while (!stopping_) {
const ssize_t n = ::recv(fd_, packet.data(), packet.size(), 0);
if (n >= 12 && (packet[0] >> 6) == 2) {
timestamps_.insert((static_cast<uint32_t>(packet[4]) << 24) |
(static_cast<uint32_t>(packet[5]) << 16) |
(static_cast<uint32_t>(packet[6]) << 8) | packet[7]);
}
}
}

int fd_ = -1;
std::string error_;
std::atomic<bool> stopping_{false};
std::set<uint32_t> timestamps_;
std::thread reader_;
};

// Empty when Insight can draw the verdicts on their video frames: it pairs a verdict with
// the frame whose RTP timestamp is within 90 ticks (1 ms) of timestamp_ms x 90. Allows 5%
// unpaired for a lost UDP packet.
std::string pairing_problem(const std::vector<MetadataJsonMessage>& messages,
const std::set<uint32_t>& frames) {
if (frames.empty()) {
return "no video RTP packets reached the video port";
}
if (messages.empty()) {
return "no verdict metadata to pair with the video";
}
std::vector<std::string> unpaired;
for (const auto& message : messages) {
const auto target = static_cast<uint32_t>(message.timestamp_ms * 90);
uint32_t nearest = UINT32_MAX;
for (const uint32_t frame : frames) {
nearest = std::min({nearest, frame - target, target - frame});
}
if (nearest > 90) {
unpaired.push_back(message.frame_id);
}
}
if (unpaired.size() * 20 > messages.size()) {
return std::to_string(unpaired.size()) + " of " + std::to_string(messages.size()) +
" verdicts have no video frame within 1 ms (first frame id " + unpaired.front() +
"), so Insight cannot draw them on their frames";
}
return "";
}

// Runs --calibrate on the bundled nominal set, then one live run with the given source
// settings. Returns the live run's stdout, or nullopt after printing a failure.
std::optional<std::string>
Expand All @@ -651,7 +758,6 @@ calibrate_and_run_live(const std::string& binary, const std::string& model_path,
}
const fs::path run_dir = fs::path(out_dir).parent_path();
const fs::path config_path = run_dir / "config.yaml";
const char* port = env_or_null("SIMANEAT_APPS_TEST_INSIGHT_VIDEO_PORT");
ConfigScalars values = {
{"model.path", model_path},
{"calibration.nominal_images_dir", "assets/datasets/patchcore/nominal"},
Expand All @@ -661,7 +767,7 @@ calibrate_and_run_live(const std::string& binary, const std::string& model_path,
{"output.dir", out_dir},
{"output.save_every", std::to_string(save_every)},
{"output.insight.host", "127.0.0.1"},
{"output.insight.video_port", port && *port ? port : "9000"},
{"output.insight.video_port", std::to_string(video_port())},
{"output.insight.metadata_port", std::to_string(metadata_port())},
{"runtime.frames", std::to_string(frames)},
};
Expand Down Expand Up @@ -740,6 +846,11 @@ int run_video_file_live_run(const std::string& binary, const std::string& model_
std::cerr << "[FAIL] video_file live run: metadata listener: " << listener.error() << "\n";
return 1;
}
VideoRtpTimestamps video_frames(video_port());
if (!video_frames.ok()) {
std::cerr << "[FAIL] video_file live run: video listener: " << video_frames.error() << "\n";
return 1;
}
std::string out_dir;
const auto stdout_text = calibrate_and_run_live(
binary, model_path, "test_video_file_live_run",
Expand Down Expand Up @@ -785,24 +896,41 @@ int run_video_file_live_run(const std::string& binary, const std::string& model_
problem = "expected verdict metadata for ~120 frames, got " + std::to_string(matched) + " " +
metadata.error;
}
if (problem.empty()) {
problem = pairing_problem(metadata.messages, video_frames.stop());
}
}
remove_dir(out_dir);
if (!problem.empty()) {
std::cerr << "[FAIL] video_file live run: " << problem << "\n";
return 1;
}
std::cout << "[OK] video_file: plain frames normal, defect frames ANOMALOUS, verdict metadata "
"sent, 3 snapshots\n";
"paired with its video frames, 3 snapshots\n";
return 0;
}

// rtsp runs the model inside the graph, scores every joined frame, streams to Insight,
// and saves snapshots at the stream's size.
// rtsp runs the model inside the graph, scores every joined frame, streams to Insight with
// each verdict paired to its video frame, and saves snapshots at the stream's size.
int run_rtsp_live_run(const std::string& binary, const std::string& model_path) {
const auto urls = rtsp_h264_urls_from_env();
if (urls.empty()) {
return skip_or_fail("SIMANEAT_TEST_RTSP_H264_URL is not set");
}
sima_examples::testing::MetadataJsonListenerOptions listener_options;
listener_options.base_port = metadata_port();
listener_options.metadata_type = "classification";
listener_options.data_array_key = "top_classes";
sima_examples::testing::MetadataJsonListener listener(listener_options);
if (!listener.ok()) {
std::cerr << "[FAIL] rtsp live run: metadata listener: " << listener.error() << "\n";
return 1;
}
VideoRtpTimestamps video_frames(video_port());
if (!video_frames.ok()) {
std::cerr << "[FAIL] rtsp live run: video listener: " << video_frames.error() << "\n";
return 1;
}
std::string out_dir;
const auto stdout_text = calibrate_and_run_live(
binary, model_path, "test_rtsp_live_run",
Expand All @@ -818,12 +946,22 @@ int run_rtsp_live_run(const std::string& binary, const std::string& model_path)
if (problem.empty()) {
problem = snapshots_problem(out_dir, 3, cv::Size());
}
if (problem.empty()) {
const auto metadata = listener.drain_pending();
if (metadata.messages.size() < 50) {
problem = "expected verdict metadata for ~60 frames, got " +
std::to_string(metadata.messages.size()) + " " + metadata.error;
} else {
problem = pairing_problem(metadata.messages, video_frames.stop());
}
}
remove_dir(out_dir);
if (!problem.empty()) {
std::cerr << "[FAIL] rtsp live run: " << problem << "\n";
return 1;
}
std::cout << "[OK] rtsp: 60 frames scored in the graph, 3 snapshots\n";
std::cout << "[OK] rtsp: 60 frames scored in the graph, verdict metadata paired with its "
"video frames, 3 snapshots\n";
return 0;
}

Expand Down
Loading
Loading