diff --git a/public/opencv-worker.js b/public/opencv-worker.js index f42cefa..b91f724 100644 --- a/public/opencv-worker.js +++ b/public/opencv-worker.js @@ -11,7 +11,7 @@ * worker → main { id, type: 'init-done', templateW, templateH, centerX, centerY } * * main → worker { id, type: 'track', imageData, searchX, searchY } - * worker → main { id, type: 'track-result', x, y } + * worker → main { id, type: 'track-result', x, y, confidence } * * main → worker { id: -1, type: 'dispose' } (no response) * @@ -68,6 +68,21 @@ function ensureOpenCV() { return cvLoadPromise; } +// Gaussian blur kernel applied to both the captured template and every search +// window before matching. A 5×5 kernel (σ ≈ 1.1 px) smooths per-frame sensor +// noise and compression artefacts without blurring object edges enough to +// degrade match precision. Because the same filter is applied at both init +// time (template) and track time (search region), the template and the search +// image remain in the same frequency domain and TM_CCOEFF_NORMED scores stay +// reliable. This is tuned for a stationary camera where the background is +// static and sensor noise is the dominant source of inter-frame variation. +function blurGray(src, dst) { + // Lazily build the Size object once cv is available. + const ksize = new cv.Size(5, 5); + cv.GaussianBlur(src, dst, ksize, 0); + ksize.delete(); +} + self.onmessage = async (event) => { const msg = event.data; const { id, type } = msg; @@ -79,6 +94,7 @@ self.onmessage = async (event) => { const { imageData, region } = msg; const frame = cv.matFromImageData(imageData); const gray = new cv.Mat(); + const blurred = new cv.Mat(); try { cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY); @@ -91,9 +107,13 @@ self.onmessage = async (event) => { throw new Error(`Invalid ROI dimensions: ${roiW}x${roiH}`); } + // Blur before cropping the ROI so the template is in the same + // frequency domain as the blurred search regions used during tracking. + blurGray(gray, blurred); + if (templateMat) templateMat.delete(); const roi = new cv.Rect(clampedX, clampedY, roiW, roiH); - templateMat = gray.roi(roi).clone(); + templateMat = blurred.roi(roi).clone(); self.postMessage({ id, @@ -106,27 +126,41 @@ self.onmessage = async (event) => { } finally { frame.delete(); gray.delete(); + blurred.delete(); } } else if (type === 'track') { if (!cv || !templateMat) { - self.postMessage({ id, type: 'track-result', x: null, y: null }); + self.postMessage({ id, type: 'track-result', x: null, y: null, confidence: 0 }); return; } const { imageData, searchX, searchY } = msg; const frame = cv.matFromImageData(imageData); const gray = new cv.Mat(); + const blurred = new cv.Mat(); const result = new cv.Mat(); try { cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY); - cv.matchTemplate(gray, templateMat, result, cv.TM_CCOEFF_NORMED); - const { maxLoc } = cv.minMaxLoc(result); + // Apply the same blur used at template-capture time so that + // TM_CCOEFF_NORMED compares apples to apples. For a stationary + // camera this also suppresses per-frame sensor noise that would + // otherwise produce spurious high-scoring locations. + blurGray(gray, blurred); + cv.matchTemplate(blurred, templateMat, result, cv.TM_CCOEFF_NORMED); + const { maxVal, maxLoc } = cv.minMaxLoc(result); const centerX = maxLoc.x + searchX + templateMat.cols / 2; const centerY = maxLoc.y + searchY + templateMat.rows / 2; - self.postMessage({ id, type: 'track-result', x: centerX, y: centerY }); + self.postMessage({ + id, + type: 'track-result', + x: centerX, + y: centerY, + confidence: maxVal, + }); } finally { frame.delete(); gray.delete(); + blurred.delete(); result.delete(); } } else if (type === 'dispose') { diff --git a/src/tracking/OpenCVTracker.ts b/src/tracking/OpenCVTracker.ts index 3d7634f..fbaea8f 100644 --- a/src/tracking/OpenCVTracker.ts +++ b/src/tracking/OpenCVTracker.ts @@ -9,6 +9,44 @@ * - Drawing the video frame onto an offscreen canvas (GPU-accelerated) * - Extracting the search-window ImageData (CPU ← GPU transfer) * - Computing the windowed search region (cheap arithmetic) + * + * ## Stationary-camera optimisations + * + * When the camera is fixed (mounted on a tripod and not moving), the + * background is completely static between frames. The only source of + * inter-frame variation is: + * 1. The tracked object itself moving through the scene. + * 2. Per-frame sensor noise and video-compression artefacts. + * + * Two mechanisms exploit these conditions: + * + * 1. **Gaussian pre-filtering** – a 5×5 Gaussian blur is applied to both the + * template at capture time and the search region on every tracking call. + * Because the same filter is used in both cases the normalised cross- + * correlation score (TM_CCOEFF_NORMED) remains accurate, while pixel-level + * sensor noise that would otherwise generate spurious match peaks is + * suppressed. This is more effective on a stationary camera than on a + * panning one because the static background means noise is the dominant + * source of false matches rather than background-content changes. + * + * 2. **Confidence threshold** – TM_CCOEFF_NORMED returns a score in [-1, 1]. + * Matches below MATCH_CONFIDENCE_THRESHOLD are silently dropped so that + * a noisy or occluded frame cannot send the tracker to a wrong location. + * The threshold is set lower (0.25) than would be appropriate for a + * moving camera because the static background makes strong false peaks + * unlikely. + * + * ## Windowed search optimisation (existing) + * + * Rather than reading back every pixel of every frame from the GPU (O(W×H) per + * frame at 30 Hz), the tracker maintains the center of the last successful match + * and restricts the next search to a window around it. Only the pixels inside + * that window are transferred from GPU to CPU via `getImageData`. For a typical + * 640×480 video and a moderate template, this reduces the pixel transfer by ~10–15×. + * + * The search window is padded by `SEARCH_PADDING_FACTOR × max(templateW, templateH)` + * on every side. If the object would exit that window between frames (very fast + * motion), the tracker falls back to a full-frame search automatically. */ import trackLab from "../TrackLabNamespace.js"; @@ -17,24 +55,12 @@ export type TrackerRegion = { x: number; y: number; w: number; h: number }; type WorkerResponse = | { id: number; type: "init-done"; templateW: number; templateH: number; centerX: number; centerY: number } - | { id: number; type: "track-result"; x: number; y: number } + | { id: number; type: "track-result"; x: number; y: number; confidence: number } | { id: number; type: "error"; message: string }; /** * Tracks a user-selected object across video frames using OpenCV template * matching running in a Web Worker. - * - * ## Windowed search optimisation - * - * Rather than reading back every pixel of every frame from the GPU (O(W×H) per - * frame at 30 Hz), the tracker maintains the center of the last successful match - * and restricts the next search to a window around it. Only the pixels inside - * that window are transferred from GPU to CPU via `getImageData`. For a typical - * 640×480 video and a moderate template, this reduces the pixel transfer by ~10–15×. - * - * The search window is padded by `SEARCH_PADDING_FACTOR × max(templateW, templateH)` - * on every side. If the object would exit that window between frames (very fast - * motion), the tracker falls back to a full-frame search automatically. */ export class OpenCVTracker { private readonly offscreen: HTMLCanvasElement; @@ -57,8 +83,18 @@ export class OpenCVTracker { private templateH = 0; private lastMatchCenter: { x: number; y: number } | null = null; + // Padding factor: search window extends this many template-lengths on each + // side. A value of 2 is well-matched to a stationary camera where the + // object's own motion is the sole source of inter-frame displacement. private static readonly SEARCH_PADDING_FACTOR = 2; + // TM_CCOEFF_NORMED confidence score below which a match is discarded. + // Set to 0.25 for a stationary camera: the static background makes strong + // false peaks rare, so a permissive threshold catches partial occlusions + // while still rejecting genuinely bad frames (compression spikes, motion + // blur on fast objects). + private static readonly MATCH_CONFIDENCE_THRESHOLD = 0.25; + /** * @param videoWidth - Pixel width of the video element (offscreen canvas size). * @param videoHeight - Pixel height of the video element. @@ -140,6 +176,10 @@ export class OpenCVTracker { * Capture the template from the current video frame inside `region` and send * it to the worker. OpenCV (WASM) is loaded inside the worker on the first * call — this may take a moment but never blocks the main thread. + * + * The worker applies a Gaussian blur to the captured ROI so that the template + * is in the same frequency domain as the blurred search regions used at + * tracking time. */ public async initFromVideo(video: HTMLVideoElement, region: TrackerRegion): Promise { this.drawVideoFrame(video); @@ -157,8 +197,13 @@ export class OpenCVTracker { /** * Match the stored template against the current video frame. * Returns the center of the best match in video-pixel coordinates, or null if - * the tracker is not ready. Runs asynchronously in the worker — the main - * thread is free while the worker executes matchTemplate. + * the tracker is not ready or the match confidence is below the threshold. + * Runs asynchronously in the worker — the main thread is free while the + * worker executes matchTemplate. + * + * For a stationary camera the search-window padding (SEARCH_PADDING_FACTOR) + * only needs to cover the object's own motion between frames; no extra margin + * for camera motion is required. */ public async track(video: HTMLVideoElement): Promise<{ x: number; y: number } | null> { if (!this.workerReady) { @@ -204,7 +249,15 @@ export class OpenCVTracker { const response = await this.send({ type: "track", imageData, searchX, searchY }); if (response.type === "track-result") { - const { x, y } = response; + const { x, y, confidence } = response; + + // Drop matches below the confidence threshold. For a stationary + // camera this catches motion-blurred frames and compression spikes + // without falsely rejecting genuine (albeit imperfect) matches. + if (confidence < OpenCVTracker.MATCH_CONFIDENCE_THRESHOLD) { + return null; + } + this.lastMatchCenter = { x, y }; return { x, y }; }