Optimize auto-track for stationary camera: Gaussian blur + confidence threshold

public/opencv-worker.js

@@ -11,7 +11,7 @@
  *   worker → main  { id, type: 'init-done', templateW, templateH, centerX, centerY }
  *
  *   main → worker  { id, type: 'track',   imageData, searchX, searchY }
- *   worker → main  { id, type: 'track-result', x, y }
+ *   worker → main  { id, type: 'track-result', x, y, confidence }
  *
  *   main → worker  { id: -1, type: 'dispose' }   (no response)
  *
@@ -68,6 +68,21 @@ function ensureOpenCV() {
   return cvLoadPromise;
 }
 
+// Gaussian blur kernel applied to both the captured template and every search
+// window before matching.  A 5×5 kernel (σ ≈ 1.1 px) smooths per-frame sensor
+// noise and compression artefacts without blurring object edges enough to
+// degrade match precision.  Because the same filter is applied at both init
+// time (template) and track time (search region), the template and the search
+// image remain in the same frequency domain and TM_CCOEFF_NORMED scores stay
+// reliable.  This is tuned for a stationary camera where the background is
+// static and sensor noise is the dominant source of inter-frame variation.
+function blurGray(src, dst) {
+  // Lazily build the Size object once cv is available.
+  const ksize = new cv.Size(5, 5);
+  cv.GaussianBlur(src, dst, ksize, 0);
+  ksize.delete();
+}
+
 self.onmessage = async (event) => {
   const msg = event.data;
   const { id, type } = msg;
@@ -79,6 +94,7 @@ self.onmessage = async (event) => {
       const { imageData, region } = msg;
       const frame = cv.matFromImageData(imageData);
       const gray = new cv.Mat();
+      const blurred = new cv.Mat();
       try {
         cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
 
@@ -91,9 +107,13 @@ self.onmessage = async (event) => {
           throw new Error(`Invalid ROI dimensions: ${roiW}x${roiH}`);
         }
 
+        // Blur before cropping the ROI so the template is in the same
+        // frequency domain as the blurred search regions used during tracking.
+        blurGray(gray, blurred);
+
         if (templateMat) templateMat.delete();
         const roi = new cv.Rect(clampedX, clampedY, roiW, roiH);
-        templateMat = gray.roi(roi).clone();
+        templateMat = blurred.roi(roi).clone();
 
         self.postMessage({
           id,
@@ -106,27 +126,41 @@ self.onmessage = async (event) => {
       } finally {
         frame.delete();
         gray.delete();
+        blurred.delete();
       }
     } else if (type === 'track') {
       if (!cv || !templateMat) {
-        self.postMessage({ id, type: 'track-result', x: null, y: null });
+        self.postMessage({ id, type: 'track-result', x: null, y: null, confidence: 0 });
         return;
       }
 
       const { imageData, searchX, searchY } = msg;
       const frame = cv.matFromImageData(imageData);
       const gray = new cv.Mat();
+      const blurred = new cv.Mat();
       const result = new cv.Mat();
       try {
         cv.cvtColor(frame, gray, cv.COLOR_RGBA2GRAY);
-        cv.matchTemplate(gray, templateMat, result, cv.TM_CCOEFF_NORMED);
-        const { maxLoc } = cv.minMaxLoc(result);
+        // Apply the same blur used at template-capture time so that
+        // TM_CCOEFF_NORMED compares apples to apples.  For a stationary
+        // camera this also suppresses per-frame sensor noise that would
+        // otherwise produce spurious high-scoring locations.
+        blurGray(gray, blurred);
+        cv.matchTemplate(blurred, templateMat, result, cv.TM_CCOEFF_NORMED);
+        const { maxVal, maxLoc } = cv.minMaxLoc(result);
         const centerX = maxLoc.x + searchX + templateMat.cols / 2;
         const centerY = maxLoc.y + searchY + templateMat.rows / 2;
-        self.postMessage({ id, type: 'track-result', x: centerX, y: centerY });
+        self.postMessage({
+          id,
+          type: 'track-result',
+          x: centerX,
+          y: centerY,
+          confidence: maxVal,
+        });
       } finally {
         frame.delete();
         gray.delete();
+        blurred.delete();
         result.delete();
       }
     } else if (type === 'dispose') {

src/tracking/OpenCVTracker.ts

@@ -9,6 +9,44 @@
  *  - Drawing the video frame onto an offscreen canvas (GPU-accelerated)
  *  - Extracting the search-window ImageData (CPU ← GPU transfer)
  *  - Computing the windowed search region (cheap arithmetic)
+ *
+ * ## Stationary-camera optimisations
+ *
+ * When the camera is fixed (mounted on a tripod and not moving), the
+ * background is completely static between frames.  The only source of
+ * inter-frame variation is:
+ *   1. The tracked object itself moving through the scene.
+ *   2. Per-frame sensor noise and video-compression artefacts.
+ *
+ * Two mechanisms exploit these conditions:
+ *
+ * 1. **Gaussian pre-filtering** – a 5×5 Gaussian blur is applied to both the
+ *    template at capture time and the search region on every tracking call.
+ *    Because the same filter is used in both cases the normalised cross-
+ *    correlation score (TM_CCOEFF_NORMED) remains accurate, while pixel-level
+ *    sensor noise that would otherwise generate spurious match peaks is
+ *    suppressed.  This is more effective on a stationary camera than on a
+ *    panning one because the static background means noise is the dominant
+ *    source of false matches rather than background-content changes.
+ *
+ * 2. **Confidence threshold** – TM_CCOEFF_NORMED returns a score in [-1, 1].
+ *    Matches below MATCH_CONFIDENCE_THRESHOLD are silently dropped so that
+ *    a noisy or occluded frame cannot send the tracker to a wrong location.
+ *    The threshold is set lower (0.25) than would be appropriate for a
+ *    moving camera because the static background makes strong false peaks
+ *    unlikely.
+ *
+ * ## Windowed search optimisation (existing)
+ *
+ * Rather than reading back every pixel of every frame from the GPU (O(W×H) per
+ * frame at 30 Hz), the tracker maintains the center of the last successful match
+ * and restricts the next search to a window around it.  Only the pixels inside
+ * that window are transferred from GPU to CPU via `getImageData`.  For a typical
+ * 640×480 video and a moderate template, this reduces the pixel transfer by ~10–15×.
+ *
+ * The search window is padded by `SEARCH_PADDING_FACTOR × max(templateW, templateH)`
+ * on every side.  If the object would exit that window between frames (very fast
+ * motion), the tracker falls back to a full-frame search automatically.
  */
 
 import trackLab from "../TrackLabNamespace.js";
@@ -17,24 +55,12 @@ export type TrackerRegion = { x: number; y: number; w: number; h: number };
 
 type WorkerResponse =
   | { id: number; type: "init-done"; templateW: number; templateH: number; centerX: number; centerY: number }
-  | { id: number; type: "track-result"; x: number; y: number }
+  | { id: number; type: "track-result"; x: number; y: number; confidence: number }
   | { id: number; type: "error"; message: string };
 
 /**
  * Tracks a user-selected object across video frames using OpenCV template
  * matching running in a Web Worker.
- *
- * ## Windowed search optimisation
- *
- * Rather than reading back every pixel of every frame from the GPU (O(W×H) per
- * frame at 30 Hz), the tracker maintains the center of the last successful match
- * and restricts the next search to a window around it.  Only the pixels inside
- * that window are transferred from GPU to CPU via `getImageData`.  For a typical
- * 640×480 video and a moderate template, this reduces the pixel transfer by ~10–15×.
- *
- * The search window is padded by `SEARCH_PADDING_FACTOR × max(templateW, templateH)`
- * on every side.  If the object would exit that window between frames (very fast
- * motion), the tracker falls back to a full-frame search automatically.
  */
 export class OpenCVTracker {
   private readonly offscreen: HTMLCanvasElement;
@@ -57,8 +83,18 @@ export class OpenCVTracker {
   private templateH = 0;
   private lastMatchCenter: { x: number; y: number } | null = null;
 
+  // Padding factor: search window extends this many template-lengths on each
+  // side.  A value of 2 is well-matched to a stationary camera where the
+  // object's own motion is the sole source of inter-frame displacement.
   private static readonly SEARCH_PADDING_FACTOR = 2;
 
+  // TM_CCOEFF_NORMED confidence score below which a match is discarded.
+  // Set to 0.25 for a stationary camera: the static background makes strong
+  // false peaks rare, so a permissive threshold catches partial occlusions
+  // while still rejecting genuinely bad frames (compression spikes, motion
+  // blur on fast objects).
+  private static readonly MATCH_CONFIDENCE_THRESHOLD = 0.25;
+
   /**
    * @param videoWidth  - Pixel width of the video element (offscreen canvas size).
    * @param videoHeight - Pixel height of the video element.
@@ -140,6 +176,10 @@ export class OpenCVTracker {
    * Capture the template from the current video frame inside `region` and send
    * it to the worker.  OpenCV (WASM) is loaded inside the worker on the first
    * call — this may take a moment but never blocks the main thread.
+   *
+   * The worker applies a Gaussian blur to the captured ROI so that the template
+   * is in the same frequency domain as the blurred search regions used at
+   * tracking time.
    */
   public async initFromVideo(video: HTMLVideoElement, region: TrackerRegion): Promise<void> {
     this.drawVideoFrame(video);
@@ -157,8 +197,13 @@ export class OpenCVTracker {
   /**
    * Match the stored template against the current video frame.
    * Returns the center of the best match in video-pixel coordinates, or null if
-   * the tracker is not ready.  Runs asynchronously in the worker — the main
-   * thread is free while the worker executes matchTemplate.
+   * the tracker is not ready or the match confidence is below the threshold.
+   * Runs asynchronously in the worker — the main thread is free while the
+   * worker executes matchTemplate.
+   *
+   * For a stationary camera the search-window padding (SEARCH_PADDING_FACTOR)
+   * only needs to cover the object's own motion between frames; no extra margin
+   * for camera motion is required.
    */
   public async track(video: HTMLVideoElement): Promise<{ x: number; y: number } | null> {
     if (!this.workerReady) {
@@ -204,7 +249,15 @@ export class OpenCVTracker {
     const response = await this.send({ type: "track", imageData, searchX, searchY });
 
     if (response.type === "track-result") {
-      const { x, y } = response;
+      const { x, y, confidence } = response;
+
+      // Drop matches below the confidence threshold.  For a stationary
+      // camera this catches motion-blurred frames and compression spikes
+      // without falsely rejecting genuine (albeit imperfect) matches.
+      if (confidence < OpenCVTracker.MATCH_CONFIDENCE_THRESHOLD) {
+        return null;
+      }
+
       this.lastMatchCenter = { x, y };
       return { x, y };
     }