From f5e18da6276872f1620e89cb93edf6d2de85915a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Castro?= Date: Mon, 26 Feb 2024 11:57:22 -0300 Subject: [PATCH 1/8] Draw AbsolutePath over canvas and overlay with frame --- demos/camera_motion/src/demo.py | 8 +- norfair/drawing/path.py | 176 ++++++++++++++++++++++++-------- 2 files changed, 140 insertions(+), 44 deletions(-) diff --git a/demos/camera_motion/src/demo.py b/demos/camera_motion/src/demo.py index 42673dd0..cafef3ca 100644 --- a/demos/camera_motion/src/demo.py +++ b/demos/camera_motion/src/demo.py @@ -163,10 +163,10 @@ def run(): help="Pass this flag to draw the paths of the objects (SLOW)", ) parser.add_argument( - "--path-history", + "--path-drawer-scale", type=int, - default=20, - help="Length of the paths", + default=3, + help="Canvas (background) scale relative to frame size for the AbsolutePath drawer", ) parser.add_argument( "--id-size", @@ -215,7 +215,7 @@ def run(): fixed_camera = FixedCamera(scale=args.fixed_camera_scale) if args.draw_paths: - path_drawer = AbsolutePaths(max_history=args.path_history, thickness=2) + path_drawer = AbsolutePaths(scale=args.path_drawer_scale) video = Video(input_path=input_path) show_or_write = ( diff --git a/norfair/drawing/path.py b/norfair/drawing/path.py index 80294413..c9e80410 100644 --- a/norfair/drawing/path.py +++ b/norfair/drawing/path.py @@ -1,8 +1,10 @@ from collections import defaultdict from typing import Callable, Optional, Sequence, Tuple +import cv2 import numpy as np +from norfair.camera_motion import HomographyTransformation, TranslationTransformation from norfair.drawing.color import Palette from norfair.drawing.drawer import Drawer from norfair.tracker import TrackedObject @@ -128,17 +130,19 @@ class AbsolutePaths: Works just like [`Paths`][norfair.drawing.Paths] but supports camera motion. - !!! warning - This drawer is not optimized so it can be stremely slow. Performance degrades linearly with - `max_history * number_of_tracked_objects`. - Parameters ---------- get_points_to_draw : Optional[Callable[[np.array], np.array]], optional - Function that takes a list of points (the `.estimate` attribute of a [`TrackedObject`][norfair.tracker.TrackedObject]) - and returns a list of points for which we want to draw their paths. - - By default it is the mean point of all the points in the tracker. + Function that takes a [`TrackedObject`][norfair.tracker.TrackedObject], and returns a list of points + (in the absolute coordinate frame) for which we want to draw their paths. + + By default we just average the points with greatest height ('feet') if the object has live points. + scale : Optional[float], optional + Norfair will draw over a background canvas in the absolute coordinates. This determines how + relatively bigger is this canvas with respect to the original frame. + After the camera moves, part of the frame might get outside the canvas if scale is not large enough. + attenuation : Optional[float], optional + How fast we forget old points in the path. (0=Draw all points, 1=Draw only most current point) thickness : Optional[int], optional Thickness of the circles representing the paths of interest. color : Optional[Tuple[int, int, int]], optional @@ -147,6 +151,12 @@ class AbsolutePaths: Radius of the circles representing the paths of interest. max_history : int, optional Number of past points to include in the path. High values make the drawing slower + path_blend_factor: Optional[float], optional + When blending the frame and the canvas (with the paths overdrawn), we do: + frame = path_blend_factor * canvas + frame_blend_factor * frame + frame_blend_factor: + When blending the frame and the canvas (with the paths overdrawn), we do: + frame = path_blend_factor * canvas + frame_blend_factor * frame Examples -------- @@ -163,70 +173,156 @@ class AbsolutePaths: def __init__( self, + scale: float = 3, + attenuation: float = 0.05, get_points_to_draw: Optional[Callable[[np.array], np.array]] = None, thickness: Optional[int] = None, color: Optional[Tuple[int, int, int]] = None, radius: Optional[int] = None, - max_history=20, + path_blend_factor=2, + frame_blend_factor=1, ): + self.scale = scale + self._background = None + self._attenuation_factor = 1 - attenuation if get_points_to_draw is None: - def get_points_to_draw(points): - return [np.mean(np.array(points), axis=0)] + def get_points_to_draw(obj): + # don't draw the object if we haven't seen it recently + if not obj.live_points.any(): + return [] + + # obtain point with greatest height (feet) + points_height = obj.estimate[:, 1] + feet_indices = np.argwhere(points_height == points_height.max()) + # average their absolute positions + try: + return np.mean( + obj.get_estimate(absolute=True)[feet_indices], axis=0 + ) + except: + return np.mean(obj.estimate[feet_indices], axis=0) self.get_points_to_draw = get_points_to_draw - self.radius = radius self.thickness = thickness self.color = color - self.past_points = defaultdict(lambda: []) - self.max_history = max_history - self.alphas = np.linspace(0.99, 0.01, max_history) + self.path_blend_factor = path_blend_factor + self.frame_blend_factor = frame_blend_factor def draw(self, frame, tracked_objects, coord_transform=None): + """ + the objects have a relative frame: frame_det + the objects have an absolute frame: frame_one + the frame passed could be either frame_det, or a new perspective where you want to draw the paths + + initialization: + 1. top_left is an arbitrary coordinate of some pixel inside background + logic: + 1. draw track.get_estimate(absolute=True) + top_left, in background + 2. transform background with the composition (coord_transform.abs_to_rel o minus_top_left_translation). If coord_transform is None, only use minus_top_left_translation. + 3. crop [:frame.width, :frame.height] from the result + 4. overlay that over frame + + Remark: + In any case, coord_transform should be the coordinate transformation between the tracker absolute coords (as abs) and frame coords (as rel) + """ + + # initialize background if necessary + if self._background is None: + original_size = ( + frame.shape[1], + frame.shape[0], + ) # OpenCV format is (width, height) + + scaled_size = tuple( + (np.array(original_size) * np.array(self.scale)).round().astype(int) + ) + self._background = np.zeros( + [scaled_size[1], scaled_size[0], frame.shape[-1]], + frame.dtype, + ) + + # this is the corner of the first passed frame (inside the background) + self.top_left = ( + np.array(self._background.shape[:2]) // 2 + - np.array(frame.shape[:2]) // 2 + ) + else: + self._background = (self._background * self._attenuation_factor).astype( + frame.dtype + ) + frame_scale = frame.shape[0] / 100 if self.radius is None: self.radius = int(max(frame_scale * 0.7, 1)) if self.thickness is None: self.thickness = int(max(frame_scale / 7, 1)) - for obj in tracked_objects: - if not obj.live_points.any(): - continue + # draw in background (each point in top_left_translation(abs_coordinate)) + for obj in tracked_objects: if self.color is None: color = Palette.choose_color(obj.id) else: color = self.color - points_to_draw = self.get_points_to_draw(obj.get_estimate(absolute=True)) + points_to_draw = self.get_points_to_draw(obj) - for point in coord_transform.abs_to_rel(points_to_draw): + for point in points_to_draw: Drawer.circle( - frame, - position=tuple(point.astype(int)), + self._background, + position=tuple((point + self.top_left).astype(int)), radius=self.radius, color=color, thickness=self.thickness, ) - last = points_to_draw - for i, past_points in enumerate(self.past_points[obj.id]): - overlay = frame.copy() - last = coord_transform.abs_to_rel(last) - for j, point in enumerate(coord_transform.abs_to_rel(past_points)): - Drawer.line( - overlay, - tuple(last[j].astype(int)), - tuple(point.astype(int)), - color=color, - thickness=self.thickness, - ) - last = past_points - - alpha = self.alphas[i] - frame = Drawer.alpha_blend(overlay, frame, alpha=alpha) - self.past_points[obj.id].insert(0, points_to_draw) - self.past_points[obj.id] = self.past_points[obj.id][: self.max_history] + # apply warp to self._background with composition abs_to_rel o -top_left_translation to background, and crop [:width, :height] to get frame overdrawn + if isinstance(coord_transform, HomographyTransformation): + minus_top_left_translation = np.array( + [[1, 0, -self.top_left[0]], [0, 1, -self.top_left[1]], [0, 0, 1]] + ) + full_transformation = ( + coord_transform.homography_matrix @ minus_top_left_translation + ) + background_size_frame = cv2.warpPerspective( + self._background, + full_transformation, + tuple(frame.shape[:2][::-1]), + cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=(0, 0, 0), + ) + elif isinstance(coord_transform, TranslationTransformation): + full_transformation = np.array( + [ + [1, 0, coord_transform.movement_vector[0] - self.top_left[0]], + [0, 1, coord_transform.movement_vector[1] - self.top_left[1]], + ] + ) + background_size_frame = cv2.warpAffine( + self._background, + full_transformation, + tuple(frame.shape[:2][::-1]), + cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=(0, 0, 0), + ) + else: + background_size_frame = self._background[ + self.top_left[1] :, self.top_left[0] : + ] + background_size_frame = background_size_frame[ + : frame.shape[0], : frame.shape[1] + ] + + frame = cv2.addWeighted( + frame, + self.frame_blend_factor, + background_size_frame, + self.path_blend_factor, + 0.0, + ) return frame From c86c7f195c07a597c40bf26cbd232b6680698b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Castro?= Date: Mon, 26 Feb 2024 12:45:09 -0300 Subject: [PATCH 2/8] Allow to used FixedCamera on homographies --- norfair/drawing/fixed_camera.py | 96 ++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 44 deletions(-) diff --git a/norfair/drawing/fixed_camera.py b/norfair/drawing/fixed_camera.py index 6c049063..c2e5b1bb 100644 --- a/norfair/drawing/fixed_camera.py +++ b/norfair/drawing/fixed_camera.py @@ -1,7 +1,9 @@ +from typing import Union + +import cv2 import numpy as np -from norfair.camera_motion import TranslationTransformation -from norfair.utils import warn_once +from norfair.camera_motion import HomographyTransformation, TranslationTransformation class FixedCamera: @@ -57,7 +59,11 @@ def __init__(self, scale: float = 2, attenuation: float = 0.05): self._attenuation_factor = 1 - attenuation def adjust_frame( - self, frame: np.ndarray, coord_transformation: TranslationTransformation + self, + frame: np.ndarray, + coord_transformation: Union[ + HomographyTransformation, TranslationTransformation + ], ) -> np.ndarray: """ Render scaled up frame. @@ -89,53 +95,55 @@ def adjust_frame( [scaled_size[1], scaled_size[0], frame.shape[-1]], frame.dtype, ) + # top_left is the anchor coordinate from where we start drawing the fame on top of the background + self.top_left = ( + np.array(self._background.shape[:2]) // 2 + - np.array(frame.shape[:2]) // 2 + ) else: self._background = (self._background * self._attenuation_factor).astype( frame.dtype ) - # top_left is the anchor coordinate from where we start drawing the fame on top of the background - # aim to draw it in the center of the background but transformations will move this point - top_left = ( - np.array(self._background.shape[:2]) // 2 - np.array(frame.shape[:2]) // 2 - ) - top_left = ( - coord_transformation.rel_to_abs(top_left[::-1]).round().astype(int)[::-1] - ) - # box of the background that will be updated and the limits of it - background_y0, background_y1 = (top_left[0], top_left[0] + frame.shape[0]) - background_x0, background_x1 = (top_left[1], top_left[1] + frame.shape[1]) - background_size_y, background_size_x = self._background.shape[:2] - - # define box of the frame that will be used - # if the scale is not enough to support the movement, warn the user but keep drawing - # cropping the frame so that the operation doesn't fail - frame_y0, frame_y1, frame_x0, frame_x1 = (0, frame.shape[0], 0, frame.shape[1]) - if ( - background_y0 < 0 - or background_x0 < 0 - or background_y1 > background_size_y - or background_x1 > background_size_x - ): - warn_once( - "moving_camera_scale is not enough to cover the range of camera movement, frame will be cropped" + # warp the frame with the following composition: + # top_left_translation o rel_to_abs + if isinstance(coord_transformation, HomographyTransformation): + top_left_translation = np.array( + [[1, 0, self.top_left[1]], [0, 1, self.top_left[0]], [0, 0, 1]] ) - # crop left or top of the frame if necessary - frame_y0 = max(-background_y0, 0) - frame_x0 = max(-background_x0, 0) - # crop right or bottom of the frame if necessary - frame_y1 = max( - min(background_size_y - background_y0, background_y1 - background_y0), 0 + full_transformation = ( + top_left_translation @ coord_transformation.inverse_homography_matrix ) - frame_x1 = max( - min(background_size_x - background_x0, background_x1 - background_x0), 0 + background_with_current_frame = cv2.warpPerspective( + frame, + full_transformation, + tuple(self._background.shape[:2][::-1]), + cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=(0, 0, 0), ) - # handle cases where the limits of the background become negative which numpy will interpret incorrectly - background_y0 = max(background_y0, 0) - background_x0 = max(background_x0, 0) - background_y1 = max(background_y1, 0) - background_x1 = max(background_x1, 0) - self._background[ - background_y0:background_y1, background_x0:background_x1, : - ] = frame[frame_y0:frame_y1, frame_x0:frame_x1, :] + elif isinstance(coord_transformation, TranslationTransformation): + + full_transformation = np.array( + [ + [1, 0, self.top_left[1] - coord_transformation.movement_vector[0]], + [0, 1, self.top_left[0] - coord_transformation.movement_vector[1]], + ] + ) + background_with_current_frame = cv2.warpAffine( + frame, + full_transformation, + tuple(self._background.shape[:2][::-1]), + cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=(0, 0, 0), + ) + + self._background = cv2.addWeighted( + self._background, + 0.5, + background_with_current_frame, + 0.5, + 0.0, + ) return self._background From bed08ed37be68c5bb51691bfc4eba33d07743699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Castro?= Date: Mon, 26 Feb 2024 14:29:36 -0300 Subject: [PATCH 3/8] Fix the order (x,y) in top left corner variables --- demos/camera_motion/src/demo.py | 2 +- norfair/drawing/fixed_camera.py | 7 ++++--- norfair/drawing/path.py | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/demos/camera_motion/src/demo.py b/demos/camera_motion/src/demo.py index cafef3ca..f0a0c791 100644 --- a/demos/camera_motion/src/demo.py +++ b/demos/camera_motion/src/demo.py @@ -142,7 +142,7 @@ def run(): "--fixed-camera-scale", type=float, default=0, - help="Scale of the fixed camera, set to 0 to disable. Note that this only works for translation", + help="Scale of the fixed camera, set to 0 to disable.", ) parser.add_argument( "--draw-absolute-grid", diff --git a/norfair/drawing/fixed_camera.py b/norfair/drawing/fixed_camera.py index c2e5b1bb..3e74d8bd 100644 --- a/norfair/drawing/fixed_camera.py +++ b/norfair/drawing/fixed_camera.py @@ -100,6 +100,7 @@ def adjust_frame( np.array(self._background.shape[:2]) // 2 - np.array(frame.shape[:2]) // 2 ) + self.top_left = self.top_left[::-1] else: self._background = (self._background * self._attenuation_factor).astype( frame.dtype @@ -109,7 +110,7 @@ def adjust_frame( # top_left_translation o rel_to_abs if isinstance(coord_transformation, HomographyTransformation): top_left_translation = np.array( - [[1, 0, self.top_left[1]], [0, 1, self.top_left[0]], [0, 0, 1]] + [[1, 0, self.top_left[0]], [0, 1, self.top_left[1]], [0, 0, 1]] ) full_transformation = ( top_left_translation @ coord_transformation.inverse_homography_matrix @@ -126,8 +127,8 @@ def adjust_frame( full_transformation = np.array( [ - [1, 0, self.top_left[1] - coord_transformation.movement_vector[0]], - [0, 1, self.top_left[0] - coord_transformation.movement_vector[1]], + [1, 0, self.top_left[0] - coord_transformation.movement_vector[0]], + [0, 1, self.top_left[1] - coord_transformation.movement_vector[1]], ] ) background_with_current_frame = cv2.warpAffine( diff --git a/norfair/drawing/path.py b/norfair/drawing/path.py index c9e80410..0b607062 100644 --- a/norfair/drawing/path.py +++ b/norfair/drawing/path.py @@ -249,6 +249,7 @@ def draw(self, frame, tracked_objects, coord_transform=None): np.array(self._background.shape[:2]) // 2 - np.array(frame.shape[:2]) // 2 ) + self.top_left = self.top_left[::-1] else: self._background = (self._background * self._attenuation_factor).astype( frame.dtype From ed273fd162a9a957ab6c475c9d4503dc7ce5dbea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Castro?= Date: Mon, 26 Feb 2024 17:43:57 -0300 Subject: [PATCH 4/8] Use the Dummy opencv to return an error if opencv not installed --- norfair/drawing/fixed_camera.py | 7 ++++++- norfair/drawing/path.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/norfair/drawing/fixed_camera.py b/norfair/drawing/fixed_camera.py index 3e74d8bd..28f4f791 100644 --- a/norfair/drawing/fixed_camera.py +++ b/norfair/drawing/fixed_camera.py @@ -1,6 +1,11 @@ from typing import Union -import cv2 +try: + import cv2 +except ImportError: + from norfair.utils import DummyOpenCVImport + + cv2 = DummyOpenCVImport() import numpy as np from norfair.camera_motion import HomographyTransformation, TranslationTransformation diff --git a/norfair/drawing/path.py b/norfair/drawing/path.py index 0b607062..b871389c 100644 --- a/norfair/drawing/path.py +++ b/norfair/drawing/path.py @@ -1,7 +1,12 @@ from collections import defaultdict from typing import Callable, Optional, Sequence, Tuple -import cv2 +try: + import cv2 +except ImportError: + from norfair.utils import DummyOpenCVImport + + cv2 = DummyOpenCVImport() import numpy as np from norfair.camera_motion import HomographyTransformation, TranslationTransformation From d8a99939069d9d9440a509e15af21feccb185ed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Castro?= Date: Mon, 26 Feb 2024 18:14:51 -0300 Subject: [PATCH 5/8] Get default scale in AbsolutePaths depending if coord_transform is passed --- demos/camera_motion/src/demo.py | 4 ++-- norfair/drawing/path.py | 15 ++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/demos/camera_motion/src/demo.py b/demos/camera_motion/src/demo.py index f0a0c791..07cd96b5 100644 --- a/demos/camera_motion/src/demo.py +++ b/demos/camera_motion/src/demo.py @@ -160,12 +160,12 @@ def run(): "--draw-paths", dest="draw_paths", action="store_true", - help="Pass this flag to draw the paths of the objects (SLOW)", + help="Pass this flag to draw the paths of the objects", ) parser.add_argument( "--path-drawer-scale", type=int, - default=3, + default=None, help="Canvas (background) scale relative to frame size for the AbsolutePath drawer", ) parser.add_argument( diff --git a/norfair/drawing/path.py b/norfair/drawing/path.py index b871389c..83a8917a 100644 --- a/norfair/drawing/path.py +++ b/norfair/drawing/path.py @@ -178,7 +178,7 @@ class AbsolutePaths: def __init__( self, - scale: float = 3, + scale: float = None, attenuation: float = 0.05, get_points_to_draw: Optional[Callable[[np.array], np.array]] = None, thickness: Optional[int] = None, @@ -236,6 +236,13 @@ def draw(self, frame, tracked_objects, coord_transform=None): # initialize background if necessary if self._background is None: + if self.scale is None: + # set the default scale, depending if coord_transform is provided or not + if coord_transform is None: + self.scale = 1 + else: + self.scale = 3 + original_size = ( frame.shape[1], frame.shape[0], @@ -318,10 +325,8 @@ def draw(self, frame, tracked_objects, coord_transform=None): ) else: background_size_frame = self._background[ - self.top_left[1] :, self.top_left[0] : - ] - background_size_frame = background_size_frame[ - : frame.shape[0], : frame.shape[1] + self.top_left[1] : self.top_left[1] + frame.shape[0], + self.top_left[0] : self.top_left[0] + frame.shape[1], ] frame = cv2.addWeighted( From e25c7c373dec837fa9989d8bb52aafd917102ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Castro?= Date: Mon, 26 Feb 2024 18:39:41 -0300 Subject: [PATCH 6/8] More consistent naming: always coord_transormations --- demos/camera_motion/src/demo.py | 2 +- norfair/drawing/fixed_camera.py | 42 ++++++++++++++++++++------------- norfair/drawing/path.py | 20 ++++++++-------- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/demos/camera_motion/src/demo.py b/demos/camera_motion/src/demo.py index 07cd96b5..b90edc58 100644 --- a/demos/camera_motion/src/demo.py +++ b/demos/camera_motion/src/demo.py @@ -286,7 +286,7 @@ def run(): if args.draw_paths: frame = path_drawer.draw( - frame, tracked_objects, coord_transform=coord_transformations + frame, tracked_objects, coord_transformations=coord_transformations ) if use_fixed_camera: diff --git a/norfair/drawing/fixed_camera.py b/norfair/drawing/fixed_camera.py index 28f4f791..4abfce89 100644 --- a/norfair/drawing/fixed_camera.py +++ b/norfair/drawing/fixed_camera.py @@ -58,7 +58,7 @@ class FixedCamera: >>> video.write(bigger_frame) """ - def __init__(self, scale: float = 2, attenuation: float = 0.05): + def __init__(self, scale: float = None, attenuation: float = 0.05): self.scale = scale self._background = None self._attenuation_factor = 1 - attenuation @@ -66,7 +66,7 @@ def __init__(self, scale: float = 2, attenuation: float = 0.05): def adjust_frame( self, frame: np.ndarray, - coord_transformation: Union[ + coord_transformations: Union[ HomographyTransformation, TranslationTransformation ], ) -> np.ndarray: @@ -77,7 +77,7 @@ def adjust_frame( ---------- frame : np.ndarray The OpenCV frame. - coord_transformation : TranslationTransformation + coord_transformations : Union[TranslationTransformation, HomographyTransformation] The coordinate transformation as returned by the [`MotionEstimator`][norfair.camera_motion.MotionEstimator] Returns @@ -88,6 +88,12 @@ def adjust_frame( # initialize background if necessary if self._background is None: + if self.scale is None: + if coord_transformations is None: + self.scale = 1 + else: + self.scale = 3 + original_size = ( frame.shape[1], frame.shape[0], @@ -113,12 +119,12 @@ def adjust_frame( # warp the frame with the following composition: # top_left_translation o rel_to_abs - if isinstance(coord_transformation, HomographyTransformation): + if isinstance(coord_transformations, HomographyTransformation): top_left_translation = np.array( [[1, 0, self.top_left[0]], [0, 1, self.top_left[1]], [0, 0, 1]] ) full_transformation = ( - top_left_translation @ coord_transformation.inverse_homography_matrix + top_left_translation @ coord_transformations.inverse_homography_matrix ) background_with_current_frame = cv2.warpPerspective( frame, @@ -128,12 +134,12 @@ def adjust_frame( borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0), ) - elif isinstance(coord_transformation, TranslationTransformation): + elif isinstance(coord_transformations, TranslationTransformation): full_transformation = np.array( [ - [1, 0, self.top_left[0] - coord_transformation.movement_vector[0]], - [0, 1, self.top_left[1] - coord_transformation.movement_vector[1]], + [1, 0, self.top_left[0] - coord_transformations.movement_vector[0]], + [0, 1, self.top_left[1] - coord_transformations.movement_vector[1]], ] ) background_with_current_frame = cv2.warpAffine( @@ -144,12 +150,14 @@ def adjust_frame( borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0), ) - - self._background = cv2.addWeighted( - self._background, - 0.5, - background_with_current_frame, - 0.5, - 0.0, - ) - return self._background + try: + self._background = cv2.addWeighted( + self._background, + 0.5, + background_with_current_frame, + 0.5, + 0.0, + ) + return self._background + except UnboundLocalError: + return frame diff --git a/norfair/drawing/path.py b/norfair/drawing/path.py index 83a8917a..3b5a0554 100644 --- a/norfair/drawing/path.py +++ b/norfair/drawing/path.py @@ -216,7 +216,7 @@ def get_points_to_draw(obj): self.path_blend_factor = path_blend_factor self.frame_blend_factor = frame_blend_factor - def draw(self, frame, tracked_objects, coord_transform=None): + def draw(self, frame, tracked_objects, coord_transformations=None): """ the objects have a relative frame: frame_det the objects have an absolute frame: frame_one @@ -226,19 +226,19 @@ def draw(self, frame, tracked_objects, coord_transform=None): 1. top_left is an arbitrary coordinate of some pixel inside background logic: 1. draw track.get_estimate(absolute=True) + top_left, in background - 2. transform background with the composition (coord_transform.abs_to_rel o minus_top_left_translation). If coord_transform is None, only use minus_top_left_translation. + 2. transform background with the composition (coord_transformations.abs_to_rel o minus_top_left_translation). If coord_transformations is None, only use minus_top_left_translation. 3. crop [:frame.width, :frame.height] from the result 4. overlay that over frame Remark: - In any case, coord_transform should be the coordinate transformation between the tracker absolute coords (as abs) and frame coords (as rel) + In any case, coord_transformations should be the coordinate transformation between the tracker absolute coords (as abs) and frame coords (as rel) """ # initialize background if necessary if self._background is None: if self.scale is None: - # set the default scale, depending if coord_transform is provided or not - if coord_transform is None: + # set the default scale, depending if coord_transformations is provided or not + if coord_transformations is None: self.scale = 1 else: self.scale = 3 @@ -293,12 +293,12 @@ def draw(self, frame, tracked_objects, coord_transform=None): ) # apply warp to self._background with composition abs_to_rel o -top_left_translation to background, and crop [:width, :height] to get frame overdrawn - if isinstance(coord_transform, HomographyTransformation): + if isinstance(coord_transformations, HomographyTransformation): minus_top_left_translation = np.array( [[1, 0, -self.top_left[0]], [0, 1, -self.top_left[1]], [0, 0, 1]] ) full_transformation = ( - coord_transform.homography_matrix @ minus_top_left_translation + coord_transformations.homography_matrix @ minus_top_left_translation ) background_size_frame = cv2.warpPerspective( self._background, @@ -308,11 +308,11 @@ def draw(self, frame, tracked_objects, coord_transform=None): borderMode=cv2.BORDER_CONSTANT, borderValue=(0, 0, 0), ) - elif isinstance(coord_transform, TranslationTransformation): + elif isinstance(coord_transformations, TranslationTransformation): full_transformation = np.array( [ - [1, 0, coord_transform.movement_vector[0] - self.top_left[0]], - [0, 1, coord_transform.movement_vector[1] - self.top_left[1]], + [1, 0, coord_transformations.movement_vector[0] - self.top_left[0]], + [0, 1, coord_transformations.movement_vector[1] - self.top_left[1]], ] ) background_size_frame = cv2.warpAffine( From 0519e17b68279b2f17c3ee3155794d3e04ca3fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Castro?= Date: Fri, 1 Mar 2024 12:55:07 -0300 Subject: [PATCH 7/8] Remove max_history argument from docstring --- norfair/drawing/path.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/norfair/drawing/path.py b/norfair/drawing/path.py index 3b5a0554..54b8d7ab 100644 --- a/norfair/drawing/path.py +++ b/norfair/drawing/path.py @@ -154,8 +154,6 @@ class AbsolutePaths: [Color][norfair.drawing.Color] of the circles representing the paths of interest. radius : Optional[int], optional Radius of the circles representing the paths of interest. - max_history : int, optional - Number of past points to include in the path. High values make the drawing slower path_blend_factor: Optional[float], optional When blending the frame and the canvas (with the paths overdrawn), we do: frame = path_blend_factor * canvas + frame_blend_factor * frame From c2274d20619bdda1e1327e1001c5fc4fdfa9371f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Agust=C3=ADn=20Castro?= Date: Mon, 4 Mar 2024 08:53:47 -0300 Subject: [PATCH 8/8] Update documentation for the new AbsolutePaths --- norfair/drawing/path.py | 62 +++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/norfair/drawing/path.py b/norfair/drawing/path.py index 54b8d7ab..da6e7beb 100644 --- a/norfair/drawing/path.py +++ b/norfair/drawing/path.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Callable, Optional, Sequence, Tuple +from typing import Callable, Optional, Sequence, Tuple, Union try: import cv2 @@ -139,7 +139,11 @@ class AbsolutePaths: ---------- get_points_to_draw : Optional[Callable[[np.array], np.array]], optional Function that takes a [`TrackedObject`][norfair.tracker.TrackedObject], and returns a list of points - (in the absolute coordinate frame) for which we want to draw their paths. + for which we want to draw their paths. + If using a MotionEstimator for the tracking, it is recommended to return the points in the absolute frame + (i.e: TrackedObject.get_estimate(absolute=True)). Otherwise, just return the points the relative frame. + Remember that the chosen coordinates will be transformed by coord_transformations (if it is not None) in the draw method, + by considering the points you returned as in absolute coordinates and the points that it will be drawing as in relative. By default we just average the points with greatest height ('feet') if the object has live points. scale : Optional[float], optional @@ -204,7 +208,7 @@ def get_points_to_draw(obj): return np.mean( obj.get_estimate(absolute=True)[feet_indices], axis=0 ) - except: + except ValueError: return np.mean(obj.estimate[feet_indices], axis=0) self.get_points_to_draw = get_points_to_draw @@ -214,22 +218,44 @@ def get_points_to_draw(obj): self.path_blend_factor = path_blend_factor self.frame_blend_factor = frame_blend_factor - def draw(self, frame, tracked_objects, coord_transformations=None): + def draw( + self, + frame, + tracked_objects, + coord_transformations: Optional[ + Union[HomographyTransformation, TranslationTransformation] + ] = None, + ): """ - the objects have a relative frame: frame_det - the objects have an absolute frame: frame_one - the frame passed could be either frame_det, or a new perspective where you want to draw the paths - - initialization: - 1. top_left is an arbitrary coordinate of some pixel inside background - logic: - 1. draw track.get_estimate(absolute=True) + top_left, in background - 2. transform background with the composition (coord_transformations.abs_to_rel o minus_top_left_translation). If coord_transformations is None, only use minus_top_left_translation. - 3. crop [:frame.width, :frame.height] from the result - 4. overlay that over frame - - Remark: - In any case, coord_transformations should be the coordinate transformation between the tracker absolute coords (as abs) and frame coords (as rel) + Draw the paths of the points interest on a frame. + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to draw on. + tracked_objects : Sequence[TrackedObject] + List of [`TrackedObject`][norfair.tracker.TrackedObject] to get the points of interest in order to update the paths. + coord_transformations=Optional[Union[HomographyTransformation, TranslationTransformation]] + Coordinate transformation between the tracker coordinates (in the coordinates returned by get_points_to_draw method) as absolute frame + and the frame coordinates as relative frame. + If no transformation is provided, we assume that the TrackedObjects coordinates coincide with their coordinates in the frame. + For example, if you are + 1. using a MotionEstimator for the tracking, then get_points_to_draw can return points in absolute coords, and + coord_transformations would be the transformation between the absolute coords and the frame. + - Particular case: if we are drawing over the original frame where the detections were done (relative frame), then + get_points_to_draw returns points in absolute coordinates, and coord_transformations is the coordinate transformation + returned by your MotionEstimator that you use for the tracking. + + 2. NOT using a MotionEstimator for the tracking, then get_points_to_draw can return points in only 'relative' coords, + and coord_transformations would be the transformation between the relative coordinates and the frame. + Particular case: if we want to draw over the original frame where the detections were done (relative frame), then + get_points_to_draw returns points in relative frame, and coord_transformations can just be None. + + + Returns + ------- + np.array + The resulting frame. """ # initialize background if necessary