From fedebb5070f3ed7803cd935ad0b6d01f1dccb661 Mon Sep 17 00:00:00 2001 From: ly28266 <2826663494@qq.com> Date: Sun, 21 Dec 2025 09:55:06 +0800 Subject: [PATCH 1/2] Update gesture_detection.py --- .../gesture_detection.py | 239 ++++++++++-------- 1 file changed, 131 insertions(+), 108 deletions(-) diff --git a/src/gesture_controll_drone/gesture_detection.py b/src/gesture_controll_drone/gesture_detection.py index f9f0768701..a59ff613c1 100644 --- a/src/gesture_controll_drone/gesture_detection.py +++ b/src/gesture_controll_drone/gesture_detection.py @@ -1,137 +1,160 @@ -import cv2 # 导入OpenCV库,用于图像处理和视频捕获 -import mediapipe as mp # 导入MediaPipe库,用于手部关键点检测 -import numpy as np # 导入numpy库,用于数值计算 +import cv2 +import mediapipe as mp +import numpy as np +from PIL import Image, ImageDraw, ImageFont # 引入Pillow库 class GestureDetector: - """手势检测类 - 基于MediaPipe实现单只手的关键点检测和基础手势识别 - 支持识别:张开手掌、握拳、食指指向、胜利手势、其他手势 - """ - - def __init__(self): - """初始化函数:创建MediaPipe手部检测实例和绘图工具""" - # 初始化MediaPipe手部检测模块 + """基于MediaPipe的手势检测类""" + # 手部关键点索引 + WRIST = 0 + THUMB_TIP = 4 + THUMB_IP = 3 + INDEX_TIP = 8 + INDEX_PIP = 6 + MIDDLE_TIP = 12 + MIDDLE_PIP = 10 + RING_TIP = 16 + RING_PIP = 14 + PINKY_TIP = 20 + PINKY_PIP = 18 + + def __init__(self, max_hands=1, detection_confidence=0.7, tracking_confidence=0.5): + # 初始化MediaPipe手部检测 self.mp_hands = mp.solutions.hands - # 创建Hands对象,配置检测参数 self.hands = self.mp_hands.Hands( - static_image_mode=False, # 动态视频模式(False),非静态图片模式 - max_num_hands=1, # 最多检测1只手 - min_detection_confidence=0.7, # 检测置信度阈值,低于0.7则认为未检测到 - min_tracking_confidence=0.5 # 跟踪置信度阈值,低于0.5则重新检测 + static_image_mode=False, + max_num_hands=max_hands, + min_detection_confidence=detection_confidence, + min_tracking_confidence=tracking_confidence ) - # 初始化MediaPipe绘图工具,用于绘制手部关键点和连接线 self.mp_drawing = mp.solutions.drawing_utils - + self.draw_spec = self.mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2, circle_radius=2) + def detect_gestures(self, frame): - """检测帧中的手势 - Args: - frame: OpenCV读取的BGR格式视频帧 - Returns: - frame: 绘制了手部关键点的帧 - gesture: 识别出的手势名称(字符串) - landmarks: 手部关键点像素坐标列表,格式[(x1,y1), (x2,y2), ...] - """ - # 将BGR格式(OpenCV默认)转换为RGB格式(MediaPipe要求) + """检测帧中的手势,返回处理后的帧、手势名称、关键点坐标""" + if frame is None or frame.size == 0: + return frame, "无效帧", None + + # 颜色转换+性能优化 rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - # 处理帧,获取手部检测结果 + rgb_frame.flags.writeable = False results = self.hands.process(rgb_frame) + rgb_frame.flags.writeable = True + + gesture = "未检测到手势" + landmarks = None - # 初始化返回值 - gesture = "未检测到手势" # 默认手势状态 - landmarks = None # 关键点坐标初始化为空 - - # 如果检测到至少一只手的关键点 if results.multi_hand_landmarks: - # 遍历检测到的每只手(此处最多1只) - for hand_landmarks in results.multi_hand_landmarks: - # 在帧上绘制手部关键点和连接线 - self.mp_drawing.draw_landmarks( - frame, hand_landmarks, self.mp_hands.HAND_CONNECTIONS) - - # 提取关键点像素坐标 - landmarks = [] - h, w, c = frame.shape # 获取帧的高度、宽度、通道数 - for lm in hand_landmarks.landmark: - # 将MediaPipe的归一化坐标(0-1)转换为像素坐标 - px = int(lm.x * w) - py = int(lm.y * h) - landmarks.append((px, py)) - - # 根据关键点坐标分类手势 - gesture = self._classify_gesture(landmarks) + hand_landmarks = results.multi_hand_landmarks[0] + # 绘制关键点 + self.mp_drawing.draw_landmarks( + frame, hand_landmarks, self.mp_hands.HAND_CONNECTIONS, + self.draw_spec, self.draw_spec + ) + # 转换坐标并识别手势 + landmarks = self._convert_landmarks_to_pixels(hand_landmarks, frame.shape) + gesture = self._classify_gesture(landmarks) - # 返回处理后的帧、手势名称、关键点坐标 return frame, gesture, landmarks + def _convert_landmarks_to_pixels(self, hand_landmarks, frame_shape): + """归一化坐标转像素坐标""" + h, w, _ = frame_shape + landmarks = [] + for lm in hand_landmarks.landmark: + x = int(np.clip(lm.x * w, 0, w-1)) + y = int(np.clip(lm.y * h, 0, h-1)) + landmarks.append((x, y)) + return landmarks + + def _is_finger_open(self, landmarks, tip_idx, pip_idx): + """判断非拇指是否张开""" + return landmarks[tip_idx][1] < landmarks[pip_idx][1] + + def _is_thumb_open(self, landmarks): + """判断拇指是否张开(适配左右手)""" + wrist_x = landmarks[self.WRIST][0] + thumb_tip_x = landmarks[self.THUMB_TIP][0] + thumb_ip_x = landmarks[self.THUMB_IP][0] + + if thumb_tip_x > wrist_x: # 右手 + return thumb_tip_x > thumb_ip_x + 10 + else: # 左手 + return thumb_tip_x < thumb_ip_x - 10 + def _classify_gesture(self, landmarks): - """根据关键点坐标分类手势(内部方法) - Args: - landmarks: 手部关键点像素坐标列表 - Returns: - str: 识别出的手势名称 - """ - # 校验关键点数量(正常手部关键点应为21个) + """分类手势""" if not landmarks or len(landmarks) < 21: return "未检测到手势" - # 提取关键部位的坐标 - thumb_tip = landmarks[4] # 拇指指尖 - index_tip = landmarks[8] # 食指指尖 - middle_tip = landmarks[12] # 中指指尖 - ring_tip = landmarks[16] # 无名指指尖 - pinky_tip = landmarks[20] # 小指指尖 - wrist = landmarks[0] # 手腕 - - # 判断每根手指是否张开,存入列表 - fingers = [] - # 拇指判断:指尖x坐标 > 拇指近节指关节x坐标 视为张开(仅适配右手) - fingers.append(thumb_tip[0] > landmarks[3][0]) + # 各手指状态 + thumb_open = self._is_thumb_open(landmarks) + index_open = self._is_finger_open(landmarks, self.INDEX_TIP, self.INDEX_PIP) + middle_open = self._is_finger_open(landmarks, self.MIDDLE_TIP, self.MIDDLE_PIP) + ring_open = self._is_finger_open(landmarks, self.RING_TIP, self.RING_PIP) + pinky_open = self._is_finger_open(landmarks, self.PINKY_TIP, self.PINKY_PIP) - # 其他手指判断:指尖y坐标 < 手腕y坐标 视为张开(逻辑较简单) - for tip in [index_tip, middle_tip, ring_tip, pinky_tip]: - fingers.append(tip[1] < wrist[1]) + finger_states = [thumb_open, index_open, middle_open, ring_open, pinky_open] - # 根据手指状态分类手势 - if all(fingers): # 所有手指都张开 - return "张开手掌" - elif not any(fingers): # 所有手指都闭合 + # 手势判断 + if not any(finger_states): return "握拳" - elif fingers[1] and not any(fingers[2:]): # 仅食指张开 + elif all(finger_states): + return "张开手掌" + elif index_open and not middle_open and not ring_open and not pinky_open: return "食指指向" - elif fingers[1] and fingers[2] and not fingers[3] and not fingers[4]: # 食指+中指张开 + elif index_open and middle_open and not ring_open and not pinky_open: return "胜利手势" - elif all(fingers[1:5]): # 除拇指外其余四指张开 - return "张开五指" - else: # 其他未匹配的手势 + else: return "其他手势" -# ------------------- 测试代码(可直接运行) ------------------- + +def put_chinese_text(frame, text, position, font_size=32, color=(0, 255, 0)): + """修复OpenCV中文显示:用Pillow绘制中文后转OpenCV格式""" + # 1. OpenCV(BGR)转PIL(RGB) + pil_frame = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + draw = ImageDraw.Draw(pil_frame) + + # 2. 加载中文字体(Windows默认路径:C:/Windows/Fonts/simhei.ttf) + try: + # 优先使用系统黑体,若不存在可替换为项目内字体路径 + font = ImageFont.truetype("simhei.ttf", font_size, encoding="utf-8") + except: + # fallback到默认字体(可能不支持中文) + font = ImageFont.load_default() + + # 3. 绘制中文 + draw.text(position, text, font=font, fill=color) + + # 4. PIL(RGB)转OpenCV(BGR) + return cv2.cvtColor(np.array(pil_frame), cv2.COLOR_RGB2BGR) + + +# 测试代码 if __name__ == "__main__": - # 创建手势检测器实例 detector = GestureDetector() - # 打开摄像头(0为默认摄像头) cap = cv2.VideoCapture(0) + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - # 循环读取视频帧 - while cap.isOpened(): - ret, frame = cap.read() - if not ret: # 读取帧失败则退出 - break - - # 检测手势 - frame, gesture, landmarks = detector.detect_gestures(frame) - - # 在帧上显示识别结果 - cv2.putText(frame, f"Gesture: {gesture}", (10, 30), - cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) - - # 显示处理后的帧 - cv2.imshow("Gesture Detection", frame) - - # 按下q键退出循环 - if cv2.waitKey(1) & 0xFF == ord('q'): - break - - # 释放资源 - cap.release() - cv2.destroyAllWindows() + try: + while cap.isOpened(): + ret, frame = cap.read() + if not ret: + print("无法读取摄像头画面,退出...") + break + + frame = cv2.flip(frame, 1) # 镜像翻转 + frame, gesture, _ = detector.detect_gestures(frame) + + # 用修复后的函数显示中文手势(替换原cv2.putText) + frame = put_chinese_text(frame, f"当前手势: {gesture}", (20, 50), font_size=32) + + cv2.imshow("手势检测(按Q退出)", frame) + if cv2.waitKey(1) & 0xFF == ord('q'): + break + except Exception as e: + print(f"运行出错: {e}") + finally: + cap.release() + cv2.destroyAllWindows() From b793865ede4d1cdeb17b96338e173818fc4971f5 Mon Sep 17 00:00:00 2001 From: ly28266 <2826663494@qq.com> Date: Sun, 21 Dec 2025 13:14:22 +0800 Subject: [PATCH 2/2] Update gesture_detection.py --- .../gesture_detection.py | 199 +++++++++++++----- 1 file changed, 142 insertions(+), 57 deletions(-) diff --git a/src/gesture_controll_drone/gesture_detection.py b/src/gesture_controll_drone/gesture_detection.py index a59ff613c1..4d05182ee6 100644 --- a/src/gesture_controll_drone/gesture_detection.py +++ b/src/gesture_controll_drone/gesture_detection.py @@ -1,160 +1,245 @@ import cv2 import mediapipe as mp import numpy as np -from PIL import Image, ImageDraw, ImageFont # 引入Pillow库 +# 引入Pillow库,用于解决OpenCV中文显示乱码问题 +from PIL import Image, ImageDraw, ImageFont class GestureDetector: - """基于MediaPipe的手势检测类""" - # 手部关键点索引 - WRIST = 0 - THUMB_TIP = 4 - THUMB_IP = 3 - INDEX_TIP = 8 - INDEX_PIP = 6 - MIDDLE_TIP = 12 - MIDDLE_PIP = 10 - RING_TIP = 16 - RING_PIP = 14 - PINKY_TIP = 20 - PINKY_PIP = 18 + """基于MediaPipe的手势检测类(修复中文显示版) + 核心功能:检测单只手关键点,识别常见手势(握拳/张开手掌/食指指向/胜利手势) + 修复点:解决OpenCV显示中文乱码(问号)问题 + """ + # 定义手部关键点索引常量(简化后续代码调用) + WRIST = 0 # 手腕关键点索引 + THUMB_TIP = 4 # 拇指指尖索引 + THUMB_IP = 3 # 拇指远节指关节索引 + INDEX_TIP = 8 # 食指指尖索引 + INDEX_PIP = 6 # 食指近节指关节索引 + MIDDLE_TIP = 12 # 中指指尖索引 + MIDDLE_PIP = 10 # 中指近节指关节索引 + RING_TIP = 16 # 无名指指尖索引 + RING_PIP = 14 # 无名指近节指关节索引 + PINKY_TIP = 20 # 小指指尖索引 + PINKY_PIP = 18 # 小指近节指关节索引 def __init__(self, max_hands=1, detection_confidence=0.7, tracking_confidence=0.5): - # 初始化MediaPipe手部检测 + """初始化手势检测器 + Args: + max_hands: 最大检测手数(默认1只) + detection_confidence: 检测置信度阈值(0-1,低于则不识别) + tracking_confidence: 跟踪置信度阈值(0-1,低于则重新检测) + """ + # 初始化MediaPipe手部检测模块 self.mp_hands = mp.solutions.hands self.hands = self.mp_hands.Hands( - static_image_mode=False, - max_num_hands=max_hands, - min_detection_confidence=detection_confidence, - min_tracking_confidence=tracking_confidence + static_image_mode=False, # 视频流模式(False),静态图片设为True + max_num_hands=max_hands, # 最多检测1只手 + min_detection_confidence=detection_confidence, # 检测置信度阈值 + min_tracking_confidence=tracking_confidence # 跟踪置信度阈值 ) + # 初始化MediaPipe绘图工具(用于绘制手部关键点) self.mp_drawing = mp.solutions.drawing_utils + # 设置绘图样式:绿色、线宽2、关键点半径2 self.draw_spec = self.mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2, circle_radius=2) def detect_gestures(self, frame): - """检测帧中的手势,返回处理后的帧、手势名称、关键点坐标""" + """核心方法:检测帧中的手势 + Args: + frame: OpenCV读取的BGR格式视频帧 + Returns: + frame: 绘制了关键点的帧 + gesture: 识别的手势名称(中文) + landmarks: 手部关键点像素坐标列表 + """ + # 异常处理:校验输入帧是否有效 if frame is None or frame.size == 0: return frame, "无效帧", None - # 颜色转换+性能优化 + # 颜色空间转换:OpenCV(BGR) → MediaPipe(RGB) + # 禁用写入权限提升处理性能 rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) rgb_frame.flags.writeable = False + # 处理帧,获取手部检测结果 results = self.hands.process(rgb_frame) + # 恢复写入权限(后续绘制关键点需要) rgb_frame.flags.writeable = True - gesture = "未检测到手势" - landmarks = None + # 初始化返回值 + gesture = "未检测到手势" # 默认未检测到 + landmarks = None # 关键点坐标初始化为空 + # 如果检测到至少一只手的关键点 if results.multi_hand_landmarks: + # 仅处理第一只手(max_num_hands=1) hand_landmarks = results.multi_hand_landmarks[0] - # 绘制关键点 + # 在帧上绘制手部关键点和连接线 self.mp_drawing.draw_landmarks( frame, hand_landmarks, self.mp_hands.HAND_CONNECTIONS, - self.draw_spec, self.draw_spec + self.draw_spec, self.draw_spec # 关键点和连接线样式 ) - # 转换坐标并识别手势 + # 将归一化坐标转换为像素坐标 landmarks = self._convert_landmarks_to_pixels(hand_landmarks, frame.shape) + # 根据关键点识别手势 gesture = self._classify_gesture(landmarks) return frame, gesture, landmarks def _convert_landmarks_to_pixels(self, hand_landmarks, frame_shape): - """归一化坐标转像素坐标""" - h, w, _ = frame_shape + """辅助方法:将MediaPipe归一化坐标转换为像素坐标 + Args: + hand_landmarks: MediaPipe手部关键点对象 + frame_shape: 帧的形状 (高度, 宽度, 通道数) + Returns: + landmarks: 像素坐标列表 [(x1,y1), (x2,y2), ...] + """ + h, w, _ = frame_shape # 获取帧的高、宽 landmarks = [] for lm in hand_landmarks.landmark: + # 归一化坐标转像素坐标,并限制在帧范围内(避免越界) x = int(np.clip(lm.x * w, 0, w-1)) y = int(np.clip(lm.y * h, 0, h-1)) landmarks.append((x, y)) return landmarks def _is_finger_open(self, landmarks, tip_idx, pip_idx): - """判断非拇指是否张开""" + """辅助方法:判断非拇指手指是否张开 + 原理:指尖y坐标 < 近节指关节y坐标 → 手指张开 + Args: + landmarks: 关键点像素坐标列表 + tip_idx: 指尖索引 + pip_idx: 近节指关节索引 + Returns: + bool: 手指是否张开 + """ return landmarks[tip_idx][1] < landmarks[pip_idx][1] def _is_thumb_open(self, landmarks): - """判断拇指是否张开(适配左右手)""" - wrist_x = landmarks[self.WRIST][0] - thumb_tip_x = landmarks[self.THUMB_TIP][0] - thumb_ip_x = landmarks[self.THUMB_IP][0] + """辅助方法:判断拇指是否张开(适配左右手) + 原理: + 右手:拇指指尖x > 拇指远节指关节x + 偏移 → 张开 + 左手:拇指指尖x < 拇指远节指关节x - 偏移 → 张开 + Args: + landmarks: 关键点像素坐标列表 + Returns: + bool: 拇指是否张开 + """ + wrist_x = landmarks[self.WRIST][0] # 手腕x坐标 + thumb_tip_x = landmarks[self.THUMB_TIP][0] # 拇指指尖x坐标 + thumb_ip_x = landmarks[self.THUMB_IP][0] # 拇指远节指关节x坐标 - if thumb_tip_x > wrist_x: # 右手 - return thumb_tip_x > thumb_ip_x + 10 - else: # 左手 + # 判断左右手:拇指指尖x > 手腕x → 右手;反之→左手 + if thumb_tip_x > wrist_x: # 右手逻辑 + return thumb_tip_x > thumb_ip_x + 10 # +10减少误判 + else: # 左手逻辑 return thumb_tip_x < thumb_ip_x - 10 def _classify_gesture(self, landmarks): - """分类手势""" + """核心方法:根据关键点分类手势 + Args: + landmarks: 关键点像素坐标列表 + Returns: + str: 手势名称(中文) + """ + # 校验关键点数量(正常应为21个) if not landmarks or len(landmarks) < 21: return "未检测到手势" - # 各手指状态 + # 逐个判断手指张开状态 thumb_open = self._is_thumb_open(landmarks) index_open = self._is_finger_open(landmarks, self.INDEX_TIP, self.INDEX_PIP) middle_open = self._is_finger_open(landmarks, self.MIDDLE_TIP, self.MIDDLE_PIP) ring_open = self._is_finger_open(landmarks, self.RING_TIP, self.RING_PIP) pinky_open = self._is_finger_open(landmarks, self.PINKY_TIP, self.PINKY_PIP) + # 整理所有手指状态 finger_states = [thumb_open, index_open, middle_open, ring_open, pinky_open] - # 手势判断 - if not any(finger_states): + # 手势分类逻辑(优先级:精准手势→通用手势) + if not any(finger_states): # 所有手指闭合 → 握拳 return "握拳" - elif all(finger_states): + elif all(finger_states): # 所有手指张开 → 张开手掌 return "张开手掌" - elif index_open and not middle_open and not ring_open and not pinky_open: + elif index_open and not middle_open and not ring_open and not pinky_open: # 仅食指张开 → 食指指向 return "食指指向" - elif index_open and middle_open and not ring_open and not pinky_open: + elif index_open and middle_open and not ring_open and not pinky_open: # 食指+中指张开 → 胜利手势 return "胜利手势" - else: + else: # 其他组合 → 其他手势 return "其他手势" def put_chinese_text(frame, text, position, font_size=32, color=(0, 255, 0)): - """修复OpenCV中文显示:用Pillow绘制中文后转OpenCV格式""" - # 1. OpenCV(BGR)转PIL(RGB) + """修复OpenCV中文显示乱码的核心函数 + 原理:OpenCV默认字体不支持中文,通过Pillow绘制中文后转回OpenCV格式 + Args: + frame: OpenCV的BGR格式帧 + text: 要显示的中文文本 + position: 文本显示位置 (x, y) + font_size: 字体大小(默认32) + color: 文本颜色(RGB格式,默认绿色) + Returns: + frame: 绘制了中文文本的BGR格式帧 + """ + # 步骤1:OpenCV(BGR) 转换为 Pillow(RGB) pil_frame = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + # 创建Pillow绘图对象 draw = ImageDraw.Draw(pil_frame) - # 2. 加载中文字体(Windows默认路径:C:/Windows/Fonts/simhei.ttf) + # 步骤2:加载中文字体(解决中文显示问题) try: - # 优先使用系统黑体,若不存在可替换为项目内字体路径 + # Windows系统默认黑体路径:C:/Windows/Fonts/simhei.ttf + # Linux/macOS需替换为对应中文字体路径(如思源黑体、文泉驿等) font = ImageFont.truetype("simhei.ttf", font_size, encoding="utf-8") - except: - # fallback到默认字体(可能不支持中文) + except Exception as e: + # 异常处理:若加载字体失败,使用默认字体(可能仍显示问号) + print(f"加载中文字体失败:{e},将使用默认字体") font = ImageFont.load_default() - # 3. 绘制中文 + # 步骤3:用Pillow绘制中文文本 draw.text(position, text, font=font, fill=color) - # 4. PIL(RGB)转OpenCV(BGR) - return cv2.cvtColor(np.array(pil_frame), cv2.COLOR_RGB2BGR) + # 步骤4:Pillow(RGB) 转回 OpenCV(BGR) + frame = cv2.cvtColor(np.array(pil_frame), cv2.COLOR_RGB2BGR) + + return frame -# 测试代码 +# 测试代码(主程序入口) if __name__ == "__main__": + # 创建手势检测器实例 detector = GestureDetector() + # 打开摄像头(0为默认摄像头,若无效可尝试1) cap = cv2.VideoCapture(0) + # 设置摄像头分辨率(640x480,平衡性能和清晰度) cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) try: + # 循环读取摄像头帧 while cap.isOpened(): ret, frame = cap.read() + # 校验帧是否读取成功 if not ret: print("无法读取摄像头画面,退出...") break - frame = cv2.flip(frame, 1) # 镜像翻转 + # 镜像翻转帧(符合人眼视觉习惯,左右操作对应) + frame = cv2.flip(frame, 1) + # 检测手势 frame, gesture, _ = detector.detect_gestures(frame) - # 用修复后的函数显示中文手势(替换原cv2.putText) + # 关键修改:用自定义函数显示中文(替换原cv2.putText) frame = put_chinese_text(frame, f"当前手势: {gesture}", (20, 50), font_size=32) + # 显示处理后的帧 cv2.imshow("手势检测(按Q退出)", frame) + # 按Q键退出循环(需小写) if cv2.waitKey(1) & 0xFF == ord('q'): break except Exception as e: - print(f"运行出错: {e}") + # 异常捕获:打印错误信息 + print(f"程序运行出错:{e}") finally: + # 释放摄像头资源 cap.release() + # 关闭所有OpenCV窗口 cv2.destroyAllWindows()