From 09ac131a896f92f3469967d256818c0c30dcec29 Mon Sep 17 00:00:00 2001 From: Arrebol01 <1245298208@qq.com> Date: Thu, 18 Dec 2025 20:17:39 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=96=B0=E6=B7=BB=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E5=A2=9E=E5=BC=BA=E8=AF=86=E5=88=AB=E7=A8=8B=E5=BA=8F=EF=BC=8C?= =?UTF-8?q?=E5=B9=B6=E4=BF=AE=E6=94=B9=E4=B8=BB=E7=A8=8B=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_detector_enhanced.py | 240 ++++++++++++++++++ src/Drone_hand_gesture_project/main.py | 201 ++++++++++++--- 2 files changed, 410 insertions(+), 31 deletions(-) create mode 100644 src/Drone_hand_gesture_project/gesture_detector_enhanced.py diff --git a/src/Drone_hand_gesture_project/gesture_detector_enhanced.py b/src/Drone_hand_gesture_project/gesture_detector_enhanced.py new file mode 100644 index 0000000000..c6c734c53c --- /dev/null +++ b/src/Drone_hand_gesture_project/gesture_detector_enhanced.py @@ -0,0 +1,240 @@ +import cv2 +import os +import numpy as np +import mediapipe as mp +from gesture_classifier import GestureClassifier, EnsembleGestureClassifier +import PIL + +try: + from PIL import Image, ImageDraw, ImageFont + import numpy as np + HAS_PIL = True +except ImportError: + HAS_PIL = False + print("⚠️ PIL未安装,中文显示不可用") + +class EnhancedGestureDetector: + """手势检测器(集成机器学习)""" + + def __init__(self, ml_model_path=None, use_ml=True): + # 基础检测器 + self.mp_hands = mp.solutions.hands + self.mp_drawing = mp.solutions.drawing_utils + self.mp_drawing_styles = mp.solutions.drawing_styles + + self.hands = self.mp_hands.Hands( + static_image_mode=False, + max_num_hands=1, + min_detection_confidence=0.5, + min_tracking_confidence=0.5 + ) + + # 机器学习模型 + self.use_ml = use_ml + self.ml_classifier = None + + if use_ml and ml_model_path: + # 检查是否是 ensemble 信息文件(很小的文件) + if os.path.exists(ml_model_path): + file_size = os.path.getsize(ml_model_path) + if file_size < 1024: # 小于1KB,可能是信息文件 + print(f"⚠️ 检测到小文件 ({file_size}字节),可能是集成模型信息文件") + print(f" 尝试加载单个模型文件...") + + # 尝试加载对应的单个模型 + base_name = os.path.basename(ml_model_path) + if "ensemble" in base_name: + # 尝试加载 svm 模型 + svm_path = ml_model_path.replace("ensemble", "svm") + if os.path.exists(svm_path): + print(f" 尝试加载: {svm_path}") + success = self.load_ml_model(svm_path) + else: + print(f" 未找到替代模型,使用规则检测") + self.use_ml = False + else: + # 正常加载 + success = self.load_ml_model(ml_model_path) + else: + self.use_ml = False + + # 手势到控制指令的映射(8个手势) + self.gesture_commands = { + "open_palm": "takeoff", + "closed_fist": "land", + "pointing_up": "up", + "pointing_down": "down", + "victory": "forward", + "thumb_up": "backward", + "thumb_down": "stop", + "ok_sign": "hover", + } + + # 中文手势名称映射 + self.chinese_gesture_names = { + "open_palm": "张开手掌", + "closed_fist": "握拳", + "victory": "胜利手势", + "thumb_up": "大拇指", + "thumb_down": "大拇指向下", + "pointing_up": "食指上指", + "pointing_down": "食指向下", + "ok_sign": "OK手势", + "none": "未检测到手" + } + + # 中文指令名称映射 + self.chinese_command_names = { + "takeoff": "起飞", + "land": "降落", + "up": "上升", + "down": "下降", + "forward": "前进", + "backward": "后退", + "stop": "停止", + "hover": "悬停", + "none": "无指令" + } + + # 中文字体 + self.chinese_font = None + if HAS_PIL: + try: + # Windows字体路径 + font_paths = [ + "C:/Windows/Fonts/msyh.ttc", # 微软雅黑 + "C:/Windows/Fonts/simhei.ttf", # 黑体 + "C:/Windows/Fonts/simsun.ttc", # 宋体 + ] + + for font_path in font_paths: + try: + if os.path.exists(font_path): + self.chinese_font = ImageFont.truetype(font_path, 30) + print(f"✅ 加载中文字体: {os.path.basename(font_path)}") + break + except: + continue + + if self.chinese_font is None: + print("⚠️ 无法加载系统字体,将使用英文显示") + except Exception as e: + print(f"⚠️ 字体初始化失败: {e}") + + + # 历史记录(用于平滑预测) + self.prediction_history = [] + self.max_history = 5 + + def load_ml_model(self, model_path): + """加载机器学习模型""" + try: + self.ml_classifier = GestureClassifier(model_path=model_path) + print(f"机器学习模型已加载: {model_path}") + return True + except Exception as e: + print(f"加载机器学习模型失败: {e}") + print("将使用规则检测器") + self.use_ml = False + return False + + def extract_landmarks_for_ml(self, hand_landmarks): + """提取关键点用于机器学习""" + landmarks = [] + for landmark in hand_landmarks.landmark: + landmarks.extend([landmark.x, landmark.y, landmark.z]) + + # 确保是63维 + if len(landmarks) < 63: + landmarks.extend([0.0] * (63 - len(landmarks))) + elif len(landmarks) > 63: + landmarks = landmarks[:63] + + return landmarks + + def detect_gestures(self, image, simulation_mode=False): + """检测手势""" + image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + results = self.hands.process(image_rgb) + + gesture = "none" # 默认none + confidence = 0.0 + landmarks_data = None + + if results.multi_hand_landmarks: + for hand_landmarks in results.multi_hand_landmarks: + # 绘制手部关键点 + self.mp_drawing.draw_landmarks( + image, + hand_landmarks, + self.mp_hands.HAND_CONNECTIONS, + self.mp_drawing_styles.get_default_hand_landmarks_style(), + self.mp_drawing_styles.get_default_hand_connections_style() + ) + + # 提取关键点 + landmarks = self.extract_landmarks_for_ml(hand_landmarks) + + # 使用机器学习或规则检测 + if self.use_ml and self.ml_classifier and len(landmarks) == 63: + # 机器学习预测 + gesture, confidence = self.ml_classifier.predict(landmarks) + + # 应用平滑滤波 + self.prediction_history.append((gesture, confidence)) + if len(self.prediction_history) > self.max_history: + self.prediction_history.pop(0) + + # 使用历史中的多数投票 + if len(self.prediction_history) >= 3: + from collections import Counter + recent_gestures = [g for g, _ in self.prediction_history[-3:]] + most_common = Counter(recent_gestures).most_common(1)[0] + if most_common[1] >= 2: # 至少2/3一致 + gesture = most_common[0] + # 计算平均置信度 + matching_confs = [c for g, c in self.prediction_history if g == gesture] + if matching_confs: + confidence = np.mean(matching_confs) + else: + # 如果未使用ML,使用简单的规则检测 + gesture, confidence = self._classify_by_rules(hand_landmarks) + + # 显示信息 + ml_info = " (ML)" if self.use_ml else " (规则)" + cv2.putText(image, f"Gesture: {gesture}{ml_info}", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) + cv2.putText(image, f"Confidence: {confidence:.2f}", (10, 70), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + + command = self.gesture_commands.get(gesture, "none") + cv2.putText(image, f"Command: {command}", (10, 110), + cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2) + + else: + # 无手部检测 + cv2.putText(image, "未检测到手部", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) + gesture = "none" + confidence = 0.0 + + return image, gesture, confidence, landmarks_data + + def _classify_by_rules(self, hand_landmarks): + """简单的规则分类器(备用)""" + # 获取关键点 + landmarks = hand_landmarks.landmark + + # 这里实现简单的规则检测逻辑 + # 例如:计算手指是否伸直等 + + # 暂时返回一个默认值 + return "open_palm", 0.5 + + def get_command(self, gesture): + """获取控制指令""" + return self.gesture_commands.get(gesture, "none") + + def release(self): + """释放资源""" + self.hands.close() \ No newline at end of file diff --git a/src/Drone_hand_gesture_project/main.py b/src/Drone_hand_gesture_project/main.py index f42ab48f6e..edcb92f282 100644 --- a/src/Drone_hand_gesture_project/main.py +++ b/src/Drone_hand_gesture_project/main.py @@ -1,8 +1,3 @@ -#!/usr/bin/env python3 -""" -手势控制无人机仿真系统 - 主程序(取消自动起飞版本) -集成:手势识别 + 无人机控制 + 3D仿真 -""" import cv2 import numpy as np import time @@ -15,7 +10,15 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__))) # 导入自定义模块 -from gesture_detector import GestureDetector +try: + from gesture_detector_enhanced import EnhancedGestureDetector + print("✅ 导入增强版手势检测器 (机器学习)") + HAS_ENHANCED_DETECTOR = True +except ImportError: + print("⚠️ 未找到增强版检测器,使用原始手势检测器") + from gesture_detector import GestureDetector + HAS_ENHANCED_DETECTOR = False + from drone_controller import DroneController from simulation_3d import Drone3DViewer @@ -42,7 +45,61 @@ def __init__(self, config=None): # 初始化模块 print("正在初始化手势检测器...") - self.gesture_detector = GestureDetector() + + # 检查可用的模型文件(按优先级排序) + model_candidates = [ + ("dataset/models/gesture_svm.pkl", "SVM模型"), + ("dataset/models/gesture_random_forest.pkl", "随机森林模型"), + ("dataset/models/gesture_mlp.pkl", "神经网络模型"), + ] + + selected_model = None + selected_model_name = None + + for model_path, model_name in model_candidates: + if os.path.exists(model_path): + file_size = os.path.getsize(model_path) + print(f"📁 找到 {model_name}: {file_size / 1024:.1f} KB") + + # 检查文件大小是否合理 + if file_size > 10 * 1024: + selected_model = model_path + selected_model_name = model_name + print(f"✅ 选择: {model_name}") + break + + if selected_model: + print(f"🎯 使用模型: {selected_model_name}") + + try: + from gesture_detector_enhanced import EnhancedGestureDetector + print("✅ 导入增强版手势检测器") + + # 使用实际的模型文件 + self.gesture_detector = EnhancedGestureDetector( + ml_model_path=selected_model, + use_ml=True + ) + + # 验证模型是否真正加载成功 + if hasattr(self.gesture_detector, 'ml_classifier') and self.gesture_detector.ml_classifier: + print(f"✅ 机器学习模型加载成功 ({selected_model_name})") + print(f" 可识别手势: {self.gesture_detector.ml_classifier.gesture_classes}") + else: + print("⚠️ 机器学习模型未加载,回退到规则检测") + self.gesture_detector = EnhancedGestureDetector(use_ml=False) + + except ImportError as e: + print(f"⚠️ 无法导入增强版检测器: {e}") + print("✅ 使用原始手势检测器") + from gesture_detector import GestureDetector + self.gesture_detector = GestureDetector() + + else: + print("⚠️ 未找到可用的机器学习模型文件") + print("✅ 使用原始手势检测器") + from gesture_detector import GestureDetector + self.gesture_detector = GestureDetector() print("正在初始化无人机控制器...") self.drone_controller = DroneController(simulation_mode=True) @@ -79,16 +136,23 @@ def __init__(self, config=None): self.command_cooldown = 1.5 # 命令冷却时间(秒),从2.0降低到1.5 # 手势识别阈值(降低以提高灵敏度) + # 如果是机器学习模式,阈值可以进一步降低 + if HAS_ENHANCED_DETECTOR and hasattr(self.gesture_detector, 'use_ml') and self.gesture_detector.use_ml: + print("✅ 使用机器学习模式,置信度阈值更低") + base_threshold = 0.55 # 机器学习可以更低 + else: + base_threshold = 0.6 # 规则检测需要高一点 + self.gesture_thresholds = { - 'open_palm': 0.6, # 降低到0.6 - 'closed_fist': 0.65, # 降低到0.65 - 'victory': 0.65, # 降低到0.65 - 'thumb_up': 0.65, # 降低到0.65 - 'thumb_down': 0.65, # 降低到0.65 - 'pointing_up': 0.6, # 降低到0.6 - 'pointing_down': 0.6, # 降低到0.6 - 'ok_sign': 0.7, # 稍微降低 - 'default': 0.6 # 默认阈值 + 'open_palm': base_threshold, + 'closed_fist': base_threshold + 0.05, + 'victory': base_threshold + 0.05, + 'thumb_up': base_threshold + 0.05, + 'thumb_down': base_threshold + 0.05, + 'pointing_up': base_threshold, + 'pointing_down': base_threshold, + 'ok_sign': base_threshold + 0.1, + 'default': base_threshold } # 初始化摄像头 @@ -98,11 +162,16 @@ def __init__(self, config=None): self.data_log = [] self.log_file = "flight_log.json" - # 不再自动起飞 - 完全由手势控制 print("无人机初始化完成,等待手势指令...") print("无人机仿真系统初始化完成 ✓") + if HAS_ENHANCED_DETECTOR and hasattr(self.gesture_detector, 'use_ml'): + if self.gesture_detector.use_ml: + print("📊 当前模式: 机器学习手势识别") + else: + print("📊 当前模式: 规则手势识别") + def _initialize_camera(self): """初始化摄像头""" # 尝试多个摄像头ID,优先使用1,如果失败则尝试0 @@ -141,6 +210,15 @@ def _gesture_recognition_loop(self): """手势识别循环""" print("手势识别线程启动...") + # 显示当前检测模式 + if HAS_ENHANCED_DETECTOR and hasattr(self.gesture_detector, 'use_ml'): + if self.gesture_detector.use_ml: + mode_text = "机器学习模式" + else: + mode_text = "规则检测模式" + else: + mode_text = "规则检测模式" + # 显示虚拟模式提示(如果摄像头未连接) if self.cap is None: print("⚠️ 使用虚拟摄像头模式,请连接摄像头进行真实手势识别") @@ -160,24 +238,34 @@ def _gesture_recognition_loop(self): frame = np.ones((480, 640, 3), dtype=np.uint8) * 255 cv2.putText(frame, "Camera Error - Virtual Mode", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) - cv2.putText(frame, "Connect camera for real gesture detection", (50, 100), + cv2.putText(frame, f"Connect camera for real gesture detection", (50, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 0), 1) + cv2.putText(frame, f"Mode: {mode_text}", (50, 140), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 100, 0), 1) else: # 虚拟模式 frame = np.ones((480, 640, 3), dtype=np.uint8) * 255 - cv2.putText(frame, "Virtual Camera Mode", (50, 50), - cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) - cv2.putText(frame, "Gesture Commands:", (50, 100), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 100, 0), 2) - cv2.putText(frame, "Open Palm - Takeoff", (50, 140), + cv2.putText(frame, "虚拟摄像头模式", (50, 50), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) + cv2.putText(frame, f"手势指令 ({mode_text}):", (50, 100), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 100, 0), 2) + cv2.putText(frame, "张开手掌 - 起飞", (50, 140), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + cv2.putText(frame, "握拳 - 降落", (50, 170), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + cv2.putText(frame, "胜利手势 - 前进", (50, 200), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + cv2.putText(frame, "大拇指 - 后退", (50, 230), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) - cv2.putText(frame, "Closed Fist - Land", (50, 170), + cv2.putText(frame, "食指上指 - 上升", (50, 260), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) - cv2.putText(frame, "Victory - Forward", (50, 200), + cv2.putText(frame, "食指向下 - 下降", (50, 290), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) - cv2.putText(frame, "Thumb Up - Backward", (50, 230), + cv2.putText(frame, "OK手势 - 悬停", (50, 320), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) - cv2.putText(frame, "Press 'q' to quit", (50, 280), + cv2.putText(frame, "大拇指向下 - 停止", (50, 350), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + cv2.putText(frame, "按 'q' 键退出", (50, 400), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) # 手势检测 @@ -195,14 +283,14 @@ def _gesture_recognition_loop(self): self._process_gesture_command(gesture, confidence) # 显示手势识别窗口 - cv2.imshow('手势控制 - Gesture Control', processed_frame) + cv2.imshow('Gesture Control', processed_frame) except Exception as e: print(f"手势检测错误: {e}") self.current_frame = frame self.current_gesture = None - # 检查退出 + # 检查退出 key = cv2.waitKey(1) & 0xFF if key == ord('q'): print("收到退出指令...") @@ -213,9 +301,48 @@ def _gesture_recognition_loop(self): self._switch_camera() elif key == ord('d'): # 调试模式 self._debug_gesture_detection() + elif key == ord('m'): # 切换模式(如果有多个模型) + self._switch_detection_mode() print("手势识别线程结束") + def _switch_detection_mode(self): + """切换检测模式(如果有多个可用模型)""" + if not HAS_ENHANCED_DETECTOR: + print("当前只有规则检测器可用") + return + + # 检查可用的模型 + model_files = [ + ("dataset/models/gesture_ensemble.pkl", "集成模型"), + ("dataset/models/gesture_svm.pkl", "SVM模型"), + ("dataset/models/gesture_random_forest.pkl", "随机森林模型"), + ("dataset/models/gesture_mlp.pkl", "神经网络模型"), + ] + + available_models = [] + for path, name in model_files: + if os.path.exists(path): + available_models.append((path, name)) + + if len(available_models) == 0: + print("未找到任何机器学习模型") + return + elif len(available_models) == 1: + print(f"只有 {available_models[0][1]} 可用") + return + + # 显示可用模型 + print("\n可用的手势识别模型:") + for i, (path, name) in enumerate(available_models, 1): + print(f" {i}. {name}") + + print("按数字键选择模型,或按其他键取消") + + # 这里简化处理,实际需要更复杂的交互 + # 暂时只记录一下 + print("注意: 需要重启程序切换模型") + def _switch_camera(self): """切换摄像头""" if self.cap: @@ -464,8 +591,20 @@ def _save_log(self): def run(self): """运行主程序""" print("=" * 60) - print(" 手势控制无人机仿真系统(完全手势控制版)") + print(" 手势控制无人机仿真系统(机器学习增强版)") print("=" * 60) + + # 显示当前检测模式 + if HAS_ENHANCED_DETECTOR and hasattr(self.gesture_detector, 'use_ml'): + if self.gesture_detector.use_ml: + mode_info = "机器学习模式 (更高精度)" + else: + mode_info = "规则检测模式 (基础)" + else: + mode_info = "规则检测模式" + + print(f"检测模式: {mode_info}") + print("系统功能:") print(" 1. 实时手势识别 (8种手势)") print(" 2. 无人机控制仿真") @@ -561,7 +700,7 @@ def load_config(): if __name__ == "__main__": - print("手势控制无人机仿真系统 - 启动(完全手势控制版)") + print("手势控制无人机仿真系统") print("=" * 60) # 检查必要的模块 From 29141dd8cb15ccb8913d788e6b4a8489c24594f9 Mon Sep 17 00:00:00 2001 From: Arrebol01 <1245298208@qq.com> Date: Fri, 19 Dec 2025 10:13:34 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E5=AE=8C=E6=95=B4README=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Drone_hand_gesture_project/README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Drone_hand_gesture_project/README.md b/src/Drone_hand_gesture_project/README.md index cb2911c8f7..6a7429a331 100644 --- a/src/Drone_hand_gesture_project/README.md +++ b/src/Drone_hand_gesture_project/README.md @@ -6,8 +6,12 @@ ├── main.py # 主程序 ├── drone_controller.py # 无人机控制 ├── simulation_3d.py # 3D仿真 + ├── physics_engine.py # 物理仿真引擎 ├── gesture_detector.py # 基础手势检测器 ├── gesture_data_collector.py # 手势图像数据收集 + ├── gesture_detector_enhanced.py # 增强手势检测器 + ├── gesture_classifier.py # 手势识别分类器 + ├── train_gesture_model.py # 训练识别模型 ├── dataset/ # 数据集目录 │ ├── raw/ # 原始数据 │ ├── processed/ # 处理后的数据 @@ -16,6 +20,7 @@ ### 运行步骤(在ubuntu或是xubuntu下) 1. 进入项目目录: + ```bash cd /mnt/hgfs/nn/src/Drone_hand_gesture_project @@ -24,7 +29,17 @@ ```bash source gesture_env/bin/activate -3. 运行主程序: +3. 收集手势图像: + + ```bash + python gesture_data_collector.py + +4. 训练学习模型: + + ```bash + python train_gesture_model.py --model_type ensemble + +5. 运行主程序: ```bash python main.py From 5a11a5fe6fa509a13046b4105a56d62f2f8b1a0b Mon Sep 17 00:00:00 2001 From: Arrebol01 <1245298208@qq.com> Date: Fri, 19 Dec 2025 14:38:39 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E5=AE=8C=E5=85=A8=E4=B8=AD=E6=96=87?= =?UTF-8?q?=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_detector_enhanced.py | 90 ++++++++++++++----- 1 file changed, 69 insertions(+), 21 deletions(-) diff --git a/src/Drone_hand_gesture_project/gesture_detector_enhanced.py b/src/Drone_hand_gesture_project/gesture_detector_enhanced.py index c6c734c53c..ff09d3871a 100644 --- a/src/Drone_hand_gesture_project/gesture_detector_enhanced.py +++ b/src/Drone_hand_gesture_project/gesture_detector_enhanced.py @@ -153,17 +153,18 @@ def extract_landmarks_for_ml(self, hand_landmarks): return landmarks def detect_gestures(self, image, simulation_mode=False): - """检测手势""" + """检测手势(支持中文显示)""" + # 转换为RGB image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) results = self.hands.process(image_rgb) - gesture = "none" # 默认none + gesture = "none" confidence = 0.0 landmarks_data = None if results.multi_hand_landmarks: for hand_landmarks in results.multi_hand_landmarks: - # 绘制手部关键点 + # 绘制手部关键点(使用OpenCV绘制) self.mp_drawing.draw_landmarks( image, hand_landmarks, @@ -172,10 +173,10 @@ def detect_gestures(self, image, simulation_mode=False): self.mp_drawing_styles.get_default_hand_connections_style() ) - # 提取关键点 + # 提取关键点用于机器学习 landmarks = self.extract_landmarks_for_ml(hand_landmarks) - # 使用机器学习或规则检测 + # 手势识别逻辑 if self.use_ml and self.ml_classifier and len(landmarks) == 63: # 机器学习预测 gesture, confidence = self.ml_classifier.predict(landmarks) @@ -197,26 +198,73 @@ def detect_gestures(self, image, simulation_mode=False): if matching_confs: confidence = np.mean(matching_confs) else: - # 如果未使用ML,使用简单的规则检测 + # 备用:规则检测 gesture, confidence = self._classify_by_rules(hand_landmarks) - # 显示信息 - ml_info = " (ML)" if self.use_ml else " (规则)" - cv2.putText(image, f"Gesture: {gesture}{ml_info}", (10, 30), - cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) - cv2.putText(image, f"Confidence: {confidence:.2f}", (10, 70), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) - - command = self.gesture_commands.get(gesture, "none") - cv2.putText(image, f"Command: {command}", (10, 110), - cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2) + # 使用PIL绘制中文文本 + try: + # 将OpenCV图像转换为PIL图像 + image_rgb_for_pil = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + image_pil = Image.fromarray(image_rgb_for_pil) + draw = ImageDraw.Draw(image_pil) + + # 设置字体 + if self.chinese_font: + font = self.chinese_font + else: + # 如果中文字体不可用,使用默认字体 + font = ImageFont.load_default() + print("⚠️ 使用默认字体,中文可能显示为方块") + + # 绘制手势信息 + chinese_gesture = self.chinese_gesture_names.get(gesture, gesture) + ml_info = " (训练模型)" if self.use_ml else " (规则)" + text = f"手势: {chinese_gesture}{ml_info}" + draw.text((10, 30), text, fill=(0, 255, 0), font=font) + + # 绘制置信度 + draw.text((10, 70), f"置信度: {confidence:.2f}", fill=(0, 255, 0), font=font) + + # 绘制指令 + command = self.gesture_commands.get(gesture, "none") + chinese_command = self.chinese_command_names.get(command, command) + draw.text((10, 110), f"指令: {chinese_command}", fill=(255, 0, 0), font=font) + + # 将PIL图像转换回OpenCV格式 + image = cv2.cvtColor(np.array(image_pil), cv2.COLOR_RGB2BGR) + + except Exception as e: + print(f"PIL绘制失败,回退到OpenCV英文显示: {e}") + # 回退到英文显示 + ml_info = " (ML)" if self.use_ml else " (Rule)" + cv2.putText(image, f"Gesture: {gesture}{ml_info}", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) + cv2.putText(image, f"Confidence: {confidence:.2f}", (10, 70), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + cv2.putText(image, f"Command: {command}", (10, 110), + cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2) else: - # 无手部检测 - cv2.putText(image, "未检测到手部", (10, 30), - cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) - gesture = "none" - confidence = 0.0 + # 没有检测到手部时也使用PIL绘制 + try: + image_rgb_for_pil = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + image_pil = Image.fromarray(image_rgb_for_pil) + draw = ImageDraw.Draw(image_pil) + + if self.chinese_font: + font = self.chinese_font + else: + font = ImageFont.load_default() + + draw.text((10, 30), "未检测到手部", fill=(0, 0, 255), font=font) + + # 转换回OpenCV格式 + image = cv2.cvtColor(np.array(image_pil), cv2.COLOR_RGB2BGR) + + except Exception as e: + print(f"PIL绘制失败: {e}") + cv2.putText(image, "No Hand Detected", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) return image, gesture, confidence, landmarks_data