diff --git a/src/Drone_hand_gesture_project/gesture_detector_enhanced.py b/src/Drone_hand_gesture_project/gesture_detector_enhanced.py new file mode 100644 index 0000000000..c6c734c53c --- /dev/null +++ b/src/Drone_hand_gesture_project/gesture_detector_enhanced.py @@ -0,0 +1,240 @@ +import cv2 +import os +import numpy as np +import mediapipe as mp +from gesture_classifier import GestureClassifier, EnsembleGestureClassifier +import PIL + +try: + from PIL import Image, ImageDraw, ImageFont + import numpy as np + HAS_PIL = True +except ImportError: + HAS_PIL = False + print("⚠️ PIL未安装,中文显示不可用") + +class EnhancedGestureDetector: + """手势检测器(集成机器学习)""" + + def __init__(self, ml_model_path=None, use_ml=True): + # 基础检测器 + self.mp_hands = mp.solutions.hands + self.mp_drawing = mp.solutions.drawing_utils + self.mp_drawing_styles = mp.solutions.drawing_styles + + self.hands = self.mp_hands.Hands( + static_image_mode=False, + max_num_hands=1, + min_detection_confidence=0.5, + min_tracking_confidence=0.5 + ) + + # 机器学习模型 + self.use_ml = use_ml + self.ml_classifier = None + + if use_ml and ml_model_path: + # 检查是否是 ensemble 信息文件(很小的文件) + if os.path.exists(ml_model_path): + file_size = os.path.getsize(ml_model_path) + if file_size < 1024: # 小于1KB,可能是信息文件 + print(f"⚠️ 检测到小文件 ({file_size}字节),可能是集成模型信息文件") + print(f" 尝试加载单个模型文件...") + + # 尝试加载对应的单个模型 + base_name = os.path.basename(ml_model_path) + if "ensemble" in base_name: + # 尝试加载 svm 模型 + svm_path = ml_model_path.replace("ensemble", "svm") + if os.path.exists(svm_path): + print(f" 尝试加载: {svm_path}") + success = self.load_ml_model(svm_path) + else: + print(f" 未找到替代模型,使用规则检测") + self.use_ml = False + else: + # 正常加载 + success = self.load_ml_model(ml_model_path) + else: + self.use_ml = False + + # 手势到控制指令的映射(8个手势) + self.gesture_commands = { + "open_palm": "takeoff", + "closed_fist": "land", + "pointing_up": "up", + "pointing_down": "down", + "victory": "forward", + "thumb_up": "backward", + "thumb_down": "stop", + "ok_sign": "hover", + } + + # 中文手势名称映射 + self.chinese_gesture_names = { + "open_palm": "张开手掌", + "closed_fist": "握拳", + "victory": "胜利手势", + "thumb_up": "大拇指", + "thumb_down": "大拇指向下", + "pointing_up": "食指上指", + "pointing_down": "食指向下", + "ok_sign": "OK手势", + "none": "未检测到手" + } + + # 中文指令名称映射 + self.chinese_command_names = { + "takeoff": "起飞", + "land": "降落", + "up": "上升", + "down": "下降", + "forward": "前进", + "backward": "后退", + "stop": "停止", + "hover": "悬停", + "none": "无指令" + } + + # 中文字体 + self.chinese_font = None + if HAS_PIL: + try: + # Windows字体路径 + font_paths = [ + "C:/Windows/Fonts/msyh.ttc", # 微软雅黑 + "C:/Windows/Fonts/simhei.ttf", # 黑体 + "C:/Windows/Fonts/simsun.ttc", # 宋体 + ] + + for font_path in font_paths: + try: + if os.path.exists(font_path): + self.chinese_font = ImageFont.truetype(font_path, 30) + print(f"✅ 加载中文字体: {os.path.basename(font_path)}") + break + except: + continue + + if self.chinese_font is None: + print("⚠️ 无法加载系统字体,将使用英文显示") + except Exception as e: + print(f"⚠️ 字体初始化失败: {e}") + + + # 历史记录(用于平滑预测) + self.prediction_history = [] + self.max_history = 5 + + def load_ml_model(self, model_path): + """加载机器学习模型""" + try: + self.ml_classifier = GestureClassifier(model_path=model_path) + print(f"机器学习模型已加载: {model_path}") + return True + except Exception as e: + print(f"加载机器学习模型失败: {e}") + print("将使用规则检测器") + self.use_ml = False + return False + + def extract_landmarks_for_ml(self, hand_landmarks): + """提取关键点用于机器学习""" + landmarks = [] + for landmark in hand_landmarks.landmark: + landmarks.extend([landmark.x, landmark.y, landmark.z]) + + # 确保是63维 + if len(landmarks) < 63: + landmarks.extend([0.0] * (63 - len(landmarks))) + elif len(landmarks) > 63: + landmarks = landmarks[:63] + + return landmarks + + def detect_gestures(self, image, simulation_mode=False): + """检测手势""" + image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + results = self.hands.process(image_rgb) + + gesture = "none" # 默认none + confidence = 0.0 + landmarks_data = None + + if results.multi_hand_landmarks: + for hand_landmarks in results.multi_hand_landmarks: + # 绘制手部关键点 + self.mp_drawing.draw_landmarks( + image, + hand_landmarks, + self.mp_hands.HAND_CONNECTIONS, + self.mp_drawing_styles.get_default_hand_landmarks_style(), + self.mp_drawing_styles.get_default_hand_connections_style() + ) + + # 提取关键点 + landmarks = self.extract_landmarks_for_ml(hand_landmarks) + + # 使用机器学习或规则检测 + if self.use_ml and self.ml_classifier and len(landmarks) == 63: + # 机器学习预测 + gesture, confidence = self.ml_classifier.predict(landmarks) + + # 应用平滑滤波 + self.prediction_history.append((gesture, confidence)) + if len(self.prediction_history) > self.max_history: + self.prediction_history.pop(0) + + # 使用历史中的多数投票 + if len(self.prediction_history) >= 3: + from collections import Counter + recent_gestures = [g for g, _ in self.prediction_history[-3:]] + most_common = Counter(recent_gestures).most_common(1)[0] + if most_common[1] >= 2: # 至少2/3一致 + gesture = most_common[0] + # 计算平均置信度 + matching_confs = [c for g, c in self.prediction_history if g == gesture] + if matching_confs: + confidence = np.mean(matching_confs) + else: + # 如果未使用ML,使用简单的规则检测 + gesture, confidence = self._classify_by_rules(hand_landmarks) + + # 显示信息 + ml_info = " (ML)" if self.use_ml else " (规则)" + cv2.putText(image, f"Gesture: {gesture}{ml_info}", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) + cv2.putText(image, f"Confidence: {confidence:.2f}", (10, 70), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + + command = self.gesture_commands.get(gesture, "none") + cv2.putText(image, f"Command: {command}", (10, 110), + cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2) + + else: + # 无手部检测 + cv2.putText(image, "未检测到手部", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) + gesture = "none" + confidence = 0.0 + + return image, gesture, confidence, landmarks_data + + def _classify_by_rules(self, hand_landmarks): + """简单的规则分类器(备用)""" + # 获取关键点 + landmarks = hand_landmarks.landmark + + # 这里实现简单的规则检测逻辑 + # 例如:计算手指是否伸直等 + + # 暂时返回一个默认值 + return "open_palm", 0.5 + + def get_command(self, gesture): + """获取控制指令""" + return self.gesture_commands.get(gesture, "none") + + def release(self): + """释放资源""" + self.hands.close() \ No newline at end of file diff --git a/src/Drone_hand_gesture_project/main.py b/src/Drone_hand_gesture_project/main.py index f42ab48f6e..edcb92f282 100644 --- a/src/Drone_hand_gesture_project/main.py +++ b/src/Drone_hand_gesture_project/main.py @@ -1,8 +1,3 @@ -#!/usr/bin/env python3 -""" -手势控制无人机仿真系统 - 主程序(取消自动起飞版本) -集成:手势识别 + 无人机控制 + 3D仿真 -""" import cv2 import numpy as np import time @@ -15,7 +10,15 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__))) # 导入自定义模块 -from gesture_detector import GestureDetector +try: + from gesture_detector_enhanced import EnhancedGestureDetector + print("✅ 导入增强版手势检测器 (机器学习)") + HAS_ENHANCED_DETECTOR = True +except ImportError: + print("⚠️ 未找到增强版检测器,使用原始手势检测器") + from gesture_detector import GestureDetector + HAS_ENHANCED_DETECTOR = False + from drone_controller import DroneController from simulation_3d import Drone3DViewer @@ -42,7 +45,61 @@ def __init__(self, config=None): # 初始化模块 print("正在初始化手势检测器...") - self.gesture_detector = GestureDetector() + + # 检查可用的模型文件(按优先级排序) + model_candidates = [ + ("dataset/models/gesture_svm.pkl", "SVM模型"), + ("dataset/models/gesture_random_forest.pkl", "随机森林模型"), + ("dataset/models/gesture_mlp.pkl", "神经网络模型"), + ] + + selected_model = None + selected_model_name = None + + for model_path, model_name in model_candidates: + if os.path.exists(model_path): + file_size = os.path.getsize(model_path) + print(f"📁 找到 {model_name}: {file_size / 1024:.1f} KB") + + # 检查文件大小是否合理 + if file_size > 10 * 1024: + selected_model = model_path + selected_model_name = model_name + print(f"✅ 选择: {model_name}") + break + + if selected_model: + print(f"🎯 使用模型: {selected_model_name}") + + try: + from gesture_detector_enhanced import EnhancedGestureDetector + print("✅ 导入增强版手势检测器") + + # 使用实际的模型文件 + self.gesture_detector = EnhancedGestureDetector( + ml_model_path=selected_model, + use_ml=True + ) + + # 验证模型是否真正加载成功 + if hasattr(self.gesture_detector, 'ml_classifier') and self.gesture_detector.ml_classifier: + print(f"✅ 机器学习模型加载成功 ({selected_model_name})") + print(f" 可识别手势: {self.gesture_detector.ml_classifier.gesture_classes}") + else: + print("⚠️ 机器学习模型未加载,回退到规则检测") + self.gesture_detector = EnhancedGestureDetector(use_ml=False) + + except ImportError as e: + print(f"⚠️ 无法导入增强版检测器: {e}") + print("✅ 使用原始手势检测器") + from gesture_detector import GestureDetector + self.gesture_detector = GestureDetector() + + else: + print("⚠️ 未找到可用的机器学习模型文件") + print("✅ 使用原始手势检测器") + from gesture_detector import GestureDetector + self.gesture_detector = GestureDetector() print("正在初始化无人机控制器...") self.drone_controller = DroneController(simulation_mode=True) @@ -79,16 +136,23 @@ def __init__(self, config=None): self.command_cooldown = 1.5 # 命令冷却时间(秒),从2.0降低到1.5 # 手势识别阈值(降低以提高灵敏度) + # 如果是机器学习模式,阈值可以进一步降低 + if HAS_ENHANCED_DETECTOR and hasattr(self.gesture_detector, 'use_ml') and self.gesture_detector.use_ml: + print("✅ 使用机器学习模式,置信度阈值更低") + base_threshold = 0.55 # 机器学习可以更低 + else: + base_threshold = 0.6 # 规则检测需要高一点 + self.gesture_thresholds = { - 'open_palm': 0.6, # 降低到0.6 - 'closed_fist': 0.65, # 降低到0.65 - 'victory': 0.65, # 降低到0.65 - 'thumb_up': 0.65, # 降低到0.65 - 'thumb_down': 0.65, # 降低到0.65 - 'pointing_up': 0.6, # 降低到0.6 - 'pointing_down': 0.6, # 降低到0.6 - 'ok_sign': 0.7, # 稍微降低 - 'default': 0.6 # 默认阈值 + 'open_palm': base_threshold, + 'closed_fist': base_threshold + 0.05, + 'victory': base_threshold + 0.05, + 'thumb_up': base_threshold + 0.05, + 'thumb_down': base_threshold + 0.05, + 'pointing_up': base_threshold, + 'pointing_down': base_threshold, + 'ok_sign': base_threshold + 0.1, + 'default': base_threshold } # 初始化摄像头 @@ -98,11 +162,16 @@ def __init__(self, config=None): self.data_log = [] self.log_file = "flight_log.json" - # 不再自动起飞 - 完全由手势控制 print("无人机初始化完成,等待手势指令...") print("无人机仿真系统初始化完成 ✓") + if HAS_ENHANCED_DETECTOR and hasattr(self.gesture_detector, 'use_ml'): + if self.gesture_detector.use_ml: + print("📊 当前模式: 机器学习手势识别") + else: + print("📊 当前模式: 规则手势识别") + def _initialize_camera(self): """初始化摄像头""" # 尝试多个摄像头ID,优先使用1,如果失败则尝试0 @@ -141,6 +210,15 @@ def _gesture_recognition_loop(self): """手势识别循环""" print("手势识别线程启动...") + # 显示当前检测模式 + if HAS_ENHANCED_DETECTOR and hasattr(self.gesture_detector, 'use_ml'): + if self.gesture_detector.use_ml: + mode_text = "机器学习模式" + else: + mode_text = "规则检测模式" + else: + mode_text = "规则检测模式" + # 显示虚拟模式提示(如果摄像头未连接) if self.cap is None: print("⚠️ 使用虚拟摄像头模式,请连接摄像头进行真实手势识别") @@ -160,24 +238,34 @@ def _gesture_recognition_loop(self): frame = np.ones((480, 640, 3), dtype=np.uint8) * 255 cv2.putText(frame, "Camera Error - Virtual Mode", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) - cv2.putText(frame, "Connect camera for real gesture detection", (50, 100), + cv2.putText(frame, f"Connect camera for real gesture detection", (50, 100), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 0), 1) + cv2.putText(frame, f"Mode: {mode_text}", (50, 140), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 100, 0), 1) else: # 虚拟模式 frame = np.ones((480, 640, 3), dtype=np.uint8) * 255 - cv2.putText(frame, "Virtual Camera Mode", (50, 50), - cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) - cv2.putText(frame, "Gesture Commands:", (50, 100), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 100, 0), 2) - cv2.putText(frame, "Open Palm - Takeoff", (50, 140), + cv2.putText(frame, "虚拟摄像头模式", (50, 50), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) + cv2.putText(frame, f"手势指令 ({mode_text}):", (50, 100), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 100, 0), 2) + cv2.putText(frame, "张开手掌 - 起飞", (50, 140), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + cv2.putText(frame, "握拳 - 降落", (50, 170), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + cv2.putText(frame, "胜利手势 - 前进", (50, 200), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + cv2.putText(frame, "大拇指 - 后退", (50, 230), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) - cv2.putText(frame, "Closed Fist - Land", (50, 170), + cv2.putText(frame, "食指上指 - 上升", (50, 260), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) - cv2.putText(frame, "Victory - Forward", (50, 200), + cv2.putText(frame, "食指向下 - 下降", (50, 290), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) - cv2.putText(frame, "Thumb Up - Backward", (50, 230), + cv2.putText(frame, "OK手势 - 悬停", (50, 320), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) - cv2.putText(frame, "Press 'q' to quit", (50, 280), + cv2.putText(frame, "大拇指向下 - 停止", (50, 350), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) + cv2.putText(frame, "按 'q' 键退出", (50, 400), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) # 手势检测 @@ -195,14 +283,14 @@ def _gesture_recognition_loop(self): self._process_gesture_command(gesture, confidence) # 显示手势识别窗口 - cv2.imshow('手势控制 - Gesture Control', processed_frame) + cv2.imshow('Gesture Control', processed_frame) except Exception as e: print(f"手势检测错误: {e}") self.current_frame = frame self.current_gesture = None - # 检查退出 + # 检查退出 key = cv2.waitKey(1) & 0xFF if key == ord('q'): print("收到退出指令...") @@ -213,9 +301,48 @@ def _gesture_recognition_loop(self): self._switch_camera() elif key == ord('d'): # 调试模式 self._debug_gesture_detection() + elif key == ord('m'): # 切换模式(如果有多个模型) + self._switch_detection_mode() print("手势识别线程结束") + def _switch_detection_mode(self): + """切换检测模式(如果有多个可用模型)""" + if not HAS_ENHANCED_DETECTOR: + print("当前只有规则检测器可用") + return + + # 检查可用的模型 + model_files = [ + ("dataset/models/gesture_ensemble.pkl", "集成模型"), + ("dataset/models/gesture_svm.pkl", "SVM模型"), + ("dataset/models/gesture_random_forest.pkl", "随机森林模型"), + ("dataset/models/gesture_mlp.pkl", "神经网络模型"), + ] + + available_models = [] + for path, name in model_files: + if os.path.exists(path): + available_models.append((path, name)) + + if len(available_models) == 0: + print("未找到任何机器学习模型") + return + elif len(available_models) == 1: + print(f"只有 {available_models[0][1]} 可用") + return + + # 显示可用模型 + print("\n可用的手势识别模型:") + for i, (path, name) in enumerate(available_models, 1): + print(f" {i}. {name}") + + print("按数字键选择模型,或按其他键取消") + + # 这里简化处理,实际需要更复杂的交互 + # 暂时只记录一下 + print("注意: 需要重启程序切换模型") + def _switch_camera(self): """切换摄像头""" if self.cap: @@ -464,8 +591,20 @@ def _save_log(self): def run(self): """运行主程序""" print("=" * 60) - print(" 手势控制无人机仿真系统(完全手势控制版)") + print(" 手势控制无人机仿真系统(机器学习增强版)") print("=" * 60) + + # 显示当前检测模式 + if HAS_ENHANCED_DETECTOR and hasattr(self.gesture_detector, 'use_ml'): + if self.gesture_detector.use_ml: + mode_info = "机器学习模式 (更高精度)" + else: + mode_info = "规则检测模式 (基础)" + else: + mode_info = "规则检测模式" + + print(f"检测模式: {mode_info}") + print("系统功能:") print(" 1. 实时手势识别 (8种手势)") print(" 2. 无人机控制仿真") @@ -561,7 +700,7 @@ def load_config(): if __name__ == "__main__": - print("手势控制无人机仿真系统 - 启动(完全手势控制版)") + print("手势控制无人机仿真系统") print("=" * 60) # 检查必要的模块