diff --git a/src/autonomus_drone_hand_gesture_project/advanced_gesture_analyzer.py b/src/autonomus_drone_hand_gesture_project/advanced_gesture_analyzer.py new file mode 100644 index 0000000000..bf90090708 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/advanced_gesture_analyzer.py @@ -0,0 +1,317 @@ +""" +高级手势分析器模块 +负责分析复杂手势和提供额外的手势识别功能 +作者: xiaoshiyuan888 +""" + +import cv2 +import numpy as np +import time +import math +from collections import deque + + +class AdvancedGestureAnalyzer: + """高级手势分析器 - 识别复杂手势""" + + def __init__(self, speech_manager=None, config=None): + self.speech_manager = speech_manager + self.config = config + + # 手势轨迹追踪 + self.trajectory_history = deque(maxlen=30) + self.fingertip_history = deque(maxlen=20) + + # 手势状态 + self.rotation_direction = None # 旋转方向: 'cw' 顺时针, 'ccw' 逆时针 + self.grab_state = False # 抓取状态 + self.photo_state = False # 拍照状态 + + # 颜色定义 + self.colors = { + 'rotation_cw': (0, 255, 255), # 黄色 - 顺时针旋转 + 'rotation_ccw': (255, 255, 0), # 青色 - 逆时针旋转 + 'grab': (255, 0, 255), # 紫色 - 抓取 + 'release': (0, 255, 0), # 绿色 - 释放 + 'photo': (255, 165, 0), # 橙色 - 拍照 + 'auto_flight': (128, 0, 128), # 紫色 - 自动飞行 + 'return_home': (0, 128, 128), # 深青色 - 返航 + } + + # 语音消息映射 + self.messages = { + 'gesture_rotate_cw': "顺时针旋转", + 'gesture_rotate_ccw': "逆时针旋转", + 'gesture_grab': "抓取", + 'gesture_release': "释放", + 'gesture_photo': "拍照", + 'gesture_auto_flight': "自动飞行模式", + 'gesture_return_home': "返航", + } + + # 初始化语音消息 + if self.speech_manager: + self.speech_manager.messages.update(self.messages) + + print("✓ 高级手势分析器已初始化") + + def analyze(self, frame, hand_data): + """分析手势数据,识别复杂手势""" + if hand_data is None: + return None + + result = {} + + # 分析旋转手势 + rotation_result = self.detect_rotation(hand_data) + if rotation_result: + result.update(rotation_result) + + # 分析抓取手势 + grab_result = self.detect_grab(hand_data) + if grab_result: + result.update(grab_result) + + # 分析拍照手势 + photo_result = self.detect_photo_gesture(hand_data) + if photo_result: + result.update(photo_result) + + # 分析复杂手势 + complex_result = self.detect_complex_gestures(hand_data) + if complex_result: + result.update(complex_result) + + return result if result else None + + def detect_rotation(self, hand_data): + """检测旋转手势""" + if 'fingertips' not in hand_data or len(hand_data['fingertips']) < 1: + return None + + # 获取食指指尖 + index_fingertip = hand_data['fingertips'][0] if len(hand_data['fingertips']) > 0 else None + + if index_fingertip is None: + return None + + # 添加指尖到历史 + self.fingertip_history.append(index_fingertip) + + if len(self.fingertip_history) < 15: + return None + + # 分析指尖运动轨迹 + trajectory = list(self.fingertip_history) + + # 计算轨迹的中心点 + points = np.array(trajectory) + center = np.mean(points, axis=0) + + # 计算每个点相对于中心的角度 + angles = [] + for point in trajectory: + dx = point[0] - center[0] + dy = point[1] - center[1] + angle = math.atan2(dy, dx) + angles.append(angle) + + # 分析角度变化趋势 + if len(angles) >= 10: + # 计算角度变化量 + angle_changes = [] + for i in range(1, len(angles)): + change = angles[i] - angles[i-1] + # 处理角度跨越2π边界的情况 + if change > math.pi: + change -= 2 * math.pi + elif change < -math.pi: + change += 2 * math.pi + angle_changes.append(change) + + avg_change = np.mean(angle_changes) + + # 判断旋转方向 + if abs(avg_change) > 0.05: # 阈值 + if avg_change > 0: + self.rotation_direction = 'ccw' + return { + 'gesture': 'RotateCCW', + 'confidence': min(0.9, abs(avg_change) * 5), + 'direction': 'ccw', + 'angle_change': avg_change + } + else: + self.rotation_direction = 'cw' + return { + 'gesture': 'RotateCW', + 'confidence': min(0.9, abs(avg_change) * 5), + 'direction': 'cw', + 'angle_change': avg_change + } + + return None + + def detect_grab(self, hand_data): + """检测抓取手势(握拳)""" + if 'contour' not in hand_data or 'bbox' not in hand_data: + return None + + contour = hand_data['contour'] + bbox = hand_data['bbox'] + + # 计算轮廓面积和边界框面积 + contour_area = cv2.contourArea(contour) + x1, y1, x2, y2 = bbox + bbox_area = (x2 - x1) * (y2 - y1) + + if bbox_area == 0: + return None + + # 计算面积比(握拳时轮廓更接近边界框) + area_ratio = contour_area / bbox_area + + # 获取手指数量 + finger_count = len(hand_data.get('fingers', [])) + + # 握拳的特征:面积比高,手指数量少 + if area_ratio > 0.7 and finger_count <= 1: + # 检查是否之前不是抓取状态 + if not self.grab_state: + self.grab_state = True + return { + 'gesture': 'Grab', + 'confidence': min(0.95, area_ratio * 1.2), + 'state': 'grab' + } + else: + if self.grab_state: + self.grab_state = False + return { + 'gesture': 'Release', + 'confidence': 0.8, + 'state': 'release' + } + + return None + + def detect_photo_gesture(self, hand_data): + """检测拍照手势(OK手势)""" + if 'fingers' not in hand_data or 'fingertips' not in hand_data: + return None + + fingers = hand_data['fingers'] + fingertips = hand_data['fingertips'] + + # OK手势的特征:拇指和食指形成圆圈,其他手指伸直 + if len(fingers) == 3 and len(fingertips) >= 2: + # 检查拇指和食指的距离 + if len(fingertips) >= 2: + thumb_tip = fingertips[0] + index_tip = fingertips[1] + + distance = math.sqrt((thumb_tip[0] - index_tip[0])**2 + (thumb_tip[1] - index_tip[1])**2) + + # 如果拇指和食指距离较近,可能是OK手势 + if distance < 50: # 阈值 + if not self.photo_state: + self.photo_state = True + return { + 'gesture': 'TakePhoto', + 'confidence': max(0.7, 1.0 - distance/100), + 'distance': distance + } + + self.photo_state = False + return None + + def detect_complex_gestures(self, hand_data): + """检测复杂手势(返航、自动飞行等)""" + if 'fingers' not in hand_data or 'fingertips' not in hand_data: + return None + + finger_count = len(hand_data.get('fingers', [])) + fingertips = hand_data.get('fingertips', []) + + # 返航手势:五指张开然后缓慢握拳 + if finger_count == 5 and len(fingertips) == 5: + # 检查是否保持这个姿势一段时间 + if hasattr(self, 'five_finger_start_time'): + elapsed = time.time() - self.five_finger_start_time + if elapsed > 2.0: # 保持2秒 + delattr(self, 'five_finger_start_time') + return { + 'gesture': 'ReturnHome', + 'confidence': 0.85, + 'hold_time': elapsed + } + else: + self.five_finger_start_time = time.time() + elif hasattr(self, 'five_finger_start_time'): + delattr(self, 'five_finger_start_time') + + # 自动飞行手势:特定手指组合 + if finger_count == 4 and len(fingertips) == 4: + # 检查是否为特定的手指组合(例如,缺少小指) + return { + 'gesture': 'AutoFlight', + 'confidence': 0.75, + 'finger_pattern': 'four_fingers' + } + + return None + + def draw_gesture_info(self, frame): + """在帧上绘制手势信息""" + h, w = frame.shape[:2] + + # 绘制旋转指示器 + if self.rotation_direction: + center_x, center_y = w - 80, 100 + radius = 30 + + if self.rotation_direction == 'cw': + color = self.colors['rotation_cw'] + # 绘制顺时针箭头 + cv2.circle(frame, (center_x, center_y), radius, color, 2) + cv2.putText(frame, "CW", (center_x - 15, center_y + 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + else: + color = self.colors['rotation_ccw'] + # 绘制逆时针箭头 + cv2.circle(frame, (center_x, center_y), radius, color, 2) + cv2.putText(frame, "CCW", (center_x - 20, center_y + 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + + # 绘制抓取状态 + if self.grab_state: + grab_x, grab_y = w - 80, 150 + color = self.colors['grab'] + cv2.rectangle(frame, (grab_x - 20, grab_y - 20), + (grab_x + 20, grab_y + 20), color, 2) + cv2.putText(frame, "抓取", (grab_x - 20, grab_y - 25), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + + # 绘制指尖轨迹 + if len(self.fingertip_history) > 1: + trajectory_color = (0, 255, 255) # 黄色 + for i in range(1, len(self.fingertip_history)): + pt1 = self.fingertip_history[i-1] + pt2 = self.fingertip_history[i] + cv2.line(frame, pt1, pt2, trajectory_color, 2) + + # 绘制当前指尖 + if self.fingertip_history: + last_point = self.fingertip_history[-1] + cv2.circle(frame, last_point, 5, (0, 0, 255), -1) + + return frame + + def get_gesture_info(self): + """获取手势信息""" + return { + 'rotation_direction': self.rotation_direction, + 'grab_state': self.grab_state, + 'photo_state': self.photo_state, + 'trajectory_length': len(self.fingertip_history) + } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/config_manager.py b/src/autonomus_drone_hand_gesture_project/config_manager.py index bfd2c46f73..7c6baff3bb 100644 --- a/src/autonomus_drone_hand_gesture_project/config_manager.py +++ b/src/autonomus_drone_hand_gesture_project/config_manager.py @@ -111,6 +111,7 @@ def __init__(self): 'show_performance_mode': True, 'show_performance_stats': True, 'show_system_resources': True, + 'show_advanced_gestures': True, # 新增:显示高级手势信息 }, 'performance': { 'target_fps': 30, diff --git a/src/autonomus_drone_hand_gesture_project/drone_controller.py b/src/autonomus_drone_hand_gesture_project/drone_controller.py index 17e8dfea36..2f1edeadb0 100644 --- a/src/autonomus_drone_hand_gesture_project/drone_controller.py +++ b/src/autonomus_drone_hand_gesture_project/drone_controller.py @@ -248,6 +248,56 @@ def move_by_gesture(self, gesture, confidence): if (self.speech_manager and self.speech_manager.enabled): self.speech_manager.speak('hovering') + elif gesture == "Grab": + # 抓取动作(模拟) + print("执行抓取动作") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_grab') + elif gesture == "Release": + # 释放动作(模拟) + print("执行释放动作") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_release') + elif gesture == "RotateCW": + # 顺时针旋转 + print("顺时针旋转") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_rotate_cw') + elif gesture == "RotateCCW": + # 逆时针旋转 + print("逆时针旋转") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_rotate_ccw') + elif gesture == "TakePhoto": + # 拍照/截图 + print("执行拍照") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_photo') + elif gesture == "ReturnHome": + # 返航 + print("执行返航") + self.client.moveToPositionAsync(0, 0, self.altitude, 5) + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_return_home') + elif gesture == "AutoFlight": + # 自动飞行模式 + print("启动自动飞行模式") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_auto_flight') if success: self.last_control_time = current_time diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index fbb45391da..0a14cccb19 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -18,11 +18,11 @@ }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 170, + "skin_upper_h": 180, "skin_lower_s": 0, - "skin_upper_s": 70, - "skin_lower_v": 0, - "skin_upper_v": 75, + "skin_upper_s": 127, + "skin_lower_v": 22, + "skin_upper_v": 178, "min_hand_area": 3000, "max_hand_area": 30000, "hand_ratio_threshold": 1.5, @@ -79,7 +79,7 @@ "show_fingertips": true, "show_palm_center": true, "show_hand_direction": true, - "show_debug_info": false, + "show_debug_info": true, "show_speech_status": true, "show_gesture_history": true, "show_stability_indicator": true, @@ -88,6 +88,9 @@ "show_performance_mode": true, "show_performance_stats": true, "show_system_resources": true, + "show_advanced_gestures": true, + "show_hotkey_hints": true, + "show_operation_hints": true, "font_size_large": 20, "font_size_medium": 16, "font_size_small": 14, @@ -168,6 +171,12 @@ "loop_playback": false, "playback_speed": 1.0 }, + "hotkeys": { + "quick_save_enabled": true, + "quick_load_enabled": true, + "hint_duration": 2.0, + "hint_fade_speed": 0.05 + }, "drone_control": { "altitude": -10.0, "base_speed": 2.0, diff --git a/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py b/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py index ff37f66b83..7f7c93840c 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py +++ b/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py @@ -72,7 +72,14 @@ def __init__(self, speech_manager=None, config=None): "Right": (0, 165, 255), "Waiting": (200, 200, 200), "Error": (255, 0, 0), - "Hover": (255, 255, 255) + "Hover": (255, 255, 255), + "Grab": (255, 0, 255), # 新增:紫色 + "Release": (0, 255, 0), # 新增:绿色 + "RotateCW": (0, 255, 255), # 新增:黄色 + "RotateCCW": (255, 255, 0), # 新增:青色 + "TakePhoto": (255, 165, 0), # 新增:橙色 + "ReturnHome": (0, 128, 128), # 新增:深青色 + "AutoFlight": (128, 0, 128), # 新增:紫色 } # 手势到语音的映射 @@ -86,6 +93,13 @@ def __init__(self, speech_manager=None, config=None): "Waiting": "gesture_waiting", "Error": "gesture_error", "Hover": "gesture_hover", + "Grab": "gesture_grab", # 新增 + "Release": "gesture_release", # 新增 + "RotateCW": "gesture_rotate_cw", # 新增 + "RotateCCW": "gesture_rotate_ccw", # 新增 + "TakePhoto": "gesture_photo", # 新增 + "ReturnHome": "gesture_return_home", # 新增 + "AutoFlight": "gesture_auto_flight", # 新增 } # 手势状态颜色 diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 0a63edfb6c..a04228c60f 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -24,6 +24,7 @@ from drone_controller import SimpleDroneController from ui_renderer import ChineseUIRenderer from trajectory_recorder import GestureTrajectoryRecorder +from advanced_gesture_analyzer import AdvancedGestureAnalyzer # 安全导入所需库 @@ -156,11 +157,11 @@ def init_camera(config): def print_welcome_message(cap, speech_manager, libs): """打印欢迎信息""" print("\n" + "=" * 80) - print("手势控制无人机系统 - 详细性能统计版") + print("手势控制无人机系统 - 增强手势识别版") print("=" * 80) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") - print(f" 手势识别: 增强的平滑处理算法") + print(f" 手势识别: 增强的平滑处理算法 + 新手势支持") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") print(f" 性能监控: 已启用") print(f" 轨迹记录: 支持录制/回放功能") @@ -181,15 +182,15 @@ def print_instructions(): print(" - 系统自动监控: FPS, CPU, 内存, 识别时间等") print("4. 性能模式选择:") print(" - 按 [O] 键循环切换性能模式: 最快(fast) → 平衡(balanced) → 最准(accurate)") - print("5. 手势控制:") - print(" - 手势识别后会有语音提示: 向上、向下、向左、向右、向前、停止") - print(" - 手势稳定性越高,识别越准确") - print(" - 手部距离摄像头适中时效果最佳") - print(" * 手势识别置信度 > 60% 时才会执行") + print("5. 新手势控制:") + print(" - 握拳手势: 抓取/释放物体 (模拟)") + print(" - 旋转手势: 顺时针旋转 (模拟)") + print(" - OK手势: 拍照/截图") + print(" - 复杂手势: 返航、自动飞行等") print("6. 轨迹记录功能:") print(" [1]开始录制 [2]停止录制 [3]保存轨迹 [4]回放轨迹 [5]清除轨迹 [6]暂停/继续") print("7. 键盘控制:") - print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止 [H]悬停 [G]返航") print("8. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") print("9. 语音控制:") @@ -203,8 +204,7 @@ def print_instructions(): def main(): """主函数""" print("=" * 60) - print("手势控制无人机系统 - 详细性能统计优化版") - print("详细性能统计优化版!") + print("手势控制无人机系统 - 增强手势识别版") print("=" * 60) # 导入库 @@ -244,6 +244,10 @@ def main(): print("初始化手势轨迹记录器...") trajectory_recorder = GestureTrajectoryRecorder(speech_manager, config) + # 高级手势分析器 + print("初始化高级手势分析器...") + advanced_gesture_analyzer = AdvancedGestureAnalyzer(speech_manager, config) + # 初始化摄像头 cap = init_camera(config) if cap and speech_manager.enabled: @@ -266,6 +270,11 @@ def main(): ord('f'): "Forward", ord('F'): "Forward", ord('x'): "Stop", ord('X'): "Stop", ord('h'): "Hover", ord('H'): "Hover", + ord('g'): "ReturnHome", ord('G'): "ReturnHome", + ord('b'): "AutoFlight", ord('B'): "AutoFlight", + ord('p'): "TakePhoto", ord('P'): "TakePhoto", + ord('r'): "RotateCW", ord('R'): "RotateCW", + ord('l'): "RotateCCW", ord('L'): "RotateCCW", } # 显示模式 @@ -293,6 +302,12 @@ def main(): # 手势识别 gesture, confidence, frame = gesture_recognizer.recognize(frame) + # 使用高级手势分析器进行补充分析 + advanced_result = advanced_gesture_analyzer.analyze(frame, gesture_recognizer.last_hand_data) + if advanced_result and advanced_result.get('confidence', 0) > confidence: + gesture = advanced_result.get('gesture', gesture) + confidence = advanced_result.get('confidence', confidence) + # 记录手势统计 if gesture and gesture != "Waiting" and gesture != "摄像头错误": performance_analyzer.record_gesture(gesture, confidence) @@ -319,6 +334,7 @@ def main(): config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_performance_stats', value=True) config.set('display', 'show_system_resources', value=True) + config.set('display', 'show_advanced_gestures', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -333,6 +349,7 @@ def main(): config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_performance_stats', value=True) config.set('display', 'show_system_resources', value=True) + config.set('display', 'show_advanced_gestures', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) @@ -345,12 +362,17 @@ def main(): config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_performance_stats', value=True) config.set('display', 'show_system_resources', value=True) + config.set('display', 'show_advanced_gestures', value=False) config.set('display', 'show_debug_info', value=False) # 绘制轨迹(如果启用) if config.get('display', 'show_trajectory'): frame = trajectory_recorder.draw_trajectory(frame) + # 绘制高级手势信息 + if config.get('display', 'show_advanced_gestures'): + frame = advanced_gesture_analyzer.draw_gesture_info(frame) + # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, @@ -375,7 +397,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像 - cv2.imshow('Gesture Controlled Drone - Performance Statistics', frame) + cv2.imshow('Gesture Controlled Drone - Enhanced Gestures', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF diff --git a/src/autonomus_drone_hand_gesture_project/performance_analyzer.py b/src/autonomus_drone_hand_gesture_project/performance_analyzer.py index 6bf7ca3d08..6ae0ab4c1c 100644 --- a/src/autonomus_drone_hand_gesture_project/performance_analyzer.py +++ b/src/autonomus_drone_hand_gesture_project/performance_analyzer.py @@ -11,7 +11,6 @@ from datetime import datetime from collections import deque, Counter - class PerformanceAnalyzer: """性能分析器 - 监控和报告系统性能""" diff --git a/src/autonomus_drone_hand_gesture_project/speech_manager.py b/src/autonomus_drone_hand_gesture_project/speech_manager.py index f2f54a7309..67f5f740c1 100644 --- a/src/autonomus_drone_hand_gesture_project/speech_manager.py +++ b/src/autonomus_drone_hand_gesture_project/speech_manager.py @@ -70,6 +70,13 @@ def __init__(self, speech_lib, config=None): 'gesture_low_confidence': "手势识别置信度低", 'gesture_good_confidence': "手势识别置信度高", 'gesture_hover': "悬停", + 'gesture_grab': "抓取", # 新增 + 'gesture_release': "释放", # 新增 + 'gesture_rotate_cw': "顺时针旋转", # 新增 + 'gesture_rotate_ccw': "逆时针旋转", # 新增 + 'gesture_photo': "拍照", # 新增 + 'gesture_auto_flight': "自动飞行模式", # 新增 + 'gesture_return_home': "返航", # 新增 # 系统相关 'program_start': "手势控制无人机系统已启动", diff --git a/src/autonomus_drone_hand_gesture_project/ui_renderer.py b/src/autonomus_drone_hand_gesture_project/ui_renderer.py index bd2e396fc4..048366bc4c 100644 --- a/src/autonomus_drone_hand_gesture_project/ui_renderer.py +++ b/src/autonomus_drone_hand_gesture_project/ui_renderer.py @@ -45,6 +45,13 @@ def __init__(self, speech_manager=None, config=None): 'memory_good': (0, 255, 0), 'memory_warning': (255, 165, 0), 'memory_critical': (255, 0, 0), + # 新增手势颜色 + 'gesture_grab': (255, 0, 255), + 'gesture_rotate_cw': (0, 255, 255), + 'gesture_rotate_ccw': (255, 255, 0), + 'gesture_photo': (255, 165, 0), + 'gesture_return_home': (0, 128, 128), + 'gesture_auto_flight': (128, 0, 128), } print("✓ 中文UI渲染器已初始化")