diff --git a/src/autonomus_drone_hand_gesture_project/config_manager.py b/src/autonomus_drone_hand_gesture_project/config_manager.py new file mode 100644 index 0000000000..bfd2c46f73 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/config_manager.py @@ -0,0 +1,313 @@ +""" +配置管理器模块 +负责加载、保存和管理系统配置 +作者: xiaoshiyuan888 +""" + +import os +import json +import numpy as np +import cv2 + + +class ConfigManager: + """配置管理器""" + + def __init__(self): + import sys + current_dir = os.path.dirname(os.path.abspath(__file__)) + self.config_file = os.path.join(current_dir, 'gesture_config.json') + + # 性能模式配置 + self.performance_modes = { + 'fast': { + 'name': '最快', + 'description': '性能优先,降低识别精度换取更高帧率', + 'detection_interval': 2, + 'smooth_frames': 3, + 'min_confidence': 0.5, + 'resize_factor': 0.5, + 'skin_detection_enabled': True, + 'background_subtraction_enabled': False, + 'contour_simplify_epsilon': 0.03, + 'history_size': 10, + 'gesture_stability_threshold': 3, + 'color': (0, 255, 0), + }, + 'balanced': { + 'name': '平衡', + 'description': '平衡性能与精度,适用于大多数场景', + 'detection_interval': 1, + 'smooth_frames': 5, + 'min_confidence': 0.6, + 'resize_factor': 0.75, + 'skin_detection_enabled': True, + 'background_subtraction_enabled': True, + 'contour_simplify_epsilon': 0.02, + 'history_size': 15, + 'gesture_stability_threshold': 5, + 'color': (255, 165, 0), + }, + 'accurate': { + 'name': '最准', + 'description': '精度优先,提供最准确的手势识别', + 'detection_interval': 1, + 'smooth_frames': 7, + 'min_confidence': 0.7, + 'resize_factor': 1.0, + 'skin_detection_enabled': True, + 'background_subtraction_enabled': True, + 'contour_simplify_epsilon': 0.01, + 'history_size': 20, + 'gesture_stability_threshold': 7, + 'color': (255, 0, 0), + } + } + + self.default_config = { + 'camera': { + 'index': 0, + 'width': 640, + 'height': 480, + 'fps': 30 + }, + 'gesture': { + 'skin_lower_h': 0, + 'skin_upper_h': 25, + 'skin_lower_s': 30, + 'skin_upper_s': 255, + 'skin_lower_v': 60, + 'skin_upper_v': 255, + 'min_hand_area': 2000, + 'max_hand_area': 30000, + 'hand_ratio_threshold': 1.5, + 'defect_distance_threshold': 20, + 'palm_circle_radius_ratio': 0.3, + 'transition_threshold': 0.3, + 'position_stability_weight': 0.4, + 'gesture_cooldown': 0.5, + }, + 'drone': { + 'velocity': 2.5, + 'duration': 0.3, + 'altitude': -10.0, + 'control_interval': 0.3 + }, + 'display': { + 'show_fps': True, + 'show_confidence': True, + 'show_help': True, + 'show_contours': True, + 'show_bbox': True, + 'show_fingertips': True, + 'show_palm_center': True, + 'show_hand_direction': True, + 'show_debug_info': False, + 'show_speech_status': True, + 'show_gesture_history': True, + 'show_stability_indicator': True, + 'show_trajectory': True, + 'show_recording_status': True, + 'show_performance_mode': True, + 'show_performance_stats': True, + 'show_system_resources': True, + }, + 'performance': { + 'target_fps': 30, + 'resize_factor': 1.0, + 'enable_multiprocessing': False, + 'mode': 'balanced', + 'current_mode_index': 1, + 'modes': ['fast', 'balanced', 'accurate'], + 'auto_report_interval': 60, + 'enable_performance_monitor': True, + }, + 'calibration': { + 'auto_calibrate_skin': True, + 'skin_calibration_frames': 30, + 'hand_size_calibration': True + }, + 'speech': { + 'enabled': True, + 'volume': 1.0, + 'rate': 150, + 'announce_gestures': True, + 'announce_connections': True, + 'announce_flight_events': True, + 'announce_gesture_changes': True, + 'announce_hand_status': True, + 'announce_performance': True, + 'announce_recording_events': True, + 'announce_performance_mode': True, + 'announce_performance_events': True, + 'min_gesture_confidence': 0.7, + 'gesture_start_threshold': 3, + 'gesture_end_threshold': 10, + }, + 'recording': { + 'auto_save_interval': 5, + 'max_trajectory_points': 1000, + 'show_trajectory': True, + 'trajectory_thickness': 2, + 'trajectory_max_length': 100, + 'default_save_dir': 'trajectories', + } + } + self.config = self.load_config() + self.skin_calibration_data = [] + self.hand_size_calibration_done = False + self.reference_hand_size = 0 + + def load_config(self): + """加载配置""" + if os.path.exists(self.config_file): + try: + with open(self.config_file, 'r', encoding='utf-8') as f: + loaded_config = json.load(f) + config = self.default_config.copy() + self._merge_config(config, loaded_config) + print("✓ 从文件加载配置") + return config + except Exception as e: + print(f"⚠ 加载配置失败: {e}, 使用默认配置") + return self.default_config.copy() + else: + print("✓ 使用默认配置") + return self.default_config.copy() + + def _merge_config(self, base, update): + """递归合并配置""" + for key, value in update.items(): + if key in base and isinstance(base[key], dict) and isinstance(value, dict): + self._merge_config(base[key], value) + else: + base[key] = value + + def save_config(self): + """保存配置""" + try: + with open(self.config_file, 'w', encoding='utf-8') as f: + json.dump(self.config, f, indent=2, ensure_ascii=False) + print("✓ 配置已保存") + except Exception as e: + print(f"⚠ 保存配置失败: {e}") + + def get(self, *keys): + """获取配置值""" + value = self.config + for key in keys: + if isinstance(value, dict) and key in value: + value = value[key] + else: + return None + return value + + def set(self, *keys, value): + """设置配置值""" + if len(keys) == 0: + return + + config = self.config + for key in keys[:-1]: + if key not in config: + config[key] = {} + config = config[key] + + config[keys[-1]] = value + self.save_config() + + def get_performance_mode_config(self, mode=None): + """获取性能模式配置""" + if mode is None: + mode = self.get('performance', 'mode') + + if mode in self.performance_modes: + return self.performance_modes[mode] + else: + return self.performance_modes['balanced'] + + def get_current_performance_mode(self): + """获取当前性能模式""" + mode = self.get('performance', 'mode') + if mode in self.performance_modes: + return mode + return 'balanced' + + def set_performance_mode(self, mode): + """设置性能模式""" + if mode in self.performance_modes: + self.set('performance', 'mode', value=mode) + + # 更新当前模式索引 + modes = self.get('performance', 'modes') + if modes and mode in modes: + index = modes.index(mode) + self.set('performance', 'current_mode_index', value=index) + + print(f"✓ 性能模式设置为: {self.performance_modes[mode]['name']}") + return True + return False + + def cycle_performance_mode(self): + """循环切换性能模式""" + modes = self.get('performance', 'modes') + if not modes: + modes = ['fast', 'balanced', 'accurate'] + + current_index = self.get('performance', 'current_mode_index') + if current_index is None: + current_index = 0 + + # 计算下一个模式索引 + next_index = (current_index + 1) % len(modes) + next_mode = modes[next_index] + + # 设置新模式 + self.set('performance', 'current_mode_index', value=next_index) + return self.set_performance_mode(next_mode) + + def calibrate_skin_color(self, frame, hand_mask): + """自动校准肤色范围""" + if not self.get('calibration', 'auto_calibrate_skin'): + return + + if len(self.skin_calibration_data) < self.get('calibration', 'skin_calibration_frames'): + # 转换到HSV + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + + # 获取肤色区域的HSV值 + skin_pixels = hsv[hand_mask > 0] + + if len(skin_pixels) > 100: # 确保有足够的像素 + self.skin_calibration_data.append(skin_pixels) + + if len(self.skin_calibration_data) == self.get('calibration', 'skin_calibration_frames'): + # 计算肤色范围 + all_skin_pixels = np.vstack(self.skin_calibration_data) + + h_min, h_max = np.percentile(all_skin_pixels[:, 0], [2, 98]) + s_min, s_max = np.percentile(all_skin_pixels[:, 1], [2, 98]) + v_min, v_max = np.percentile(all_skin_pixels[:, 2], [2, 98]) + + # 更新配置 + self.set('gesture', 'skin_lower_h', value=int(max(0, h_min - 5))) + self.set('gesture', 'skin_upper_h', value=int(min(180, h_max + 5))) + self.set('gesture', 'skin_lower_s', value=int(max(0, s_min - 10))) + self.set('gesture', 'skin_upper_s', value=int(min(255, s_max + 10))) + self.set('gesture', 'skin_lower_v', value=int(max(0, v_min - 10))) + self.set('gesture', 'skin_upper_v', value=int(min(255, v_max + 10))) + + print("✓ 肤色校准完成") + print(f" 肤色范围: H[{self.get('gesture', 'skin_lower_h')}-{self.get('gesture', 'skin_upper_h')}], " + f"S[{self.get('gesture', 'skin_lower_s')}-{self.get('gesture', 'skin_upper_s')}], " + f"V[{self.get('gesture', 'skin_lower_v')}-{self.get('gesture', 'skin_upper_v')}]") + + def calibrate_hand_size(self, hand_area): + """校准手部大小""" + if not self.get('calibration', 'hand_size_calibration') or self.hand_size_calibration_done: + return + + if hand_area > 0: + self.reference_hand_size = hand_area + self.hand_size_calibration_done = True + print(f"✓ 手部大小校准完成: {self.reference_hand_size:.0f} 像素") \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/drone_controller.py b/src/autonomus_drone_hand_gesture_project/drone_controller.py new file mode 100644 index 0000000000..17e8dfea36 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/drone_controller.py @@ -0,0 +1,282 @@ +""" +无人机控制器模块 +负责控制无人机的连接、起飞、降落和移动 +作者: xiaoshiyuan888 +""" + +import time + + +class SimpleDroneController: + """简单的无人机控制器""" + + def __init__(self, airsim_module, speech_manager=None, config=None): + self.airsim = airsim_module + self.client = None + self.connected = False + self.flying = False + self.speech_manager = speech_manager + self.config = config + + # 控制参数 + self.velocity = config.get('drone', 'velocity') + self.duration = config.get('drone', 'duration') + self.altitude = config.get('drone', 'altitude') + self.control_interval = config.get('drone', 'control_interval') + + # 控制状态 + self.last_control_time = 0 + self.last_gesture = None + + # 上次语音提示状态 + self.last_connection_announced = False + self.last_takeoff_announced = False + self.last_land_announced = False + + print("✓ 简单的无人机控制器已初始化") + + def connect(self): + """连接AirSim无人机""" + if self.connected: + return True + + # 语音提示:正在连接 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('connecting') + + if self.airsim is None: + print("⚠ AirSim不可用,使用模拟模式") + + # 语音提示:模拟模式 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('simulation_mode') + + self.connected = True + return True + + print("连接AirSim...") + + try: + self.client = self.airsim.MultirotorClient() + self.client.confirmConnection() + print("✅ 已连接AirSim!") + + # 语音提示:连接成功 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('connected') + + self.client.enableApiControl(True) + print("✅ API控制已启用") + + self.client.armDisarm(True) + print("✅ 无人机已武装") + + self.connected = True + return True + + except Exception as e: + print(f"❌ 连接失败: {e}") + + # 语音提示:连接失败 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('connection_failed') + + print("\n使用模拟模式继续? (y/n)") + choice = input().strip().lower() + if choice == 'y': + self.connected = True + print("✅ 使用模拟模式") + + # 语音提示:模拟模式 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('simulation_mode') + + return True + + return False + + def takeoff(self): + """起飞""" + if not self.connected: + return False + + # 语音提示:正在起飞 + if (self.speech_manager and + self.speech_manager.enabled and + not self.last_takeoff_announced): + self.speech_manager.speak('taking_off') + self.last_takeoff_announced = True + self.last_land_announced = False + + try: + if self.airsim is None or self.client is None: + print("✅ 模拟起飞") + self.flying = True + + # 语音提示:起飞成功 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('takeoff_success') + + return True + + print("起飞中...") + self.client.takeoffAsync().join() + time.sleep(1) + + # 上升到指定高度 + self.client.moveToZAsync(self.altitude, 3).join() + + self.flying = True + print("✅ 无人机成功起飞") + + # 语音提示:起飞成功 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('takeoff_success') + + return True + except Exception as e: + print(f"❌ 起飞失败: {e}") + + # 语音提示:起飞失败 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('takeoff_failed') + + return False + + def land(self): + """降落""" + if not self.connected: + return False + + # 语音提示:正在降落 + if (self.speech_manager and + self.speech_manager.enabled and + not self.last_land_announced): + self.speech_manager.speak('landing') + self.last_land_announced = True + self.last_takeoff_announced = False + + try: + if self.airsim is None or self.client is None: + print("✅ 模拟降落") + self.flying = False + + # 语音提示:降落成功 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('land_success') + + return True + + print("降落中...") + self.client.landAsync().join() + self.flying = False + print("✅ 无人机已降落") + + # 语音提示:降落成功 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('land_success') + + return True + except Exception as e: + print(f"降落失败: {e}") + return False + + def move_by_gesture(self, gesture, confidence): + """根据手势移动""" + if not self.connected or not self.flying: + return False + + # 检查控制间隔 + current_time = time.time() + if current_time - self.last_control_time < self.control_interval: + return False + + # 检查置信度阈值 + min_confidence = self.config.get('gesture', 'min_confidence') + if confidence < min_confidence: + # 低置信度语音提示 + if (self.speech_manager and + self.speech_manager.enabled and + confidence < min_confidence * 0.8): + self.speech_manager.speak('gesture_low_confidence') + return False + + try: + if self.airsim is None or self.client is None: + print(f"模拟移动: {gesture}") + self.last_control_time = current_time + self.last_gesture = gesture + return True + + success = False + + if gesture == "Up": + self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) + success = True + elif gesture == "Down": + self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) + success = True + elif gesture == "Left": + self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) + success = True + elif gesture == "Right": + self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) + success = True + elif gesture == "Forward": + self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) + success = True + elif gesture == "Stop": + self.client.hoverAsync() + success = True + # 悬停语音提示 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('hovering') + elif gesture == "Hover": + self.client.hoverAsync() + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('hovering') + + if success: + self.last_control_time = current_time + self.last_gesture = gesture + + return success + except Exception as e: + print(f"控制命令失败: {e}") + return False + + def emergency_stop(self): + """紧急停止""" + if self.connected: + try: + if self.flying and self.client is not None: + print("紧急降落...") + + # 语音提示:紧急停止 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('emergency_stop') + + self.land() + if self.client is not None: + self.client.armDisarm(False) + self.client.enableApiControl(False) + print("✅ 紧急停止完成") + except: + pass + + self.connected = False + self.flying = False \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index 3cdf48a5cb..fbb45391da 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -18,11 +18,11 @@ }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 176, + "skin_upper_h": 170, "skin_lower_s": 0, - "skin_upper_s": 116, + "skin_upper_s": 70, "skin_lower_v": 0, - "skin_upper_v": 92, + "skin_upper_v": 75, "min_hand_area": 3000, "max_hand_area": 30000, "hand_ratio_threshold": 1.5, diff --git a/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py b/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py new file mode 100644 index 0000000000..ff37f66b83 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py @@ -0,0 +1,952 @@ +""" +手势识别器模块 +负责识别和分析手势 +作者: xiaoshiyuan888 +""" + +import time +import math +import numpy as np +import cv2 +from collections import deque, Counter + + +class EnhancedGestureRecognizer: + """增强的手势识别器 - 支持性能模式选择""" + + def __init__(self, speech_manager=None, config=None): + self.speech_manager = speech_manager + self.config = config + + # 加载性能模式配置 + self.performance_mode = config.get_current_performance_mode() + self.mode_config = config.get_performance_mode_config(self.performance_mode) + + # 根据性能模式初始化参数 + self.history_size = self.mode_config['history_size'] + self.detection_interval = self.mode_config['detection_interval'] + self.smooth_frames = self.mode_config['smooth_frames'] + self.min_confidence = self.mode_config['min_confidence'] + self.resize_factor = self.mode_config['resize_factor'] + + # 增强的手势历史和平滑 + self.gesture_history = deque(maxlen=self.history_size) + self.confidence_history = deque(maxlen=self.history_size) + self.position_history = deque(maxlen=self.history_size) + self.current_gesture = "Waiting" + self.current_confidence = 0.0 + + # 手势状态追踪 + self.gesture_state = "none" + self.gesture_stability_counter = 0 + self.last_stable_gesture = "Waiting" + self.gesture_active_frames = 0 + self.last_gesture_change_time = 0 + + # 记录上次播报的手势 + self.last_announced_gesture = None + self.last_announced_time = 0 + self.last_hand_status_time = 0 + self.gesture_announce_interval = 2.0 + + # 手部跟踪和状态 + self.last_hand_position = None + self.hand_tracking = False + self.track_window = None + self.hand_states = deque(maxlen=15) + self.hand_detected_frames = 0 + self.hand_lost_frames = 0 + + # 性能统计 + self.process_times = deque(maxlen=30) + self.frame_counter = 0 + self.last_performance_report = 0 + + # 手势颜色映射 + self.gesture_colors = { + "Stop": (0, 0, 255), + "Forward": (0, 255, 0), + "Up": (255, 255, 0), + "Down": (255, 0, 255), + "Left": (255, 165, 0), + "Right": (0, 165, 255), + "Waiting": (200, 200, 200), + "Error": (255, 0, 0), + "Hover": (255, 255, 255) + } + + # 手势到语音的映射 + self.gesture_speech_map = { + "Stop": "gesture_stop", + "Forward": "gesture_forward", + "Up": "gesture_up", + "Down": "gesture_down", + "Left": "gesture_left", + "Right": "gesture_right", + "Waiting": "gesture_waiting", + "Error": "gesture_error", + "Hover": "gesture_hover", + } + + # 手势状态颜色 + self.state_colors = { + "none": (100, 100, 100), + "starting": (255, 165, 0), + "active": (0, 255, 0), + "ending": (255, 0, 0), + } + + # 根据性能模式初始化背景减除器 + self.bg_subtractor = None + if self.mode_config['background_subtraction_enabled']: + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( + history=100, varThreshold=25, detectShadows=True + ) + + # 形态学操作核 + self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) + + # 性能监控 + self.avg_process_time = 0 + self.frame_rate = 0 + self.last_fps_check = time.time() + + # 存储手部数据用于轨迹记录 + self.last_hand_data = None + + # 性能模式信息 + self.performance_mode_color = self.mode_config['color'] + self.performance_mode_name = self.mode_config['name'] + + print(f"✓ 增强的手势识别器已初始化 - 性能模式: {self.performance_mode_name}") + + def set_performance_mode(self, mode): + """设置性能模式""" + self.performance_mode = mode + self.mode_config = self.config.get_performance_mode_config(mode) + + # 更新参数 + self.history_size = self.mode_config['history_size'] + self.detection_interval = self.mode_config['detection_interval'] + self.smooth_frames = self.mode_config['smooth_frames'] + self.min_confidence = self.mode_config['min_confidence'] + self.resize_factor = self.mode_config['resize_factor'] + + # 更新背景减除器 + if self.mode_config['background_subtraction_enabled'] and self.bg_subtractor is None: + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( + history=100, varThreshold=25, detectShadows=True + ) + elif not self.mode_config['background_subtraction_enabled']: + self.bg_subtractor = None + + # 更新队列大小 + self.gesture_history = deque(maxlen=self.history_size) + self.confidence_history = deque(maxlen=self.history_size) + self.position_history = deque(maxlen=self.history_size) + + # 更新显示信息 + self.performance_mode_color = self.mode_config['color'] + self.performance_mode_name = self.mode_config['name'] + + print(f"✓ 切换到性能模式: {self.performance_mode_name}") + + def get_skin_mask(self, frame): + """获取肤色掩码""" + h_low = self.config.get('gesture', 'skin_lower_h') + h_high = self.config.get('gesture', 'skin_upper_h') + s_low = self.config.get('gesture', 'skin_lower_s') + s_high = self.config.get('gesture', 'skin_upper_s') + v_low = self.config.get('gesture', 'skin_lower_v') + v_high = self.config.get('gesture', 'skin_upper_v') + + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + lower_skin = np.array([h_low, s_low, v_low], dtype=np.uint8) + upper_skin = np.array([h_high, s_high, v_high], dtype=np.uint8) + skin_mask = cv2.inRange(hsv, lower_skin, upper_skin) + + return skin_mask, hsv + + def enhance_skin_detection(self, frame, skin_mask): + """增强肤色检测""" + if not self.mode_config['skin_detection_enabled']: + return skin_mask + + if self.bg_subtractor is not None: + fg_mask = self.bg_subtractor.apply(frame) + combined_mask = cv2.bitwise_and(skin_mask, fg_mask) + else: + combined_mask = skin_mask + + # 根据性能模式决定形态学操作次数 + if self.performance_mode == 'accurate': + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) + combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) + elif self.performance_mode == 'balanced': + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=1) + combined_mask = cv2.GaussianBlur(combined_mask, (3, 3), 0) + + return combined_mask + + def preprocess_frame(self, frame): + """预处理帧""" + if self.resize_factor != 1.0: + new_width = int(frame.shape[1] * self.resize_factor) + new_height = int(frame.shape[0] * self.resize_factor) + resized_frame = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_AREA) + return resized_frame + return frame + + def find_best_hand_contour(self, mask, frame): + """找到最佳的手部轮廓""" + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + if not contours: + return None, 0.0 + + contours = sorted(contours, key=cv2.contourArea, reverse=True) + best_contour = None + best_score = 0.0 + min_area = self.config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = self.config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) + + # 根据性能模式调整检查的轮廓数量 + max_contours = 3 if self.performance_mode != 'fast' else 1 + + for contour in contours[:max_contours]: + area = cv2.contourArea(contour) + + if area < min_area or area > max_area: + continue + + score = self.rate_contour(contour, frame.shape) + + if score > best_score: + best_score = score + best_contour = contour + + return best_contour, best_score + + def rate_contour(self, contour, frame_shape): + """评估轮廓作为手部的可能性""" + score = 0.0 + area = cv2.contourArea(contour) + min_area = self.config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = self.config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) + + if min_area < area < max_area: + area_ratio = min(area / max_area, 1.0) + score += area_ratio * 0.3 + + perimeter = cv2.arcLength(contour, True) + if perimeter > 100: + score += 0.2 + + if area > 0: + compactness = perimeter ** 2 / area + if 12 < compactness < 25: + compactness_score = 1.0 - abs(compactness - 18) / 6 + score += compactness_score * 0.3 + + x, y, w, h = cv2.boundingRect(contour) + if h > 0: + aspect_ratio = w / h + if 0.4 < aspect_ratio < 2.5: + aspect_score = 1.0 - abs(aspect_ratio - 1.0) / 1.5 + score += aspect_score * 0.2 + + return score + + def analyze_hand_features(self, contour, frame_shape): + """分析手部特征""" + if contour is None: + return None, 0.0 + + area = cv2.contourArea(contour) + M = cv2.moments(contour) + if M["m00"] == 0: + return None, 0.0 + + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + x, y, w, h = cv2.boundingRect(contour) + bbox_area = w * h + palm_center = (cx, cy) + palm_radius = int(w * self.config.get('gesture', 'palm_circle_radius_ratio')) + + # 根据性能模式调整轮廓简化程度 + epsilon = self.mode_config['contour_simplify_epsilon'] * cv2.arcLength(contour, True) + fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius, epsilon) + + direction = self.calculate_hand_direction(contour, cx, cy) + h_img, w_img = frame_shape[:2] + norm_x = cx / w_img + norm_y = cy / h_img + confidence = self.calculate_confidence(area, len(fingers), len(contour), bbox_area) + + result = { + 'contour': contour, + 'center': (cx, cy), + 'bbox': (x, y, x + w, y + h), + 'fingers': fingers, + 'fingertips': fingertips, + 'defects': defects, + 'palm_center': palm_center, + 'palm_radius': palm_radius, + 'direction': direction, + 'area': area, + 'position': (norm_x, norm_y), + 'bbox_size': (w, h), + 'confidence': confidence + } + + return result, confidence + + def analyze_fingers(self, contour, palm_center, palm_radius, epsilon): + """分析手指""" + approx = cv2.approxPolyDP(contour, epsilon, True) + hull = cv2.convexHull(approx, returnPoints=False) + + if hull is None or len(hull) < 3: + return [], [], [] + + defects = cv2.convexityDefects(approx, hull) + fingers = [] + fingertips = [] + defect_points = [] + + if defects is not None: + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(approx[s][0]) + end = tuple(approx[e][0]) + far = tuple(approx[f][0]) + + start_dist = np.linalg.norm(np.array(start) - np.array(palm_center)) + end_dist = np.linalg.norm(np.array(end) - np.array(palm_center)) + far_dist = np.linalg.norm(np.array(far) - np.array(palm_center)) + + if (start_dist > palm_radius * 1.2 and + end_dist > palm_radius * 1.2 and + d > self.config.get('gesture', 'defect_distance_threshold') * 256): + + a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + + if b * c != 0: + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + angle_degrees = math.degrees(angle) + + if angle_degrees < 90: + finger = { + 'start': start, + 'end': end, + 'far': far, + 'depth': d, + 'angle': angle_degrees + } + fingers.append(finger) + + for point in [start, end]: + if point not in fingertips: + point_dist = np.linalg.norm(np.array(point) - np.array(palm_center)) + if point_dist > palm_radius * 1.5: + fingertips.append(point) + + defect_points.append((start, end, far, d)) + + if len(fingertips) == 0: + hull_points = cv2.convexHull(approx, returnPoints=True) + if len(hull_points) > 0: + hull_points = hull_points.reshape(-1, 2) + + for point in hull_points: + point_tuple = tuple(point) + point_dist = np.linalg.norm(point - np.array(palm_center)) + if point_dist > palm_radius * 1.5 and point_tuple not in fingertips: + fingertips.append(point_tuple) + + fingertips = fingertips[:5] + + return fingers, fingertips, defect_points + + def calculate_hand_direction(self, contour, cx, cy): + """计算手部方向""" + if len(contour) < 5: + return 0.0 + + points = contour.reshape(-1, 2).astype(np.float32) + mean = np.empty((0)) + mean, eigenvectors, eigenvalues = cv2.PCACompute2(points, mean) + direction = math.degrees(math.atan2(eigenvectors[0, 1], eigenvectors[0, 0])) + + return direction + + def calculate_confidence(self, area, finger_count, contour_length, bbox_area): + """计算手势置信度""" + confidence = 0.5 + min_area = self.config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = self.config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) + + if min_area < area < max_area: + area_norm = (area - min_area) / (max_area - min_area) + confidence += area_norm * 0.2 + + if 0 <= finger_count <= 5: + confidence += 0.2 + + if contour_length > 200: + confidence += 0.1 + + if bbox_area > 0: + fill_ratio = area / bbox_area + if 0.2 < fill_ratio < 0.8: + fill_score = 1.0 - abs(fill_ratio - 0.5) / 0.3 + confidence += fill_score * 0.1 + + return min(confidence, 1.0) + + def recognize_gesture_improved(self, hand_data): + """改进的手势识别逻辑""" + if hand_data is None: + return "Waiting", 0.3 + + finger_count = len(hand_data.get('fingers', [])) + fingertips = hand_data.get('fingertips', []) + norm_x, norm_y = hand_data['position'] + direction = hand_data.get('direction', 0.0) + confidence = hand_data['confidence'] + w, h = hand_data['bbox_size'] + aspect_ratio = w / h if h > 0 else 1.0 + + # 根据手指数量分类 + if finger_count == 0: + if len(fingertips) == 0: + return "Stop", confidence * 0.9 + else: + return "Stop", confidence * 0.7 + + elif finger_count == 1: + if len(fingertips) >= 1: + cx, cy = hand_data['center'] + fingertip = fingertips[0] + dx = fingertip[0] - cx + dy = fingertip[1] - cy + + if abs(dx) > abs(dy): + if dx > 0: + return "Right", confidence * 0.8 + else: + return "Left", confidence * 0.8 + else: + if dy < 0: + return "Up", confidence * 0.8 + else: + return "Forward", confidence * 0.8 + return "Forward", confidence * 0.7 + + elif finger_count == 2: + return "Forward", confidence * 0.7 + + elif finger_count == 3: + if -45 <= direction <= 45: + if direction > 0: + return "Right", confidence * 0.7 + else: + return "Left", confidence * 0.7 + else: + if direction > 0: + return "Down", confidence * 0.7 + else: + return "Up", confidence * 0.7 + + elif finger_count >= 4: + if norm_x < 0.4: + return "Left", confidence * 0.8 + elif norm_x > 0.6: + return "Right", confidence * 0.8 + elif norm_y < 0.4: + return "Up", confidence * 0.8 + elif norm_y > 0.6: + return "Down", confidence * 0.8 + else: + if -45 <= direction <= 45: + return "Forward", confidence * 0.7 + else: + return "Stop", confidence * 0.7 + + return "Waiting", confidence * 0.5 + + def smooth_gesture_enhanced(self, new_gesture, new_confidence, hand_data): + """增强的手势平滑处理""" + current_time = time.time() + + # 检查手势冷却时间 + if current_time - self.last_gesture_change_time < self.config.get('gesture', 'gesture_cooldown'): + return self.current_gesture, self.current_confidence + + # 添加到历史 + self.gesture_history.append(new_gesture) + self.confidence_history.append(new_confidence) + + if hand_data is not None: + self.position_history.append(hand_data['position']) + + # 计算手势稳定性 + if len(self.gesture_history) >= 3: + recent_gestures = list(self.gesture_history)[-3:] + gesture_counter = Counter(recent_gestures) + most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] + + # 计算位置稳定性 + position_stability = 1.0 + if len(self.position_history) >= 2 and hand_data is not None: + current_pos = hand_data['position'] + prev_pos = self.position_history[-2] if len(self.position_history) >= 2 else current_pos + position_diff = math.sqrt((current_pos[0] - prev_pos[0]) ** 2 + (current_pos[1] - prev_pos[1]) ** 2) + position_stability = max(0, 1.0 - position_diff * 5.0) + + # 增强的稳定性检查 + stability_threshold = self.mode_config['gesture_stability_threshold'] + transition_threshold = self.config.get('gesture', 'transition_threshold') + position_weight = self.config.get('gesture', 'position_stability_weight') + + # 计算综合稳定性得分 + gesture_stability = most_common_count / 3.0 + overall_stability = gesture_stability * (1.0 - position_weight) + position_stability * position_weight + + # 手势状态转换逻辑 + if overall_stability >= transition_threshold: + if most_common_gesture != self.last_stable_gesture: + self.gesture_stability_counter += 1 + else: + self.gesture_stability_counter = max(0, self.gesture_stability_counter - 1) + + if self.gesture_stability_counter >= stability_threshold: + self.current_gesture = most_common_gesture + self.current_confidence = np.mean(list(self.confidence_history)[-3:]) + self.last_stable_gesture = most_common_gesture + self.gesture_stability_counter = 0 + self.last_gesture_change_time = current_time + else: + self.gesture_stability_counter = max(0, self.gesture_stability_counter - 2) + + return self.current_gesture, self.current_confidence + + def update_gesture_state(self, hand_data, gesture, confidence): + """更新手势状态""" + current_time = time.time() + + if hand_data is None: + # 手部丢失 + self.hand_lost_frames += 1 + self.hand_detected_frames = max(0, self.hand_detected_frames - 1) + + if self.hand_lost_frames > 10 and self.gesture_state != "none": + self.gesture_state = "none" + if (self.speech_manager and + self.speech_manager.enabled and + current_time - self.last_hand_status_time > 3.0): + self.speech_manager.speak('hand_lost', immediate=True) + self.last_hand_status_time = current_time + return + + # 手部检测到 + self.hand_detected_frames += 1 + self.hand_lost_frames = 0 + + # 检查手部大小和位置 + hand_area = hand_data['area'] + min_area = self.config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = self.config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) + + # 提供手部位置反馈 + if (self.speech_manager and + self.speech_manager.enabled and + current_time - self.last_hand_status_time > 5.0): + + if hand_area < min_area * 0.8: + self.speech_manager.speak('move_closer', immediate=True) + self.last_hand_status_time = current_time + elif hand_area > max_area * 1.2: + self.speech_manager.speak('move_away', immediate=True) + self.last_hand_status_time = current_time + elif self.hand_detected_frames == 5: + self.speech_manager.speak('hand_detected', immediate=True) + self.last_hand_status_time = current_time + + # 手势状态转换 + if self.gesture_state == "none" and gesture != "Waiting" and confidence > 0.6: + self.gesture_state = "starting" + self.gesture_active_frames = 0 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_start', immediate=True) + + elif self.gesture_state == "starting": + self.gesture_active_frames += 1 + if self.gesture_active_frames >= self.config.get('speech', 'gesture_start_threshold'): + self.gesture_state = "active" + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_stable', immediate=True) + + elif self.gesture_state == "active": + if gesture == "Waiting" or confidence < 0.5: + self.gesture_active_frames = max(0, self.gesture_active_frames - 2) + if self.gesture_active_frames <= 0: + self.gesture_state = "ending" + else: + self.gesture_active_frames = min(20, self.gesture_active_frames + 1) + + elif self.gesture_state == "ending": + self.gesture_active_frames -= 1 + if self.gesture_active_frames <= 0: + self.gesture_state = "none" + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_end', immediate=True) + + def visualize_detection(self, frame, hand_data, gesture, confidence): + """可视化检测结果""" + if hand_data is None: + return frame + + show_contours = self.config.get('display', 'show_contours') + show_bbox = self.config.get('display', 'show_bbox') + show_fingertips = self.config.get('display', 'show_fingertips') + show_palm_center = self.config.get('display', 'show_palm_center') + show_hand_direction = self.config.get('display', 'show_hand_direction') + show_debug_info = self.config.get('display', 'show_debug_info') + show_gesture_history = self.config.get('display', 'show_gesture_history') + show_stability_indicator = self.config.get('display', 'show_stability_indicator') + + # 绘制轮廓 + if show_contours and 'contour' in hand_data: + cv2.drawContours(frame, [hand_data['contour']], -1, (0, 255, 0), 2) + + # 绘制边界框 + if show_bbox and 'bbox' in hand_data: + x1, y1, x2, y2 = hand_data['bbox'] + color = self.gesture_colors.get(gesture, (255, 255, 255)) + + # 根据手势状态调整边界框颜色 + state_color = self.state_colors.get(self.gesture_state, color) + if self.gesture_state != "none": + color = state_color + + cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) + + # 显示手势标签 + label = f"{gesture}" + if self.config.get('display', 'show_confidence'): + label += f" ({confidence:.0%})" + + # 显示手势状态 + if self.gesture_state != "none": + state_text = {"starting": "开始", "active": "活跃", "ending": "结束"} + label += f" [{state_text.get(self.gesture_state, '')}]" + + # 计算文本大小 + (text_width, text_height), baseline = cv2.getTextSize( + label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 + ) + + # 绘制文本背景 + cv2.rectangle(frame, + (x1, y1 - text_height - 10), + (x1 + text_width, y1), + color, -1) + + # 绘制文本 + cv2.putText(frame, label, (x1, y1 - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) + + # 绘制手掌中心 + if show_palm_center and 'palm_center' in hand_data: + cx, cy = hand_data['palm_center'] + palm_radius = hand_data.get('palm_radius', 20) + cv2.circle(frame, (cx, cy), 5, (0, 0, 255), -1) + cv2.circle(frame, (cx, cy), palm_radius, (0, 0, 255), 1) + cv2.putText(frame, "Palm", (cx + 10, cy), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) + + # 绘制指尖 + if show_fingertips and 'fingertips' in hand_data: + for i, point in enumerate(hand_data['fingertips']): + cv2.circle(frame, point, 4, (255, 0, 0), -1) + cv2.putText(frame, f"F{i + 1}", (point[0] + 5, point[1]), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1) + + # 绘制手部方向 + if show_hand_direction and 'direction' in hand_data and 'center' in hand_data: + cx, cy = hand_data['center'] + direction = hand_data['direction'] + length = 50 + + dx = length * math.cos(math.radians(direction)) + dy = length * math.sin(math.radians(direction)) + + end_point = (int(cx + dx), int(cy + dy)) + cv2.arrowedLine(frame, (cx, cy), end_point, (255, 255, 0), 2) + + angle_text = f"Dir: {direction:.0f}°" + cv2.putText(frame, angle_text, (cx, cy - 20), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 1) + + # 绘制稳定性指示器 + if show_stability_indicator: + h, w = frame.shape[:2] + indicator_x = w - 100 + indicator_y = 30 + + # 绘制稳定性背景 + cv2.rectangle(frame, (indicator_x, indicator_y), + (indicator_x + 80, indicator_y + 15), (50, 50, 50), -1) + + # 计算稳定性指示条长度 + stability_level = min(1.0, + self.gesture_stability_counter / self.mode_config['gesture_stability_threshold']) + bar_length = int(70 * stability_level) + + # 根据稳定性级别选择颜色 + if stability_level > 0.7: + bar_color = (0, 255, 0) + elif stability_level > 0.4: + bar_color = (255, 165, 0) + else: + bar_color = (255, 0, 0) + + # 绘制稳定性指示条 + cv2.rectangle(frame, (indicator_x + 5, indicator_y + 5), + (indicator_x + 5 + bar_length, indicator_y + 10), bar_color, -1) + + # 绘制稳定性文本 + cv2.putText(frame, "稳定度", (indicator_x, indicator_y - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 绘制手势历史 + if show_gesture_history and len(self.gesture_history) > 0: + h, w = frame.shape[:2] + history_y = h - 50 + + # 绘制历史背景 + cv2.rectangle(frame, (10, history_y - 20), (200, history_y + 10), (0, 0, 0), -1) + cv2.putText(frame, "手势历史:", (15, history_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 显示最近几个手势 + recent_gestures = list(self.gesture_history)[-5:] if len(self.gesture_history) >= 5 else list( + self.gesture_history) + for i, gest in enumerate(recent_gestures): + color = self.gesture_colors.get(gest, (255, 255, 255)) + cv2.putText(frame, gest[0], (85 + i * 20, history_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + + # 显示调试信息 + if show_debug_info: + finger_count = len(hand_data.get('fingers', [])) + finger_text = f"Fingers: {finger_count}" + cv2.putText(frame, finger_text, (10, frame.shape[0] - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + pos_text = f"Pos: ({hand_data['position'][0]:.2f}, {hand_data['position'][1]:.2f})" + cv2.putText(frame, pos_text, (10, frame.shape[0] - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + state_text = f"State: {self.gesture_state}" + cv2.putText(frame, state_text, (150, frame.shape[0] - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + stability_text = f"Stability: {self.gesture_stability_counter}" + cv2.putText(frame, stability_text, (150, frame.shape[0] - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + return frame + + def recognize(self, frame): + """识别手势""" + start_time = time.time() + + try: + # 预处理帧 + processed_frame = cv2.flip(frame, 1) + + # 根据性能模式调整图像大小 + original_frame = processed_frame.copy() + if self.resize_factor != 1.0: + processed_frame = self.preprocess_frame(processed_frame) + + # 每隔几帧检测一次以提高性能 + if self.frame_counter % self.detection_interval != 0: + self.frame_counter += 1 + return self.current_gesture, self.current_confidence, original_frame + + # 获取肤色掩码 + skin_mask, hsv = self.get_skin_mask(processed_frame) + + # 增强肤色检测 + enhanced_mask = self.enhance_skin_detection(processed_frame, skin_mask) + + # 找到最佳的手部轮廓 + hand_contour, contour_score = self.find_best_hand_contour(enhanced_mask, processed_frame) + + # 分析手部特征 + hand_data, confidence = self.analyze_hand_features(hand_contour, processed_frame.shape) + + # 保存手部数据用于轨迹记录 + self.last_hand_data = hand_data + + # 识别手势 + if hand_data is not None: + # 校准肤色 + self.config.calibrate_skin_color(processed_frame, enhanced_mask) + + # 校准手部大小 + self.config.calibrate_hand_size(hand_data['area']) + + # 识别手势 + gesture, raw_confidence = self.recognize_gesture_improved(hand_data) + confidence = max(confidence, raw_confidence) + + # 更新手势状态 + self.update_gesture_state(hand_data, gesture, confidence) + + # 增强的手势平滑 + final_gesture, final_confidence = self.smooth_gesture_enhanced(gesture, confidence, hand_data) + else: + gesture, confidence = "Waiting", 0.3 + self.update_gesture_state(None, gesture, confidence) + final_gesture, final_confidence = gesture, confidence + + # 手势语音播报 + if (self.speech_manager and + self.speech_manager.enabled): + + current_time = time.time() + + # 根据置信度提供反馈 + if confidence >= 0.8 and current_time - self.last_hand_status_time > 5.0: + self.speech_manager.speak('gesture_good_confidence', immediate=True) + self.last_hand_status_time = current_time + elif confidence < 0.5 and current_time - self.last_hand_status_time > 5.0: + self.speech_manager.speak('gesture_low_confidence', immediate=True) + self.last_hand_status_time = current_time + + # 手势语音播报 + if (final_gesture != "Waiting" and + final_gesture != "Error" and + final_gesture != "摄像头错误" and + final_confidence >= self.min_confidence and + current_time - self.last_announced_time > self.gesture_announce_interval): + + if final_gesture in self.gesture_speech_map: + speech_key = self.gesture_speech_map[final_gesture] + self.speech_manager.speak(speech_key) + else: + self.speech_manager.speak_direct(f"手势{final_gesture}") + + self.last_announced_gesture = final_gesture + self.last_announced_time = current_time + + # 性能报告 + current_time = time.time() + if current_time - self.last_performance_report > 30.0: + if (self.speech_manager and + self.speech_manager.enabled): + + if self.avg_process_time < 20: + self.speech_manager.speak('performance_good', immediate=True) + elif self.avg_process_time > 50: + self.speech_manager.speak('performance_warning', immediate=True) + + self.last_performance_report = current_time + + # 可视化结果 + if hand_data is not None: + # 需要将坐标转换回原始图像大小 + if self.resize_factor != 1.0: + scale_factor = 1.0 / self.resize_factor + if 'center' in hand_data: + hand_data['center'] = (int(hand_data['center'][0] * scale_factor), + int(hand_data['center'][1] * scale_factor)) + if 'bbox' in hand_data: + x1, y1, x2, y2 = hand_data['bbox'] + hand_data['bbox'] = (int(x1 * scale_factor), int(y1 * scale_factor), + int(x2 * scale_factor), int(y2 * scale_factor)) + if 'fingertips' in hand_data: + hand_data['fingertips'] = [(int(x * scale_factor), int(y * scale_factor)) + for (x, y) in hand_data['fingertips']] + if 'palm_center' in hand_data: + hand_data['palm_center'] = (int(hand_data['palm_center'][0] * scale_factor), + int(hand_data['palm_center'][1] * scale_factor)) + + original_frame = self.visualize_detection( + original_frame, hand_data, final_gesture, final_confidence + ) + else: + original_frame = self.visualize_detection( + original_frame, None, final_gesture, final_confidence + ) + + # 更新计数器 + self.frame_counter += 1 + + # 计算处理时间 + process_time = (time.time() - start_time) * 1000 + self.process_times.append(process_time) + + # 更新平均处理时间 + if len(self.process_times) > 0: + self.avg_process_time = np.mean(list(self.process_times)) + + # 更新帧率 + current_time = time.time() + if current_time - self.last_fps_check >= 1.0: + self.frame_rate = self.frame_counter + self.frame_counter = 0 + self.last_fps_check = current_time + + return final_gesture, final_confidence, original_frame + + except Exception as e: + print(f"⚠ 手势识别错误: {e}") + return "Error", 0.0, frame + + def get_performance_stats(self): + """获取性能统计""" + if len(self.process_times) == 0: + return 0.0, self.frame_rate + + return np.mean(list(self.process_times)), self.frame_rate + + def get_performance_mode_info(self): + """获取性能模式信息""" + return { + 'name': self.performance_mode_name, + 'mode': self.performance_mode, + 'color': self.performance_mode_color, + 'detection_interval': self.detection_interval, + 'resize_factor': self.resize_factor, + 'smooth_frames': self.smooth_frames, + 'min_confidence': self.min_confidence + } + + def set_simulated_gesture(self, gesture): + """设置模拟的手势""" + self.current_gesture = gesture + self.current_confidence = 0.9 + + # 模拟手势也触发语音提示 + if (self.speech_manager and + self.speech_manager.enabled): + + if gesture in self.gesture_speech_map: + self.speech_manager.speak(self.gesture_speech_map[gesture]) + else: + self.speech_manager.speak_direct(f"手势{gesture}") + + self.last_announced_gesture = gesture + self.last_announced_time = time.time() \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 7e4ae44fbf..0a63edfb6c 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 详细性能统计优化版 -新增完整的性能监控、统计和报告功能 +手势控制无人机系统 - 主入口模块 +协调所有子模块,实现完整的手势控制无人机系统 作者: xiaoshiyuan888 """ @@ -9,67 +9,59 @@ import os import time import traceback -import json -import math -import threading -import tempfile -import pickle -from datetime import datetime -from PIL import Image, ImageDraw, ImageFont import cv2 import numpy as np -from collections import deque, Counter -import psutil # 新增:用于系统资源监控 -import csv # 新增:用于性能日志导出 -print("=" * 60) -print("Gesture Controlled Drone - Performance Statistics Enhanced") -print("详细性能统计优化版!") -print("=" * 60) - -# ========== 修复导入路径 ========== +# 添加当前目录到路径 current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) +# 导入自定义模块 +from config_manager import ConfigManager +from performance_analyzer import PerformanceAnalyzer +from speech_manager import EnhancedSpeechFeedbackManager +from gesture_recognizer import EnhancedGestureRecognizer +from drone_controller import SimpleDroneController +from ui_renderer import ChineseUIRenderer +from trajectory_recorder import GestureTrajectoryRecorder -# ========== 核心模块导入 ========== -def safe_import(): - """安全导入所有模块""" - modules_status = {} - try: - from PIL import Image, ImageDraw, ImageFont - modules_status['PIL'] = True - print("[PIL] ✓ 图像处理库就绪") - except Exception as e: - modules_status['PIL'] = False - print(f"[PIL] ✗ 导入失败: {e}") - return None, modules_status +# 安全导入所需库 +def safe_import_libs(): + """安全导入所有需要的库""" + libs = {} + status = {} try: import cv2 import numpy as np - modules_status['OpenCV'] = True + libs['cv2'] = cv2 + libs['np'] = np + status['OpenCV'] = True print("[OpenCV] ✓ 计算机视觉库就绪") except Exception as e: - modules_status['OpenCV'] = False + status['OpenCV'] = False print(f"[OpenCV] ✗ 导入失败: {e}") - return None, modules_status + return None, status try: - import psutil - modules_status['psutil'] = True - print("[psutil] ✓ 系统资源监控库就绪") + from PIL import Image, ImageDraw, ImageFont + libs['PIL'] = {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont} + status['PIL'] = True + print("[PIL] ✓ 图像处理库就绪") except Exception as e: - modules_status['psutil'] = False - print(f"[psutil] ✗ 导入失败: {e}") - # 继续运行,psutil不是必需的 + status['PIL'] = False + print(f"[PIL] ✗ 导入失败: {e}") + return None, status + # 尝试导入AirSim airsim_module = None try: - airsim_module = __import__('airsim') - modules_status['AirSim'] = True - print(f"[AirSim] ✓ 成功导入") + import airsim + airsim_module = airsim + libs['airsim'] = airsim_module + status['AirSim'] = True + print("[AirSim] ✓ 成功导入") except ImportError: print("\n" + "!" * 60) print("⚠ AirSim库未找到!") @@ -86,3202 +78,56 @@ def safe_import(): if choice != 'y': sys.exit(1) - # 尝试导入语音合成库 + # 尝试导入语音库 speech_module = None try: - # 尝试导入pyttsx3(离线TTS) import pyttsx3 speech_module = pyttsx3 - modules_status['Speech'] = True + status['Speech'] = True print("[Speech] ✓ pyttsx3语音库就绪 (离线)") except ImportError: print("\n" + "!" * 60) print("⚠ pyttsx3语音库未找到!") print("!" * 60) - print("安装语音库 (使用清华大学镜像源):") - print("1. 安装离线语音库: pip install pyttsx3 -i https://pypi.tuna.tsinghua.edu.cn/simple") - print("2. 或者安装在线语音库: pip install gtts pygame -i https://pypi.tuna.tsinghua.edu.cn/simple") + print("安装语音库:") + print("1. 安装离线语音库: pip install pyttsx3") + print("2. 或者安装在线语音库: pip install gtts pygame") print("!" * 60) - # 尝试其他语音库 try: - # 尝试使用gTTS(需要网络) from gtts import gTTS speech_module = {'gTTS': gTTS, 'type': 'gtts'} - modules_status['Speech'] = True + status['Speech'] = True print("[Speech] ✓ gTTS语音库就绪 (需要网络连接)") - # 尝试导入音频播放库 try: import pygame pygame.mixer.init() speech_module['pygame'] = pygame print("[Speech] ✓ pygame音频播放库就绪") except ImportError: - # 尝试其他播放方式 - try: - import pydub - from pydub import AudioSegment - from pydub.playback import play - speech_module['pydub'] = pydub - speech_module['AudioSegment'] = AudioSegment - speech_module['play'] = play - print("[Speech] ✓ pydub音频播放库就绪") - except ImportError: - # 最后尝试使用系统命令 - if os.name == 'nt': # Windows - speech_module['play_method'] = 'windows' - print("[Speech] ✓ 使用Windows系统命令播放音频") - elif os.name == 'posix': # Linux/Mac - speech_module['play_method'] = 'posix' - print("[Speech] ✓ 使用系统命令播放音频") - else: - print("[Speech] ✗ 所有音频播放库导入失败,语音功能将不可用") - speech_module = None - modules_status['Speech'] = False - + speech_module['play_method'] = 'system' except ImportError: print("[Speech] ✗ 所有语音库导入失败,语音功能将不可用") speech_module = None - modules_status['Speech'] = False - - return { - 'cv2': cv2, - 'np': np, - 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, - 'airsim': airsim_module, - 'speech': speech_module, - 'psutil': psutil if 'psutil' in modules_status and modules_status['psutil'] else None - }, modules_status - - -# 执行导入 -libs, status = safe_import() -if not status.get('OpenCV', False) or not status.get('PIL', False): - print("\n❌ 核心库缺失,无法启动。") - input("按回车键退出...") - sys.exit(1) - -print("-" * 60) -print("✅ 环境检查通过,正在初始化...") -print("-" * 60) - -# 解包库 -cv2, np = libs['cv2'], libs['np'] -Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] - - -# ========== 性能分析器类 ========== -class PerformanceAnalyzer: - """性能分析器 - 监控和报告系统性能""" - - def __init__(self, speech_manager=None): - self.speech_manager = speech_manager - self.start_time = time.time() - self.session_start_time = time.time() - - # 帧率统计 - self.frame_times = deque(maxlen=300) # 保留最近300帧的时间戳 - self.frame_count = 0 - self.fps_history = deque(maxlen=100) # FPS历史记录 - - # 手势识别性能 - self.gesture_recognition_times = deque(maxlen=100) - self.avg_recognition_time = 0 - self.max_recognition_time = 0 - - # 系统资源监控 - self.cpu_usage_history = deque(maxlen=100) - self.memory_usage_history = deque(maxlen=100) - - # 性能事件记录 - self.performance_events = [] - self.performance_snapshots = [] - - # 手势统计 - self.gesture_counts = {} - self.gesture_confidence_sum = {} - self.gesture_confidence_count = {} - - # 错误统计 - self.error_count = 0 - self.warning_count = 0 - - # 无人机控制统计 - self.drone_commands = 0 - self.successful_commands = 0 - self.failed_commands = 0 - - # 轨迹记录统计 - self.recording_sessions = 0 - self.total_trajectory_points = 0 - - # 性能日志 - self.performance_log = [] - self.log_file = os.path.join(current_dir, 'performance_log.csv') - - # 性能阈值 - self.performance_thresholds = { - 'fps_warning': 15, - 'fps_critical': 5, - 'cpu_warning': 80, - 'cpu_critical': 90, - 'memory_warning': 85, - 'memory_critical': 95, - 'recognition_warning': 50, # ms - 'recognition_critical': 100 # ms - } - - # 性能状态 - self.performance_status = "良好" - self.last_performance_report = 0 - self.auto_report_interval = 60 # 自动报告间隔(秒) - - print("✓ 性能分析器已初始化") - - def update_frame(self): - """更新帧统计""" - current_time = time.time() - self.frame_times.append(current_time) - self.frame_count += 1 - - # 计算当前FPS - if len(self.frame_times) > 1: - time_span = self.frame_times[-1] - self.frame_times[0] - if time_span > 0: - current_fps = (len(self.frame_times) - 1) / time_span - self.fps_history.append(current_fps) - - def update_gesture_recognition_time(self, recognition_time_ms): - """更新手势识别时间""" - self.gesture_recognition_times.append(recognition_time_ms) - - # 更新平均识别时间 - if len(self.gesture_recognition_times) > 0: - self.avg_recognition_time = np.mean(list(self.gesture_recognition_times)) - self.max_recognition_time = max(self.max_recognition_time, recognition_time_ms) - - def update_system_resources(self): - """更新系统资源使用情况""" - try: - if libs['psutil']: - cpu_percent = libs['psutil'].cpu_percent(interval=0.1) - memory_percent = libs['psutil'].virtual_memory().percent - - self.cpu_usage_history.append(cpu_percent) - self.memory_usage_history.append(memory_percent) - - # 检查性能问题 - self.check_performance_issues(cpu_percent, memory_percent) - except: - pass # psutil不可用时跳过 - - def check_performance_issues(self, cpu_percent, memory_percent): - """检查性能问题""" - issues = [] - - # 检查FPS - if len(self.fps_history) > 0: - avg_fps = np.mean(list(self.fps_history[-10:])) if len(self.fps_history) >= 10 else self.fps_history[-1] - - if avg_fps < self.performance_thresholds['fps_critical']: - issues.append(("严重", f"帧率过低: {avg_fps:.1f} FPS")) - self.performance_status = "严重" - elif avg_fps < self.performance_thresholds['fps_warning']: - issues.append(("警告", f"帧率较低: {avg_fps:.1f} FPS")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 检查CPU使用率 - if cpu_percent > self.performance_thresholds['cpu_critical']: - issues.append(("严重", f"CPU使用率过高: {cpu_percent:.1f}%")) - self.performance_status = "严重" - elif cpu_percent > self.performance_thresholds['cpu_warning']: - issues.append(("警告", f"CPU使用率较高: {cpu_percent:.1f}%")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 检查内存使用率 - if memory_percent > self.performance_thresholds['memory_critical']: - issues.append(("严重", f"内存使用率过高: {memory_percent:.1f}%")) - self.performance_status = "严重" - elif memory_percent > self.performance_thresholds['memory_warning']: - issues.append(("警告", f"内存使用率较高: {memory_percent:.1f}%")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 检查手势识别时间 - if self.avg_recognition_time > self.performance_thresholds['recognition_critical']: - issues.append(("严重", f"手势识别时间过长: {self.avg_recognition_time:.1f}ms")) - self.performance_status = "严重" - elif self.avg_recognition_time > self.performance_thresholds['recognition_warning']: - issues.append(("警告", f"手势识别时间较长: {self.avg_recognition_time:.1f}ms")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 记录性能事件 - if issues: - for level, message in issues: - self.add_performance_event(level, message) - - # 语音提示(仅在状态变化时) - if (self.speech_manager and - self.speech_manager.enabled and - level == "严重"): - current_time = time.time() - if current_time - self.last_performance_report > 10: # 避免频繁提示 - self.speech_manager.speak_direct(f"性能{level}: {message}") - self.last_performance_report = current_time - - def add_performance_event(self, level, message): - """添加性能事件""" - event = { - 'timestamp': time.time(), - 'level': level, - 'message': message, - 'session_time': time.time() - self.session_start_time - } - self.performance_events.append(event) - - # 记录到日志 - self.log_performance_event(event) - - if level == "警告": - self.warning_count += 1 - elif level == "严重": - self.error_count += 1 - - def log_performance_event(self, event): - """记录性能事件到日志""" - log_entry = { - 'timestamp': datetime.fromtimestamp(event['timestamp']).strftime('%Y-%m-%d %H:%M:%S'), - 'session_time': f"{event['session_time']:.1f}", - 'level': event['level'], - 'message': event['message'] - } - self.performance_log.append(log_entry) - - def record_gesture(self, gesture, confidence): - """记录手势统计""" - if gesture not in self.gesture_counts: - self.gesture_counts[gesture] = 0 - self.gesture_confidence_sum[gesture] = 0 - self.gesture_confidence_count[gesture] = 0 - - self.gesture_counts[gesture] += 1 - self.gesture_confidence_sum[gesture] += confidence - self.gesture_confidence_count[gesture] += 1 - - def record_drone_command(self, success=True): - """记录无人机命令""" - self.drone_commands += 1 - if success: - self.successful_commands += 1 - else: - self.failed_commands += 1 - - def record_recording_session(self, points_count=0): - """记录录制会话""" - self.recording_sessions += 1 - self.total_trajectory_points += points_count - - def take_snapshot(self, label=""): - """拍摄性能快照""" - snapshot = { - 'timestamp': time.time(), - 'label': label, - 'fps': self.get_current_fps(), - 'avg_fps': self.get_average_fps(), - 'avg_recognition_time': self.avg_recognition_time, - 'max_recognition_time': self.max_recognition_time, - 'cpu_usage': self.get_current_cpu_usage(), - 'memory_usage': self.get_current_memory_usage(), - 'gesture_counts': dict(self.gesture_counts), - 'performance_status': self.performance_status, - 'frame_count': self.frame_count, - 'session_duration': time.time() - self.session_start_time - } - self.performance_snapshots.append(snapshot) - - print(f"📸 性能快照已保存: {label}") - return snapshot - - def get_current_fps(self): - """获取当前FPS""" - if len(self.fps_history) > 0: - return self.fps_history[-1] - return 0 - - def get_average_fps(self): - """获取平均FPS""" - if len(self.fps_history) > 0: - return np.mean(list(self.fps_history)) - return 0 - - def get_current_cpu_usage(self): - """获取当前CPU使用率""" - if len(self.cpu_usage_history) > 0: - return self.cpu_usage_history[-1] - return 0 - - def get_current_memory_usage(self): - """获取当前内存使用率""" - if len(self.memory_usage_history) > 0: - return self.memory_usage_history[-1] - return 0 - - def generate_report(self, detailed=True): - """生成性能报告""" - report_time = time.time() - session_duration = report_time - self.session_start_time - - # 基础报告 - report = { - '生成时间': datetime.fromtimestamp(report_time).strftime('%Y-%m-%d %H:%M:%S'), - '会话时长': f"{session_duration:.1f}秒", - '总帧数': self.frame_count, - '平均FPS': f"{self.get_average_fps():.1f}", - '当前FPS': f"{self.get_current_fps():.1f}", - '平均手势识别时间': f"{self.avg_recognition_time:.1f}ms", - '最大手势识别时间': f"{self.max_recognition_time:.1f}ms", - '当前CPU使用率': f"{self.get_current_cpu_usage():.1f}%", - '当前内存使用率': f"{self.get_current_memory_usage():.1f}%", - '性能状态': self.performance_status, - '警告数量': self.warning_count, - '错误数量': self.error_count, - '无人机命令': { - '总数': self.drone_commands, - '成功': self.successful_commands, - '失败': self.failed_commands, - '成功率': f"{(self.successful_commands / self.drone_commands * 100 if self.drone_commands > 0 else 0):.1f}%" - }, - '录制统计': { - '会话数': self.recording_sessions, - '总轨迹点数': self.total_trajectory_points - } - } - - # 详细报告 - if detailed: - # 手势统计 - gesture_stats = {} - for gesture in self.gesture_counts: - count = self.gesture_counts[gesture] - if gesture in self.gesture_confidence_count and self.gesture_confidence_count[gesture] > 0: - avg_confidence = self.gesture_confidence_sum[gesture] / self.gesture_confidence_count[gesture] - else: - avg_confidence = 0 - - gesture_stats[gesture] = { - '次数': count, - '占比': f"{(count / self.frame_count * 100 if self.frame_count > 0 else 0):.1f}%", - '平均置信度': f"{avg_confidence:.1%}" - } - - report['手势统计'] = gesture_stats - - # 性能事件 - if self.performance_events: - recent_events = list(self.performance_events)[-10:] # 最近10个事件 - report['最近性能事件'] = [ - { - '时间': datetime.fromtimestamp(e['timestamp']).strftime('%H:%M:%S'), - '级别': e['level'], - '消息': e['message'] - } - for e in recent_events - ] - - # 性能快照 - if self.performance_snapshots: - report['性能快照数'] = len(self.performance_snapshots) - - # 系统建议 - suggestions = self.generate_suggestions() - if suggestions: - report['优化建议'] = suggestions - - return report - - def generate_suggestions(self): - """生成优化建议""" - suggestions = [] - - # 检查FPS - avg_fps = self.get_average_fps() - if avg_fps < self.performance_thresholds['fps_warning']: - suggestions.append(f"帧率较低({avg_fps:.1f}FPS),建议切换到'最快'性能模式") - - # 检查CPU - cpu_usage = self.get_current_cpu_usage() - if cpu_usage > self.performance_thresholds['cpu_warning']: - suggestions.append(f"CPU使用率较高({cpu_usage:.1f}%),请关闭其他占用CPU的程序") - - # 检查内存 - memory_usage = self.get_current_memory_usage() - if memory_usage > self.performance_thresholds['memory_warning']: - suggestions.append(f"内存使用率较高({memory_usage:.1f}%),请关闭不必要的程序") - - # 检查识别时间 - if self.avg_recognition_time > self.performance_thresholds['recognition_warning']: - suggestions.append(f"手势识别时间较长({self.avg_recognition_time:.1f}ms),建议调整摄像头位置或光线") - - return suggestions - - def print_report(self, detailed=True): - """打印性能报告""" - report = self.generate_report(detailed) - - print("\n" + "=" * 80) - print("📊 性能分析报告") - print("=" * 80) - - # 基础信息 - print(f"生成时间: {report['生成时间']}") - print(f"会话时长: {report['会话时长']}") - print(f"总帧数: {report['总帧数']}") - print(f"平均FPS: {report['平均FPS']}") - print(f"当前FPS: {report['当前FPS']}") - print(f"平均手势识别时间: {report['平均手势识别时间']}") - print(f"最大手势识别时间: {report['最大手势识别时间']}") - print(f"当前CPU使用率: {report['当前CPU使用率']}") - print(f"当前内存使用率: {report['当前内存使用率']}") - print(f"性能状态: {report['性能状态']}") - - # 无人机命令统计 - cmd_stats = report['无人机命令'] - print(f"\n无人机命令统计:") - print(f" 总数: {cmd_stats['总数']}") - print(f" 成功: {cmd_stats['成功']}") - print(f" 失败: {cmd_stats['失败']}") - print(f" 成功率: {cmd_stats['成功率']}") - - # 录制统计 - rec_stats = report['录制统计'] - print(f"\n录制统计:") - print(f" 会话数: {rec_stats['会话数']}") - print(f" 总轨迹点数: {rec_stats['总轨迹点数']}") - - # 详细报告 - if detailed and '手势统计' in report: - print(f"\n手势统计:") - for gesture, stats in report['手势统计'].items(): - print(f" {gesture}: {stats['次数']}次 ({stats['占比']}), 平均置信度: {stats['平均置信度']}") - - # 性能事件 - if detailed and '最近性能事件' in report and report['最近性能事件']: - print(f"\n最近性能事件:") - for event in report['最近性能事件']: - print(f" [{event['时间']}] {event['级别']}: {event['消息']}") - - # 优化建议 - if detailed and '优化建议' in report and report['优化建议']: - print(f"\n优化建议:") - for i, suggestion in enumerate(report['优化建议'], 1): - print(f" {i}. {suggestion}") - - print("=" * 80) - - # 语音播报摘要 - if self.speech_manager and self.speech_manager.enabled: - summary = (f"性能报告: 平均帧率{report['平均FPS']},识别时间{report['平均手势识别时间']}," - f"性能状态{report['性能状态']},无人机命令成功率{cmd_stats['成功率']}") - self.speech_manager.speak_direct(summary) - - def export_log(self, filename=None): - """导出性能日志""" - if filename is None: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - filename = os.path.join(current_dir, f'performance_log_{timestamp}.csv') - - try: - with open(filename, 'w', newline='', encoding='utf-8') as f: - if self.performance_log: - fieldnames = self.performance_log[0].keys() - writer = csv.DictWriter(f, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(self.performance_log) - - print(f"📤 性能日志已导出到: {filename}") - return True - except Exception as e: - print(f"❌ 导出性能日志失败: {e}") - return False - - def auto_report(self): - """自动性能报告(定期执行)""" - current_time = time.time() - if current_time - self.last_performance_report > self.auto_report_interval: - # 生成简要报告 - report = self.generate_report(detailed=False) - - # 检查是否需要报告 - if (self.performance_status == "严重" or - self.warning_count > 5 or - self.error_count > 0): - - print(f"⚠ 自动性能检查: {report['性能状态']}, FPS: {report['当前FPS']}, " - f"CPU: {report['当前CPU使用率']}, 内存: {report['当前内存使用率']}") - - # 语音提示 - if (self.speech_manager and - self.speech_manager.enabled and - self.performance_status == "严重"): - self.speech_manager.speak_direct(f"系统性能{self.performance_status},建议检查") - - self.last_performance_report = current_time - - def reset_session(self): - """重置会话统计""" - self.session_start_time = time.time() - self.performance_events = [] - self.performance_snapshots = [] - self.gesture_counts = {} - self.gesture_confidence_sum = {} - self.gesture_confidence_count = {} - self.error_count = 0 - self.warning_count = 0 - self.drone_commands = 0 - self.successful_commands = 0 - self.failed_commands = 0 - self.performance_status = "良好" - - print("✓ 性能统计会话已重置") - - def get_stats_summary(self): - """获取统计摘要""" - return { - 'fps': self.get_current_fps(), - 'avg_fps': self.get_average_fps(), - 'recognition_time': self.avg_recognition_time, - 'cpu_usage': self.get_current_cpu_usage(), - 'memory_usage': self.get_current_memory_usage(), - 'performance_status': self.performance_status, - 'gesture_count': sum(self.gesture_counts.values()), - 'unique_gestures': len(self.gesture_counts) - } - - -# ========== 增强语音反馈管理器 ========== -class EnhancedSpeechFeedbackManager: - """增强的语音反馈管理器""" - - def __init__(self, speech_lib): - self.speech_lib = speech_lib - self.enabled = True - self.volume = 1.0 - self.rate = 150 - self.voice_id = None - self.last_speech_time = {} - self.min_interval = 1.5 # 缩短相同语音的最小间隔(秒) - - # 语音队列,避免语音重叠 - self.speech_queue = [] - self.is_speaking = False - self.queue_thread = None - - # 音频播放方法 - self.audio_method = None - - # 新增:手势状态追踪 - self.last_gesture_state = "none" # 记录上次手势状态 - self.gesture_active_time = 0 # 手势持续活跃时间 - - # 增强的语音消息映射 - self.messages = { - # 连接相关 - 'connecting': "正在连接无人机,请稍候", - 'connected': "无人机连接成功", - 'connection_failed': "无人机连接失败,进入模拟模式", - - # 飞行相关 - 'taking_off': "无人机正在起飞", - 'takeoff_success': "起飞成功", - 'takeoff_failed': "起飞失败", - 'landing': "无人机正在降落", - 'land_success': "降落成功", - 'emergency_stop': "紧急停止,无人机已降落", - 'hovering': "无人机悬停中", - - # 手势相关 - 增强 - 'gesture_detected': "手势识别就绪,请开始手势", - 'gesture_start': "开始识别手势", - 'gesture_end': "手势识别结束", - 'gesture_stop': "停止", - 'gesture_up': "向上", - 'gesture_down': "向下", - 'gesture_left': "向左", - 'gesture_right': "向右", - 'gesture_forward': "向前", - 'gesture_backward': "向后", - 'gesture_waiting': "等待手势", - 'gesture_error': "手势识别错误", - 'gesture_stable': "手势稳定", - 'gesture_change': "手势变化", - 'gesture_low_confidence': "手势识别置信度低", - 'gesture_good_confidence': "手势识别置信度高", - 'gesture_hover': "悬停", - - # 系统相关 - 'program_start': "手势控制无人机系统已启动", - 'program_exit': "程序退出,感谢使用", - 'camera_error': "摄像头错误,请检查连接", - 'camera_ready': "摄像头就绪", - 'system_ready': "系统准备就绪", - - # 模式相关 - 'simulation_mode': "进入模拟模式", - 'debug_mode_on': "调试模式已开启", - 'debug_mode_off': "调试模式已关闭", - 'display_mode_changed': "显示模式已切换", - 'help_toggled': "帮助信息已切换", - 'performance_mode_fast': "切换到最快性能模式", - 'performance_mode_balanced': "切换到平衡性能模式", - 'performance_mode_accurate': "切换到最准确性能模式", - - # 新增:性能相关 - 'performance_good': "系统运行流畅", - 'performance_warning': "系统性能警告", - 'performance_critical': "系统性能严重警告", - 'performance_report': "性能报告生成完成", - 'performance_snapshot': "性能快照已保存", - 'performance_log_exported': "性能日志已导出", - - # 新增:手势指导 - 'move_closer': "请将手靠近摄像头", - 'move_away': "请将手移远一些", - 'good_position': "手部位置良好", - 'hand_detected': "手部已检测到", - 'hand_lost': "手部丢失,请重新放置", - - # 新增:录制相关 - 'recording_start': "开始录制手势轨迹", - 'recording_stop': "停止录制", - 'recording_saved': "轨迹已保存", - 'recording_loaded': "轨迹已加载", - 'recording_playback_start': "开始回放手势轨迹", - 'recording_playback_stop': "回放结束", - 'recording_cleared': "轨迹已清除", - 'recording_paused': "回放已暂停", - 'recording_resumed': "回放继续", - 'recording_not_found': "未找到轨迹数据", - 'recording_frame_count': "轨迹帧数", - } - - # 初始化语音引擎 - self.init_speech_engine() - - def init_speech_engine(self): - """初始化语音引擎""" - if self.speech_lib is None: - print("⚠ 语音库未找到,语音功能禁用") - self.enabled = False - return - - try: - if hasattr(self.speech_lib, 'init'): # pyttsx3 - self.engine = self.speech_lib.init() - self.audio_method = 'pyttsx3' - - # 设置语音参数 - voices = self.engine.getProperty('voices') - - # 尝试寻找中文语音 - for voice in voices: - # 检查语音名称是否包含中文相关标识 - if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower() or 'zh_CN' in voice.name.lower(): - self.engine.setProperty('voice', voice.id) - self.voice_id = voice.id - print(f"[Speech] 使用中文语音: {voice.name}") - break - - # 如果没找到中文语音,使用第一个可用语音 - if self.voice_id is None and len(voices) > 0: - self.engine.setProperty('voice', voices[0].id) - print(f"[Speech] 使用默认语音: {voices[0].name}") - - # 设置语速和音量 - self.engine.setProperty('rate', self.rate) - self.engine.setProperty('volume', self.volume) - - print("✅ 语音引擎初始化成功 (pyttsx3)") - - elif isinstance(self.speech_lib, dict) and self.speech_lib.get('type') == 'gtts': - print("✅ 语音引擎初始化成功 (gTTS,需要网络连接)") - self.audio_method = 'gtts' - - # 确定播放方法 - if 'pygame' in self.speech_lib: - self.audio_method = 'gtts_pygame' - print("✅ 使用pygame播放音频") - elif 'pydub' in self.speech_lib: - self.audio_method = 'gtts_pydub' - print("✅ 使用pydub播放音频") - elif 'play_method' in self.speech_lib: - self.audio_method = f"gtts_{self.speech_lib['play_method']}" - print(f"✅ 使用系统命令播放音频") - else: - self.audio_method = 'gtts_system' - print("✅ 使用默认系统播放器") - - else: - print("⚠ 未知语音库类型,语音功能可能不正常") - self.enabled = False - - except Exception as e: - print(f"⚠ 语音引擎初始化失败: {e}") - self.enabled = False - - def play_audio_file(self, audio_file): - """播放音频文件(根据可用库选择方法)""" - try: - if self.audio_method == 'gtts_pygame' and 'pygame' in self.speech_lib: - pygame = self.speech_lib['pygame'] - pygame.mixer.music.load(audio_file) - pygame.mixer.music.play() - - # 等待播放完成 - while pygame.mixer.music.get_busy(): - pygame.time.Clock().tick(10) - - elif self.audio_method == 'gtts_pydub' and 'pydub' in self.speech_lib: - AudioSegment = self.speech_lib['AudioSegment'] - play = self.speech_lib['play'] - - audio = AudioSegment.from_mp3(audio_file) - play(audio) - - elif self.audio_method == 'gtts_windows': - # Windows系统命令 - os.startfile(audio_file) - # 等待播放完成(简单等待) - time.sleep(1.5) - - elif self.audio_method == 'gtts_posix': - # Linux/Mac系统命令 - import subprocess - if sys.platform == 'darwin': # macOS - subprocess.call(['afplay', audio_file]) - else: # Linux - subprocess.call(['xdg-open', audio_file]) - - else: - # 通用方法:使用系统默认播放器 - import subprocess - if sys.platform == 'win32': - os.startfile(audio_file) - elif sys.platform == 'darwin': - subprocess.call(['open', audio_file]) - else: - subprocess.call(['xdg-open', audio_file]) - - return True - - except Exception as e: - print(f"⚠ 音频播放失败: {e}") - return False - - def speak(self, message_key, force=False, immediate=False): - """播放语音""" - if not self.enabled: - return - - # 检查是否在最小间隔内 - current_time = time.time() - if not force and message_key in self.last_speech_time: - if current_time - self.last_speech_time[message_key] < self.min_interval: - return - - # 获取消息文本 - if message_key in self.messages: - text = self.messages[message_key] - else: - text = message_key # 直接使用传入的文本 - - # 如果是立即播放,直接在新线程中播放 - if immediate: - self.speak_direct(text) - else: - # 添加到语音队列 - self.speech_queue.append(text) - - # 如果没有在播放,启动播放线程 - if not self.is_speaking and self.queue_thread is None: - self.queue_thread = threading.Thread(target=self._process_speech_queue) - self.queue_thread.daemon = True - self.queue_thread.start() - - # 更新时间戳 - self.last_speech_time[message_key] = current_time - - def _process_speech_queue(self): - """处理语音队列""" - while self.speech_queue and self.enabled: - self.is_speaking = True - - text = self.speech_queue.pop(0) - - try: - if self.audio_method == 'pyttsx3': - # 清除之前的语音 - self.engine.stop() - # 播报新语音 - self.engine.say(text) - self.engine.runAndWait() - - elif self.audio_method.startswith('gtts'): - # gTTS需要网络连接 - tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') - - # 保存临时文件 - with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: - temp_file = f.name - tts.save(temp_file) - - # 播放音频 - self.play_audio_file(temp_file) - - # 删除临时文件 - try: - os.unlink(temp_file) - except: - pass # 忽略删除错误 - - except Exception as e: - print(f"⚠ 语音播放失败: {e}") - - time.sleep(0.05) # 减少等待时间,避免卡顿 - - self.is_speaking = False - self.queue_thread = None - - def speak_direct(self, text): - """直接播放文本(不通过消息映射)""" - if not self.enabled: - return - - # 在新线程中播放 - thread = threading.Thread(target=self._speak_thread, args=(text,)) - thread.daemon = True - thread.start() - - def _speak_thread(self, text): - """语音播放线程""" - try: - if self.audio_method == 'pyttsx3': - self.engine.say(text) - self.engine.runAndWait() - - elif self.audio_method.startswith('gtts'): - tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') - - with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: - temp_file = f.name - tts.save(temp_file) - - self.play_audio_file(temp_file) - - try: - os.unlink(temp_file) - except: - pass - - except Exception as e: - print(f"⚠ 直接语音播放失败: {e}") - - def stop(self): - """停止所有语音""" - if hasattr(self, 'engine'): - self.engine.stop() - - self.speech_queue.clear() - self.is_speaking = False - - def set_enabled(self, enabled): - """启用/禁用语音""" - self.enabled = enabled - if not enabled: - self.stop() - - def toggle_enabled(self): - """切换语音启用状态""" - self.enabled = not self.enabled - status = "启用" if self.enabled else "禁用" - self.speak_direct(f"语音反馈已{status}") - return self.enabled - - def get_status(self): - """获取语音状态""" - return { - 'enabled': self.enabled, - 'engine': 'pyttsx3' if self.audio_method == 'pyttsx3' else - 'gTTS' if self.audio_method.startswith('gtts') else - 'None', - 'queue_size': len(self.speech_queue), - 'is_speaking': self.is_speaking, - 'audio_method': self.audio_method - } - - -# ========== 手势轨迹记录器 ========== -class GestureTrajectoryRecorder: - """手势轨迹记录器 - 记录、保存、加载和回放手势轨迹""" - - def __init__(self, speech_manager=None): - self.speech_manager = speech_manager - self.trajectory_data = [] - self.is_recording = False - self.is_playing = False - self.playback_index = 0 - self.playback_paused = False - self.max_trajectory_points = 1000 # 最大轨迹点数 - self.recording_start_time = 0 - self.last_save_time = 0 - self.save_interval = 5 # 自动保存间隔(秒) - - # 轨迹文件路径 - self.trajectory_dir = os.path.join(current_dir, 'trajectories') - if not os.path.exists(self.trajectory_dir): - os.makedirs(self.trajectory_dir) - - # 默认轨迹文件名 - self.default_filename = os.path.join(self.trajectory_dir, - f'trajectory_{datetime.now().strftime("%Y%m%d_%H%M%S")}.pkl') - - # 轨迹可视化设置 - self.trajectory_colors = [ - (255, 0, 0), # 红色 - 起点 - (0, 255, 0), # 绿色 - 正常点 - (0, 0, 255), # 蓝色 - 终点 - (255, 255, 0), # 青色 - 特殊点 - (255, 0, 255) # 紫色 - 特殊点 - ] - - self.show_trajectory = True - self.trajectory_thickness = 2 - self.trajectory_max_length = 100 # 显示的最大轨迹长度 - - print("✓ 手势轨迹记录器已初始化") - - def start_recording(self): - """开始录制手势轨迹""" - if self.is_recording: - return False - - self.trajectory_data = [] - self.is_recording = True - self.recording_start_time = time.time() - self.last_save_time = time.time() - - print("🎬 开始录制手势轨迹") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_start', immediate=True) - - return True - - def stop_recording(self): - """停止录制手势轨迹""" - if not self.is_recording: - return False - - self.is_recording = False - recording_duration = time.time() - self.recording_start_time - - print(f"⏹️ 停止录制手势轨迹") - print(f" 录制时长: {recording_duration:.1f}秒") - print(f" 轨迹点数: {len(self.trajectory_data)}") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_stop', immediate=True) - if len(self.trajectory_data) > 0: - self.speech_manager.speak_direct(f"录制了{len(self.trajectory_data)}个轨迹点") - - return True - - def add_trajectory_point(self, hand_data, gesture, confidence, frame_shape): - """添加轨迹点""" - if not self.is_recording or len(self.trajectory_data) >= self.max_trajectory_points: - return False - - if hand_data is None: - return False - - # 创建轨迹点数据 - trajectory_point = { - 'timestamp': time.time(), - 'hand_position': hand_data['position'] if 'position' in hand_data else (0.5, 0.5), - 'hand_center': hand_data['center'] if 'center' in hand_data else (0, 0), - 'gesture': gesture, - 'confidence': confidence, - 'fingertips': hand_data.get('fingertips', []), - 'frame_shape': frame_shape - } - - self.trajectory_data.append(trajectory_point) - - # 自动保存检查 - current_time = time.time() - if current_time - self.last_save_time >= self.save_interval and len(self.trajectory_data) > 10: - self.auto_save() - self.last_save_time = current_time - - return True - - def auto_save(self): - """自动保存轨迹(临时文件)""" - if len(self.trajectory_data) == 0: - return - - temp_file = os.path.join(self.trajectory_dir, 'trajectory_temp.pkl') - try: - with open(temp_file, 'wb') as f: - pickle.dump(self.trajectory_data, f) - print(f"💾 自动保存轨迹到临时文件 ({len(self.trajectory_data)}个点)") - except Exception as e: - print(f"⚠ 自动保存轨迹失败: {e}") - - def save_trajectory(self, filename=None): - """保存轨迹到文件""" - if len(self.trajectory_data) == 0: - print("⚠ 没有轨迹数据可保存") - return False - - if filename is None: - filename = self.default_filename - - try: - with open(filename, 'wb') as f: - pickle.dump(self.trajectory_data, f) - - print(f"💾 轨迹已保存到: {filename}") - print(f" 轨迹点数: {len(self.trajectory_data)}") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_saved', immediate=True) - self.speech_manager.speak_direct(f"保存了{len(self.trajectory_data)}个轨迹点") - - return True - except Exception as e: - print(f"❌ 保存轨迹失败: {e}") - return False - - def load_trajectory(self, filename): - """从文件加载轨迹""" - try: - if not os.path.exists(filename): - print(f"❌ 轨迹文件不存在: {filename}") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_not_found', immediate=True) - - return False - - with open(filename, 'rb') as f: - self.trajectory_data = pickle.load(f) - - print(f"📂 轨迹已从文件加载: {filename}") - print(f" 轨迹点数: {len(self.trajectory_data)}") + status['Speech'] = False - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_loaded', immediate=True) - self.speech_manager.speak_direct(f"加载了{len(self.trajectory_data)}个轨迹点") - - return True - except Exception as e: - print(f"❌ 加载轨迹失败: {e}") - return False - - def start_playback(self): - """开始回放轨迹""" - if len(self.trajectory_data) == 0: - print("⚠ 没有轨迹数据可回放") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_not_found', immediate=True) - - return False - - self.is_playing = True - self.playback_index = 0 - self.playback_paused = False - - print(f"▶️ 开始回放手势轨迹") - print(f" 总帧数: {len(self.trajectory_data)}") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_playback_start', immediate=True) - - return True - - def stop_playback(self): - """停止回放轨迹""" - if not self.is_playing: - return False - - self.is_playing = False - self.playback_paused = False - - print("⏹️ 停止回放手势轨迹") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_playback_stop', immediate=True) - - return True - - def pause_playback(self): - """暂停/继续回放""" - self.playback_paused = not self.playback_paused - - status = "暂停" if self.playback_paused else "继续" - print(f"⏸️ 回放已{status}") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - if self.playback_paused: - self.speech_manager.speak('recording_paused', immediate=True) - else: - self.speech_manager.speak('recording_resumed', immediate=True) - - return self.playback_paused - - def get_next_playback_point(self): - """获取下一个回放点""" - if not self.is_playing or self.playback_paused or len(self.trajectory_data) == 0: - return None - - if self.playback_index >= len(self.trajectory_data): - self.stop_playback() - return None - - point = self.trajectory_data[self.playback_index] - self.playback_index += 1 - - # 检查是否到达末尾 - if self.playback_index >= len(self.trajectory_data): - self.stop_playback() - - return point - - def clear_trajectory(self): - """清除轨迹数据""" - self.trajectory_data = [] - self.is_recording = False - self.is_playing = False - self.playback_index = 0 - - print("🗑️ 轨迹数据已清除") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_cleared', immediate=True) - - return True - - def draw_trajectory(self, frame): - """在帧上绘制轨迹""" - if not self.show_trajectory or len(self.trajectory_data) == 0: - return frame - - h, w = frame.shape[:2] - - # 限制显示的轨迹点数 - display_points = min(len(self.trajectory_data), self.trajectory_max_length) - start_idx = max(0, len(self.trajectory_data) - display_points) - - # 绘制轨迹线 - for i in range(start_idx, len(self.trajectory_data) - 1): - point1 = self.trajectory_data[i] - point2 = self.trajectory_data[i + 1] - - # 获取手部中心位置(转换为图像坐标) - if 'hand_center' in point1 and 'hand_center' in point2: - x1, y1 = point1['hand_center'] - x2, y2 = point2['hand_center'] - - # 确保坐标在图像范围内 - x1 = max(0, min(w - 1, x1)) - y1 = max(0, min(h - 1, y1)) - x2 = max(0, min(w - 1, x2)) - y2 = max(0, min(h - 1, y2)) - - # 根据索引计算颜色(渐变色) - color_idx = int((i - start_idx) / display_points * (len(self.trajectory_colors) - 1)) - color = self.trajectory_colors[color_idx] - - # 绘制线条 - cv2.line(frame, (int(x1), int(y1)), (int(x2), int(y2)), - color, self.trajectory_thickness) - - # 绘制当前点(如果正在录制或回放) - if self.is_recording or self.is_playing: - current_idx = len(self.trajectory_data) - 1 if self.is_recording else self.playback_index - 1 - if 0 <= current_idx < len(self.trajectory_data): - point = self.trajectory_data[current_idx] - if 'hand_center' in point: - x, y = point['hand_center'] - x = max(0, min(w - 1, x)) - y = max(0, min(h - 1, y)) - - # 绘制当前点 - cv2.circle(frame, (int(x), int(y)), 8, (0, 255, 255), -1) - cv2.circle(frame, (int(x), int(y)), 8, (0, 0, 0), 2) - - return frame - - def get_status(self): - """获取录制状态""" - return { - 'is_recording': self.is_recording, - 'is_playing': self.is_playing, - 'playback_paused': self.playback_paused, - 'trajectory_points': len(self.trajectory_data), - 'playback_index': self.playback_index, - 'playback_total': len(self.trajectory_data), - 'recording_duration': time.time() - self.recording_start_time if self.is_recording else 0 - } - - def list_saved_trajectories(self): - """列出保存的轨迹文件""" - try: - files = [f for f in os.listdir(self.trajectory_dir) if f.endswith('.pkl')] - return sorted(files, reverse=True) - except: - return [] - - -# ========== 配置管理器 ========== -class ConfigManager: - """配置管理器""" - - def __init__(self): - self.config_file = os.path.join(current_dir, 'gesture_config.json') - - # 性能模式配置 - self.performance_modes = { - 'fast': { - 'name': '最快', - 'description': '性能优先,降低识别精度换取更高帧率', - 'detection_interval': 2, # 每2帧检测一次 - 'smooth_frames': 3, # 平滑帧数 - 'min_confidence': 0.5, # 最小置信度阈值 - 'resize_factor': 0.5, # 图像缩放因子 - 'skin_detection_enabled': True, - 'background_subtraction_enabled': False, - 'contour_simplify_epsilon': 0.03, - 'history_size': 10, - 'gesture_stability_threshold': 3, - 'color': (0, 255, 0), # 绿色 - }, - 'balanced': { - 'name': '平衡', - 'description': '平衡性能与精度,适用于大多数场景', - 'detection_interval': 1, # 每帧检测 - 'smooth_frames': 5, # 平滑帧数 - 'min_confidence': 0.6, # 最小置信度阈值 - 'resize_factor': 0.75, # 图像缩放因子 - 'skin_detection_enabled': True, - 'background_subtraction_enabled': True, - 'contour_simplify_epsilon': 0.02, - 'history_size': 15, - 'gesture_stability_threshold': 5, - 'color': (255, 165, 0), # 橙色 - }, - 'accurate': { - 'name': '最准', - 'description': '精度优先,提供最准确的手势识别', - 'detection_interval': 1, # 每帧检测 - 'smooth_frames': 7, # 平滑帧数 - 'min_confidence': 0.7, # 最小置信度阈值 - 'resize_factor': 1.0, # 不缩放 - 'skin_detection_enabled': True, - 'background_subtraction_enabled': True, - 'contour_simplify_epsilon': 0.01, - 'history_size': 20, - 'gesture_stability_threshold': 7, - 'color': (255, 0, 0), # 红色 - } - } - - self.default_config = { - 'camera': { - 'index': 0, - 'width': 640, - 'height': 480, - 'fps': 30 - }, - 'gesture': { - 'skin_lower_h': 0, - 'skin_upper_h': 25, - 'skin_lower_s': 30, - 'skin_upper_s': 255, - 'skin_lower_v': 60, - 'skin_upper_v': 255, - 'min_hand_area': 2000, - 'max_hand_area': 30000, - 'hand_ratio_threshold': 1.5, - 'defect_distance_threshold': 20, - 'palm_circle_radius_ratio': 0.3, - 'transition_threshold': 0.3, - 'position_stability_weight': 0.4, - 'gesture_cooldown': 0.5, - }, - 'drone': { - 'velocity': 2.5, - 'duration': 0.3, - 'altitude': -10.0, - 'control_interval': 0.3 - }, - 'display': { - 'show_fps': True, - 'show_confidence': True, - 'show_help': True, - 'show_contours': True, - 'show_bbox': True, - 'show_fingertips': True, - 'show_palm_center': True, - 'show_hand_direction': True, - 'show_debug_info': False, - 'show_speech_status': True, - 'show_gesture_history': True, - 'show_stability_indicator': True, - 'show_trajectory': True, - 'show_recording_status': True, - 'show_performance_mode': True, # 新增:显示性能模式 - 'show_performance_stats': True, # 新增:显示性能统计 - 'show_system_resources': True, # 新增:显示系统资源 - }, - 'performance': { - 'target_fps': 30, - 'resize_factor': 1.0, - 'enable_multiprocessing': False, - 'mode': 'balanced', # 新增:性能模式选择 - 'current_mode_index': 1, # 当前模式索引 - 'modes': ['fast', 'balanced', 'accurate'], # 可用模式列表 - 'auto_report_interval': 60, # 自动报告间隔 - 'enable_performance_monitor': True, # 启用性能监控 - }, - 'calibration': { - 'auto_calibrate_skin': True, - 'skin_calibration_frames': 30, - 'hand_size_calibration': True - }, - 'speech': { - 'enabled': True, - 'volume': 1.0, - 'rate': 150, - 'announce_gestures': True, - 'announce_connections': True, - 'announce_flight_events': True, - 'announce_gesture_changes': True, - 'announce_hand_status': True, - 'announce_performance': True, - 'announce_recording_events': True, - 'announce_performance_mode': True, # 新增:播报性能模式切换 - 'announce_performance_events': True, # 新增:播报性能事件 - 'min_gesture_confidence': 0.7, - 'gesture_start_threshold': 3, - 'gesture_end_threshold': 10, - }, - 'recording': { - 'auto_save_interval': 5, - 'max_trajectory_points': 1000, - 'show_trajectory': True, - 'trajectory_thickness': 2, - 'trajectory_max_length': 100, - 'default_save_dir': 'trajectories', - } - } - self.config = self.load_config() - self.skin_calibration_data = [] - self.hand_size_calibration_done = False - self.reference_hand_size = 0 - - def load_config(self): - """加载配置""" - if os.path.exists(self.config_file): - try: - with open(self.config_file, 'r', encoding='utf-8') as f: - loaded_config = json.load(f) - config = self.default_config.copy() - self._merge_config(config, loaded_config) - print("✓ 从文件加载配置") - return config - except Exception as e: - print(f"⚠ 加载配置失败: {e}, 使用默认配置") - return self.default_config.copy() - else: - print("✓ 使用默认配置") - return self.default_config.copy() - - def _merge_config(self, base, update): - """递归合并配置""" - for key, value in update.items(): - if key in base and isinstance(base[key], dict) and isinstance(value, dict): - self._merge_config(base[key], value) - else: - base[key] = value - - def save_config(self): - """保存配置""" - try: - with open(self.config_file, 'w', encoding='utf-8') as f: - json.dump(self.config, f, indent=2, ensure_ascii=False) - print("✓ 配置已保存") - except Exception as e: - print(f"⚠ 保存配置失败: {e}") - - def get(self, *keys): - """获取配置值""" - value = self.config - for key in keys: - if isinstance(value, dict) and key in value: - value = value[key] - else: - return None - return value - - def set(self, *keys, value): - """设置配置值""" - if len(keys) == 0: - return - - config = self.config - for key in keys[:-1]: - if key not in config: - config[key] = {} - config = config[key] - - config[keys[-1]] = value - self.save_config() - - def get_performance_mode_config(self, mode=None): - """获取性能模式配置""" - if mode is None: - mode = self.get('performance', 'mode') - - if mode in self.performance_modes: - return self.performance_modes[mode] - else: - return self.performance_modes['balanced'] - - def get_current_performance_mode(self): - """获取当前性能模式""" - mode = self.get('performance', 'mode') - if mode in self.performance_modes: - return mode - return 'balanced' - - def set_performance_mode(self, mode): - """设置性能模式""" - if mode in self.performance_modes: - self.set('performance', 'mode', value=mode) - - # 更新当前模式索引 - modes = self.get('performance', 'modes') - if modes and mode in modes: - index = modes.index(mode) - self.set('performance', 'current_mode_index', value=index) - - print(f"✓ 性能模式设置为: {self.performance_modes[mode]['name']}") - return True - return False - - def cycle_performance_mode(self): - """循环切换性能模式""" - modes = self.get('performance', 'modes') - if not modes: - modes = ['fast', 'balanced', 'accurate'] - - current_index = self.get('performance', 'current_mode_index') - if current_index is None: - current_index = 0 - - # 计算下一个模式索引 - next_index = (current_index + 1) % len(modes) - next_mode = modes[next_index] - - # 设置新模式 - self.set('performance', 'current_mode_index', value=next_index) - return self.set_performance_mode(next_mode) - - def calibrate_skin_color(self, frame, hand_mask): - """自动校准肤色范围""" - if not self.get('calibration', 'auto_calibrate_skin'): - return - - if len(self.skin_calibration_data) < self.get('calibration', 'skin_calibration_frames'): - # 转换到HSV - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - - # 获取肤色区域的HSV值 - skin_pixels = hsv[hand_mask > 0] - - if len(skin_pixels) > 100: # 确保有足够的像素 - self.skin_calibration_data.append(skin_pixels) - - if len(self.skin_calibration_data) == self.get('calibration', 'skin_calibration_frames'): - # 计算肤色范围 - all_skin_pixels = np.vstack(self.skin_calibration_data) - - h_min, h_max = np.percentile(all_skin_pixels[:, 0], [2, 98]) - s_min, s_max = np.percentile(all_skin_pixels[:, 1], [2, 98]) - v_min, v_max = np.percentile(all_skin_pixels[:, 2], [2, 98]) - - # 更新配置 - self.set('gesture', 'skin_lower_h', value=int(max(0, h_min - 5))) - self.set('gesture', 'skin_upper_h', value=int(min(180, h_max + 5))) - self.set('gesture', 'skin_lower_s', value=int(max(0, s_min - 10))) - self.set('gesture', 'skin_upper_s', value=int(min(255, s_max + 10))) - self.set('gesture', 'skin_lower_v', value=int(max(0, v_min - 10))) - self.set('gesture', 'skin_upper_v', value=int(min(255, v_max + 10))) - - print("✓ 肤色校准完成") - print(f" 肤色范围: H[{self.get('gesture', 'skin_lower_h')}-{self.get('gesture', 'skin_upper_h')}], " - f"S[{self.get('gesture', 'skin_lower_s')}-{self.get('gesture', 'skin_upper_s')}], " - f"V[{self.get('gesture', 'skin_lower_v')}-{self.get('gesture', 'skin_upper_v')}]") - - def calibrate_hand_size(self, hand_area): - """校准手部大小""" - if not self.get('calibration', 'hand_size_calibration') or self.hand_size_calibration_done: - return - - if hand_area > 0: - self.reference_hand_size = hand_area - self.hand_size_calibration_done = True - print(f"✓ 手部大小校准完成: {self.reference_hand_size:.0f} 像素") - - -config = ConfigManager() - - -# ========== 改进的手势识别器(支持性能模式) ========== -class EnhancedGestureRecognizer: - """增强的手势识别器 - 支持性能模式选择""" - - def __init__(self, speech_manager=None): - self.speech_manager = speech_manager - - # 加载性能模式配置 - self.performance_mode = config.get_current_performance_mode() - self.mode_config = config.get_performance_mode_config(self.performance_mode) - - # 根据性能模式初始化参数 - self.history_size = self.mode_config['history_size'] - self.detection_interval = self.mode_config['detection_interval'] - self.smooth_frames = self.mode_config['smooth_frames'] - self.min_confidence = self.mode_config['min_confidence'] - self.resize_factor = self.mode_config['resize_factor'] - - # 增强的手势历史和平滑 - self.gesture_history = deque(maxlen=self.history_size) - self.confidence_history = deque(maxlen=self.history_size) - self.position_history = deque(maxlen=self.history_size) - self.current_gesture = "Waiting" - self.current_confidence = 0.0 - - # 新增:手势状态追踪 - self.gesture_state = "none" - self.gesture_stability_counter = 0 - self.last_stable_gesture = "Waiting" - self.gesture_active_frames = 0 - self.last_gesture_change_time = 0 - - # 记录上次播报的手势 - self.last_announced_gesture = None - self.last_announced_time = 0 - self.last_hand_status_time = 0 - self.gesture_announce_interval = 2.0 - - # 手部跟踪和状态 - self.last_hand_position = None - self.hand_tracking = False - self.track_window = None - self.hand_states = deque(maxlen=15) - self.hand_detected_frames = 0 - self.hand_lost_frames = 0 - - # 性能统计 - self.process_times = deque(maxlen=30) - self.frame_counter = 0 - self.last_performance_report = 0 - - # 手势颜色映射 - self.gesture_colors = { - "Stop": (0, 0, 255), - "Forward": (0, 255, 0), - "Up": (255, 255, 0), - "Down": (255, 0, 255), - "Left": (255, 165, 0), - "Right": (0, 165, 255), - "Waiting": (200, 200, 200), - "Error": (255, 0, 0), - "Hover": (255, 255, 255) - } - - # 手势到语音的映射 - self.gesture_speech_map = { - "Stop": "gesture_stop", - "Forward": "gesture_forward", - "Up": "gesture_up", - "Down": "gesture_down", - "Left": "gesture_left", - "Right": "gesture_right", - "Waiting": "gesture_waiting", - "Error": "gesture_error", - "Hover": "gesture_hover", - } - - # 手势状态颜色 - self.state_colors = { - "none": (100, 100, 100), - "starting": (255, 165, 0), - "active": (0, 255, 0), - "ending": (255, 0, 0), - } - - # 根据性能模式初始化背景减除器 - self.bg_subtractor = None - if self.mode_config['background_subtraction_enabled']: - self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( - history=100, varThreshold=25, detectShadows=True - ) - - # 形态学操作核 - self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - - # 性能监控 - self.avg_process_time = 0 - self.frame_rate = 0 - self.last_fps_check = time.time() - - # 存储手部数据用于轨迹记录 - self.last_hand_data = None - - # 性能模式信息 - self.performance_mode_color = self.mode_config['color'] - self.performance_mode_name = self.mode_config['name'] - - print(f"✓ 增强的手势识别器已初始化 - 性能模式: {self.performance_mode_name}") - - def set_performance_mode(self, mode): - """设置性能模式""" - self.performance_mode = mode - self.mode_config = config.get_performance_mode_config(mode) - - # 更新参数 - self.history_size = self.mode_config['history_size'] - self.detection_interval = self.mode_config['detection_interval'] - self.smooth_frames = self.mode_config['smooth_frames'] - self.min_confidence = self.mode_config['min_confidence'] - self.resize_factor = self.mode_config['resize_factor'] - - # 更新背景减除器 - if self.mode_config['background_subtraction_enabled'] and self.bg_subtractor is None: - self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( - history=100, varThreshold=25, detectShadows=True - ) - elif not self.mode_config['background_subtraction_enabled']: - self.bg_subtractor = None - - # 更新队列大小 - self.gesture_history = deque(maxlen=self.history_size) - self.confidence_history = deque(maxlen=self.history_size) - self.position_history = deque(maxlen=self.history_size) - - # 更新显示信息 - self.performance_mode_color = self.mode_config['color'] - self.performance_mode_name = self.mode_config['name'] - - print(f"✓ 切换到性能模式: {self.performance_mode_name}") - - def get_skin_mask(self, frame): - """获取肤色掩码""" - h_low = config.get('gesture', 'skin_lower_h') - h_high = config.get('gesture', 'skin_upper_h') - s_low = config.get('gesture', 'skin_lower_s') - s_high = config.get('gesture', 'skin_upper_s') - v_low = config.get('gesture', 'skin_lower_v') - v_high = config.get('gesture', 'skin_upper_v') - - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - lower_skin = np.array([h_low, s_low, v_low], dtype=np.uint8) - upper_skin = np.array([h_high, s_high, v_high], dtype=np.uint8) - skin_mask = cv2.inRange(hsv, lower_skin, upper_skin) - - return skin_mask, hsv - - def enhance_skin_detection(self, frame, skin_mask): - """增强肤色检测""" - if not self.mode_config['skin_detection_enabled']: - return skin_mask - - if self.bg_subtractor is not None: - fg_mask = self.bg_subtractor.apply(frame) - combined_mask = cv2.bitwise_and(skin_mask, fg_mask) - else: - combined_mask = skin_mask - - # 根据性能模式决定形态学操作次数 - if self.performance_mode == 'accurate': - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) - elif self.performance_mode == 'balanced': - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=1) - combined_mask = cv2.GaussianBlur(combined_mask, (3, 3), 0) - # fast模式不进行额外的形态学操作 - - return combined_mask - - def preprocess_frame(self, frame): - """预处理帧(根据性能模式调整大小)""" - if self.resize_factor != 1.0: - new_width = int(frame.shape[1] * self.resize_factor) - new_height = int(frame.shape[0] * self.resize_factor) - resized_frame = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_AREA) - return resized_frame - return frame - - def find_best_hand_contour(self, mask, frame): - """找到最佳的手部轮廓""" - contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - - if not contours: - return None, 0.0 - - contours = sorted(contours, key=cv2.contourArea, reverse=True) - best_contour = None - best_score = 0.0 - min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) - max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) - - # 根据性能模式调整检查的轮廓数量 - max_contours = 3 if self.performance_mode != 'fast' else 1 - - for contour in contours[:max_contours]: - area = cv2.contourArea(contour) - - if area < min_area or area > max_area: - continue - - score = self.rate_contour(contour, frame.shape) - - if score > best_score: - best_score = score - best_contour = contour - - return best_contour, best_score - - def rate_contour(self, contour, frame_shape): - """评估轮廓作为手部的可能性""" - score = 0.0 - area = cv2.contourArea(contour) - min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) - max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) - - if min_area < area < max_area: - area_ratio = min(area / max_area, 1.0) - score += area_ratio * 0.3 - - perimeter = cv2.arcLength(contour, True) - if perimeter > 100: - score += 0.2 - - if area > 0: - compactness = perimeter ** 2 / area - if 12 < compactness < 25: - compactness_score = 1.0 - abs(compactness - 18) / 6 - score += compactness_score * 0.3 - - x, y, w, h = cv2.boundingRect(contour) - if h > 0: - aspect_ratio = w / h - if 0.4 < aspect_ratio < 2.5: - aspect_score = 1.0 - abs(aspect_ratio - 1.0) / 1.5 - score += aspect_score * 0.2 - - return score - - def analyze_hand_features(self, contour, frame_shape): - """分析手部特征""" - if contour is None: - return None, 0.0 - - area = cv2.contourArea(contour) - M = cv2.moments(contour) - if M["m00"] == 0: - return None, 0.0 - - cx = int(M["m10"] / M["m00"]) - cy = int(M["m01"] / M["m00"]) - x, y, w, h = cv2.boundingRect(contour) - bbox_area = w * h - palm_center = (cx, cy) - palm_radius = int(w * config.get('gesture', 'palm_circle_radius_ratio')) - - # 根据性能模式调整轮廓简化程度 - epsilon = self.mode_config['contour_simplify_epsilon'] * cv2.arcLength(contour, True) - fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius, epsilon) - - direction = self.calculate_hand_direction(contour, cx, cy) - h_img, w_img = frame_shape[:2] - norm_x = cx / w_img - norm_y = cy / h_img - confidence = self.calculate_confidence(area, len(fingers), len(contour), bbox_area) - - result = { - 'contour': contour, - 'center': (cx, cy), - 'bbox': (x, y, x + w, y + h), - 'fingers': fingers, - 'fingertips': fingertips, - 'defects': defects, - 'palm_center': palm_center, - 'palm_radius': palm_radius, - 'direction': direction, - 'area': area, - 'position': (norm_x, norm_y), - 'bbox_size': (w, h), - 'confidence': confidence - } - - return result, confidence - - def analyze_fingers(self, contour, palm_center, palm_radius, epsilon): - """分析手指""" - approx = cv2.approxPolyDP(contour, epsilon, True) - hull = cv2.convexHull(approx, returnPoints=False) - - if hull is None or len(hull) < 3: - return [], [], [] - - defects = cv2.convexityDefects(approx, hull) - fingers = [] - fingertips = [] - defect_points = [] - - if defects is not None: - for i in range(defects.shape[0]): - s, e, f, d = defects[i, 0] - start = tuple(approx[s][0]) - end = tuple(approx[e][0]) - far = tuple(approx[f][0]) - - start_dist = np.linalg.norm(np.array(start) - np.array(palm_center)) - end_dist = np.linalg.norm(np.array(end) - np.array(palm_center)) - far_dist = np.linalg.norm(np.array(far) - np.array(palm_center)) - - if (start_dist > palm_radius * 1.2 and - end_dist > palm_radius * 1.2 and - d > config.get('gesture', 'defect_distance_threshold') * 256): - - a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) - b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) - c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) - - if b * c != 0: - angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) - angle_degrees = math.degrees(angle) - - if angle_degrees < 90: - finger = { - 'start': start, - 'end': end, - 'far': far, - 'depth': d, - 'angle': angle_degrees - } - fingers.append(finger) - - for point in [start, end]: - if point not in fingertips: - point_dist = np.linalg.norm(np.array(point) - np.array(palm_center)) - if point_dist > palm_radius * 1.5: - fingertips.append(point) - - defect_points.append((start, end, far, d)) - - if len(fingertips) == 0: - hull_points = cv2.convexHull(approx, returnPoints=True) - if len(hull_points) > 0: - hull_points = hull_points.reshape(-1, 2) - - for point in hull_points: - point_tuple = tuple(point) - point_dist = np.linalg.norm(point - np.array(palm_center)) - if point_dist > palm_radius * 1.5 and point_tuple not in fingertips: - fingertips.append(point_tuple) - - fingertips = fingertips[:5] - - return fingers, fingertips, defect_points - - def calculate_hand_direction(self, contour, cx, cy): - """计算手部方向""" - if len(contour) < 5: - return 0.0 - - points = contour.reshape(-1, 2).astype(np.float32) - mean = np.empty((0)) - mean, eigenvectors, eigenvalues = cv2.PCACompute2(points, mean) - direction = math.degrees(math.atan2(eigenvectors[0, 1], eigenvectors[0, 0])) - - return direction - - def calculate_confidence(self, area, finger_count, contour_length, bbox_area): - """计算手势置信度""" - confidence = 0.5 - min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) - max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) - - if min_area < area < max_area: - area_norm = (area - min_area) / (max_area - min_area) - confidence += area_norm * 0.2 - - if 0 <= finger_count <= 5: - confidence += 0.2 - - if contour_length > 200: - confidence += 0.1 - - if bbox_area > 0: - fill_ratio = area / bbox_area - if 0.2 < fill_ratio < 0.8: - fill_score = 1.0 - abs(fill_ratio - 0.5) / 0.3 - confidence += fill_score * 0.1 - - return min(confidence, 1.0) - - def recognize_gesture_improved(self, hand_data): - """改进的手势识别逻辑""" - if hand_data is None: - return "Waiting", 0.3 - - finger_count = len(hand_data.get('fingers', [])) - fingertips = hand_data.get('fingertips', []) - norm_x, norm_y = hand_data['position'] - direction = hand_data.get('direction', 0.0) - confidence = hand_data['confidence'] - w, h = hand_data['bbox_size'] - aspect_ratio = w / h if h > 0 else 1.0 - - # 根据手指数量分类 - if finger_count == 0: - if len(fingertips) == 0: - return "Stop", confidence * 0.9 - else: - return "Stop", confidence * 0.7 - - elif finger_count == 1: - if len(fingertips) >= 1: - cx, cy = hand_data['center'] - fingertip = fingertips[0] - dx = fingertip[0] - cx - dy = fingertip[1] - cy - - if abs(dx) > abs(dy): - if dx > 0: - return "Right", confidence * 0.8 - else: - return "Left", confidence * 0.8 - else: - if dy < 0: - return "Up", confidence * 0.8 - else: - return "Forward", confidence * 0.8 - return "Forward", confidence * 0.7 - - elif finger_count == 2: - return "Forward", confidence * 0.7 - - elif finger_count == 3: - if -45 <= direction <= 45: - if direction > 0: - return "Right", confidence * 0.7 - else: - return "Left", confidence * 0.7 - else: - if direction > 0: - return "Down", confidence * 0.7 - else: - return "Up", confidence * 0.7 - - elif finger_count >= 4: - if norm_x < 0.4: - return "Left", confidence * 0.8 - elif norm_x > 0.6: - return "Right", confidence * 0.8 - elif norm_y < 0.4: - return "Up", confidence * 0.8 - elif norm_y > 0.6: - return "Down", confidence * 0.8 - else: - if -45 <= direction <= 45: - return "Forward", confidence * 0.7 - else: - return "Stop", confidence * 0.7 - - return "Waiting", confidence * 0.5 - - def smooth_gesture_enhanced(self, new_gesture, new_confidence, hand_data): - """增强的手势平滑处理""" - current_time = time.time() - - # 检查手势冷却时间 - if current_time - self.last_gesture_change_time < config.get('gesture', 'gesture_cooldown'): - return self.current_gesture, self.current_confidence - - # 添加到历史 - self.gesture_history.append(new_gesture) - self.confidence_history.append(new_confidence) - - if hand_data is not None: - self.position_history.append(hand_data['position']) - - # 计算手势稳定性 - if len(self.gesture_history) >= 3: - # 检查最近N个手势是否一致 - recent_gestures = list(self.gesture_history)[-3:] - gesture_counter = Counter(recent_gestures) - most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] - - # 计算位置稳定性 - position_stability = 1.0 - if len(self.position_history) >= 2 and hand_data is not None: - current_pos = hand_data['position'] - prev_pos = self.position_history[-2] if len(self.position_history) >= 2 else current_pos - position_diff = math.sqrt((current_pos[0] - prev_pos[0]) ** 2 + (current_pos[1] - prev_pos[1]) ** 2) - position_stability = max(0, 1.0 - position_diff * 5.0) - - # 增强的稳定性检查 - stability_threshold = self.mode_config['gesture_stability_threshold'] - transition_threshold = config.get('gesture', 'transition_threshold') - position_weight = config.get('gesture', 'position_stability_weight') - - # 计算综合稳定性得分 - gesture_stability = most_common_count / 3.0 - overall_stability = gesture_stability * (1.0 - position_weight) + position_stability * position_weight - - # 手势状态转换逻辑 - if overall_stability >= transition_threshold: - if most_common_gesture != self.last_stable_gesture: - self.gesture_stability_counter += 1 - else: - self.gesture_stability_counter = max(0, self.gesture_stability_counter - 1) - - if self.gesture_stability_counter >= stability_threshold: - # 手势稳定,更新当前手势 - self.current_gesture = most_common_gesture - self.current_confidence = np.mean(list(self.confidence_history)[-3:]) - self.last_stable_gesture = most_common_gesture - self.gesture_stability_counter = 0 - self.last_gesture_change_time = current_time - else: - # 手势不稳定,重置计数器 - self.gesture_stability_counter = max(0, self.gesture_stability_counter - 2) - - return self.current_gesture, self.current_confidence - - def update_gesture_state(self, hand_data, gesture, confidence): - """更新手势状态""" - current_time = time.time() - - if hand_data is None: - # 手部丢失 - self.hand_lost_frames += 1 - self.hand_detected_frames = max(0, self.hand_detected_frames - 1) - - if self.hand_lost_frames > 10 and self.gesture_state != "none": - self.gesture_state = "none" - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_hand_status') and - current_time - self.last_hand_status_time > 3.0): - self.speech_manager.speak('hand_lost', immediate=True) - self.last_hand_status_time = current_time - return - - # 手部检测到 - self.hand_detected_frames += 1 - self.hand_lost_frames = 0 - - # 检查手部大小和位置 - hand_area = hand_data['area'] - min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) - max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) - - # 提供手部位置反馈 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_hand_status') and - current_time - self.last_hand_status_time > 5.0): - - if hand_area < min_area * 0.8: - self.speech_manager.speak('move_closer', immediate=True) - self.last_hand_status_time = current_time - elif hand_area > max_area * 1.2: - self.speech_manager.speak('move_away', immediate=True) - self.last_hand_status_time = current_time - elif self.hand_detected_frames == 5: # 首次稳定检测 - self.speech_manager.speak('hand_detected', immediate=True) - self.last_hand_status_time = current_time - - # 手势状态转换 - if self.gesture_state == "none" and gesture != "Waiting" and confidence > 0.6: - # 手势开始 - self.gesture_state = "starting" - self.gesture_active_frames = 0 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gesture_changes')): - self.speech_manager.speak('gesture_start', immediate=True) - - elif self.gesture_state == "starting": - self.gesture_active_frames += 1 - if self.gesture_active_frames >= config.get('speech', 'gesture_start_threshold'): - self.gesture_state = "active" - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gesture_changes')): - self.speech_manager.speak('gesture_stable', immediate=True) - - elif self.gesture_state == "active": - if gesture == "Waiting" or confidence < 0.5: - self.gesture_active_frames = max(0, self.gesture_active_frames - 2) - if self.gesture_active_frames <= 0: - self.gesture_state = "ending" - else: - self.gesture_active_frames = min(20, self.gesture_active_frames + 1) - - elif self.gesture_state == "ending": - self.gesture_active_frames -= 1 - if self.gesture_active_frames <= 0: - self.gesture_state = "none" - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gesture_changes')): - self.speech_manager.speak('gesture_end', immediate=True) - - def visualize_detection(self, frame, hand_data, gesture, confidence): - """可视化检测结果""" - if hand_data is None: - return frame - - show_contours = config.get('display', 'show_contours') - show_bbox = config.get('display', 'show_bbox') - show_fingertips = config.get('display', 'show_fingertips') - show_palm_center = config.get('display', 'show_palm_center') - show_hand_direction = config.get('display', 'show_hand_direction') - show_debug_info = config.get('display', 'show_debug_info') - show_gesture_history = config.get('display', 'show_gesture_history') - show_stability_indicator = config.get('display', 'show_stability_indicator') - - # 绘制轮廓 - if show_contours and 'contour' in hand_data: - cv2.drawContours(frame, [hand_data['contour']], -1, (0, 255, 0), 2) - - # 绘制边界框 - if show_bbox and 'bbox' in hand_data: - x1, y1, x2, y2 = hand_data['bbox'] - color = self.gesture_colors.get(gesture, (255, 255, 255)) - - # 根据手势状态调整边界框颜色 - state_color = self.state_colors.get(self.gesture_state, color) - if self.gesture_state != "none": - color = state_color - - cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) - - # 显示手势标签 - label = f"{gesture}" - if config.get('display', 'show_confidence'): - label += f" ({confidence:.0%})" - - # 显示手势状态 - if self.gesture_state != "none": - state_text = {"starting": "开始", "active": "活跃", "ending": "结束"} - label += f" [{state_text.get(self.gesture_state, '')}]" - - # 计算文本大小 - (text_width, text_height), baseline = cv2.getTextSize( - label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 - ) - - # 绘制文本背景 - cv2.rectangle(frame, - (x1, y1 - text_height - 10), - (x1 + text_width, y1), - color, -1) - - # 绘制文本 - cv2.putText(frame, label, (x1, y1 - 5), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) - - # 绘制手掌中心 - if show_palm_center and 'palm_center' in hand_data: - cx, cy = hand_data['palm_center'] - palm_radius = hand_data.get('palm_radius', 20) - cv2.circle(frame, (cx, cy), 5, (0, 0, 255), -1) - cv2.circle(frame, (cx, cy), palm_radius, (0, 0, 255), 1) - cv2.putText(frame, "Palm", (cx + 10, cy), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) - - # 绘制指尖 - if show_fingertips and 'fingertips' in hand_data: - for i, point in enumerate(hand_data['fingertips']): - cv2.circle(frame, point, 4, (255, 0, 0), -1) - cv2.putText(frame, f"F{i + 1}", (point[0] + 5, point[1]), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1) - - # 绘制手部方向 - if show_hand_direction and 'direction' in hand_data and 'center' in hand_data: - cx, cy = hand_data['center'] - direction = hand_data['direction'] - length = 50 - - dx = length * math.cos(math.radians(direction)) - dy = length * math.sin(math.radians(direction)) - - end_point = (int(cx + dx), int(cy + dy)) - cv2.arrowedLine(frame, (cx, cy), end_point, (255, 255, 0), 2) - - angle_text = f"Dir: {direction:.0f}°" - cv2.putText(frame, angle_text, (cx, cy - 20), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 1) - - # 绘制稳定性指示器 - if show_stability_indicator: - h, w = frame.shape[:2] - indicator_x = w - 100 - indicator_y = 30 - - # 绘制稳定性背景 - cv2.rectangle(frame, (indicator_x, indicator_y), - (indicator_x + 80, indicator_y + 15), (50, 50, 50), -1) - - # 计算稳定性指示条长度 - stability_level = min(1.0, - self.gesture_stability_counter / self.mode_config['gesture_stability_threshold']) - bar_length = int(70 * stability_level) - - # 根据稳定性级别选择颜色 - if stability_level > 0.7: - bar_color = (0, 255, 0) # 绿色 - elif stability_level > 0.4: - bar_color = (255, 165, 0) # 橙色 - else: - bar_color = (255, 0, 0) # 红色 - - # 绘制稳定性指示条 - cv2.rectangle(frame, (indicator_x + 5, indicator_y + 5), - (indicator_x + 5 + bar_length, indicator_y + 10), bar_color, -1) - - # 绘制稳定性文本 - cv2.putText(frame, "稳定度", (indicator_x, indicator_y - 5), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - # 绘制手势历史(如果启用) - if show_gesture_history and len(self.gesture_history) > 0: - h, w = frame.shape[:2] - history_y = h - 50 - - # 绘制历史背景 - cv2.rectangle(frame, (10, history_y - 20), (200, history_y + 10), (0, 0, 0), -1) - cv2.putText(frame, "手势历史:", (15, history_y), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - # 显示最近几个手势 - recent_gestures = list(self.gesture_history)[-5:] if len(self.gesture_history) >= 5 else list( - self.gesture_history) - for i, gest in enumerate(recent_gestures): - color = self.gesture_colors.get(gest, (255, 255, 255)) - cv2.putText(frame, gest[0], (85 + i * 20, history_y), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) - - # 显示调试信息 - if show_debug_info: - # 显示手指数量 - finger_count = len(hand_data.get('fingers', [])) - finger_text = f"Fingers: {finger_count}" - cv2.putText(frame, finger_text, (10, frame.shape[0] - 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - # 显示手部位置 - pos_text = f"Pos: ({hand_data['position'][0]:.2f}, {hand_data['position'][1]:.2f})" - cv2.putText(frame, pos_text, (10, frame.shape[0] - 10), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - # 显示手势状态 - state_text = f"State: {self.gesture_state}" - cv2.putText(frame, state_text, (150, frame.shape[0] - 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - # 显示稳定性计数器 - stability_text = f"Stability: {self.gesture_stability_counter}" - cv2.putText(frame, stability_text, (150, frame.shape[0] - 10), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - return frame - - def recognize(self, frame): - """识别手势""" - start_time = time.time() - - try: - # 预处理帧 - processed_frame = cv2.flip(frame, 1) - - # 根据性能模式调整图像大小 - original_frame = processed_frame.copy() - if self.resize_factor != 1.0: - processed_frame = self.preprocess_frame(processed_frame) - - # 每隔几帧检测一次以提高性能 - if self.frame_counter % self.detection_interval != 0: - self.frame_counter += 1 - return self.current_gesture, self.current_confidence, original_frame - - # 获取肤色掩码 - skin_mask, hsv = self.get_skin_mask(processed_frame) - - # 增强肤色检测 - enhanced_mask = self.enhance_skin_detection(processed_frame, skin_mask) - - # 找到最佳的手部轮廓 - hand_contour, contour_score = self.find_best_hand_contour(enhanced_mask, processed_frame) - - # 分析手部特征 - hand_data, confidence = self.analyze_hand_features(hand_contour, processed_frame.shape) - - # 保存手部数据用于轨迹记录 - self.last_hand_data = hand_data - - # 识别手势 - if hand_data is not None: - # 校准肤色(如果需要) - config.calibrate_skin_color(processed_frame, enhanced_mask) - - # 校准手部大小(如果需要) - config.calibrate_hand_size(hand_data['area']) - - # 识别手势 - gesture, raw_confidence = self.recognize_gesture_improved(hand_data) - confidence = max(confidence, raw_confidence) - - # 更新手势状态 - self.update_gesture_state(hand_data, gesture, confidence) - - # 增强的手势平滑 - final_gesture, final_confidence = self.smooth_gesture_enhanced(gesture, confidence, hand_data) - else: - gesture, confidence = "Waiting", 0.3 - self.update_gesture_state(None, gesture, confidence) - final_gesture, final_confidence = gesture, confidence - - # 手势语音播报 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gestures')): - - current_time = time.time() - - # 根据置信度提供反馈 - if confidence >= 0.8 and current_time - self.last_hand_status_time > 5.0: - self.speech_manager.speak('gesture_good_confidence', immediate=True) - self.last_hand_status_time = current_time - elif confidence < 0.5 and current_time - self.last_hand_status_time > 5.0: - self.speech_manager.speak('gesture_low_confidence', immediate=True) - self.last_hand_status_time = current_time - - # 手势语音播报 - if (final_gesture != "Waiting" and - final_gesture != "Error" and - final_gesture != "摄像头错误" and - final_confidence >= self.min_confidence and - current_time - self.last_announced_time > self.gesture_announce_interval): - - if final_gesture in self.gesture_speech_map: - speech_key = self.gesture_speech_map[final_gesture] - self.speech_manager.speak(speech_key) - else: - self.speech_manager.speak_direct(f"手势{final_gesture}") - - self.last_announced_gesture = final_gesture - self.last_announced_time = current_time - - # 性能报告 - current_time = time.time() - if current_time - self.last_performance_report > 30.0: - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_performance')): - - if self.avg_process_time < 20: - self.speech_manager.speak('performance_good', immediate=True) - elif self.avg_process_time > 50: - self.speech_manager.speak('performance_warning', immediate=True) - - self.last_performance_report = current_time - - # 可视化结果(在原图上) - if hand_data is not None: - # 需要将坐标转换回原始图像大小 - if self.resize_factor != 1.0: - # 缩放坐标 - scale_factor = 1.0 / self.resize_factor - if 'center' in hand_data: - hand_data['center'] = (int(hand_data['center'][0] * scale_factor), - int(hand_data['center'][1] * scale_factor)) - if 'bbox' in hand_data: - x1, y1, x2, y2 = hand_data['bbox'] - hand_data['bbox'] = (int(x1 * scale_factor), int(y1 * scale_factor), - int(x2 * scale_factor), int(y2 * scale_factor)) - if 'fingertips' in hand_data: - hand_data['fingertips'] = [(int(x * scale_factor), int(y * scale_factor)) - for (x, y) in hand_data['fingertips']] - if 'palm_center' in hand_data: - hand_data['palm_center'] = (int(hand_data['palm_center'][0] * scale_factor), - int(hand_data['palm_center'][1] * scale_factor)) - - original_frame = self.visualize_detection( - original_frame, hand_data, final_gesture, final_confidence - ) - else: - original_frame = self.visualize_detection( - original_frame, None, final_gesture, final_confidence - ) - - # 更新计数器 - self.frame_counter += 1 - - # 计算处理时间 - process_time = (time.time() - start_time) * 1000 - self.process_times.append(process_time) - - # 更新平均处理时间 - if len(self.process_times) > 0: - self.avg_process_time = np.mean(list(self.process_times)) - - # 更新帧率 - current_time = time.time() - if current_time - self.last_fps_check >= 1.0: - self.frame_rate = self.frame_counter - self.frame_counter = 0 - self.last_fps_check = current_time - - return final_gesture, final_confidence, original_frame - - except Exception as e: - print(f"⚠ 手势识别错误: {e}") - return "Error", 0.0, frame - - def get_performance_stats(self): - """获取性能统计""" - if len(self.process_times) == 0: - return 0.0, self.frame_rate - - return np.mean(list(self.process_times)), self.frame_rate - - def get_performance_mode_info(self): - """获取性能模式信息""" - return { - 'name': self.performance_mode_name, - 'mode': self.performance_mode, - 'color': self.performance_mode_color, - 'detection_interval': self.detection_interval, - 'resize_factor': self.resize_factor, - 'smooth_frames': self.smooth_frames, - 'min_confidence': self.min_confidence - } - - def set_simulated_gesture(self, gesture): - """设置模拟的手势""" - self.current_gesture = gesture - self.current_confidence = 0.9 - - # 模拟手势也触发语音提示 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gestures')): - - if gesture in self.gesture_speech_map: - self.speech_manager.speak(self.gesture_speech_map[gesture]) - else: - self.speech_manager.speak_direct(f"手势{gesture}") - - self.last_announced_gesture = gesture - self.last_announced_time = time.time() - - -# ========== 简单的无人机控制器 ========== -class SimpleDroneController: - """简单的无人机控制器""" - - def __init__(self, airsim_module, speech_manager=None): - self.airsim = airsim_module - self.client = None - self.connected = False - self.flying = False - self.speech_manager = speech_manager - - # 控制参数 - self.velocity = config.get('drone', 'velocity') - self.duration = config.get('drone', 'duration') - self.altitude = config.get('drone', 'altitude') - self.control_interval = config.get('drone', 'control_interval') - - # 控制状态 - self.last_control_time = 0 - self.last_gesture = None - - # 上次语音提示状态 - self.last_connection_announced = False - self.last_takeoff_announced = False - self.last_land_announced = False - - print("✓ 简单的无人机控制器已初始化") - - def connect(self): - """连接AirSim无人机""" - if self.connected: - return True - - # 语音提示:正在连接 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_connections')): - self.speech_manager.speak('connecting') - - if self.airsim is None: - print("⚠ AirSim不可用,使用模拟模式") - - # 语音提示:模拟模式 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_connections')): - self.speech_manager.speak('simulation_mode') - - self.connected = True - return True - - print("连接AirSim...") - - try: - self.client = self.airsim.MultirotorClient() - self.client.confirmConnection() - print("✅ 已连接AirSim!") - - # 语音提示:连接成功 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_connections')): - self.speech_manager.speak('connected') - - self.client.enableApiControl(True) - print("✅ API控制已启用") - - self.client.armDisarm(True) - print("✅ 无人机已武装") - - self.connected = True - return True - - except Exception as e: - print(f"❌ 连接失败: {e}") - - # 语音提示:连接失败 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_connections')): - self.speech_manager.speak('connection_failed') - - print("\n使用模拟模式继续? (y/n)") - choice = input().strip().lower() - if choice == 'y': - self.connected = True - print("✅ 使用模拟模式") - - # 语音提示:模拟模式 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_connections')): - self.speech_manager.speak('simulation_mode') - - return True - - return False - - def takeoff(self): - """起飞""" - if not self.connected: - return False - - # 语音提示:正在起飞 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events') and - not self.last_takeoff_announced): - self.speech_manager.speak('taking_off') - self.last_takeoff_announced = True - self.last_land_announced = False - - try: - if self.airsim is None or self.client is None: - print("✅ 模拟起飞") - self.flying = True - - # 语音提示:起飞成功 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('takeoff_success') - - return True - - print("起飞中...") - self.client.takeoffAsync().join() - time.sleep(1) - - # 上升到指定高度 - self.client.moveToZAsync(self.altitude, 3).join() - - self.flying = True - print("✅ 无人机成功起飞") - - # 语音提示:起飞成功 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('takeoff_success') - - return True - except Exception as e: - print(f"❌ 起飞失败: {e}") - - # 语音提示:起飞失败 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('takeoff_failed') - - return False - - def land(self): - """降落""" - if not self.connected: - return False - - # 语音提示:正在降落 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events') and - not self.last_land_announced): - self.speech_manager.speak('landing') - self.last_land_announced = True - self.last_takeoff_announced = False - - try: - if self.airsim is None or self.client is None: - print("✅ 模拟降落") - self.flying = False - - # 语音提示:降落成功 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('land_success') - - return True - - print("降落中...") - self.client.landAsync().join() - self.flying = False - print("✅ 无人机已降落") - - # 语音提示:降落成功 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('land_success') - - return True - except Exception as e: - print(f"降落失败: {e}") - return False - - def move_by_gesture(self, gesture, confidence): - """根据手势移动""" - if not self.connected or not self.flying: - return False - - # 检查控制间隔 - current_time = time.time() - if current_time - self.last_control_time < self.control_interval: - return False - - # 检查置信度阈值 - min_confidence = config.get('gesture', 'min_confidence') - if confidence < min_confidence: - # 低置信度语音提示 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gestures') and - confidence < min_confidence * 0.8): - self.speech_manager.speak('gesture_low_confidence') - return False - - try: - if self.airsim is None or self.client is None: - print(f"模拟移动: {gesture}") - self.last_control_time = current_time - self.last_gesture = gesture - return True - - success = False - - if gesture == "Up": - self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) - success = True - elif gesture == "Down": - self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) - success = True - elif gesture == "Left": - self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) - success = True - elif gesture == "Right": - self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) - success = True - elif gesture == "Forward": - self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) - success = True - elif gesture == "Stop": - self.client.hoverAsync() - success = True - # 悬停语音提示 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('hovering') - elif gesture == "Hover": - self.client.hoverAsync() - success = True - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('hovering') - - if success: - self.last_control_time = current_time - self.last_gesture = gesture - - return success - except Exception as e: - print(f"控制命令失败: {e}") - return False - - def emergency_stop(self): - """紧急停止""" - if self.connected: - try: - if self.flying and self.client is not None: - print("紧急降落...") - - # 语音提示:紧急停止 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('emergency_stop') - - self.land() - if self.client is not None: - self.client.armDisarm(False) - self.client.enableApiControl(False) - print("✅ 紧急停止完成") - except: - pass - - self.connected = False - self.flying = False - - -# ========== 中文UI渲染器(增强版) ========== -class ChineseUIRenderer: - """中文UI渲染器""" - - def __init__(self, speech_manager=None): - self.fonts = {} - self.speech_manager = speech_manager - self.load_fonts() - - # 颜色定义 - self.colors = { - 'title': (0, 255, 255), - 'connected': (0, 255, 0), - 'disconnected': (0, 0, 255), - 'flying': (0, 255, 0), - 'landed': (255, 165, 0), - 'warning': (0, 165, 255), - 'info': (255, 255, 255), - 'help': (255, 200, 100), - 'speech_enabled': (0, 255, 0), - 'speech_disabled': (255, 0, 0), - 'performance_good': (0, 255, 0), - 'performance_warning': (255, 165, 0), - 'performance_bad': (255, 0, 0), - 'recording': (255, 50, 50), - 'playback': (50, 50, 255), - 'paused': (255, 255, 0), - 'performance_fast': (0, 255, 0), # 绿色 - 'performance_balanced': (255, 165, 0), # 橙色 - 'performance_accurate': (255, 0, 0), # 红色 - 'cpu_good': (0, 255, 0), # CPU使用率颜色 - 'cpu_warning': (255, 165, 0), - 'cpu_critical': (255, 0, 0), - 'memory_good': (0, 255, 0), # 内存使用率颜色 - 'memory_warning': (255, 165, 0), - 'memory_critical': (255, 0, 0), - } - - print("✓ 中文UI渲染器已初始化") - - def load_fonts(self): - """加载字体""" - font_paths = [ - 'simhei.ttf', - 'C:/Windows/Fonts/simhei.ttf', - 'C:/Windows/Fonts/msyh.ttc', - '/System/Library/Fonts/PingFang.ttc', - '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' - ] - - for path in font_paths: - try: - self.fonts[14] = ImageFont.truetype(path, 14) - self.fonts[16] = ImageFont.truetype(path, 16) - self.fonts[18] = ImageFont.truetype(path, 18) - self.fonts[20] = ImageFont.truetype(path, 20) - self.fonts[24] = ImageFont.truetype(path, 24) - print(f"✓ 字体已加载: {path}") - return - except: - continue - - print("⚠ 未找到字体,使用默认") - - def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): - """在图像上绘制文本""" - try: - img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - pil_img = Image.fromarray(img_rgb) - draw = ImageDraw.Draw(pil_img) - - font = self.fonts.get(size, self.fonts.get(16)) - - # 绘制阴影 - shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 1, pos[1] + 1) - draw.text(shadow_pos, text, font=font, fill=shadow_color) - - # 绘制文字 - rgb_color = color[::-1] # BGR to RGB - draw.text(pos, text, font=font, fill=rgb_color) - - return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) - except: - # 备用方案:使用OpenCV绘制英文 - cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, - size / 25, color, 1) - return frame - - def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, - trajectory_recorder=None, gesture_recognizer=None, performance_analyzer=None): - """绘制状态栏(增强版)""" - h, w = frame.shape[:2] - - # 绘制半透明背景 - overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (w, 130), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) - - # 标题 - title = "手势控制无人机系统 - 性能统计版" - frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) - - # 连接状态 - status_color = self.colors['connected'] if drone_controller.connected else self.colors['disconnected'] - status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" - frame = self.draw_text(frame, status_text, (10, 40), size=16, color=status_color) - - # 飞行状态 - flight_color = self.colors['flying'] if drone_controller.flying else self.colors['landed'] - flight_text = f"飞行状态: {'飞行中' if drone_controller.flying else '已降落'}" - frame = self.draw_text(frame, flight_text, (10, 65), size=16, color=flight_color) - - # 手势信息 - if confidence > 0.7: - gesture_color = (0, 255, 0) - elif confidence > 0.5: - gesture_color = (255, 165, 0) - else: - gesture_color = (200, 200, 200) - - gesture_text = f"当前手势: {gesture}" - if config.get('display', 'show_confidence'): - gesture_text += f" ({confidence:.0%})" - - frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) - - # 性能模式显示 - if gesture_recognizer and config.get('display', 'show_performance_mode'): - mode_info = gesture_recognizer.get_performance_mode_info() - - # 根据模式选择颜色 - if mode_info['mode'] == 'fast': - mode_color = self.colors['performance_fast'] - elif mode_info['mode'] == 'balanced': - mode_color = self.colors['performance_balanced'] - else: # accurate - mode_color = self.colors['performance_accurate'] - - mode_text = f"性能模式: {mode_info['name']}" - frame = self.draw_text(frame, mode_text, (w // 2, 65), size=14, color=mode_color) - - # 录制/回放状态 - elif trajectory_recorder and config.get('display', 'show_recording_status'): - recorder_status = trajectory_recorder.get_status() - - if recorder_status['is_recording']: - status_color = self.colors['recording'] - status_text = f"录制中: {recorder_status['trajectory_points']}点" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) - elif recorder_status['is_playing']: - if recorder_status['playback_paused']: - status_color = self.colors['paused'] - status_text = f"回放暂停: {recorder_status['playback_index']}/{recorder_status['playback_total']}" - else: - status_color = self.colors['playback'] - status_text = f"回放中: {recorder_status['playback_index']}/{recorder_status['playback_total']}" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) - else: - status_text = f"轨迹点: {recorder_status['trajectory_points']}" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=12, color=self.colors['info']) - - # 性能统计显示 - if performance_analyzer and config.get('display', 'show_performance_stats'): - perf_stats = performance_analyzer.get_stats_summary() - - # 性能状态颜色 - if perf_stats['performance_status'] == "良好": - perf_color = self.colors['performance_good'] - elif perf_stats['performance_status'] == "警告": - perf_color = self.colors['performance_warning'] - else: - perf_color = self.colors['performance_bad'] - - perf_text = f"性能: {perf_stats['performance_status']} ({perf_stats['fps']:.0f}FPS)" - frame = self.draw_text(frame, perf_text, (w // 2, 90), size=12, color=perf_color) - - # 系统资源显示 - if performance_analyzer and config.get('display', 'show_system_resources'): - cpu_usage = performance_analyzer.get_current_cpu_usage() - memory_usage = performance_analyzer.get_current_memory_usage() - - # CPU颜色 - if cpu_usage < 70: - cpu_color = self.colors['cpu_good'] - elif cpu_usage < 85: - cpu_color = self.colors['cpu_warning'] - else: - cpu_color = self.colors['cpu_critical'] - - # 内存颜色 - if memory_usage < 70: - mem_color = self.colors['memory_good'] - elif memory_usage < 85: - mem_color = self.colors['memory_warning'] - else: - mem_color = self.colors['memory_critical'] - - # 绘制系统资源信息 - sys_text = f"CPU:{cpu_usage:.0f}% 内存:{memory_usage:.0f}%" - frame = self.draw_text(frame, sys_text, (w // 2, 110), size=11, color=self.colors['info']) - - # 语音状态 - if config.get('display', 'show_speech_status') and self.speech_manager: - speech_status = self.speech_manager.get_status() - speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] - speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" - frame = self.draw_text(frame, speech_text, (w - 200, 90), size=12, color=speech_color) - - # 性能信息 - if config.get('display', 'show_fps'): - perf_text = f"帧率: {fps:.1f}" - if process_time > 0: - perf_text += f" | 延迟: {process_time:.1f}ms" - - # 根据处理时间选择颜色 - if process_time < 20: - perf_color = self.colors['performance_good'] - elif process_time < 50: - perf_color = self.colors['performance_warning'] - else: - perf_color = self.colors['performance_bad'] - else: - perf_color = self.colors['info'] - - frame = self.draw_text(frame, perf_text, (w - 200, 65), size=12, color=perf_color) - - # 控制提示 - control_text = "提示: 确保手部完全进入画面,保持稳定手势" - frame = self.draw_text(frame, control_text, (10, 90), size=12, color=self.colors['info']) - - # 性能快捷键提示 - if performance_analyzer: - perf_tip = "P:报告 L:快照 K:导出" - frame = self.draw_text(frame, perf_tip, (10, 110), size=11, color=self.colors['help']) - - return frame - - def draw_help_bar(self, frame): - """绘制帮助栏""" - if not config.get('display', 'show_help'): - return frame - - h, w = frame.shape[:2] - - # 绘制底部帮助栏 - cv2.rectangle(frame, (0, h - 120), (w, h), (0, 0, 0), -1) - - # 帮助文本(增强版,包含性能统计快捷键) - help_lines = [ - "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", - "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音 P:性能报告 L:性能快照 K:导出日志", - "O:切换性能模式 P:详细报告 R:重置统计", - "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" - ] - - for i, line in enumerate(help_lines): - y_pos = h - 105 + i * 20 - frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) - - return frame - - def draw_warning(self, frame, message): - """绘制警告信息""" - h, w = frame.shape[:2] - - # 在顶部绘制警告 - warning_bg = np.zeros((40, w, 3), dtype=np.uint8) - warning_bg[:, :] = (0, 69, 255) - - frame[130:170, 0:w] = cv2.addWeighted( - frame[130:170, 0:w], 0.3, - warning_bg, 0.7, 0 - ) - - # 绘制警告文本 - frame = self.draw_text(frame, message, (10, 145), - size=16, color=self.colors['warning']) - - return frame - - def draw_performance_chart(self, frame, performance_analyzer): - """绘制性能图表""" - if not performance_analyzer: - return frame - - h, w = frame.shape[:2] - - # 绘制图表区域背景 - chart_x = w - 250 - chart_y = 130 - chart_width = 240 - chart_height = 150 - - cv2.rectangle(frame, (chart_x, chart_y), - (chart_x + chart_width, chart_y + chart_height), - (30, 30, 30), -1) - cv2.rectangle(frame, (chart_x, chart_y), - (chart_x + chart_width, chart_y + chart_height), - (100, 100, 100), 1) - - # 绘制标题 - frame = self.draw_text(frame, "性能监控", (chart_x + 10, chart_y + 20), - size=14, color=self.colors['title']) - - # 绘制FPS图表 - if len(performance_analyzer.fps_history) > 1: - fps_values = list(performance_analyzer.fps_history)[-20:] # 最近20个值 - - # 计算最大值和最小值 - if fps_values: - max_fps = max(fps_values) - min_fps = min(fps_values) - if max_fps == min_fps: - max_fps = min_fps + 1 - - # 绘制FPS曲线 - points = [] - for i, fps in enumerate(fps_values): - x = chart_x + 10 + i * (chart_width - 20) / len(fps_values) - y = chart_y + chart_height - 30 - (fps - min_fps) * (chart_height - 50) / (max_fps - min_fps) - points.append((int(x), int(y))) - - # 连接点形成曲线 - for i in range(len(points) - 1): - cv2.line(frame, points[i], points[i + 1], (0, 255, 0), 2) - - # 显示当前FPS值 - current_fps = fps_values[-1] if fps_values else 0 - fps_color = self.colors['performance_good'] if current_fps > 20 else \ - self.colors['performance_warning'] if current_fps > 10 else \ - self.colors['performance_bad'] - - frame = self.draw_text(frame, f"FPS: {current_fps:.1f}", - (chart_x + 120, chart_y + 40), - size=12, color=fps_color) - - return frame - - -# ========== 主程序 ========== -def main(): - """主函数""" - # 初始化语音管理器 - print("初始化语音反馈系统...") - speech_manager = EnhancedSpeechFeedbackManager(libs['speech']) - - # 程序启动语音提示 - if speech_manager.enabled: - speech_manager.speak('program_start', force=True, immediate=True) - speech_manager.speak('system_ready', immediate=True) - - # 初始化组件 - print("初始化组件...") + # 尝试导入psutil + try: + import psutil + libs['psutil'] = psutil + status['psutil'] = True + print("[psutil] ✓ 系统资源监控库就绪") + except ImportError: + print("[psutil] ⚠ 未找到,性能监控功能受限") + libs['psutil'] = None - gesture_recognizer = EnhancedGestureRecognizer(speech_manager) - drone_controller = SimpleDroneController(libs['airsim'], speech_manager) - ui_renderer = ChineseUIRenderer(speech_manager) + libs['speech'] = speech_module + return libs, status - # 初始化性能分析器 - print("初始化性能分析器...") - performance_analyzer = PerformanceAnalyzer(speech_manager) - # 手势轨迹记录器 - print("初始化手势轨迹记录器...") - trajectory_recorder = GestureTrajectoryRecorder(speech_manager) - - # 初始化摄像头 +def init_camera(config): + """初始化摄像头""" cap = None try: cap = cv2.VideoCapture(0) @@ -3290,7 +136,6 @@ def main(): cap.set(cv2.CAP_PROP_FRAME_HEIGHT, config.get('camera', 'height')) cap.set(cv2.CAP_PROP_FPS, config.get('camera', 'fps')) - # 获取实际参数 actual_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) actual_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) actual_fps = int(cap.get(cv2.CAP_PROP_FPS)) @@ -3298,28 +143,18 @@ def main(): print(f"✓ 摄像头已初始化") print(f" 分辨率: {actual_width}x{actual_height}") print(f" 帧率: {actual_fps}") - - # 摄像头就绪语音提示 - if speech_manager.enabled: - speech_manager.speak('camera_ready', immediate=True) else: print("❌ 摄像头不可用,使用模拟模式") - - # 摄像头错误语音提示 - if speech_manager.enabled: - speech_manager.speak('camera_error', immediate=True) - cap = None except Exception as e: print(f"⚠ 摄像头初始化失败: {e}") + cap = None - # 摄像头错误语音提示 - if speech_manager.enabled: - speech_manager.speak('camera_error', immediate=True) + return cap - cap = None - # 显示欢迎信息 +def print_welcome_message(cap, speech_manager, libs): + """打印欢迎信息""" print("\n" + "=" * 80) print("手势控制无人机系统 - 详细性能统计版") print("=" * 80) @@ -3327,13 +162,14 @@ def main(): print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") - print(f" 性能模式: {gesture_recognizer.performance_mode_name}") - print(f" 性能监控: {'已启用' if config.get('performance', 'enable_performance_monitor') else '已禁用'}") + print(f" 性能监控: 已启用") print(f" 轨迹记录: 支持录制/回放功能") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 80) - # 显示操作说明 + +def print_instructions(): + """打印操作说明""" print("\n操作说明:") print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") @@ -3363,6 +199,64 @@ def main(): print("程序启动成功!") print("-" * 80) + +def main(): + """主函数""" + print("=" * 60) + print("手势控制无人机系统 - 详细性能统计优化版") + print("详细性能统计优化版!") + print("=" * 60) + + # 导入库 + libs, status = safe_import_libs() + if not status.get('OpenCV', False) or not status.get('PIL', False): + print("\n❌ 核心库缺失,无法启动。") + input("按回车键退出...") + sys.exit(1) + + print("-" * 60) + print("✅ 环境检查通过,正在初始化...") + print("-" * 60) + + # 初始化配置管理器 + config = ConfigManager() + + # 初始化语音管理器 + print("初始化语音反馈系统...") + speech_manager = EnhancedSpeechFeedbackManager(libs['speech'], config) + + # 程序启动语音提示 + if speech_manager.enabled: + speech_manager.speak('program_start', force=True, immediate=True) + speech_manager.speak('system_ready', immediate=True) + + # 初始化其他组件 + print("初始化组件...") + gesture_recognizer = EnhancedGestureRecognizer(speech_manager, config) + drone_controller = SimpleDroneController(libs['airsim'], speech_manager, config) + ui_renderer = ChineseUIRenderer(speech_manager, config) + + # 初始化性能分析器 + print("初始化性能分析器...") + performance_analyzer = PerformanceAnalyzer(speech_manager, libs.get('psutil'), config) + + # 手势轨迹记录器 + print("初始化手势轨迹记录器...") + trajectory_recorder = GestureTrajectoryRecorder(speech_manager, config) + + # 初始化摄像头 + cap = init_camera(config) + if cap and speech_manager.enabled: + speech_manager.speak('camera_ready', immediate=True) + elif not cap and speech_manager.enabled: + speech_manager.speak('camera_error', immediate=True) + + # 显示欢迎信息 + print_welcome_message(cap, speech_manager, libs) + + # 显示操作说明 + print_instructions() + # 键盘手势映射 key_to_gesture = { ord('w'): "Up", ord('W'): "Up", @@ -3480,7 +374,7 @@ def main(): warning_msg = f"⚠ 性能状态: {performance_analyzer.performance_status}" frame = ui_renderer.draw_warning(frame, warning_msg) - # 显示图像(窗口标题用英文) + # 显示图像 cv2.imshow('Gesture Controlled Drone - Performance Statistics', frame) # ========== 键盘控制 ========== @@ -3515,7 +409,7 @@ def main(): elif key == ord('r') or key == ord('R'): # 重置手势识别 print("重置手势识别...") - gesture_recognizer = EnhancedGestureRecognizer(speech_manager) + gesture_recognizer = EnhancedGestureRecognizer(speech_manager, config) print("✓ 手势识别已重置") # 语音提示 @@ -3576,7 +470,7 @@ def main(): elif key == ord('l') or key == ord('L'): # 性能快照 - snapshot_label = f"快照_{datetime.now().strftime('%H:%M:%S')}" + snapshot_label = f"快照_{time.strftime('%H:%M:%S')}" snapshot = performance_analyzer.take_snapshot(snapshot_label) print(f"📸 性能快照已保存: {snapshot_label}") @@ -3617,7 +511,7 @@ def main(): print(f"✓ 已切换到性能模式: {gesture_recognizer.performance_mode_name}") - # 新增:重置性能统计 + # 重置性能统计 elif key == ord('R') and chr(key).isupper(): # 大写R print("重置性能统计...") performance_analyzer.reset_session() @@ -3627,7 +521,7 @@ def main(): if speech_manager.enabled: speech_manager.speak_direct("性能统计已重置") - # 新增:轨迹记录控制 + # 轨迹记录控制 elif key == ord('1'): # 开始录制轨迹 if trajectory_recorder.start_recording(): @@ -3755,6 +649,5 @@ def main(): input("按回车键退出...") -# ========== 程序入口 ========== if __name__ == "__main__": main() \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/performance_analyzer.py b/src/autonomus_drone_hand_gesture_project/performance_analyzer.py new file mode 100644 index 0000000000..6bf7ca3d08 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/performance_analyzer.py @@ -0,0 +1,506 @@ +""" +性能分析器模块 +负责监控和报告系统性能 +作者: xiaoshiyuan888 +""" + +import time +import os +import csv +import numpy as np +from datetime import datetime +from collections import deque, Counter + + +class PerformanceAnalyzer: + """性能分析器 - 监控和报告系统性能""" + + def __init__(self, speech_manager=None, psutil_lib=None, config=None): + self.speech_manager = speech_manager + self.psutil_lib = psutil_lib + self.config = config + self.start_time = time.time() + self.session_start_time = time.time() + + # 帧率统计 + self.frame_times = deque(maxlen=300) + self.frame_count = 0 + self.fps_history = deque(maxlen=100) + + # 手势识别性能 + self.gesture_recognition_times = deque(maxlen=100) + self.avg_recognition_time = 0 + self.max_recognition_time = 0 + + # 系统资源监控 + self.cpu_usage_history = deque(maxlen=100) + self.memory_usage_history = deque(maxlen=100) + + # 性能事件记录 + self.performance_events = [] + self.performance_snapshots = [] + + # 手势统计 + self.gesture_counts = {} + self.gesture_confidence_sum = {} + self.gesture_confidence_count = {} + + # 错误统计 + self.error_count = 0 + self.warning_count = 0 + + # 无人机控制统计 + self.drone_commands = 0 + self.successful_commands = 0 + self.failed_commands = 0 + + # 轨迹记录统计 + self.recording_sessions = 0 + self.total_trajectory_points = 0 + + # 性能日志 + self.performance_log = [] + self.log_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'performance_log.csv') + + # 性能阈值 + self.performance_thresholds = { + 'fps_warning': 15, + 'fps_critical': 5, + 'cpu_warning': 80, + 'cpu_critical': 90, + 'memory_warning': 85, + 'memory_critical': 95, + 'recognition_warning': 50, + 'recognition_critical': 100 + } + + # 性能状态 + self.performance_status = "良好" + self.last_performance_report = 0 + self.auto_report_interval = 60 + + print("✓ 性能分析器已初始化") + + def update_frame(self): + """更新帧统计""" + current_time = time.time() + self.frame_times.append(current_time) + self.frame_count += 1 + + # 计算当前FPS + if len(self.frame_times) > 1: + time_span = self.frame_times[-1] - self.frame_times[0] + if time_span > 0: + current_fps = (len(self.frame_times) - 1) / time_span + self.fps_history.append(current_fps) + + def update_gesture_recognition_time(self, recognition_time_ms): + """更新手势识别时间""" + self.gesture_recognition_times.append(recognition_time_ms) + + # 更新平均识别时间 + if len(self.gesture_recognition_times) > 0: + self.avg_recognition_time = np.mean(list(self.gesture_recognition_times)) + self.max_recognition_time = max(self.max_recognition_time, recognition_time_ms) + + def update_system_resources(self): + """更新系统资源使用情况""" + try: + if self.psutil_lib: + cpu_percent = self.psutil_lib.cpu_percent(interval=0.1) + memory_percent = self.psutil_lib.virtual_memory().percent + + self.cpu_usage_history.append(cpu_percent) + self.memory_usage_history.append(memory_percent) + + # 检查性能问题 + self.check_performance_issues(cpu_percent, memory_percent) + except: + pass + + def check_performance_issues(self, cpu_percent, memory_percent): + """检查性能问题""" + issues = [] + + # 检查FPS + if len(self.fps_history) > 0: + avg_fps = np.mean(list(self.fps_history[-10:])) if len(self.fps_history) >= 10 else self.fps_history[-1] + + if avg_fps < self.performance_thresholds['fps_critical']: + issues.append(("严重", f"帧率过低: {avg_fps:.1f} FPS")) + self.performance_status = "严重" + elif avg_fps < self.performance_thresholds['fps_warning']: + issues.append(("警告", f"帧率较低: {avg_fps:.1f} FPS")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查CPU使用率 + if cpu_percent > self.performance_thresholds['cpu_critical']: + issues.append(("严重", f"CPU使用率过高: {cpu_percent:.1f}%")) + self.performance_status = "严重" + elif cpu_percent > self.performance_thresholds['cpu_warning']: + issues.append(("警告", f"CPU使用率较高: {cpu_percent:.1f}%")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查内存使用率 + if memory_percent > self.performance_thresholds['memory_critical']: + issues.append(("严重", f"内存使用率过高: {memory_percent:.1f}%")) + self.performance_status = "严重" + elif memory_percent > self.performance_thresholds['memory_warning']: + issues.append(("警告", f"内存使用率较高: {memory_percent:.1f}%")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查手势识别时间 + if self.avg_recognition_time > self.performance_thresholds['recognition_critical']: + issues.append(("严重", f"手势识别时间过长: {self.avg_recognition_time:.1f}ms")) + self.performance_status = "严重" + elif self.avg_recognition_time > self.performance_thresholds['recognition_warning']: + issues.append(("警告", f"手势识别时间较长: {self.avg_recognition_time:.1f}ms")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 记录性能事件 + if issues: + for level, message in issues: + self.add_performance_event(level, message) + + # 语音提示(仅在状态变化时) + if (self.speech_manager and + self.speech_manager.enabled and + level == "严重"): + current_time = time.time() + if current_time - self.last_performance_report > 10: + self.speech_manager.speak_direct(f"性能{level}: {message}") + self.last_performance_report = current_time + + def add_performance_event(self, level, message): + """添加性能事件""" + event = { + 'timestamp': time.time(), + 'level': level, + 'message': message, + 'session_time': time.time() - self.session_start_time + } + self.performance_events.append(event) + + # 记录到日志 + self.log_performance_event(event) + + if level == "警告": + self.warning_count += 1 + elif level == "严重": + self.error_count += 1 + + def log_performance_event(self, event): + """记录性能事件到日志""" + log_entry = { + 'timestamp': datetime.fromtimestamp(event['timestamp']).strftime('%Y-%m-%d %H:%M:%S'), + 'session_time': f"{event['session_time']:.1f}", + 'level': event['level'], + 'message': event['message'] + } + self.performance_log.append(log_entry) + + def record_gesture(self, gesture, confidence): + """记录手势统计""" + if gesture not in self.gesture_counts: + self.gesture_counts[gesture] = 0 + self.gesture_confidence_sum[gesture] = 0 + self.gesture_confidence_count[gesture] = 0 + + self.gesture_counts[gesture] += 1 + self.gesture_confidence_sum[gesture] += confidence + self.gesture_confidence_count[gesture] += 1 + + def record_drone_command(self, success=True): + """记录无人机命令""" + self.drone_commands += 1 + if success: + self.successful_commands += 1 + else: + self.failed_commands += 1 + + def record_recording_session(self, points_count=0): + """记录录制会话""" + self.recording_sessions += 1 + self.total_trajectory_points += points_count + + def take_snapshot(self, label=""): + """拍摄性能快照""" + snapshot = { + 'timestamp': time.time(), + 'label': label, + 'fps': self.get_current_fps(), + 'avg_fps': self.get_average_fps(), + 'avg_recognition_time': self.avg_recognition_time, + 'max_recognition_time': self.max_recognition_time, + 'cpu_usage': self.get_current_cpu_usage(), + 'memory_usage': self.get_current_memory_usage(), + 'gesture_counts': dict(self.gesture_counts), + 'performance_status': self.performance_status, + 'frame_count': self.frame_count, + 'session_duration': time.time() - self.session_start_time + } + self.performance_snapshots.append(snapshot) + + print(f"📸 性能快照已保存: {label}") + return snapshot + + def get_current_fps(self): + """获取当前FPS""" + if len(self.fps_history) > 0: + return self.fps_history[-1] + return 0 + + def get_average_fps(self): + """获取平均FPS""" + if len(self.fps_history) > 0: + return np.mean(list(self.fps_history)) + return 0 + + def get_current_cpu_usage(self): + """获取当前CPU使用率""" + if len(self.cpu_usage_history) > 0: + return self.cpu_usage_history[-1] + return 0 + + def get_current_memory_usage(self): + """获取当前内存使用率""" + if len(self.memory_usage_history) > 0: + return self.memory_usage_history[-1] + return 0 + + def generate_report(self, detailed=True): + """生成性能报告""" + report_time = time.time() + session_duration = report_time - self.session_start_time + + # 基础报告 + report = { + '生成时间': datetime.fromtimestamp(report_time).strftime('%Y-%m-%d %H:%M:%S'), + '会话时长': f"{session_duration:.1f}秒", + '总帧数': self.frame_count, + '平均FPS': f"{self.get_average_fps():.1f}", + '当前FPS': f"{self.get_current_fps():.1f}", + '平均手势识别时间': f"{self.avg_recognition_time:.1f}ms", + '最大手势识别时间': f"{self.max_recognition_time:.1f}ms", + '当前CPU使用率': f"{self.get_current_cpu_usage():.1f}%", + '当前内存使用率': f"{self.get_current_memory_usage():.1f}%", + '性能状态': self.performance_status, + '警告数量': self.warning_count, + '错误数量': self.error_count, + '无人机命令': { + '总数': self.drone_commands, + '成功': self.successful_commands, + '失败': self.failed_commands, + '成功率': f"{(self.successful_commands / self.drone_commands * 100 if self.drone_commands > 0 else 0):.1f}%" + }, + '录制统计': { + '会话数': self.recording_sessions, + '总轨迹点数': self.total_trajectory_points + } + } + + # 详细报告 + if detailed: + # 手势统计 + gesture_stats = {} + for gesture in self.gesture_counts: + count = self.gesture_counts[gesture] + if gesture in self.gesture_confidence_count and self.gesture_confidence_count[gesture] > 0: + avg_confidence = self.gesture_confidence_sum[gesture] / self.gesture_confidence_count[gesture] + else: + avg_confidence = 0 + + gesture_stats[gesture] = { + '次数': count, + '占比': f"{(count / self.frame_count * 100 if self.frame_count > 0 else 0):.1f}%", + '平均置信度': f"{avg_confidence:.1%}" + } + + report['手势统计'] = gesture_stats + + # 性能事件 + if self.performance_events: + recent_events = list(self.performance_events)[-10:] + report['最近性能事件'] = [ + { + '时间': datetime.fromtimestamp(e['timestamp']).strftime('%H:%M:%S'), + '级别': e['level'], + '消息': e['message'] + } + for e in recent_events + ] + + # 性能快照 + if self.performance_snapshots: + report['性能快照数'] = len(self.performance_snapshots) + + # 系统建议 + suggestions = self.generate_suggestions() + if suggestions: + report['优化建议'] = suggestions + + return report + + def generate_suggestions(self): + """生成优化建议""" + suggestions = [] + + # 检查FPS + avg_fps = self.get_average_fps() + if avg_fps < self.performance_thresholds['fps_warning']: + suggestions.append(f"帧率较低({avg_fps:.1f}FPS),建议切换到'最快'性能模式") + + # 检查CPU + cpu_usage = self.get_current_cpu_usage() + if cpu_usage > self.performance_thresholds['cpu_warning']: + suggestions.append(f"CPU使用率较高({cpu_usage:.1f}%),请关闭其他占用CPU的程序") + + # 检查内存 + memory_usage = self.get_current_memory_usage() + if memory_usage > self.performance_thresholds['memory_warning']: + suggestions.append(f"内存使用率较高({memory_usage:.1f}%),请关闭不必要的程序") + + # 检查识别时间 + if self.avg_recognition_time > self.performance_thresholds['recognition_warning']: + suggestions.append(f"手势识别时间较长({self.avg_recognition_time:.1f}ms),建议调整摄像头位置或光线") + + return suggestions + + def print_report(self, detailed=True): + """打印性能报告""" + report = self.generate_report(detailed) + + print("\n" + "=" * 80) + print("📊 性能分析报告") + print("=" * 80) + + # 基础信息 + print(f"生成时间: {report['生成时间']}") + print(f"会话时长: {report['会话时长']}") + print(f"总帧数: {report['总帧数']}") + print(f"平均FPS: {report['平均FPS']}") + print(f"当前FPS: {report['当前FPS']}") + print(f"平均手势识别时间: {report['平均手势识别时间']}") + print(f"最大手势识别时间: {report['最大手势识别时间']}") + print(f"当前CPU使用率: {report['当前CPU使用率']}") + print(f"当前内存使用率: {report['当前内存使用率']}") + print(f"性能状态: {report['性能状态']}") + + # 无人机命令统计 + cmd_stats = report['无人机命令'] + print(f"\n无人机命令统计:") + print(f" 总数: {cmd_stats['总数']}") + print(f" 成功: {cmd_stats['成功']}") + print(f" 失败: {cmd_stats['失败']}") + print(f" 成功率: {cmd_stats['成功率']}") + + # 录制统计 + rec_stats = report['录制统计'] + print(f"\n录制统计:") + print(f" 会话数: {rec_stats['会话数']}") + print(f" 总轨迹点数: {rec_stats['总轨迹点数']}") + + # 详细报告 + if detailed and '手势统计' in report: + print(f"\n手势统计:") + for gesture, stats in report['手势统计'].items(): + print(f" {gesture}: {stats['次数']}次 ({stats['占比']}), 平均置信度: {stats['平均置信度']}") + + # 性能事件 + if detailed and '最近性能事件' in report and report['最近性能事件']: + print(f"\n最近性能事件:") + for event in report['最近性能事件']: + print(f" [{event['时间']}] {event['级别']}: {event['消息']}") + + # 优化建议 + if detailed and '优化建议' in report and report['优化建议']: + print(f"\n优化建议:") + for i, suggestion in enumerate(report['优化建议'], 1): + print(f" {i}. {suggestion}") + + print("=" * 80) + + # 语音播报摘要 + if self.speech_manager and self.speech_manager.enabled: + summary = (f"性能报告: 平均帧率{report['平均FPS']},识别时间{report['平均手势识别时间']}," + f"性能状态{report['性能状态']},无人机命令成功率{cmd_stats['成功率']}") + self.speech_manager.speak_direct(summary) + + def export_log(self, filename=None): + """导出性能日志""" + import os + current_dir = os.path.dirname(os.path.abspath(__file__)) + if filename is None: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = os.path.join(current_dir, f'performance_log_{timestamp}.csv') + + try: + with open(filename, 'w', newline='', encoding='utf-8') as f: + if self.performance_log: + fieldnames = self.performance_log[0].keys() + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(self.performance_log) + + print(f"📤 性能日志已导出到: {filename}") + return True + except Exception as e: + print(f"❌ 导出性能日志失败: {e}") + return False + + def auto_report(self): + """自动性能报告(定期执行)""" + current_time = time.time() + if current_time - self.last_performance_report > self.auto_report_interval: + # 生成简要报告 + report = self.generate_report(detailed=False) + + # 检查是否需要报告 + if (self.performance_status == "严重" or + self.warning_count > 5 or + self.error_count > 0): + + print(f"⚠ 自动性能检查: {report['性能状态']}, FPS: {report['当前FPS']}, " + f"CPU: {report['当前CPU使用率']}, 内存: {report['当前内存使用率']}") + + # 语音提示 + if (self.speech_manager and + self.speech_manager.enabled and + self.performance_status == "严重"): + self.speech_manager.speak_direct(f"系统性能{self.performance_status},建议检查") + + self.last_performance_report = current_time + + def reset_session(self): + """重置会话统计""" + self.session_start_time = time.time() + self.performance_events = [] + self.performance_snapshots = [] + self.gesture_counts = {} + self.gesture_confidence_sum = {} + self.gesture_confidence_count = {} + self.error_count = 0 + self.warning_count = 0 + self.drone_commands = 0 + self.successful_commands = 0 + self.failed_commands = 0 + self.performance_status = "良好" + + print("✓ 性能统计会话已重置") + + def get_stats_summary(self): + """获取统计摘要""" + return { + 'fps': self.get_current_fps(), + 'avg_fps': self.get_average_fps(), + 'recognition_time': self.avg_recognition_time, + 'cpu_usage': self.get_current_cpu_usage(), + 'memory_usage': self.get_current_memory_usage(), + 'performance_status': self.performance_status, + 'gesture_count': sum(self.gesture_counts.values()), + 'unique_gestures': len(self.gesture_counts) + } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/speech_manager.py b/src/autonomus_drone_hand_gesture_project/speech_manager.py new file mode 100644 index 0000000000..f2f54a7309 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/speech_manager.py @@ -0,0 +1,361 @@ +""" +语音反馈管理器模块 +负责语音提示和音频播放 +作者: xiaoshiyuan888 +""" + +import threading +import time +import os +import sys +import tempfile + + +class EnhancedSpeechFeedbackManager: + """增强的语音反馈管理器""" + + def __init__(self, speech_lib, config=None): + self.speech_lib = speech_lib + self.config = config + self.enabled = True + self.volume = 1.0 + self.rate = 150 + self.voice_id = None + self.last_speech_time = {} + self.min_interval = 1.5 + + # 语音队列 + self.speech_queue = [] + self.is_speaking = False + self.queue_thread = None + + # 音频播放方法 + self.audio_method = None + + # 手势状态追踪 + self.last_gesture_state = "none" + self.gesture_active_time = 0 + + # 语音消息映射 + self.messages = { + # 连接相关 + 'connecting': "正在连接无人机,请稍候", + 'connected': "无人机连接成功", + 'connection_failed': "无人机连接失败,进入模拟模式", + + # 飞行相关 + 'taking_off': "无人机正在起飞", + 'takeoff_success': "起飞成功", + 'takeoff_failed': "起飞失败", + 'landing': "无人机正在降落", + 'land_success': "降落成功", + 'emergency_stop': "紧急停止,无人机已降落", + 'hovering': "无人机悬停中", + + # 手势相关 + 'gesture_detected': "手势识别就绪,请开始手势", + 'gesture_start': "开始识别手势", + 'gesture_end': "手势识别结束", + 'gesture_stop': "停止", + 'gesture_up': "向上", + 'gesture_down': "向下", + 'gesture_left': "向左", + 'gesture_right': "向右", + 'gesture_forward': "向前", + 'gesture_backward': "向后", + 'gesture_waiting': "等待手势", + 'gesture_error': "手势识别错误", + 'gesture_stable': "手势稳定", + 'gesture_change': "手势变化", + 'gesture_low_confidence': "手势识别置信度低", + 'gesture_good_confidence': "手势识别置信度高", + 'gesture_hover': "悬停", + + # 系统相关 + 'program_start': "手势控制无人机系统已启动", + 'program_exit': "程序退出,感谢使用", + 'camera_error': "摄像头错误,请检查连接", + 'camera_ready': "摄像头就绪", + 'system_ready': "系统准备就绪", + + # 模式相关 + 'simulation_mode': "进入模拟模式", + 'debug_mode_on': "调试模式已开启", + 'debug_mode_off': "调试模式已关闭", + 'display_mode_changed': "显示模式已切换", + 'help_toggled': "帮助信息已切换", + 'performance_mode_fast': "切换到最快性能模式", + 'performance_mode_balanced': "切换到平衡性能模式", + 'performance_mode_accurate': "切换到最准确性能模式", + + # 性能相关 + 'performance_good': "系统运行流畅", + 'performance_warning': "系统性能警告", + 'performance_critical': "系统性能严重警告", + 'performance_report': "性能报告生成完成", + 'performance_snapshot': "性能快照已保存", + 'performance_log_exported': "性能日志已导出", + + # 手势指导 + 'move_closer': "请将手靠近摄像头", + 'move_away': "请将手移远一些", + 'good_position': "手部位置良好", + 'hand_detected': "手部已检测到", + 'hand_lost': "手部丢失,请重新放置", + + # 录制相关 + 'recording_start': "开始录制手势轨迹", + 'recording_stop': "停止录制", + 'recording_saved': "轨迹已保存", + 'recording_loaded': "轨迹已加载", + 'recording_playback_start': "开始回放手势轨迹", + 'recording_playback_stop': "回放结束", + 'recording_cleared': "轨迹已清除", + 'recording_paused': "回放已暂停", + 'recording_resumed': "回放继续", + 'recording_not_found': "未找到轨迹数据", + 'recording_frame_count': "轨迹帧数", + } + + # 初始化语音引擎 + self.init_speech_engine() + + def init_speech_engine(self): + """初始化语音引擎""" + if self.speech_lib is None: + print("⚠ 语音库未找到,语音功能禁用") + self.enabled = False + return + + try: + if hasattr(self.speech_lib, 'init'): # pyttsx3 + self.engine = self.speech_lib.init() + self.audio_method = 'pyttsx3' + + # 设置语音参数 + voices = self.engine.getProperty('voices') + + # 尝试寻找中文语音 + for voice in voices: + if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower() or 'zh_CN' in voice.name.lower(): + self.engine.setProperty('voice', voice.id) + self.voice_id = voice.id + print(f"[Speech] 使用中文语音: {voice.name}") + break + + # 如果没找到中文语音,使用第一个可用语音 + if self.voice_id is None and len(voices) > 0: + self.engine.setProperty('voice', voices[0].id) + print(f"[Speech] 使用默认语音: {voices[0].name}") + + # 设置语速和音量 + self.engine.setProperty('rate', self.rate) + self.engine.setProperty('volume', self.volume) + + print("✅ 语音引擎初始化成功 (pyttsx3)") + + elif isinstance(self.speech_lib, dict) and self.speech_lib.get('type') == 'gtts': + print("✅ 语音引擎初始化成功 (gTTS,需要网络连接)") + self.audio_method = 'gtts' + + # 确定播放方法 + if 'pygame' in self.speech_lib: + self.audio_method = 'gtts_pygame' + print("✅ 使用pygame播放音频") + elif 'pydub' in self.speech_lib: + self.audio_method = 'gtts_pydub' + print("✅ 使用pydub播放音频") + elif 'play_method' in self.speech_lib: + self.audio_method = f"gtts_{self.speech_lib['play_method']}" + print(f"✅ 使用系统命令播放音频") + else: + self.audio_method = 'gtts_system' + print("✅ 使用默认系统播放器") + + else: + print("⚠ 未知语音库类型,语音功能可能不正常") + self.enabled = False + + except Exception as e: + print(f"⚠ 语音引擎初始化失败: {e}") + self.enabled = False + + def play_audio_file(self, audio_file): + """播放音频文件""" + try: + if self.audio_method == 'gtts_pygame' and 'pygame' in self.speech_lib: + pygame = self.speech_lib['pygame'] + pygame.mixer.music.load(audio_file) + pygame.mixer.music.play() + + # 等待播放完成 + while pygame.mixer.music.get_busy(): + pygame.time.Clock().tick(10) + + elif self.audio_method == 'gtts_pydub' and 'pydub' in self.speech_lib: + AudioSegment = self.speech_lib['AudioSegment'] + play = self.speech_lib['play'] + + audio = AudioSegment.from_mp3(audio_file) + play(audio) + + elif self.audio_method == 'gtts_windows': + # Windows系统命令 + os.startfile(audio_file) + time.sleep(1.5) + + elif self.audio_method == 'gtts_posix': + # Linux/Mac系统命令 + import subprocess + if sys.platform == 'darwin': + subprocess.call(['afplay', audio_file]) + else: + subprocess.call(['xdg-open', audio_file]) + + else: + # 通用方法 + import subprocess + if sys.platform == 'win32': + os.startfile(audio_file) + elif sys.platform == 'darwin': + subprocess.call(['open', audio_file]) + else: + subprocess.call(['xdg-open', audio_file]) + + return True + + except Exception as e: + print(f"⚠ 音频播放失败: {e}") + return False + + def speak(self, message_key, force=False, immediate=False): + """播放语音""" + if not self.enabled: + return + + # 检查是否在最小间隔内 + current_time = time.time() + if not force and message_key in self.last_speech_time: + if current_time - self.last_speech_time[message_key] < self.min_interval: + return + + # 获取消息文本 + if message_key in self.messages: + text = self.messages[message_key] + else: + text = message_key + + # 立即播放或加入队列 + if immediate: + self.speak_direct(text) + else: + self.speech_queue.append(text) + + if not self.is_speaking and self.queue_thread is None: + self.queue_thread = threading.Thread(target=self._process_speech_queue) + self.queue_thread.daemon = True + self.queue_thread.start() + + # 更新时间戳 + self.last_speech_time[message_key] = current_time + + def _process_speech_queue(self): + """处理语音队列""" + while self.speech_queue and self.enabled: + self.is_speaking = True + + text = self.speech_queue.pop(0) + + try: + if self.audio_method == 'pyttsx3': + self.engine.stop() + self.engine.say(text) + self.engine.runAndWait() + + elif self.audio_method.startswith('gtts'): + tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') + + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: + temp_file = f.name + tts.save(temp_file) + + self.play_audio_file(temp_file) + + try: + os.unlink(temp_file) + except: + pass + + except Exception as e: + print(f"⚠ 语音播放失败: {e}") + + time.sleep(0.05) + + self.is_speaking = False + self.queue_thread = None + + def speak_direct(self, text): + """直接播放文本""" + if not self.enabled: + return + + thread = threading.Thread(target=self._speak_thread, args=(text,)) + thread.daemon = True + thread.start() + + def _speak_thread(self, text): + """语音播放线程""" + try: + if self.audio_method == 'pyttsx3': + self.engine.say(text) + self.engine.runAndWait() + + elif self.audio_method.startswith('gtts'): + tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') + + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: + temp_file = f.name + tts.save(temp_file) + + self.play_audio_file(temp_file) + + try: + os.unlink(temp_file) + except: + pass + + except Exception as e: + print(f"⚠ 直接语音播放失败: {e}") + + def stop(self): + """停止所有语音""" + if hasattr(self, 'engine'): + self.engine.stop() + + self.speech_queue.clear() + self.is_speaking = False + + def set_enabled(self, enabled): + """启用/禁用语音""" + self.enabled = enabled + if not enabled: + self.stop() + + def toggle_enabled(self): + """切换语音启用状态""" + self.enabled = not self.enabled + status = "启用" if self.enabled else "禁用" + self.speak_direct(f"语音反馈已{status}") + return self.enabled + + def get_status(self): + """获取语音状态""" + return { + 'enabled': self.enabled, + 'engine': 'pyttsx3' if self.audio_method == 'pyttsx3' else + 'gTTS' if self.audio_method.startswith('gtts') else + 'None', + 'queue_size': len(self.speech_queue), + 'is_speaking': self.is_speaking, + 'audio_method': self.audio_method + } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/trajectory_recorder.py b/src/autonomus_drone_hand_gesture_project/trajectory_recorder.py new file mode 100644 index 0000000000..ffeabf09d5 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/trajectory_recorder.py @@ -0,0 +1,347 @@ +""" +手势轨迹记录器模块 +负责录制、保存、加载和回放手势轨迹 +作者: xiaoshiyuan888 +""" + +import os +import time +import pickle +import cv2 +from datetime import datetime + + +class GestureTrajectoryRecorder: + """手势轨迹记录器 - 记录、保存、加载和回放手势轨迹""" + + def __init__(self, speech_manager=None, config=None): + self.speech_manager = speech_manager + self.config = config + self.trajectory_data = [] + self.is_recording = False + self.is_playing = False + self.playback_index = 0 + self.playback_paused = False + self.max_trajectory_points = 1000 + self.recording_start_time = 0 + self.last_save_time = 0 + self.save_interval = 5 + + # 轨迹文件路径 + current_dir = os.path.dirname(os.path.abspath(__file__)) + self.trajectory_dir = os.path.join(current_dir, 'trajectories') + if not os.path.exists(self.trajectory_dir): + os.makedirs(self.trajectory_dir) + + # 默认轨迹文件名 + self.default_filename = os.path.join(self.trajectory_dir, + f'trajectory_{datetime.now().strftime("%Y%m%d_%H%M%S")}.pkl') + + # 轨迹可视化设置 + self.trajectory_colors = [ + (255, 0, 0), # 红色 - 起点 + (0, 255, 0), # 绿色 - 正常点 + (0, 0, 255), # 蓝色 - 终点 + (255, 255, 0), # 青色 - 特殊点 + (255, 0, 255) # 紫色 - 特殊点 + ] + + self.show_trajectory = True + self.trajectory_thickness = 2 + self.trajectory_max_length = 100 + + print("✓ 手势轨迹记录器已初始化") + + def start_recording(self): + """开始录制手势轨迹""" + if self.is_recording: + return False + + self.trajectory_data = [] + self.is_recording = True + self.recording_start_time = time.time() + self.last_save_time = time.time() + + print("🎬 开始录制手势轨迹") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_start', immediate=True) + + return True + + def stop_recording(self): + """停止录制手势轨迹""" + if not self.is_recording: + return False + + self.is_recording = False + recording_duration = time.time() - self.recording_start_time + + print(f"⏹️ 停止录制手势轨迹") + print(f" 录制时长: {recording_duration:.1f}秒") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_stop', immediate=True) + if len(self.trajectory_data) > 0: + self.speech_manager.speak_direct(f"录制了{len(self.trajectory_data)}个轨迹点") + + return True + + def add_trajectory_point(self, hand_data, gesture, confidence, frame_shape): + """添加轨迹点""" + if not self.is_recording or len(self.trajectory_data) >= self.max_trajectory_points: + return False + + if hand_data is None: + return False + + # 创建轨迹点数据 + trajectory_point = { + 'timestamp': time.time(), + 'hand_position': hand_data['position'] if 'position' in hand_data else (0.5, 0.5), + 'hand_center': hand_data['center'] if 'center' in hand_data else (0, 0), + 'gesture': gesture, + 'confidence': confidence, + 'fingertips': hand_data.get('fingertips', []), + 'frame_shape': frame_shape + } + + self.trajectory_data.append(trajectory_point) + + # 自动保存检查 + current_time = time.time() + if current_time - self.last_save_time >= self.save_interval and len(self.trajectory_data) > 10: + self.auto_save() + self.last_save_time = current_time + + return True + + def auto_save(self): + """自动保存轨迹(临时文件)""" + if len(self.trajectory_data) == 0: + return + + temp_file = os.path.join(self.trajectory_dir, 'trajectory_temp.pkl') + try: + with open(temp_file, 'wb') as f: + pickle.dump(self.trajectory_data, f) + print(f"💾 自动保存轨迹到临时文件 ({len(self.trajectory_data)}个点)") + except Exception as e: + print(f"⚠ 自动保存轨迹失败: {e}") + + def save_trajectory(self, filename=None): + """保存轨迹到文件""" + if len(self.trajectory_data) == 0: + print("⚠ 没有轨迹数据可保存") + return False + + if filename is None: + filename = self.default_filename + + try: + with open(filename, 'wb') as f: + pickle.dump(self.trajectory_data, f) + + print(f"💾 轨迹已保存到: {filename}") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_saved', immediate=True) + self.speech_manager.speak_direct(f"保存了{len(self.trajectory_data)}个轨迹点") + + return True + except Exception as e: + print(f"❌ 保存轨迹失败: {e}") + return False + + def load_trajectory(self, filename): + """从文件加载轨迹""" + try: + if not os.path.exists(filename): + print(f"❌ 轨迹文件不存在: {filename}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_not_found', immediate=True) + + return False + + with open(filename, 'rb') as f: + self.trajectory_data = pickle.load(f) + + print(f"📂 轨迹已从文件加载: {filename}") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_loaded', immediate=True) + self.speech_manager.speak_direct(f"加载了{len(self.trajectory_data)}个轨迹点") + + return True + except Exception as e: + print(f"❌ 加载轨迹失败: {e}") + return False + + def start_playback(self): + """开始回放轨迹""" + if len(self.trajectory_data) == 0: + print("⚠ 没有轨迹数据可回放") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_not_found', immediate=True) + + return False + + self.is_playing = True + self.playback_index = 0 + self.playback_paused = False + + print(f"▶️ 开始回放手势轨迹") + print(f" 总帧数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_playback_start', immediate=True) + + return True + + def stop_playback(self): + """停止回放轨迹""" + if not self.is_playing: + return False + + self.is_playing = False + self.playback_paused = False + + print("⏹️ 停止回放手势轨迹") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_playback_stop', immediate=True) + + return True + + def pause_playback(self): + """暂停/继续回放""" + self.playback_paused = not self.playback_paused + + status = "暂停" if self.playback_paused else "继续" + print(f"⏸️ 回放已{status}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + if self.playback_paused: + self.speech_manager.speak('recording_paused', immediate=True) + else: + self.speech_manager.speak('recording_resumed', immediate=True) + + return self.playback_paused + + def get_next_playback_point(self): + """获取下一个回放点""" + if not self.is_playing or self.playback_paused or len(self.trajectory_data) == 0: + return None + + if self.playback_index >= len(self.trajectory_data): + self.stop_playback() + return None + + point = self.trajectory_data[self.playback_index] + self.playback_index += 1 + + # 检查是否到达末尾 + if self.playback_index >= len(self.trajectory_data): + self.stop_playback() + + return point + + def clear_trajectory(self): + """清除轨迹数据""" + self.trajectory_data = [] + self.is_recording = False + self.is_playing = False + self.playback_index = 0 + + print("🗑️ 轨迹数据已清除") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_cleared', immediate=True) + + return True + + def draw_trajectory(self, frame): + """在帧上绘制轨迹""" + if not self.show_trajectory or len(self.trajectory_data) == 0: + return frame + + h, w = frame.shape[:2] + + # 限制显示的轨迹点数 + display_points = min(len(self.trajectory_data), self.trajectory_max_length) + start_idx = max(0, len(self.trajectory_data) - display_points) + + # 绘制轨迹线 + for i in range(start_idx, len(self.trajectory_data) - 1): + point1 = self.trajectory_data[i] + point2 = self.trajectory_data[i + 1] + + # 获取手部中心位置 + if 'hand_center' in point1 and 'hand_center' in point2: + x1, y1 = point1['hand_center'] + x2, y2 = point2['hand_center'] + + # 确保坐标在图像范围内 + x1 = max(0, min(w - 1, x1)) + y1 = max(0, min(h - 1, y1)) + x2 = max(0, min(w - 1, x2)) + y2 = max(0, min(h - 1, y2)) + + # 根据索引计算颜色 + color_idx = int((i - start_idx) / display_points * (len(self.trajectory_colors) - 1)) + color = self.trajectory_colors[color_idx] + + # 绘制线条 + cv2.line(frame, (int(x1), int(y1)), (int(x2), int(y2)), + color, self.trajectory_thickness) + + # 绘制当前点 + if self.is_recording or self.is_playing: + current_idx = len(self.trajectory_data) - 1 if self.is_recording else self.playback_index - 1 + if 0 <= current_idx < len(self.trajectory_data): + point = self.trajectory_data[current_idx] + if 'hand_center' in point: + x, y = point['hand_center'] + x = max(0, min(w - 1, x)) + y = max(0, min(h - 1, y)) + + # 绘制当前点 + cv2.circle(frame, (int(x), int(y)), 8, (0, 255, 255), -1) + cv2.circle(frame, (int(x), int(y)), 8, (0, 0, 0), 2) + + return frame + + def get_status(self): + """获取录制状态""" + return { + 'is_recording': self.is_recording, + 'is_playing': self.is_playing, + 'playback_paused': self.playback_paused, + 'trajectory_points': len(self.trajectory_data), + 'playback_index': self.playback_index, + 'playback_total': len(self.trajectory_data), + 'recording_duration': time.time() - self.recording_start_time if self.is_recording else 0 + } + + def list_saved_trajectories(self): + """列出保存的轨迹文件""" + try: + files = [f for f in os.listdir(self.trajectory_dir) if f.endswith('.pkl')] + return sorted(files, reverse=True) + except: + return [] \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/ui_renderer.py b/src/autonomus_drone_hand_gesture_project/ui_renderer.py new file mode 100644 index 0000000000..bd2e396fc4 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/ui_renderer.py @@ -0,0 +1,350 @@ +""" +用户界面渲染器模块 +负责绘制中文界面和可视化元素 +作者: xiaoshiyuan888 +""" + +import cv2 +import numpy as np +from PIL import Image, ImageDraw, ImageFont + + +class ChineseUIRenderer: + """中文UI渲染器""" + + def __init__(self, speech_manager=None, config=None): + self.fonts = {} + self.speech_manager = speech_manager + self.config = config + self.load_fonts() + + # 颜色定义 + self.colors = { + 'title': (0, 255, 255), + 'connected': (0, 255, 0), + 'disconnected': (0, 0, 255), + 'flying': (0, 255, 0), + 'landed': (255, 165, 0), + 'warning': (0, 165, 255), + 'info': (255, 255, 255), + 'help': (255, 200, 100), + 'speech_enabled': (0, 255, 0), + 'speech_disabled': (255, 0, 0), + 'performance_good': (0, 255, 0), + 'performance_warning': (255, 165, 0), + 'performance_bad': (255, 0, 0), + 'recording': (255, 50, 50), + 'playback': (50, 50, 255), + 'paused': (255, 255, 0), + 'performance_fast': (0, 255, 0), + 'performance_balanced': (255, 165, 0), + 'performance_accurate': (255, 0, 0), + 'cpu_good': (0, 255, 0), + 'cpu_warning': (255, 165, 0), + 'cpu_critical': (255, 0, 0), + 'memory_good': (0, 255, 0), + 'memory_warning': (255, 165, 0), + 'memory_critical': (255, 0, 0), + } + + print("✓ 中文UI渲染器已初始化") + + def load_fonts(self): + """加载字体""" + font_paths = [ + 'simhei.ttf', + 'C:/Windows/Fonts/simhei.ttf', + 'C:/Windows/Fonts/msyh.ttc', + '/System/Library/Fonts/PingFang.ttc', + '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' + ] + + for path in font_paths: + try: + self.fonts[14] = ImageFont.truetype(path, 14) + self.fonts[16] = ImageFont.truetype(path, 16) + self.fonts[18] = ImageFont.truetype(path, 18) + self.fonts[20] = ImageFont.truetype(path, 20) + self.fonts[24] = ImageFont.truetype(path, 24) + print(f"✓ 字体已加载: {path}") + return + except: + continue + + print("⚠ 未找到字体,使用默认") + + def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): + """在图像上绘制文本""" + try: + img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + pil_img = Image.fromarray(img_rgb) + draw = ImageDraw.Draw(pil_img) + + font = self.fonts.get(size, self.fonts.get(16)) + + # 绘制阴影 + shadow_color = (0, 0, 0) + shadow_pos = (pos[0] + 1, pos[1] + 1) + draw.text(shadow_pos, text, font=font, fill=shadow_color) + + # 绘制文字 + rgb_color = color[::-1] # BGR to RGB + draw.text(pos, text, font=font, fill=rgb_color) + + return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) + except: + # 备用方案 + cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, + size / 25, color, 1) + return frame + + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, + trajectory_recorder=None, gesture_recognizer=None, performance_analyzer=None): + """绘制状态栏(增强版)""" + h, w = frame.shape[:2] + + # 绘制半透明背景 + overlay = frame.copy() + cv2.rectangle(overlay, (0, 0), (w, 130), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + + # 标题 + title = "手势控制无人机系统 - 性能统计版" + frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) + + # 连接状态 + status_color = self.colors['connected'] if drone_controller.connected else self.colors['disconnected'] + status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" + frame = self.draw_text(frame, status_text, (10, 40), size=16, color=status_color) + + # 飞行状态 + flight_color = self.colors['flying'] if drone_controller.flying else self.colors['landed'] + flight_text = f"飞行状态: {'飞行中' if drone_controller.flying else '已降落'}" + frame = self.draw_text(frame, flight_text, (10, 65), size=16, color=flight_color) + + # 手势信息 + if confidence > 0.7: + gesture_color = (0, 255, 0) + elif confidence > 0.5: + gesture_color = (255, 165, 0) + else: + gesture_color = (200, 200, 200) + + gesture_text = f"当前手势: {gesture}" + if self.config.get('display', 'show_confidence'): + gesture_text += f" ({confidence:.0%})" + + frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) + + # 性能模式显示 + if gesture_recognizer and self.config.get('display', 'show_performance_mode'): + mode_info = gesture_recognizer.get_performance_mode_info() + + # 根据模式选择颜色 + if mode_info['mode'] == 'fast': + mode_color = self.colors['performance_fast'] + elif mode_info['mode'] == 'balanced': + mode_color = self.colors['performance_balanced'] + else: # accurate + mode_color = self.colors['performance_accurate'] + + mode_text = f"性能模式: {mode_info['name']}" + frame = self.draw_text(frame, mode_text, (w // 2, 65), size=14, color=mode_color) + + # 录制/回放状态 + elif trajectory_recorder and self.config.get('display', 'show_recording_status'): + recorder_status = trajectory_recorder.get_status() + + if recorder_status['is_recording']: + status_color = self.colors['recording'] + status_text = f"录制中: {recorder_status['trajectory_points']}点" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) + elif recorder_status['is_playing']: + if recorder_status['playback_paused']: + status_color = self.colors['paused'] + status_text = f"回放暂停: {recorder_status['playback_index']}/{recorder_status['playback_total']}" + else: + status_color = self.colors['playback'] + status_text = f"回放中: {recorder_status['playback_index']}/{recorder_status['playback_total']}" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) + else: + status_text = f"轨迹点: {recorder_status['trajectory_points']}" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=12, color=self.colors['info']) + + # 性能统计显示 + if performance_analyzer and self.config.get('display', 'show_performance_stats'): + perf_stats = performance_analyzer.get_stats_summary() + + # 性能状态颜色 + if perf_stats['performance_status'] == "良好": + perf_color = self.colors['performance_good'] + elif perf_stats['performance_status'] == "警告": + perf_color = self.colors['performance_warning'] + else: + perf_color = self.colors['performance_bad'] + + perf_text = f"性能: {perf_stats['performance_status']} ({perf_stats['fps']:.0f}FPS)" + frame = self.draw_text(frame, perf_text, (w // 2, 90), size=12, color=perf_color) + + # 系统资源显示 + if performance_analyzer and self.config.get('display', 'show_system_resources'): + cpu_usage = performance_analyzer.get_current_cpu_usage() + memory_usage = performance_analyzer.get_current_memory_usage() + + # CPU颜色 + if cpu_usage < 70: + cpu_color = self.colors['cpu_good'] + elif cpu_usage < 85: + cpu_color = self.colors['cpu_warning'] + else: + cpu_color = self.colors['cpu_critical'] + + # 内存颜色 + if memory_usage < 70: + mem_color = self.colors['memory_good'] + elif memory_usage < 85: + mem_color = self.colors['memory_warning'] + else: + mem_color = self.colors['memory_critical'] + + # 绘制系统资源信息 + sys_text = f"CPU:{cpu_usage:.0f}% 内存:{memory_usage:.0f}%" + frame = self.draw_text(frame, sys_text, (w // 2, 110), size=11, color=self.colors['info']) + + # 语音状态 + if self.config.get('display', 'show_speech_status') and self.speech_manager: + speech_status = self.speech_manager.get_status() + speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] + speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" + frame = self.draw_text(frame, speech_text, (w - 200, 90), size=12, color=speech_color) + + # 性能信息 + if self.config.get('display', 'show_fps'): + perf_text = f"帧率: {fps:.1f}" + if process_time > 0: + perf_text += f" | 延迟: {process_time:.1f}ms" + + # 根据处理时间选择颜色 + if process_time < 20: + perf_color = self.colors['performance_good'] + elif process_time < 50: + perf_color = self.colors['performance_warning'] + else: + perf_color = self.colors['performance_bad'] + else: + perf_color = self.colors['info'] + + frame = self.draw_text(frame, perf_text, (w - 200, 65), size=12, color=perf_color) + + # 控制提示 + control_text = "提示: 确保手部完全进入画面,保持稳定手势" + frame = self.draw_text(frame, control_text, (10, 90), size=12, color=self.colors['info']) + + # 性能快捷键提示 + if performance_analyzer: + perf_tip = "P:报告 L:快照 K:导出" + frame = self.draw_text(frame, perf_tip, (10, 110), size=11, color=self.colors['help']) + + return frame + + def draw_help_bar(self, frame): + """绘制帮助栏""" + if not self.config.get('display', 'show_help'): + return frame + + h, w = frame.shape[:2] + + # 绘制底部帮助栏 + cv2.rectangle(frame, (0, h - 120), (w, h), (0, 0, 0), -1) + + # 帮助文本 + help_lines = [ + "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", + "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", + "V:切换语音反馈 M:测试语音 P:性能报告 L:性能快照 K:导出日志", + "O:切换性能模式 P:详细报告 R:重置统计", + "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" + ] + + for i, line in enumerate(help_lines): + y_pos = h - 105 + i * 20 + frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) + + return frame + + def draw_warning(self, frame, message): + """绘制警告信息""" + h, w = frame.shape[:2] + + # 在顶部绘制警告 + warning_bg = np.zeros((40, w, 3), dtype=np.uint8) + warning_bg[:, :] = (0, 69, 255) + + frame[130:170, 0:w] = cv2.addWeighted( + frame[130:170, 0:w], 0.3, + warning_bg, 0.7, 0 + ) + + # 绘制警告文本 + frame = self.draw_text(frame, message, (10, 145), + size=16, color=self.colors['warning']) + + return frame + + def draw_performance_chart(self, frame, performance_analyzer): + """绘制性能图表""" + if not performance_analyzer: + return frame + + h, w = frame.shape[:2] + + # 绘制图表区域背景 + chart_x = w - 250 + chart_y = 130 + chart_width = 240 + chart_height = 150 + + cv2.rectangle(frame, (chart_x, chart_y), + (chart_x + chart_width, chart_y + chart_height), + (30, 30, 30), -1) + cv2.rectangle(frame, (chart_x, chart_y), + (chart_x + chart_width, chart_y + chart_height), + (100, 100, 100), 1) + + # 绘制标题 + frame = self.draw_text(frame, "性能监控", (chart_x + 10, chart_y + 20), + size=14, color=self.colors['title']) + + # 绘制FPS图表 + if len(performance_analyzer.fps_history) > 1: + fps_values = list(performance_analyzer.fps_history)[-20:] # 最近20个值 + + # 计算最大值和最小值 + if fps_values: + max_fps = max(fps_values) + min_fps = min(fps_values) + if max_fps == min_fps: + max_fps = min_fps + 1 + + # 绘制FPS曲线 + points = [] + for i, fps in enumerate(fps_values): + x = chart_x + 10 + i * (chart_width - 20) / len(fps_values) + y = chart_y + chart_height - 30 - (fps - min_fps) * (chart_height - 50) / (max_fps - min_fps) + points.append((int(x), int(y))) + + # 连接点形成曲线 + for i in range(len(points) - 1): + cv2.line(frame, points[i], points[i + 1], (0, 255, 0), 2) + + # 显示当前FPS值 + current_fps = fps_values[-1] if fps_values else 0 + fps_color = self.colors['performance_good'] if current_fps > 20 else \ + self.colors['performance_warning'] if current_fps > 10 else \ + self.colors['performance_bad'] + + frame = self.draw_text(frame, f"FPS: {current_fps:.1f}", + (chart_x + 120, chart_y + 40), + size=12, color=fps_color) + + return frame \ No newline at end of file