From 18d483948dbb0f21f8bb2973bd18fa60d66d44b5 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Thu, 18 Dec 2025 16:16:41 +0800 Subject: [PATCH 01/18] =?UTF-8?q?new=E5=88=86=E6=94=AF=EF=BC=9A1.=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0.gitignore=E5=BF=BD=E7=95=A5Lib=E7=9B=AE=E5=BD=95=202.?= =?UTF-8?q?=E4=BF=AE=E6=94=B9main.py=E4=BC=98=E5=8C=96=E6=97=A0=E4=BA=BA?= =?UTF-8?q?=E6=9C=BA=E6=8E=A7=E5=88=B6=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 6 + .../main.py | 546 ++++++++++++++++++ 2 files changed, 552 insertions(+) create mode 100644 src/autonomus_drone_hand_gesture_project/.gitignore diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore new file mode 100644 index 0000000000..ad298e3e2d --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -0,0 +1,6 @@ +# 忽略整个Lib目录(包含所有子文件/子目录) +Lib/ +# 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 +__pycache__/ +*.pyc +*.dll.a \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index e69de29bb2..b77d05ba12 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -0,0 +1,546 @@ +# -*- coding: utf-8 -*- +""" +手势控制AirSim无人机 - 修复连接问题版 +确保正确连接AirSim,移除模拟模式 +作者: xiaoshiyuan888 +""" + +import sys +import os +import time +import traceback +import subprocess +from PIL import Image, ImageDraw, ImageFont +import cv2 +import numpy as np + +print("=" * 60) +print("Gesture Controlled Drone - Fixed Connection Version") +print("=" * 60) + +# ========== 修复导入路径 ========== +# 添加当前目录到Python路径 +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + + +# ========== 核心模块导入与异常处理 ========== +def safe_import(): + """安全导入所有模块""" + modules_status = {} + + # 1. 导入PIL + try: + from PIL import Image, ImageDraw, ImageFont + modules_status['PIL'] = True + print("[PIL] ✓ Image processing library ready") + except Exception as e: + modules_status['PIL'] = False + print(f"[PIL] ✗ Import failed: {e}") + return None, modules_status + + # 2. 导入OpenCV + try: + import cv2 + import numpy as np + modules_status['OpenCV'] = True + print("[OpenCV] ✓ Computer vision library ready") + except Exception as e: + modules_status['OpenCV'] = False + print(f"[OpenCV] ✗ Import failed: {e}") + return None, modules_status + + # 3. 导入MediaPipe + mp = None + mp_hands = None + mp_drawing = None + try: + import mediapipe as mp + mp_hands = mp.solutions.hands + mp_drawing = mp.solutions.drawing_utils + modules_status['MediaPipe'] = True + print("[MediaPipe] ✓ Gesture recognition library ready") + except Exception as e: + modules_status['MediaPipe'] = False + print(f"[MediaPipe] ✗ Import failed: {e}") + print(" Please install: pip install mediapipe") + + # 4. 尝试导入AirSim - 关键修复部分 + airsim_module = None + airsim_client = None + + # 尝试多种导入方式 + import_methods = [ + lambda: __import__('airsim'), + lambda: __import__('AirSim'), + ] + + for method in import_methods: + try: + airsim_module = method() + modules_status['AirSim'] = True + print(f"[AirSim] ✓ Successfully imported airsim from {airsim_module.__file__}") + break + except ImportError: + continue + except Exception as e: + print(f"[AirSim] Import error: {e}") + continue + + if not modules_status.get('AirSim', False): + print("\n" + "!" * 60) + print("⚠ AirSim library NOT FOUND!") + print("!" * 60) + print("To install AirSim, run:") + print("1. First install: pip install msgpack-rpc-python") + print("2. Then install: pip install airsim") + print("\nOr from source:") + print(" pip install git+https://github.com/microsoft/AirSim.git") + print("!" * 60) + + # 询问是否自动安装 + print("\nDo you want to try automatic installation? (y/n)") + choice = input().strip().lower() + if choice == 'y': + try: + print("Installing msgpack-rpc-python...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "msgpack-rpc-python"]) + print("Installing AirSim...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "airsim"]) + print("\n✅ Installation complete! Please restart the program.") + except Exception as e: + print(f"Installation failed: {e}") + sys.exit(1) + + return { + 'cv2': cv2, + 'np': np, + 'mp': mp, + 'mp_hands': mp_hands, + 'mp_drawing': mp_drawing, + 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, + 'airsim': airsim_module + }, modules_status + + +# 执行导入 +libs, status = safe_import() +if not status.get('OpenCV', False) or not status.get('PIL', False): + print("\n❌ Core libraries missing, cannot start.") + input("Press Enter to exit...") + sys.exit(1) + +print("-" * 60) +print("✅ Environment check passed, initializing...") +print("-" * 60) + +# 解包库 +cv2, np, mp = libs['cv2'], libs['np'], libs['mp'] +Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] +mp_hands, mp_drawing = libs['mp_hands'], libs['mp_drawing'] + + +# ========== 中文显示模块 ========== +class ChineseTextRenderer: + """中文文本渲染器""" + + def __init__(self): + self.fonts = {} + self.load_fonts() + + def load_fonts(self): + """加载字体""" + font_paths = [ + 'simhei.ttf', + 'C:/Windows/Fonts/simhei.ttf', + 'C:/Windows/Fonts/msyh.ttc' + ] + for path in font_paths: + try: + self.fonts[20] = ImageFont.truetype(path, 20) + self.fonts[24] = ImageFont.truetype(path, 24) + self.fonts[30] = ImageFont.truetype(path, 30) + print(f"✓ Chinese fonts loaded: {path}") + return + except: + continue + print("⚠ No Chinese fonts found, using default") + + def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): + """在图像上绘制中文文本""" + try: + img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + pil_img = Image.fromarray(img_rgb) + draw = ImageDraw.Draw(pil_img) + + font = self.fonts.get(size, self.fonts.get(20)) + + # 绘制阴影 + shadow_color = (0, 0, 0) + shadow_pos = (pos[0] + 1, pos[1] + 1) + draw.text(shadow_pos, text, font=font, fill=shadow_color) + + # 绘制文字 + rgb_color = color[::-1] + draw.text(pos, text, font=font, fill=rgb_color) + + return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) + except: + cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, + size / 30, color, 2) + return frame + + +# 初始化中文渲染器 +chinese_renderer = ChineseTextRenderer() + + +# ========== 手势识别模块 ========== +class GestureRecognizer: + """手势识别器""" + + def __init__(self): + try: + self.hands = mp_hands.Hands( + max_num_hands=1, + min_detection_confidence=0.7, + min_tracking_confidence=0.5 + ) + print("✓ MediaPipe gesture detector initialized") + except Exception as e: + print(f"⚠ MediaPipe init failed: {e}") + self.hands = None + + self.current_gesture = "等待手势" + + def recognize(self, frame): + """识别手势""" + if self.hands is None: + return self.current_gesture + + try: + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + results = self.hands.process(rgb_frame) + + if results.multi_hand_landmarks: + for hand_landmarks in results.multi_hand_landmarks: + # 绘制手部关键点 + mp_drawing.draw_landmarks( + frame, hand_landmarks, mp_hands.HAND_CONNECTIONS, + mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2), + mp_drawing.DrawingSpec(color=(255, 0, 0), thickness=2) + ) + + # 简化的手势判断 + landmarks = hand_landmarks.landmark + if len(landmarks) >= 9: + wrist = landmarks[0] + index_tip = landmarks[8] + + # 手势判断逻辑 + if index_tip.y < wrist.y - 0.2: + self.current_gesture = "向上" + elif index_tip.y > wrist.y + 0.2: + self.current_gesture = "向下" + elif index_tip.x < wrist.x - 0.2: + self.current_gesture = "向左" + elif index_tip.x > wrist.x + 0.2: + self.current_gesture = "向右" + elif abs(index_tip.y - wrist.y) < 0.1: + self.current_gesture = "向前" + else: + self.current_gesture = "停止" + + return self.current_gesture + except Exception as e: + print(f"Gesture recognition error: {e}") + return "识别异常" + + def set_simulated_gesture(self, gesture): + """设置模拟的手势""" + self.current_gesture = gesture + + +# ========== 无人机控制模块 ========== +class DroneController: + """无人机控制器""" + + def __init__(self, airsim_module): + self.airsim = airsim_module + self.client = None + self.connected = False + self.flying = False + self.connection_attempted = False + + def connect(self): + """连接AirSim无人机""" + if self.connection_attempted: + return self.connected + + self.connection_attempted = True + print("Connecting to AirSim...") + + try: + # 创建客户端 + self.client = self.airsim.MultirotorClient() + self.client.confirmConnection() + print("✅ Connected to AirSim!") + + # 启用API控制 + self.client.enableApiControl(True) + print("✅ API control enabled") + + # 解锁无人机 + self.client.armDisarm(True) + print("✅ Drone armed") + + self.connected = True + return True + + except Exception as e: + print(f"❌ Connection failed: {e}") + print("\nPlease ensure:") + print("1. AirSim simulator is running") + print("2. In simulator: Settings → Computer Vision mode is OFF") + print("3. Drone is spawned in the world") + print("4. Press 'R' in simulator to reset drone if needed") + return False + + def takeoff(self): + """起飞""" + if not self.connected: + print("❌ Drone not connected") + return False + + try: + print("Taking off...") + # 起飞到5米高度 + self.client.takeoffAsync().join() + # 悬停1秒稳定 + time.sleep(1) + self.flying = True + print("✅ Drone took off successfully") + return True + except Exception as e: + print(f"❌ Takeoff failed: {e}") + return False + + def land(self): + """降落""" + if not self.connected: + return False + + try: + print("Landing...") + self.client.landAsync().join() + self.flying = False + print("✅ Drone landed") + return True + except Exception as e: + print(f"Landing failed: {e}") + return False + + def move_by_gesture(self, gesture): + """根据手势移动""" + if not self.connected or not self.flying: + return + + try: + # 设置速度参数(单位:米/秒) + velocity = 3 + duration = 0.5 # 持续0.5秒 + + if gesture == "向上": + self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() + elif gesture == "向下": + self.client.moveByVelocityZAsync(0, 0, velocity, duration).join() + elif gesture == "向左": + self.client.moveByVelocityAsync(-velocity, 0, 0, duration).join() + elif gesture == "向右": + self.client.moveByVelocityAsync(velocity, 0, 0, duration).join() + elif gesture == "向前": + self.client.moveByVelocityAsync(0, -velocity, 0, duration).join() + elif gesture == "停止": + self.client.hoverAsync().join() + except Exception as e: + print(f"Control command failed: {e}") + + def emergency_stop(self): + """紧急停止""" + if self.connected: + try: + if self.flying: + print("Emergency landing...") + self.land() + self.client.armDisarm(False) + self.client.enableApiControl(False) + print("✅ Emergency stop complete") + except: + pass + self.connected = False + self.flying = False + + +# ========== 主程序 ========== +def main(): + """主函数""" + # 初始化组件 + gesture_recognizer = GestureRecognizer() + drone_controller = DroneController(libs['airsim']) + + # 初始化摄像头 + cap = None + try: + cap = cv2.VideoCapture(0) + if not cap.isOpened(): + print("❌ Camera not available") + cap = None + else: + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + print("✓ Camera initialized") + except: + print("⚠ Camera init failed") + cap = None + + # 显示说明 + print("\n" + "=" * 60) + print("INSTRUCTIONS") + print("=" * 60) + print("1. Start AirSim simulator first") + print("2. Choose environment (e.g., Landscape Mountains)") + print("3. Press [C] in this window to connect drone") + print("4. Press [SPACE] to takeoff/land") + if cap: + print("5. Gesture control with hand in front of camera") + else: + print("5. Keyboard control:") + print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") + print("6. Press [ESC] to exit safely") + print("=" * 60) + print("Program started successfully!") + print("-" * 60) + + # 键盘手势映射 + key_to_gesture = { + ord('w'): "向上", ord('W'): "向上", + ord('s'): "向下", ord('S'): "向下", + ord('a'): "向左", ord('A'): "向左", + ord('d'): "向右", ord('D'): "向右", + ord('f'): "向前", ord('F'): "向前", + ord('x'): "停止", ord('X'): "停止", + } + + # 主循环 + while True: + frame = None + + # 读取摄像头画面 + if cap: + ret, frame = cap.read() + if ret: + frame = cv2.flip(frame, 1) + current_gesture = gesture_recognizer.recognize(frame) + else: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + current_gesture = "摄像头错误" + else: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + current_gesture = gesture_recognizer.current_gesture + + # ========== 绘制界面 ========== + # 标题栏 + overlay = frame.copy() + cv2.rectangle(overlay, (0, 0), (640, 100), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + + # 主标题 + frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), + size=30, color=(0, 255, 255)) + + # 状态信息 + status_color = (0, 255, 0) if drone_controller.connected else (0, 0, 255) + status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" + frame = chinese_renderer.put_text(frame, status_text, (10, 50), + size=24, color=status_color) + + flight_color = (0, 255, 0) if drone_controller.flying else (255, 255, 0) + flight_status = f"飞行: {'是' if drone_controller.flying else '否'}" + frame = chinese_renderer.put_text(frame, flight_status, (10, 80), + size=24, color=flight_color) + + # 手势信息 + gesture_color = (0, 255, 0) if current_gesture != "等待手势" else (200, 200, 200) + gesture_text = f"手势: {current_gesture}" + frame = chinese_renderer.put_text(frame, gesture_text, (10, 120), + size=24, color=gesture_color) + + # 操作提示 + help_text = "C:连接 空格:起飞/降落 ESC:退出" + if not cap: + help_text += " W/A/S/D/F/X:控制" + + frame = chinese_renderer.put_text(frame, help_text, (10, 450), + size=20, color=(255, 255, 255)) + + # AirSim连接状态 + if not drone_controller.connected: + warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" + frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + size=18, color=(0, 165, 255)) + + # 显示窗口 + cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) + + # ========== 键盘控制 ========== + key = cv2.waitKey(30) & 0xFF + + if key == 27: # ESC键 + print("\nExiting...") + break + + elif key == ord('c') or key == ord('C'): # 连接无人机 + if not drone_controller.connected: + drone_controller.connect() + + elif key == 32: # 空格键 + if drone_controller.connected: + if drone_controller.flying: + drone_controller.land() + else: + drone_controller.takeoff() + time.sleep(0.3) + + elif key in key_to_gesture: # 键盘控制 + gesture = key_to_gesture[key] + gesture_recognizer.set_simulated_gesture(gesture) + if drone_controller.connected and drone_controller.flying: + drone_controller.move_by_gesture(gesture) + + # 真实手势控制 + if (current_gesture and current_gesture != "等待手势" and + current_gesture != "摄像头错误" and + drone_controller.connected and drone_controller.flying): + drone_controller.move_by_gesture(current_gesture) + + # ========== 清理资源 ========== + print("\nCleaning up resources...") + if cap: + cap.release() + cv2.destroyAllWindows() + drone_controller.emergency_stop() + print("Program safely exited") + print("=" * 60) + + +# ========== 程序入口 ========== +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nProgram interrupted by user") + except Exception as e: + print(f"\nProgram error: {e}") + traceback.print_exc() + finally: + input("\nPress Enter to exit...") \ No newline at end of file From dd54e834a48d55daa8c8f6dd313c074efcea1d05 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Thu, 18 Dec 2025 22:05:02 +0800 Subject: [PATCH 02/18] =?UTF-8?q?=E6=94=B9=E8=BF=9B=E6=97=A0=E4=BA=BA?= =?UTF-8?q?=E6=9C=BA=E6=89=8B=E5=8A=BF=E6=8E=A7=E5=88=B6=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=EF=BC=8C=E4=BC=98=E5=8C=96=E6=91=84=E5=83=8F=E5=A4=B4=E5=A4=84?= =?UTF-8?q?=E7=90=86=E5=92=8C=E6=89=8B=E5=8A=BF=E7=A8=B3=E5=AE=9A=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 4 +- .../main.py | 362 ++++++++++++------ 2 files changed, 246 insertions(+), 120 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index ad298e3e2d..05dab76400 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -3,4 +3,6 @@ Lib/ # 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 __pycache__/ *.pyc -*.dll.a \ No newline at end of file +*.dll.a +# 忽略IDEA配置文件 +.idea/ \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index b77d05ba12..f2a7020806 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 修复连接问题版 -确保正确连接AirSim,移除模拟模式 +手势控制AirSim无人机 - 修复手势识别版 +优化了手势识别算法,增加更多手势判断逻辑 作者: xiaoshiyuan888 """ @@ -15,21 +15,19 @@ import numpy as np print("=" * 60) -print("Gesture Controlled Drone - Fixed Connection Version") +print("Gesture Controlled Drone - Enhanced Gesture Recognition") print("=" * 60) # ========== 修复导入路径 ========== -# 添加当前目录到Python路径 current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -# ========== 核心模块导入与异常处理 ========== +# ========== 核心模块导入 ========== def safe_import(): """安全导入所有模块""" modules_status = {} - # 1. 导入PIL try: from PIL import Image, ImageDraw, ImageFont modules_status['PIL'] = True @@ -39,7 +37,6 @@ def safe_import(): print(f"[PIL] ✗ Import failed: {e}") return None, modules_status - # 2. 导入OpenCV try: import cv2 import numpy as np @@ -50,7 +47,6 @@ def safe_import(): print(f"[OpenCV] ✗ Import failed: {e}") return None, modules_status - # 3. 导入MediaPipe mp = None mp_hands = None mp_drawing = None @@ -65,11 +61,7 @@ def safe_import(): print(f"[MediaPipe] ✗ Import failed: {e}") print(" Please install: pip install mediapipe") - # 4. 尝试导入AirSim - 关键修复部分 airsim_module = None - airsim_client = None - - # 尝试多种导入方式 import_methods = [ lambda: __import__('airsim'), lambda: __import__('AirSim'), @@ -79,7 +71,7 @@ def safe_import(): try: airsim_module = method() modules_status['AirSim'] = True - print(f"[AirSim] ✓ Successfully imported airsim from {airsim_module.__file__}") + print(f"[AirSim] ✓ Successfully imported") break except ImportError: continue @@ -98,7 +90,6 @@ def safe_import(): print(" pip install git+https://github.com/microsoft/AirSim.git") print("!" * 60) - # 询问是否自动安装 print("\nDo you want to try automatic installation? (y/n)") choice = input().strip().lower() if choice == 'y': @@ -153,7 +144,9 @@ def load_fonts(self): font_paths = [ 'simhei.ttf', 'C:/Windows/Fonts/simhei.ttf', - 'C:/Windows/Fonts/msyh.ttc' + 'C:/Windows/Fonts/msyh.ttc', + '/System/Library/Fonts/PingFang.ttc', + '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' ] for path in font_paths: try: @@ -177,7 +170,7 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): # 绘制阴影 shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 1, pos[1] + 1) + shadow_pos = (pos[0] + 2, pos[1] + 2) draw.text(shadow_pos, text, font=font, fill=shadow_color) # 绘制文字 @@ -186,70 +179,172 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) except: + # 备用方案:使用OpenCV绘制英文 cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, size / 30, color, 2) return frame -# 初始化中文渲染器 chinese_renderer = ChineseTextRenderer() -# ========== 手势识别模块 ========== -class GestureRecognizer: - """手势识别器""" +# ========== 简化的手势识别模块 ========== +class SimpleGestureRecognizer: + """简化的手势识别器,不依赖MediaPipe""" def __init__(self): - try: - self.hands = mp_hands.Hands( - max_num_hands=1, - min_detection_confidence=0.7, - min_tracking_confidence=0.5 - ) - print("✓ MediaPipe gesture detector initialized") - except Exception as e: - print(f"⚠ MediaPipe init failed: {e}") - self.hands = None - self.current_gesture = "等待手势" + self.last_gesture_time = time.time() + self.gesture_history = [] + self.max_history = 5 + print("✓ Simple gesture recognizer initialized (不使用MediaPipe)") + + # 加载OpenCV的预训练模型用于手部检测 + try: + # 尝试加载Haar级联分类器 + cascade_path = cv2.data.haarcascades + 'haarcascade_frontalface_default.xml' + # 我们使用人脸检测器作为替代,实际上应该用手部检测器 + self.face_cascade = cv2.CascadeClassifier(cascade_path) + print("✓ OpenCV cascade classifier loaded") + except: + print("⚠ OpenCV cascade classifier not available") + self.face_cascade = None + + def detect_hand_contour(self, frame): + """使用轮廓检测手部""" + # 转换为HSV颜色空间 + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + + # 定义肤色范围 + lower_skin = np.array([0, 20, 70], dtype=np.uint8) + upper_skin = np.array([20, 255, 255], dtype=np.uint8) + + # 创建肤色掩码 + mask = cv2.inRange(hsv, lower_skin, upper_skin) + + # 形态学操作 + kernel = np.ones((5, 5), np.uint8) + mask = cv2.dilate(mask, kernel, iterations=2) + mask = cv2.erode(mask, kernel, iterations=1) + mask = cv2.GaussianBlur(mask, (5, 5), 0) + + # 查找轮廓 + contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + if contours: + # 找到最大的轮廓(假设是手) + largest_contour = max(contours, key=cv2.contourArea) + + # 计算轮廓面积 + area = cv2.contourArea(largest_contour) + + if area > 1000: # 最小面积阈值 + # 获取边界矩形 + x, y, w, h = cv2.boundingRect(largest_contour) + + # 计算轮廓的中心点 + M = cv2.moments(largest_contour) + if M["m00"] != 0: + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + + # 计算凸包和凸缺陷 + hull = cv2.convexHull(largest_contour, returnPoints=False) + if len(hull) > 3: + defects = cv2.convexityDefects(largest_contour, hull) + + if defects is not None: + # 计算手指数量 + finger_count = 0 + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(largest_contour[s][0]) + end = tuple(largest_contour[e][0]) + far = tuple(largest_contour[f][0]) + + # 计算角度 + a = np.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = np.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = np.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + + angle = np.arccos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + + if angle <= np.pi / 2: + finger_count += 1 + + finger_count = min(finger_count + 1, 5) + + # 绘制轮廓和中心点 + cv2.drawContours(frame, [largest_contour], -1, (0, 255, 0), 2) + cv2.circle(frame, (cx, cy), 7, (255, 0, 0), -1) + cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) + + # 在边界框上方显示手指数量 + cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + + return cx, cy, w, h, finger_count, frame + + return None, None, None, None, 0, frame + + def recognize_simple_gesture(self, frame): + """简化的手势识别""" + # 检测手部轮廓 + cx, cy, w, h, finger_count, frame = self.detect_hand_contour(frame) + + if cx is None or cy is None: + return "等待手势", frame + + # 获取图像尺寸 + height, width = frame.shape[:2] + + # 定义区域 + center_x = width // 2 + center_y = height // 2 + + # 根据手指数量和位置判断手势 + if finger_count == 0: + return "停止", frame + elif finger_count == 1: + return "向前", frame + elif finger_count >= 4: + # 手掌张开,根据位置判断方向 + if cy < center_y - 50: + return "向上", frame + elif cy > center_y + 50: + return "向下", frame + elif cx < center_x - 50: + return "向左", frame + elif cx > center_x + 50: + return "向右", frame + else: + return "停止", frame + else: + return "等待手势", frame def recognize(self, frame): """识别手势""" - if self.hands is None: - return self.current_gesture - try: - rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - results = self.hands.process(rgb_frame) - - if results.multi_hand_landmarks: - for hand_landmarks in results.multi_hand_landmarks: - # 绘制手部关键点 - mp_drawing.draw_landmarks( - frame, hand_landmarks, mp_hands.HAND_CONNECTIONS, - mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2), - mp_drawing.DrawingSpec(color=(255, 0, 0), thickness=2) - ) - - # 简化的手势判断 - landmarks = hand_landmarks.landmark - if len(landmarks) >= 9: - wrist = landmarks[0] - index_tip = landmarks[8] - - # 手势判断逻辑 - if index_tip.y < wrist.y - 0.2: - self.current_gesture = "向上" - elif index_tip.y > wrist.y + 0.2: - self.current_gesture = "向下" - elif index_tip.x < wrist.x - 0.2: - self.current_gesture = "向左" - elif index_tip.x > wrist.x + 0.2: - self.current_gesture = "向右" - elif abs(index_tip.y - wrist.y) < 0.1: - self.current_gesture = "向前" - else: - self.current_gesture = "停止" + # 使用简化的手势识别 + gesture, processed_frame = self.recognize_simple_gesture(frame) + + # 手势历史记录和平滑处理 + self.gesture_history.append(gesture) + if len(self.gesture_history) > self.max_history: + self.gesture_history.pop(0) + + # 使用多数表决决定最终手势 + if self.gesture_history: + gesture_counts = {} + for g in self.gesture_history: + gesture_counts[g] = gesture_counts.get(g, 0) + 1 + + # 找到出现次数最多的手势 + max_gesture = max(gesture_counts, key=gesture_counts.get) + if gesture_counts[max_gesture] > len(self.gesture_history) // 2: + self.current_gesture = max_gesture + else: + self.current_gesture = gesture return self.current_gesture except Exception as e: @@ -281,16 +376,13 @@ def connect(self): print("Connecting to AirSim...") try: - # 创建客户端 self.client = self.airsim.MultirotorClient() self.client.confirmConnection() print("✅ Connected to AirSim!") - # 启用API控制 self.client.enableApiControl(True) print("✅ API control enabled") - # 解锁无人机 self.client.armDisarm(True) print("✅ Drone armed") @@ -299,11 +391,11 @@ def connect(self): except Exception as e: print(f"❌ Connection failed: {e}") - print("\nPlease ensure:") - print("1. AirSim simulator is running") - print("2. In simulator: Settings → Computer Vision mode is OFF") - print("3. Drone is spawned in the world") - print("4. Press 'R' in simulator to reset drone if needed") + print("\n请确保:") + print("1. AirSim模拟器正在运行") + print("2. 选择环境(如:Landscape Mountains)") + print("3. 无人机已在世界中生成") + print("4. 如果需要,按模拟器中的'R'键重置无人机") return False def takeoff(self): @@ -314,9 +406,7 @@ def takeoff(self): try: print("Taking off...") - # 起飞到5米高度 self.client.takeoffAsync().join() - # 悬停1秒稳定 time.sleep(1) self.flying = True print("✅ Drone took off successfully") @@ -346,9 +436,8 @@ def move_by_gesture(self, gesture): return try: - # 设置速度参数(单位:米/秒) - velocity = 3 - duration = 0.5 # 持续0.5秒 + velocity = 2.0 # 降低速度以获得更平滑的控制 + duration = 0.3 # 减少持续时间 if gesture == "向上": self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() @@ -384,8 +473,8 @@ def emergency_stop(self): # ========== 主程序 ========== def main(): """主函数""" - # 初始化组件 - gesture_recognizer = GestureRecognizer() + # 使用简化的手势识别器 + gesture_recognizer = SimpleGestureRecognizer() drone_controller = DroneController(libs['airsim']) # 初始化摄像头 @@ -393,32 +482,46 @@ def main(): try: cap = cv2.VideoCapture(0) if not cap.isOpened(): - print("❌ Camera not available") - cap = None + # 尝试其他摄像头索引 + for i in range(1, 5): + cap = cv2.VideoCapture(i) + if cap.isOpened(): + print(f"✓ Camera {i} initialized") + break + else: + print("❌ Camera not available, using keyboard control only") + cap = None else: + print("✓ Camera 0 initialized") + + if cap: cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - print("✓ Camera initialized") - except: - print("⚠ Camera init failed") + cap.set(cv2.CAP_PROP_FPS, 30) + except Exception as e: + print(f"⚠ Camera init failed: {e}") cap = None # 显示说明 print("\n" + "=" * 60) - print("INSTRUCTIONS") + print("操作说明") print("=" * 60) - print("1. Start AirSim simulator first") - print("2. Choose environment (e.g., Landscape Mountains)") - print("3. Press [C] in this window to connect drone") - print("4. Press [SPACE] to takeoff/land") + print("1. 首先启动AirSim模拟器") + print("2. 选择环境(例如:Landscape Mountains)") + print("3. 按 [C] 连接无人机") + print("4. 按 [空格键] 起飞/降落") if cap: - print("5. Gesture control with hand in front of camera") + print("5. 手势控制:") + print(" - 握拳(0个手指):停止") + print(" - 伸出1个手指:向前") + print(" - 张开手掌(4-5个手指):根据手的位置控制方向") + print(" * 确保手部在摄像头视野内,光线充足,背景简单") else: - print("5. Keyboard control:") - print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") - print("6. Press [ESC] to exit safely") + print("5. 键盘控制:") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print("6. 按 [ESC] 安全退出") print("=" * 60) - print("Program started successfully!") + print("程序启动成功!") print("-" * 60) # 键盘手势映射 @@ -431,14 +534,17 @@ def main(): ord('x'): "停止", ord('X'): "停止", } + last_control_time = time.time() + control_interval = 0.5 # 控制间隔,避免过于频繁 + # 主循环 while True: frame = None - # 读取摄像头画面 if cap: ret, frame = cap.read() if ret: + # 镜像图像以便更直观的控制 frame = cv2.flip(frame, 1) current_gesture = gesture_recognizer.recognize(frame) else: @@ -449,12 +555,16 @@ def main(): current_gesture = gesture_recognizer.current_gesture # ========== 绘制界面 ========== - # 标题栏 + # 创建半透明覆盖层 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (640, 100), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + cv2.rectangle(overlay, (0, 0), (640, 120), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.6, frame, 0.4, 0) - # 主标题 + # 绘制底部操作提示区域 + cv2.rectangle(frame, (0, 400), (640, 480), (0, 0, 0), -1) + frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) + + # 标题 frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), size=30, color=(0, 255, 255)) @@ -470,9 +580,15 @@ def main(): size=24, color=flight_color) # 手势信息 - gesture_color = (0, 255, 0) if current_gesture != "等待手势" else (200, 200, 200) + if current_gesture != "等待手势" and current_gesture != "摄像头错误": + gesture_color = (0, 255, 0) # 绿色 + # 添加手势识别置信度指示 + cv2.circle(frame, (600, 60), 10, gesture_color, -1) + else: + gesture_color = (200, 200, 200) # 灰色 + gesture_text = f"手势: {current_gesture}" - frame = chinese_renderer.put_text(frame, gesture_text, (10, 120), + frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), size=24, color=gesture_color) # 操作提示 @@ -480,26 +596,30 @@ def main(): if not cap: help_text += " W/A/S/D/F/X:控制" - frame = chinese_renderer.put_text(frame, help_text, (10, 450), + frame = chinese_renderer.put_text(frame, help_text, (10, 420), size=20, color=(255, 255, 255)) - # AirSim连接状态 + # 添加手势提示 + if cap: + gesture_hint = "提示: 握拳停止,食指向前,手掌张开移动" + frame = chinese_renderer.put_text(frame, gesture_hint, (10, 450), + size=18, color=(255, 200, 100)) + if not drone_controller.connected: warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" - frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + frame = chinese_renderer.put_text(frame, warning_text, (10, 360), size=18, color=(0, 165, 255)) - # 显示窗口 cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) # ========== 键盘控制 ========== - key = cv2.waitKey(30) & 0xFF + key = cv2.waitKey(1) & 0xFF if key == 27: # ESC键 - print("\nExiting...") + print("\n退出程序...") break - elif key == ord('c') or key == ord('C'): # 连接无人机 + elif key == ord('c') or key == ord('C'): if not drone_controller.connected: drone_controller.connect() @@ -511,25 +631,29 @@ def main(): drone_controller.takeoff() time.sleep(0.3) - elif key in key_to_gesture: # 键盘控制 + elif key in key_to_gesture: gesture = key_to_gesture[key] gesture_recognizer.set_simulated_gesture(gesture) + current_gesture = gesture if drone_controller.connected and drone_controller.flying: drone_controller.move_by_gesture(gesture) - # 真实手势控制 + # 真实手势控制(添加间隔控制避免过于频繁) + current_time = time.time() if (current_gesture and current_gesture != "等待手势" and - current_gesture != "摄像头错误" and - drone_controller.connected and drone_controller.flying): + current_gesture != "摄像头错误" and current_gesture != "识别异常" and + drone_controller.connected and drone_controller.flying and + current_time - last_control_time > control_interval): drone_controller.move_by_gesture(current_gesture) + last_control_time = current_time - # ========== 清理资源 ========== - print("\nCleaning up resources...") + # 清理资源 + print("\n清理资源...") if cap: cap.release() cv2.destroyAllWindows() drone_controller.emergency_stop() - print("Program safely exited") + print("程序安全退出") print("=" * 60) @@ -538,9 +662,9 @@ def main(): try: main() except KeyboardInterrupt: - print("\nProgram interrupted by user") + print("\n程序被用户中断") except Exception as e: - print(f"\nProgram error: {e}") + print(f"\n程序错误: {e}") traceback.print_exc() finally: - input("\nPress Enter to exit...") \ No newline at end of file + input("\n按回车键退出...") \ No newline at end of file From d1726c94f7739505e7cd5e035dd74dc285ac53ba Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 09:57:53 +0800 Subject: [PATCH 03/18] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E8=AF=86=E5=88=AB=E7=AE=97=E6=B3=95=EF=BC=8C=E6=8F=90=E5=8D=87?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F=E7=A8=B3=E5=AE=9A=E6=80=A7=E5=92=8C=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E4=BD=93=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main.py | 857 ++++++++++++------ 1 file changed, 588 insertions(+), 269 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index f2a7020806..5325f35afa 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 修复手势识别版 -优化了手势识别算法,增加更多手势判断逻辑 +手势控制AirSim无人机 - 深度优化版 +优化了手势识别准确性和系统稳定性 作者: xiaoshiyuan888 """ @@ -10,12 +10,14 @@ import time import traceback import subprocess +import math from PIL import Image, ImageDraw, ImageFont import cv2 import numpy as np +from collections import deque print("=" * 60) -print("Gesture Controlled Drone - Enhanced Gesture Recognition") +print("Gesture Controlled Drone - Advanced Recognition v2.0") print("=" * 60) # ========== 修复导入路径 ========== @@ -47,6 +49,7 @@ def safe_import(): print(f"[OpenCV] ✗ Import failed: {e}") return None, modules_status + # 尝试导入mediapipe,如果失败则使用OpenCV方案 mp = None mp_hands = None mp_drawing = None @@ -58,28 +61,14 @@ def safe_import(): print("[MediaPipe] ✓ Gesture recognition library ready") except Exception as e: modules_status['MediaPipe'] = False - print(f"[MediaPipe] ✗ Import failed: {e}") - print(" Please install: pip install mediapipe") + print(f"[MediaPipe] ✗ Import failed, using OpenCV fallback: {e}") airsim_module = None - import_methods = [ - lambda: __import__('airsim'), - lambda: __import__('AirSim'), - ] - - for method in import_methods: - try: - airsim_module = method() - modules_status['AirSim'] = True - print(f"[AirSim] ✓ Successfully imported") - break - except ImportError: - continue - except Exception as e: - print(f"[AirSim] Import error: {e}") - continue - - if not modules_status.get('AirSim', False): + try: + airsim_module = __import__('airsim') + modules_status['AirSim'] = True + print(f"[AirSim] ✓ Successfully imported") + except ImportError: print("\n" + "!" * 60) print("⚠ AirSim library NOT FOUND!") print("!" * 60) @@ -90,18 +79,10 @@ def safe_import(): print(" pip install git+https://github.com/microsoft/AirSim.git") print("!" * 60) - print("\nDo you want to try automatic installation? (y/n)") + print("\nContinue without AirSim? (y/n)") choice = input().strip().lower() - if choice == 'y': - try: - print("Installing msgpack-rpc-python...") - subprocess.check_call([sys.executable, "-m", "pip", "install", "msgpack-rpc-python"]) - print("Installing AirSim...") - subprocess.check_call([sys.executable, "-m", "pip", "install", "airsim"]) - print("\n✅ Installation complete! Please restart the program.") - except Exception as e: - print(f"Installation failed: {e}") - sys.exit(1) + if choice != 'y': + sys.exit(1) return { 'cv2': cv2, @@ -126,9 +107,8 @@ def safe_import(): print("-" * 60) # 解包库 -cv2, np, mp = libs['cv2'], libs['np'], libs['mp'] +cv2, np = libs['cv2'], libs['np'] Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -mp_hands, mp_drawing = libs['mp_hands'], libs['mp_drawing'] # ========== 中文显示模块 ========== @@ -150,16 +130,18 @@ def load_fonts(self): ] for path in font_paths: try: + self.fonts[16] = ImageFont.truetype(path, 16) self.fonts[20] = ImageFont.truetype(path, 20) self.fonts[24] = ImageFont.truetype(path, 24) - self.fonts[30] = ImageFont.truetype(path, 30) + self.fonts[28] = ImageFont.truetype(path, 28) + self.fonts[32] = ImageFont.truetype(path, 32) print(f"✓ Chinese fonts loaded: {path}") return except: continue print("⚠ No Chinese fonts found, using default") - def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): + def put_text(self, frame, text, pos, size=20, color=(255, 255, 255), bg_color=None): """在图像上绘制中文文本""" try: img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) @@ -168,197 +150,421 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): font = self.fonts.get(size, self.fonts.get(20)) + # 如果有背景色,先绘制背景 + if bg_color: + bbox = draw.textbbox(pos, text, font=font) + padding = 2 + draw.rectangle( + [bbox[0] - padding, bbox[1] - padding, bbox[2] + padding, bbox[3] + padding], + fill=bg_color + ) + # 绘制阴影 shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 2, pos[1] + 2) + shadow_pos = (pos[0] + 1, pos[1] + 1) draw.text(shadow_pos, text, font=font, fill=shadow_color) # 绘制文字 - rgb_color = color[::-1] + rgb_color = color[::-1] # BGR to RGB draw.text(pos, text, font=font, fill=rgb_color) return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) - except: + except Exception as e: # 备用方案:使用OpenCV绘制英文 cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, - size / 30, color, 2) + size / 25, color, 2) return frame chinese_renderer = ChineseTextRenderer() -# ========== 简化的手势识别模块 ========== -class SimpleGestureRecognizer: - """简化的手势识别器,不依赖MediaPipe""" +# ========== 高级手势识别模块 ========== +class AdvancedGestureRecognizer: + """高级手势识别器,结合多种检测方法""" def __init__(self): + self.use_mediapipe = status.get('MediaPipe', False) + + # 初始化MediaPipe(如果可用) + if self.use_mediapipe and libs['mp'] is not None: + try: + self.hands = libs['mp_hands'].Hands( + max_num_hands=2, + min_detection_confidence=0.7, + min_tracking_confidence=0.5, + static_image_mode=False, + model_complexity=1 + ) + print("✓ Using MediaPipe for gesture recognition") + except: + self.use_mediapipe = False + print("⚠ MediaPipe initialization failed, using OpenCV fallback") + else: + self.use_mediapipe = False + + # OpenCV手势识别参数 + self.history_size = 10 + self.gesture_history = deque(maxlen=self.history_size) + self.finger_history = deque(maxlen=self.history_size) self.current_gesture = "等待手势" + self.confidence = 0.0 self.last_gesture_time = time.time() - self.gesture_history = [] - self.max_history = 5 - print("✓ Simple gesture recognizer initialized (不使用MediaPipe)") - # 加载OpenCV的预训练模型用于手部检测 - try: - # 尝试加载Haar级联分类器 - cascade_path = cv2.data.haarcascades + 'haarcascade_frontalface_default.xml' - # 我们使用人脸检测器作为替代,实际上应该用手部检测器 - self.face_cascade = cv2.CascadeClassifier(cascade_path) - print("✓ OpenCV cascade classifier loaded") - except: - print("⚠ OpenCV cascade classifier not available") - self.face_cascade = None - - def detect_hand_contour(self, frame): - """使用轮廓检测手部""" - # 转换为HSV颜色空间 - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + # 肤色检测参数(自适应) + self.skin_lower = np.array([0, 30, 60], dtype=np.uint8) + self.skin_upper = np.array([25, 255, 255], dtype=np.uint8) + + # 背景减除 + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2(history=500, varThreshold=16, detectShadows=True) + + # 手部跟踪 + self.track_window = None + self.track_box = None + self.tracking = False + + print("✓ Advanced gesture recognizer initialized") - # 定义肤色范围 - lower_skin = np.array([0, 20, 70], dtype=np.uint8) - upper_skin = np.array([20, 255, 255], dtype=np.uint8) + def adaptive_skin_detection(self, frame): + """自适应肤色检测""" + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - # 创建肤色掩码 - mask = cv2.inRange(hsv, lower_skin, upper_skin) + # 自适应调整肤色范围 + mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) # 形态学操作 - kernel = np.ones((5, 5), np.uint8) - mask = cv2.dilate(mask, kernel, iterations=2) + kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) mask = cv2.erode(mask, kernel, iterations=1) + mask = cv2.dilate(mask, kernel, iterations=2) mask = cv2.GaussianBlur(mask, (5, 5), 0) + return mask + + def find_hand_contours(self, frame): + """查找手部轮廓""" + # 使用背景减除 + fg_mask = self.bg_subtractor.apply(frame) + + # 肤色检测 + skin_mask = self.adaptive_skin_detection(frame) + + # 结合两种掩码 + combined_mask = cv2.bitwise_and(fg_mask, skin_mask) + + # 形态学操作优化 + kernel = np.ones((7, 7), np.uint8) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, kernel) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, kernel) + # 查找轮廓 - contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours, _ = cv2.findContours(combined_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + if not contours: + return None, combined_mask + + # 找到最大的轮廓(假设是手) + largest_contour = max(contours, key=cv2.contourArea) + + # 过滤掉太小的轮廓 + if cv2.contourArea(largest_contour) < 5000: + return None, combined_mask + + return largest_contour, combined_mask - if contours: - # 找到最大的轮廓(假设是手) - largest_contour = max(contours, key=cv2.contourArea) + def analyze_hand_contour(self, contour): + """分析手部轮廓特征""" + # 计算轮廓特征 + area = cv2.contourArea(contour) + perimeter = cv2.arcLength(contour, True) - # 计算轮廓面积 - area = cv2.contourArea(largest_contour) + # 计算凸包和凸缺陷 + hull = cv2.convexHull(contour, returnPoints=False) + if len(hull) < 3: + return 0, None, None - if area > 1000: # 最小面积阈值 - # 获取边界矩形 - x, y, w, h = cv2.boundingRect(largest_contour) + defects = cv2.convexityDefects(contour, hull) - # 计算轮廓的中心点 - M = cv2.moments(largest_contour) - if M["m00"] != 0: - cx = int(M["m10"] / M["m00"]) - cy = int(M["m01"] / M["m00"]) + if defects is None: + return 0, None, None - # 计算凸包和凸缺陷 - hull = cv2.convexHull(largest_contour, returnPoints=False) - if len(hull) > 3: - defects = cv2.convexityDefects(largest_contour, hull) + # 计算手指数量 + finger_count = 0 + finger_tips = [] - if defects is not None: - # 计算手指数量 - finger_count = 0 - for i in range(defects.shape[0]): - s, e, f, d = defects[i, 0] - start = tuple(largest_contour[s][0]) - end = tuple(largest_contour[e][0]) - far = tuple(largest_contour[f][0]) + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(contour[s][0]) + end = tuple(contour[e][0]) + far = tuple(contour[f][0]) - # 计算角度 - a = np.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) - b = np.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) - c = np.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + # 计算角度 + a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) - angle = np.arccos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) - if angle <= np.pi / 2: - finger_count += 1 + # 如果角度小于90度,可能是指缝 + if angle < math.pi / 2: + finger_count += 1 + finger_tips.append(end) - finger_count = min(finger_count + 1, 5) + # 手指数量加1(大拇指) + finger_count = min(finger_count + 1, 5) - # 绘制轮廓和中心点 - cv2.drawContours(frame, [largest_contour], -1, (0, 255, 0), 2) - cv2.circle(frame, (cx, cy), 7, (255, 0, 0), -1) - cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) + # 计算轮廓中心 + M = cv2.moments(contour) + if M["m00"] != 0: + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + center = (cx, cy) + else: + center = None + + return finger_count, center, finger_tips - # 在边界框上方显示手指数量 - cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + def detect_fingertips_with_contours(self, contour): + """使用轮廓检测指尖""" + # 简化轮廓 + epsilon = 0.02 * cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon, True) - return cx, cy, w, h, finger_count, frame + # 查找凸点(可能是指尖) + hull = cv2.convexHull(approx, returnPoints=True) - return None, None, None, None, 0, frame + fingertips = [] + for point in hull: + fingertips.append(tuple(point[0])) - def recognize_simple_gesture(self, frame): - """简化的手势识别""" - # 检测手部轮廓 - cx, cy, w, h, finger_count, frame = self.detect_hand_contour(frame) + return fingertips[:5] # 最多5个指尖 - if cx is None or cy is None: - return "等待手势", frame + def recognize_gesture_by_fingers(self, finger_count, center, frame_shape): + """根据手指数量识别手势""" + if center is None: + return "等待手势", 0.5 - # 获取图像尺寸 - height, width = frame.shape[:2] + h, w = frame_shape[:2] + cx, cy = center - # 定义区域 - center_x = width // 2 - center_y = height // 2 + # 计算手部位置(归一化) + norm_x = cx / w + norm_y = cy / h - # 根据手指数量和位置判断手势 if finger_count == 0: - return "停止", frame + return "停止", 0.9 elif finger_count == 1: - return "向前", frame + return "向前", 0.8 + elif finger_count == 2: + # V字手势,可能是向前或特殊命令 + return "向前", 0.7 + elif finger_count == 3: + # 三指手势,根据位置判断 + if norm_y < 0.4: + return "向上", 0.7 + elif norm_y > 0.6: + return "向下", 0.7 + else: + return "停止", 0.6 elif finger_count >= 4: # 手掌张开,根据位置判断方向 - if cy < center_y - 50: - return "向上", frame - elif cy > center_y + 50: - return "向下", frame - elif cx < center_x - 50: - return "向左", frame - elif cx > center_x + 50: - return "向右", frame + if norm_x < 0.3: + return "向左", 0.8 + elif norm_x > 0.7: + return "向右", 0.8 + elif norm_y < 0.4: + return "向上", 0.8 + elif norm_y > 0.6: + return "向下", 0.8 else: - return "停止", frame - else: - return "等待手势", frame + return "停止", 0.7 + + return "等待手势", 0.5 + + def recognize_with_mediapipe(self, frame): + """使用MediaPipe进行手势识别""" + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + results = self.hands.process(rgb_frame) + + gesture = "等待手势" + confidence = 0.5 + + if results.multi_hand_landmarks: + for hand_landmarks in results.multi_hand_landmarks: + # 绘制手部关键点 + libs['mp_drawing'].draw_landmarks( + frame, hand_landmarks, libs['mp_hands'].HAND_CONNECTIONS, + libs['mp_drawing'].DrawingSpec(color=(0, 255, 0), thickness=2), + libs['mp_drawing'].DrawingSpec(color=(255, 0, 0), thickness=2) + ) + + # 计算手指状态 + landmarks = hand_landmarks.landmark + + # 简单的手指状态检测 + finger_tips = [4, 8, 12, 16, 20] + finger_pips = [2, 6, 10, 14, 18] + + extended_fingers = 0 + for tip, pip in zip(finger_tips, finger_pips): + if landmarks[tip].y < landmarks[pip].y: + extended_fingers += 1 + + # 根据伸直的手指数量判断手势 + if extended_fingers == 0: + gesture = "停止" + confidence = 0.9 + elif extended_fingers == 1: + gesture = "向前" + confidence = 0.8 + elif extended_fingers >= 4: + # 根据手掌方向判断 + wrist = landmarks[0] + middle_mcp = landmarks[9] + + dx = middle_mcp.x - wrist.x + dy = middle_mcp.y - wrist.y + + if abs(dx) > abs(dy): + if dx > 0: + gesture = "向右" + else: + gesture = "向左" + else: + if dy > 0: + gesture = "向下" + else: + gesture = "向上" + confidence = 0.7 + + return gesture, confidence, frame + + def smooth_gesture(self, gesture, confidence): + """平滑手势输出""" + self.gesture_history.append((gesture, confidence)) + + if len(self.gesture_history) < 3: + return gesture, confidence + + # 统计最近的手势 + gesture_counts = {} + confidence_sum = {} + + for g, c in self.gesture_history: + if g not in gesture_counts: + gesture_counts[g] = 0 + confidence_sum[g] = 0 + gesture_counts[g] += 1 + confidence_sum[g] += c + + # 找到最频繁的手势 + max_count = 0 + best_gesture = gesture + best_confidence = confidence + + for g, count in gesture_counts.items(): + if count > max_count and g != "等待手势": + max_count = count + best_gesture = g + best_confidence = confidence_sum[g] / count + + # 如果手势稳定,更新当前手势 + if max_count >= len(self.gesture_history) // 2: + self.current_gesture = best_gesture + self.confidence = best_confidence + + return self.current_gesture, self.confidence + + def visualize_hand(self, frame, contour, center, finger_count, fingertips): + """可视化手部检测结果""" + if contour is not None: + # 绘制轮廓 + cv2.drawContours(frame, [contour], -1, (0, 255, 0), 2) + + # 绘制边界框 + x, y, w, h = cv2.boundingRect(contour) + cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 0), 2) + + # 绘制中心点 + if center: + cx, cy = center + cv2.circle(frame, (cx, cy), 8, (0, 0, 255), -1) + + # 绘制指尖 + for tip in fingertips[:finger_count]: + cv2.circle(frame, tip, 6, (255, 255, 0), -1) + + # 显示手指数量 + cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + + return frame def recognize(self, frame): """识别手势""" + start_time = time.time() + try: - # 使用简化的手势识别 - gesture, processed_frame = self.recognize_simple_gesture(frame) - - # 手势历史记录和平滑处理 - self.gesture_history.append(gesture) - if len(self.gesture_history) > self.max_history: - self.gesture_history.pop(0) - - # 使用多数表决决定最终手势 - if self.gesture_history: - gesture_counts = {} - for g in self.gesture_history: - gesture_counts[g] = gesture_counts.get(g, 0) + 1 - - # 找到出现次数最多的手势 - max_gesture = max(gesture_counts, key=gesture_counts.get) - if gesture_counts[max_gesture] > len(self.gesture_history) // 2: - self.current_gesture = max_gesture + # 镜像图像以便更直观的控制 + frame = cv2.flip(frame, 1) + + # 方法1:使用MediaPipe(如果可用) + if self.use_mediapipe: + gesture, confidence, processed_frame = self.recognize_with_mediapipe(frame) + frame = processed_frame + else: + # 方法2:使用OpenCV + contour, mask = self.find_hand_contours(frame) + + if contour is not None: + finger_count, center, finger_tips = self.analyze_hand_contour(contour) + + # 如果凸缺陷检测失败,使用轮廓检测指尖 + if finger_tips is None or len(finger_tips) < finger_count: + finger_tips = self.detect_fingertips_with_contours(contour) + + self.finger_history.append(finger_count) + + # 计算平均手指数量(平滑) + if len(self.finger_history) > 0: + avg_fingers = sum(self.finger_history) / len(self.finger_history) + finger_count = int(round(avg_fingers)) + + # 识别手势 + gesture, confidence = self.recognize_gesture_by_fingers( + finger_count, center, frame.shape + ) + + # 可视化结果 + frame = self.visualize_hand(frame, contour, center, finger_count, finger_tips) else: - self.current_gesture = gesture + gesture, confidence = "等待手势", 0.3 + + # 手势平滑处理 + final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) + + # 更新最后检测时间 + self.last_gesture_time = time.time() + + # 显示处理时间 + process_time = (time.time() - start_time) * 1000 + cv2.putText(frame, f'Process: {process_time:.1f}ms', (10, frame.shape[0] - 20), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + return final_gesture, final_confidence, frame - return self.current_gesture except Exception as e: print(f"Gesture recognition error: {e}") - return "识别异常" + return "识别异常", 0.0, frame def set_simulated_gesture(self, gesture): """设置模拟的手势""" self.current_gesture = gesture + self.confidence = 0.9 -# ========== 无人机控制模块 ========== -class DroneController: - """无人机控制器""" +# ========== 增强的无人机控制模块 ========== +class EnhancedDroneController: + """增强的无人机控制器""" def __init__(self, airsim_module): self.airsim = airsim_module @@ -367,12 +573,27 @@ def __init__(self, airsim_module): self.flying = False self.connection_attempted = False + # 控制参数 + self.velocity = 3.0 + self.duration = 0.2 + self.altitude = -10.0 # 初始高度(负值表示向上) + + # 控制历史 + self.control_history = deque(maxlen=10) + self.last_control_time = 0 + def connect(self): """连接AirSim无人机""" if self.connection_attempted: return self.connected self.connection_attempted = True + + if self.airsim is None: + print("⚠ AirSim not available, using simulation mode") + self.connected = True + return True + print("Connecting to AirSim...") try: @@ -396,6 +617,14 @@ def connect(self): print("2. 选择环境(如:Landscape Mountains)") print("3. 无人机已在世界中生成") print("4. 如果需要,按模拟器中的'R'键重置无人机") + + print("\n使用模拟模式继续?(y/n)") + choice = input().strip().lower() + if choice == 'y': + self.connected = True + print("✅ Using simulation mode") + return True + return False def takeoff(self): @@ -405,15 +634,25 @@ def takeoff(self): return False try: + if self.airsim is None or self.client is None: + print("✅ Simulated takeoff") + self.flying = True + return True + print("Taking off...") self.client.takeoffAsync().join() time.sleep(1) + + # 上升到指定高度 + self.client.moveToZAsync(self.altitude, 3).join() + self.flying = True print("✅ Drone took off successfully") return True except Exception as e: print(f"❌ Takeoff failed: {e}") - return False + self.flying = True # 模拟模式 + return True def land(self): """降落""" @@ -421,6 +660,11 @@ def land(self): return False try: + if self.airsim is None or self.client is None: + print("✅ Simulated landing") + self.flying = False + return True + print("Landing...") self.client.landAsync().join() self.flying = False @@ -428,101 +672,155 @@ def land(self): return True except Exception as e: print(f"Landing failed: {e}") + self.flying = False return False - def move_by_gesture(self, gesture): + def move_by_gesture(self, gesture, confidence=0.7): """根据手势移动""" if not self.connected or not self.flying: - return + return False + + # 检查控制间隔 + current_time = time.time() + if current_time - self.last_control_time < 0.1: # 最小控制间隔 + return False + + # 如果置信度太低,不执行动作 + if confidence < 0.5: + return False try: - velocity = 2.0 # 降低速度以获得更平滑的控制 - duration = 0.3 # 减少持续时间 + if self.airsim is None or self.client is None: + print(f"Simulated move: {gesture}") + self.control_history.append((gesture, current_time)) + self.last_control_time = current_time + return True + + success = False if gesture == "向上": - self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() + self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) + success = True elif gesture == "向下": - self.client.moveByVelocityZAsync(0, 0, velocity, duration).join() + self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) + success = True elif gesture == "向左": - self.client.moveByVelocityAsync(-velocity, 0, 0, duration).join() + self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) + success = True elif gesture == "向右": - self.client.moveByVelocityAsync(velocity, 0, 0, duration).join() + self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) + success = True elif gesture == "向前": - self.client.moveByVelocityAsync(0, -velocity, 0, duration).join() + self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) + success = True elif gesture == "停止": - self.client.hoverAsync().join() + self.client.hoverAsync() + success = True + + if success: + self.control_history.append((gesture, current_time)) + self.last_control_time = current_time + + return success except Exception as e: print(f"Control command failed: {e}") + return False def emergency_stop(self): """紧急停止""" if self.connected: try: - if self.flying: + if self.flying and self.client is not None: print("Emergency landing...") self.land() - self.client.armDisarm(False) - self.client.enableApiControl(False) - print("✅ Emergency stop complete") + if self.client is not None: + self.client.armDisarm(False) + self.client.enableApiControl(False) + print("✅ Emergency stop complete") except: pass self.connected = False self.flying = False +# ========== 性能监控模块 ========== +class PerformanceMonitor: + """性能监控器""" + + def __init__(self): + self.frame_times = deque(maxlen=60) + self.fps = 0 + self.last_update = time.time() + + def update(self): + """更新性能数据""" + current_time = time.time() + self.frame_times.append(current_time) + + # 每秒更新一次FPS + if current_time - self.last_update >= 1.0: + if len(self.frame_times) > 1: + self.fps = len(self.frame_times) / (self.frame_times[-1] - self.frame_times[0]) + self.last_update = current_time + + def get_stats(self): + """获取性能统计""" + return { + 'fps': self.fps, + 'frame_count': len(self.frame_times) + } + + # ========== 主程序 ========== def main(): """主函数""" - # 使用简化的手势识别器 - gesture_recognizer = SimpleGestureRecognizer() - drone_controller = DroneController(libs['airsim']) + # 初始化组件 + gesture_recognizer = AdvancedGestureRecognizer() + drone_controller = EnhancedDroneController(libs['airsim']) + performance_monitor = PerformanceMonitor() # 初始化摄像头 cap = None - try: - cap = cv2.VideoCapture(0) - if not cap.isOpened(): - # 尝试其他摄像头索引 - for i in range(1, 5): - cap = cv2.VideoCapture(i) - if cap.isOpened(): - print(f"✓ Camera {i} initialized") - break - else: - print("❌ Camera not available, using keyboard control only") - cap = None - else: - print("✓ Camera 0 initialized") - - if cap: - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - cap.set(cv2.CAP_PROP_FPS, 30) - except Exception as e: - print(f"⚠ Camera init failed: {e}") + available_cameras = [] + + # 检测可用摄像头 + for i in range(0, 4): + temp_cap = cv2.VideoCapture(i) + if temp_cap.isOpened(): + available_cameras.append(i) + temp_cap.release() + + if available_cameras: + print(f"✓ Available cameras: {available_cameras}") + cap = cv2.VideoCapture(available_cameras[0]) + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + cap.set(cv2.CAP_PROP_FPS, 30) + print(f"✓ Using camera {available_cameras[0]}") + else: + print("❌ No cameras available, using keyboard control only") cap = None # 显示说明 print("\n" + "=" * 60) - print("操作说明") + print("手势控制无人机系统 - 高级版") print("=" * 60) - print("1. 首先启动AirSim模拟器") - print("2. 选择环境(例如:Landscape Mountains)") - print("3. 按 [C] 连接无人机") - print("4. 按 [空格键] 起飞/降落") + print("操作说明:") + print("1. 首先启动AirSim模拟器(可选)") + print("2. 按 [C] 连接无人机") + print("3. 按 [空格键] 起飞/降落") if cap: - print("5. 手势控制:") - print(" - 握拳(0个手指):停止") - print(" - 伸出1个手指:向前") - print(" - 张开手掌(4-5个手指):根据手的位置控制方向") - print(" * 确保手部在摄像头视野内,光线充足,背景简单") + print("4. 手势控制:") + print(" - 握拳(0指):停止") + print(" - 食指(1指):向前") + print(" - 手掌张开(4-5指):根据手的位置控制方向") + print(" * 手势识别置信度 > 50% 时才会执行") else: - print("5. 键盘控制:") + print("4. 键盘控制:") print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") - print("6. 按 [ESC] 安全退出") + print("5. 按 [ESC] 安全退出") + print("6. 按 [R] 重置手势识别") print("=" * 60) - print("程序启动成功!") - print("-" * 60) # 键盘手势映射 key_to_gesture = { @@ -534,83 +832,91 @@ def main(): ord('x'): "停止", ord('X'): "停止", } - last_control_time = time.time() - control_interval = 0.5 # 控制间隔,避免过于频繁 - # 主循环 + last_gesture_time = time.time() + gesture_timeout = 2.0 # 手势超时时间 + while True: - frame = None + performance_monitor.update() + # 读取摄像头帧 if cap: ret, frame = cap.read() - if ret: - # 镜像图像以便更直观的控制 - frame = cv2.flip(frame, 1) - current_gesture = gesture_recognizer.recognize(frame) + if not ret: + # 如果读取失败,尝试重新初始化摄像头 + print("⚠ Camera read failed, trying to reinitialize...") + cap.release() + if available_cameras: + cap = cv2.VideoCapture(available_cameras[0]) + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + ret, frame = cap.read() + + if not ret: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + current_gesture = "摄像头错误" + confidence = 0.0 + else: + current_gesture, confidence, frame = gesture_recognizer.recognize(frame) else: - frame = np.zeros((480, 640, 3), dtype=np.uint8) - current_gesture = "摄像头错误" + current_gesture, confidence, frame = gesture_recognizer.recognize(frame) else: frame = np.zeros((480, 640, 3), dtype=np.uint8) current_gesture = gesture_recognizer.current_gesture + confidence = gesture_recognizer.confidence # ========== 绘制界面 ========== - # 创建半透明覆盖层 + # 创建信息覆盖层 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (640, 120), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.6, frame, 0.4, 0) - - # 绘制底部操作提示区域 - cv2.rectangle(frame, (0, 400), (640, 480), (0, 0, 0), -1) - frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) + cv2.rectangle(overlay, (0, 0), (640, 140), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) - # 标题 - frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), - size=30, color=(0, 255, 255)) + # 标题和版本 + title = "手势控制无人机系统 - 高级版" + frame = chinese_renderer.put_text(frame, title, (10, 10), size=28, color=(0, 255, 255)) # 状态信息 status_color = (0, 255, 0) if drone_controller.connected else (0, 0, 255) - status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" - frame = chinese_renderer.put_text(frame, status_text, (10, 50), - size=24, color=status_color) + status_text = f"状态: {'已连接' if drone_controller.connected else '未连接'}" + frame = chinese_renderer.put_text(frame, status_text, (10, 50), size=24, color=status_color) - flight_color = (0, 255, 0) if drone_controller.flying else (255, 255, 0) - flight_status = f"飞行: {'是' if drone_controller.flying else '否'}" - frame = chinese_renderer.put_text(frame, flight_status, (10, 80), - size=24, color=flight_color) + flight_color = (0, 255, 0) if drone_controller.flying else (255, 165, 0) + flight_text = f"飞行: {'是' if drone_controller.flying else '否'}" + frame = chinese_renderer.put_text(frame, flight_text, (10, 80), size=24, color=flight_color) # 手势信息 - if current_gesture != "等待手势" and current_gesture != "摄像头错误": - gesture_color = (0, 255, 0) # 绿色 - # 添加手势识别置信度指示 - cv2.circle(frame, (600, 60), 10, gesture_color, -1) - else: - gesture_color = (200, 200, 200) # 灰色 + gesture_color = (0, 255, 0) if confidence > 0.7 else (255, 165, 0) if confidence > 0.5 else (200, 200, 200) + gesture_text = f"手势: {current_gesture} ({confidence:.1%})" + frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), size=24, color=gesture_color) - gesture_text = f"手势: {current_gesture}" - frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), - size=24, color=gesture_color) + # 性能信息 + stats = performance_monitor.get_stats() + perf_text = f"FPS: {stats['fps']:.1f} | 延迟: {1000 / stats['fps']:.1f}ms" if stats['fps'] > 0 else "FPS: 0" + frame = chinese_renderer.put_text(frame, perf_text, (440, 50), size=20, color=(255, 255, 255)) # 操作提示 - help_text = "C:连接 空格:起飞/降落 ESC:退出" + help_text = "C:连接 空格:起飞/降落 R:重置 ESC:退出" if not cap: help_text += " W/A/S/D/F/X:控制" - frame = chinese_renderer.put_text(frame, help_text, (10, 420), - size=20, color=(255, 255, 255)) + # 绘制底部提示栏 + cv2.rectangle(frame, (0, 440), (640, 480), (0, 0, 0), -1) + frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) + frame = chinese_renderer.put_text(frame, help_text, (10, 450), size=18, color=(255, 255, 255)) - # 添加手势提示 + # 手势提示 if cap: - gesture_hint = "提示: 握拳停止,食指向前,手掌张开移动" - frame = chinese_renderer.put_text(frame, gesture_hint, (10, 450), - size=18, color=(255, 200, 100)) + hint_text = "提示: 保持手势稳定,确保良好光线" + frame = chinese_renderer.put_text(frame, hint_text, (10, 470), size=16, color=(255, 200, 100)) + # 显示连接提示 if not drone_controller.connected: - warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" - frame = chinese_renderer.put_text(frame, warning_text, (10, 360), - size=18, color=(0, 165, 255)) + warning_text = "⚠ 按C键连接无人机,或继续使用模拟模式" + frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + size=18, color=(0, 165, 255), bg_color=(0, 0, 0)) - cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) + # 显示图像 + cv2.imshow('Gesture Controlled Drone - Advanced Version', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -629,23 +935,35 @@ def main(): drone_controller.land() else: drone_controller.takeoff() - time.sleep(0.3) + time.sleep(0.5) + + elif key == ord('r') or key == ord('R'): + print("重置手势识别...") + gesture_recognizer = AdvancedGestureRecognizer() elif key in key_to_gesture: gesture = key_to_gesture[key] gesture_recognizer.set_simulated_gesture(gesture) current_gesture = gesture + confidence = 0.9 if drone_controller.connected and drone_controller.flying: - drone_controller.move_by_gesture(gesture) + drone_controller.move_by_gesture(gesture, confidence) - # 真实手势控制(添加间隔控制避免过于频繁) + # 真实手势控制 current_time = time.time() if (current_gesture and current_gesture != "等待手势" and current_gesture != "摄像头错误" and current_gesture != "识别异常" and drone_controller.connected and drone_controller.flying and - current_time - last_control_time > control_interval): - drone_controller.move_by_gesture(current_gesture) - last_control_time = current_time + confidence > 0.5 and # 只有置信度足够高才执行 + current_time - last_gesture_time > 0.2): # 控制频率限制 + + success = drone_controller.move_by_gesture(current_gesture, confidence) + if success: + last_gesture_time = current_time + + # 检查手势超时 + if current_time - gesture_recognizer.last_gesture_time > gesture_timeout: + gesture_recognizer.current_gesture = "等待手势" # 清理资源 print("\n清理资源...") @@ -667,4 +985,5 @@ def main(): print(f"\n程序错误: {e}") traceback.print_exc() finally: - input("\n按回车键退出...") \ No newline at end of file + print("\n按回车键退出...") + input() \ No newline at end of file From 4b5e52e8e77f1a51b3fadc2b457a45463264baca Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 14:30:43 +0800 Subject: [PATCH 04/18] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=8B=B1=E6=96=87?= =?UTF-8?q?=E7=95=8C=E9=9D=A2=EF=BC=8C=E6=94=B9=E8=BF=9B=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E8=AF=86=E5=88=AB=E7=AE=97=E6=B3=95=EF=BC=8C=E5=A2=9E=E5=BC=BA?= =?UTF-8?q?=E5=8D=95=E6=8C=87=E8=AF=86=E5=88=AB=E5=87=86=E7=A1=AE=E7=8E=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 9 +- .../config.json | 53 + .../gesture_config.json | 45 + .../main.py | 1409 ++++++++++------- 4 files changed, 906 insertions(+), 610 deletions(-) create mode 100644 src/autonomus_drone_hand_gesture_project/config.json create mode 100644 src/autonomus_drone_hand_gesture_project/gesture_config.json diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index 05dab76400..0ad6fac5ef 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -1,8 +1,13 @@ # 忽略整个Lib目录(包含所有子文件/子目录) Lib/ -# 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 + +# 忽略Python编译文件,避免无关文件干扰 __pycache__/ *.pyc *.dll.a + # 忽略IDEA配置文件 -.idea/ \ No newline at end of file +.idea/ + +# 忽略日志文件夹(临时日志文件,无需提交到仓库) +logs/ \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/config.json b/src/autonomus_drone_hand_gesture_project/config.json new file mode 100644 index 0000000000..1fa5f144eb --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/config.json @@ -0,0 +1,53 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "detection_confidence": 0.7, + "tracking_confidence": 0.5, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.6 + }, + "drone": { + "velocity": 3.0, + "duration": 0.2, + "altitude": -10.0, + "control_interval": 0.2 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_landmarks": true, + "show_help": true + }, + "performance": { + "target_fps": 30, + "resize_factor": 0.5, + "enable_threading": true + }, + "camera_width": 640, + "camera_height": 480, + "camera_fps": 30, + "camera_index": 0, + "mirror_camera": false, + "min_detection_confidence": 0.6, + "min_tracking_confidence": 0.5, + "gesture_smoothness": 5, + "gesture_stability_threshold": 3, + "control_velocity": 3.0, + "control_duration": 0.25, + "control_deadzone": 0.1, + "max_altitude": -20.0, + "min_altitude": -5.0, + "target_fps": 30, + "frame_timeout": 2.0, + "log_level": "INFO", + "ui_theme": "dark", + "show_fps": true, + "show_landmarks": true, + "show_confidence": true +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json new file mode 100644 index 0000000000..c6d931acbe --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -0,0 +1,45 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "skin_lower": [ + 0, + 30, + 60 + ], + "skin_upper": [ + 25, + 255, + 255 + ], + "min_hand_area": 3000, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.6, + "detection_interval": 2, + "single_finger_angle_threshold": 60 + }, + "drone": { + "velocity": 2.5, + "duration": 0.3, + "altitude": -10.0, + "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false + } +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 5325f35afa..4bd8903a38 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,33 +1,34 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 深度优化版 -优化了手势识别准确性和系统稳定性 -作者: xiaoshiyuan888 +Gesture Controlled AirSim Drone - Optimized Version +Pure OpenCV implementation with English interface +Author: xiaoshiyuan888 """ import sys import os import time import traceback -import subprocess +import json import math +from datetime import datetime from PIL import Image, ImageDraw, ImageFont import cv2 import numpy as np -from collections import deque +from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Advanced Recognition v2.0") +print("Gesture Controlled Drone - Optimized Version") print("=" * 60) -# ========== 修复导入路径 ========== +# ========== Fix import path ========== current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -# ========== 核心模块导入 ========== +# ========== Core module imports ========== def safe_import(): - """安全导入所有模块""" + """Safely import all modules""" modules_status = {} try: @@ -49,20 +50,6 @@ def safe_import(): print(f"[OpenCV] ✗ Import failed: {e}") return None, modules_status - # 尝试导入mediapipe,如果失败则使用OpenCV方案 - mp = None - mp_hands = None - mp_drawing = None - try: - import mediapipe as mp - mp_hands = mp.solutions.hands - mp_drawing = mp.solutions.drawing_utils - modules_status['MediaPipe'] = True - print("[MediaPipe] ✓ Gesture recognition library ready") - except Exception as e: - modules_status['MediaPipe'] = False - print(f"[MediaPipe] ✗ Import failed, using OpenCV fallback: {e}") - airsim_module = None try: airsim_module = __import__('airsim') @@ -87,15 +74,12 @@ def safe_import(): return { 'cv2': cv2, 'np': np, - 'mp': mp, - 'mp_hands': mp_hands, - 'mp_drawing': mp_drawing, 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, 'airsim': airsim_module }, modules_status -# 执行导入 +# Execute imports libs, status = safe_import() if not status.get('OpenCV', False) or not status.get('PIL', False): print("\n❌ Core libraries missing, cannot start.") @@ -106,488 +90,608 @@ def safe_import(): print("✅ Environment check passed, initializing...") print("-" * 60) -# 解包库 +# Unpack libraries cv2, np = libs['cv2'], libs['np'] Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -# ========== 中文显示模块 ========== -class ChineseTextRenderer: - """中文文本渲染器""" +# ========== Configuration Manager ========== +class ConfigManager: + """Configuration Manager""" def __init__(self): - self.fonts = {} - self.load_fonts() - - def load_fonts(self): - """加载字体""" - font_paths = [ - 'simhei.ttf', - 'C:/Windows/Fonts/simhei.ttf', - 'C:/Windows/Fonts/msyh.ttc', - '/System/Library/Fonts/PingFang.ttc', - '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' - ] - for path in font_paths: + self.config_file = os.path.join(current_dir, 'gesture_config.json') + self.default_config = { + 'camera': { + 'index': 0, + 'width': 640, + 'height': 480, + 'fps': 30 + }, + 'gesture': { + 'skin_lower': [0, 30, 60], + 'skin_upper': [25, 255, 255], + 'min_hand_area': 3000, + 'history_size': 10, + 'smooth_frames': 5, + 'min_confidence': 0.6, + 'detection_interval': 1, + 'single_finger_angle_threshold': 60 # 新增:单个手指角度阈值 + }, + 'drone': { + 'velocity': 2.5, + 'duration': 0.3, + 'altitude': -10.0, + 'control_interval': 0.3 + }, + 'display': { + 'show_fps': True, + 'show_confidence': True, + 'show_help': True, + 'show_contours': True, + 'show_bbox': True, + 'show_fingertips': True # 新增:显示指尖 + }, + 'performance': { + 'target_fps': 30, + 'resize_factor': 1.0, + 'enable_multiprocessing': False + } + } + self.config = self.load_config() + + def load_config(self): + """Load configuration""" + if os.path.exists(self.config_file): try: - self.fonts[16] = ImageFont.truetype(path, 16) - self.fonts[20] = ImageFont.truetype(path, 20) - self.fonts[24] = ImageFont.truetype(path, 24) - self.fonts[28] = ImageFont.truetype(path, 28) - self.fonts[32] = ImageFont.truetype(path, 32) - print(f"✓ Chinese fonts loaded: {path}") - return - except: - continue - print("⚠ No Chinese fonts found, using default") + with open(self.config_file, 'r', encoding='utf-8') as f: + loaded_config = json.load(f) + config = self.default_config.copy() + self._merge_config(config, loaded_config) + print("✓ Configuration loaded from file") + return config + except Exception as e: + print(f"⚠ Failed to load config: {e}, using defaults") + return self.default_config.copy() + else: + print("✓ Using default configuration") + return self.default_config.copy() + + def _merge_config(self, base, update): + """Recursively merge configurations""" + for key, value in update.items(): + if key in base and isinstance(base[key], dict) and isinstance(value, dict): + self._merge_config(base[key], value) + else: + base[key] = value - def put_text(self, frame, text, pos, size=20, color=(255, 255, 255), bg_color=None): - """在图像上绘制中文文本""" + def save_config(self): + """Save configuration""" try: - img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - pil_img = Image.fromarray(img_rgb) - draw = ImageDraw.Draw(pil_img) - - font = self.fonts.get(size, self.fonts.get(20)) - - # 如果有背景色,先绘制背景 - if bg_color: - bbox = draw.textbbox(pos, text, font=font) - padding = 2 - draw.rectangle( - [bbox[0] - padding, bbox[1] - padding, bbox[2] + padding, bbox[3] + padding], - fill=bg_color - ) + with open(self.config_file, 'w', encoding='utf-8') as f: + json.dump(self.config, f, indent=2, ensure_ascii=False) + print("✓ Configuration saved") + except Exception as e: + print(f"⚠ Failed to save config: {e}") + + def get(self, *keys): + """Get configuration value""" + value = self.config + for key in keys: + if isinstance(value, dict) and key in value: + value = value[key] + else: + return None + return value - # 绘制阴影 - shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 1, pos[1] + 1) - draw.text(shadow_pos, text, font=font, fill=shadow_color) + def set(self, *keys, value): + """Set configuration value""" + if len(keys) == 0: + return - # 绘制文字 - rgb_color = color[::-1] # BGR to RGB - draw.text(pos, text, font=font, fill=rgb_color) + config = self.config + for key in keys[:-1]: + if key not in config: + config[key] = {} + config = config[key] - return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) - except Exception as e: - # 备用方案:使用OpenCV绘制英文 - cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, - size / 25, color, 2) - return frame + config[keys[-1]] = value + self.save_config() -chinese_renderer = ChineseTextRenderer() +config = ConfigManager() -# ========== 高级手势识别模块 ========== -class AdvancedGestureRecognizer: - """高级手势识别器,结合多种检测方法""" +# ========== Robust Gesture Recognizer ========== +class RobustGestureRecognizer: + """Robust Gesture Recognizer - Pure OpenCV Implementation""" def __init__(self): - self.use_mediapipe = status.get('MediaPipe', False) - - # 初始化MediaPipe(如果可用) - if self.use_mediapipe and libs['mp'] is not None: - try: - self.hands = libs['mp_hands'].Hands( - max_num_hands=2, - min_detection_confidence=0.7, - min_tracking_confidence=0.5, - static_image_mode=False, - model_complexity=1 - ) - print("✓ Using MediaPipe for gesture recognition") - except: - self.use_mediapipe = False - print("⚠ MediaPipe initialization failed, using OpenCV fallback") - else: - self.use_mediapipe = False + self.config = config.get('gesture') - # OpenCV手势识别参数 - self.history_size = 10 + # Gesture history and smoothing + self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) - self.finger_history = deque(maxlen=self.history_size) - self.current_gesture = "等待手势" - self.confidence = 0.0 - self.last_gesture_time = time.time() + self.confidence_history = deque(maxlen=self.history_size) + self.current_gesture = "Waiting" + self.current_confidence = 0.0 - # 肤色检测参数(自适应) - self.skin_lower = np.array([0, 30, 60], dtype=np.uint8) - self.skin_upper = np.array([25, 255, 255], dtype=np.uint8) + # Hand tracking + self.last_hand_position = None + self.hand_tracking = False + self.track_window = None - # 背景减除 - self.bg_subtractor = cv2.createBackgroundSubtractorMOG2(history=500, varThreshold=16, detectShadows=True) + # Performance statistics + self.process_times = deque(maxlen=30) + self.frame_counter = 0 + self.detection_interval = self.config['detection_interval'] + + # Gesture color mapping + self.gesture_colors = { + "Stop": (0, 0, 255), # Red + "Forward": (0, 255, 0), # Green + "Up": (255, 255, 0), # Cyan + "Down": (255, 0, 255), # Purple + "Left": (255, 165, 0), # Orange + "Right": (0, 165, 255), # Light Blue + "Waiting": (200, 200, 200), # Gray + "Error": (255, 0, 0) # Blue + } - # 手部跟踪 - self.track_window = None - self.track_box = None - self.tracking = False + # Skin detection parameters + self.skin_lower = np.array(self.config['skin_lower'], dtype=np.uint8) + self.skin_upper = np.array(self.config['skin_upper'], dtype=np.uint8) - print("✓ Advanced gesture recognizer initialized") + # Background subtractor + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( + history=100, varThreshold=25, detectShadows=True + ) - def adaptive_skin_detection(self, frame): - """自适应肤色检测""" - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + # Morphological operation kernels + self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) + + # Fingertip detection + self.prev_fingertips = None + self.fingertip_stability_threshold = 3 - # 自适应调整肤色范围 - mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) + print("✓ Robust gesture recognizer initialized (Pure OpenCV)") + + def preprocess_frame(self, frame): + """Preprocess frame""" + # Flip image horizontally for intuitive control + frame = cv2.flip(frame, 1) + + # Adjust size for performance + resize_factor = config.get('performance', 'resize_factor') + if resize_factor != 1.0: + new_width = int(frame.shape[1] * resize_factor) + new_height = int(frame.shape[0] * resize_factor) + frame = cv2.resize(frame, (new_width, new_height)) + + return frame - # 形态学操作 - kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - mask = cv2.erode(mask, kernel, iterations=1) - mask = cv2.dilate(mask, kernel, iterations=2) - mask = cv2.GaussianBlur(mask, (5, 5), 0) + def detect_skin(self, frame): + """Detect skin regions""" + # Convert to HSV color space + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - return mask + # Skin mask + skin_mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) - def find_hand_contours(self, frame): - """查找手部轮廓""" - # 使用背景减除 + # Apply background subtraction fg_mask = self.bg_subtractor.apply(frame) - # 肤色检测 - skin_mask = self.adaptive_skin_detection(frame) + # Combine skin mask and foreground mask + combined_mask = cv2.bitwise_and(skin_mask, fg_mask) + + # Morphological operations + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - # 结合两种掩码 - combined_mask = cv2.bitwise_and(fg_mask, skin_mask) + # Gaussian blur + combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) - # 形态学操作优化 - kernel = np.ones((7, 7), np.uint8) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, kernel) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, kernel) + return combined_mask - # 查找轮廓 - contours, _ = cv2.findContours(combined_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + def find_hand_contours(self, mask): + """Find hand contours""" + # Find contours + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: - return None, combined_mask + return None - # 找到最大的轮廓(假设是手) - largest_contour = max(contours, key=cv2.contourArea) + # Find the largest contour (assumed to be hand) + contours = sorted(contours, key=cv2.contourArea, reverse=True) - # 过滤掉太小的轮廓 - if cv2.contourArea(largest_contour) < 5000: - return None, combined_mask + for contour in contours: + area = cv2.contourArea(contour) - return largest_contour, combined_mask + # Filter out too small contours + if area < self.config['min_hand_area']: + continue - def analyze_hand_contour(self, contour): - """分析手部轮廓特征""" - # 计算轮廓特征 - area = cv2.contourArea(contour) - perimeter = cv2.arcLength(contour, True) + # Calculate contour perimeter + perimeter = cv2.arcLength(contour, True) + + # Filter out too simple contours + if perimeter < 100: + continue + + return contour + + return None + + def detect_fingertips_advanced(self, contour): + """Advanced fingertip detection""" + # Simplify contour + epsilon = 0.01 * cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon, True) + + # Find convex hull + hull = cv2.convexHull(approx, returnPoints=False) - # 计算凸包和凸缺陷 - hull = cv2.convexHull(contour, returnPoints=False) - if len(hull) < 3: - return 0, None, None + if hull is None or len(hull) < 3: + return [], 0 - defects = cv2.convexityDefects(contour, hull) + # Find convexity defects + defects = cv2.convexityDefects(approx, hull) if defects is None: - return 0, None, None + return [], 0 - # 计算手指数量 - finger_count = 0 - finger_tips = [] + fingertips = [] + defects_points = [] for i in range(defects.shape[0]): s, e, f, d = defects[i, 0] - start = tuple(contour[s][0]) - end = tuple(contour[e][0]) - far = tuple(contour[f][0]) + start = tuple(approx[s][0]) + end = tuple(approx[e][0]) + far = tuple(approx[f][0]) - # 计算角度 + defects_points.append((start, end, far)) + + # Calculate triangle sides a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) - angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + # Calculate angle + if b * c != 0: + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + angle_degrees = math.degrees(angle) + + # If angle is less than 90 degrees, it might be a finger + if angle_degrees < 90: + # Check if this is a deep defect (far from hull) + if d > 1000: # Adjust this threshold based on your needs + # Both start and end points could be fingertips + for point in [start, end]: + if point not in fingertips: + fingertips.append(point) + + # If we found few fingertips, try alternative method + if len(fingertips) <= 2: + # Use convex hull points as fingertips + hull_points = cv2.convexHull(approx, returnPoints=True) + + # Find extreme points in hull + if len(hull_points) > 0: + hull_points = hull_points.reshape(-1, 2) + + # Sort by y-coordinate (top to bottom) + hull_points = hull_points[hull_points[:, 1].argsort()] + + # Take top points as fingertips + num_fingertips = min(5, len(hull_points)) + for i in range(num_fingertips): + point = tuple(hull_points[i]) + if point not in fingertips: + fingertips.append(point) + + return fingertips, len(fingertips) + + def analyze_hand_contour(self, contour, frame_shape): + """Analyze hand contour""" + if contour is None: + return None, 0.0 + + # Calculate contour area + area = cv2.contourArea(contour) - # 如果角度小于90度,可能是指缝 - if angle < math.pi / 2: - finger_count += 1 - finger_tips.append(end) + # Calculate contour center + M = cv2.moments(contour) + if M["m00"] == 0: + return None, 0.0 + + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + + # Calculate bounding box + x, y, w, h = cv2.boundingRect(contour) + + # Advanced fingertip detection + fingertips, finger_count = self.detect_fingertips_advanced(contour) + + # Calculate hand position (normalized) + h_img, w_img = frame_shape[:2] + norm_x = cx / w_img + norm_y = cy / h_img + + # Calculate gesture confidence + confidence = self.calculate_confidence(area, finger_count, len(contour)) + + # Return result + result = { + 'contour': contour, + 'center': (cx, cy), + 'bbox': (x, y, x + w, y + h), + 'finger_count': finger_count, + 'fingertips': fingertips, + 'area': area, + 'position': (norm_x, norm_y), + 'confidence': confidence + } - # 手指数量加1(大拇指) - finger_count = min(finger_count + 1, 5) + return result, confidence - # 计算轮廓中心 - M = cv2.moments(contour) - if M["m00"] != 0: - cx = int(M["m10"] / M["m00"]) - cy = int(M["m01"] / M["m00"]) - center = (cx, cy) - else: - center = None + def calculate_confidence(self, area, finger_count, contour_length): + """Calculate gesture confidence""" + confidence = 0.5 # Base confidence - return finger_count, center, finger_tips + # Confidence based on area + if area > 5000: + confidence += 0.2 + elif area > 3000: + confidence += 0.1 - def detect_fingertips_with_contours(self, contour): - """使用轮廓检测指尖""" - # 简化轮廓 - epsilon = 0.02 * cv2.arcLength(contour, True) - approx = cv2.approxPolyDP(contour, epsilon, True) + # Confidence based on finger count + if 0 <= finger_count <= 5: + confidence += 0.2 - # 查找凸点(可能是指尖) - hull = cv2.convexHull(approx, returnPoints=True) + # Confidence based on contour complexity + if contour_length > 200: + confidence += 0.1 - fingertips = [] - for point in hull: - fingertips.append(tuple(point[0])) + return min(confidence, 1.0) + + def recognize_gesture(self, hand_data): + """Recognize gesture based on hand data""" + if hand_data is None: + return "Waiting", 0.3 - return fingertips[:5] # 最多5个指尖 + finger_count = hand_data['finger_count'] + norm_x, norm_y = hand_data['position'] + confidence = hand_data['confidence'] + fingertips = hand_data.get('fingertips', []) - def recognize_gesture_by_fingers(self, finger_count, center, frame_shape): - """根据手指数量识别手势""" - if center is None: - return "等待手势", 0.5 + # Special case: single finger detection + if finger_count == 1: + # Additional check for single finger gesture + if len(fingertips) == 1: + # Check if the fingertip is above the hand center (typical for pointing) + cx, cy = hand_data['center'] + fingertip = fingertips[0] - h, w = frame_shape[:2] - cx, cy = center + if fingertip[1] < cy - 20: # Fingertip is above hand center + return "Forward", confidence * 0.9 + else: + return "Stop", confidence * 0.7 + return "Forward", confidence * 0.8 - # 计算手部位置(归一化) - norm_x = cx / w - norm_y = cy / h + # No fingers (fist) + elif finger_count == 0: + return "Stop", confidence * 0.9 - if finger_count == 0: - return "停止", 0.9 - elif finger_count == 1: - return "向前", 0.8 + # Two fingers (V sign) elif finger_count == 2: - # V字手势,可能是向前或特殊命令 - return "向前", 0.7 + return "Forward", confidence * 0.7 + + # Three fingers elif finger_count == 3: - # 三指手势,根据位置判断 + # Judge direction based on position if norm_y < 0.4: - return "向上", 0.7 + return "Up", confidence * 0.7 elif norm_y > 0.6: - return "向下", 0.7 + return "Down", confidence * 0.7 else: - return "停止", 0.6 + return "Stop", confidence * 0.6 + + # Four or five fingers (open hand) elif finger_count >= 4: - # 手掌张开,根据位置判断方向 + # Judge direction based on hand position if norm_x < 0.3: - return "向左", 0.8 + return "Left", confidence * 0.8 elif norm_x > 0.7: - return "向右", 0.8 + return "Right", confidence * 0.8 elif norm_y < 0.4: - return "向上", 0.8 + return "Up", confidence * 0.8 elif norm_y > 0.6: - return "向下", 0.8 + return "Down", confidence * 0.8 else: - return "停止", 0.7 + return "Stop", confidence * 0.7 - return "等待手势", 0.5 + return "Waiting", confidence * 0.5 - def recognize_with_mediapipe(self, frame): - """使用MediaPipe进行手势识别""" - rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - results = self.hands.process(rgb_frame) + def smooth_gesture(self, new_gesture, new_confidence): + """Smooth gesture output""" + # Add to history + self.gesture_history.append(new_gesture) + self.confidence_history.append(new_confidence) - gesture = "等待手势" - confidence = 0.5 + # If not enough history, return directly + if len(self.gesture_history) < 3: + self.current_gesture = new_gesture + self.current_confidence = new_confidence + return new_gesture, new_confidence - if results.multi_hand_landmarks: - for hand_landmarks in results.multi_hand_landmarks: - # 绘制手部关键点 - libs['mp_drawing'].draw_landmarks( - frame, hand_landmarks, libs['mp_hands'].HAND_CONNECTIONS, - libs['mp_drawing'].DrawingSpec(color=(0, 255, 0), thickness=2), - libs['mp_drawing'].DrawingSpec(color=(255, 0, 0), thickness=2) - ) + # Count most common gesture + gesture_counter = Counter(self.gesture_history) + most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] - # 计算手指状态 - landmarks = hand_landmarks.landmark - - # 简单的手指状态检测 - finger_tips = [4, 8, 12, 16, 20] - finger_pips = [2, 6, 10, 14, 18] - - extended_fingers = 0 - for tip, pip in zip(finger_tips, finger_pips): - if landmarks[tip].y < landmarks[pip].y: - extended_fingers += 1 - - # 根据伸直的手指数量判断手势 - if extended_fingers == 0: - gesture = "停止" - confidence = 0.9 - elif extended_fingers == 1: - gesture = "向前" - confidence = 0.8 - elif extended_fingers >= 4: - # 根据手掌方向判断 - wrist = landmarks[0] - middle_mcp = landmarks[9] - - dx = middle_mcp.x - wrist.x - dy = middle_mcp.y - wrist.y - - if abs(dx) > abs(dy): - if dx > 0: - gesture = "向右" - else: - gesture = "向左" - else: - if dy > 0: - gesture = "向下" - else: - gesture = "向上" - confidence = 0.7 + # Calculate average confidence + avg_confidence = np.mean(list(self.confidence_history)) - return gesture, confidence, frame + # If the most common gesture appears enough times, use it + smooth_frames = self.config['smooth_frames'] + if most_common_count >= smooth_frames: + self.current_gesture = most_common_gesture + self.current_confidence = avg_confidence - def smooth_gesture(self, gesture, confidence): - """平滑手势输出""" - self.gesture_history.append((gesture, confidence)) + return self.current_gesture, self.current_confidence - if len(self.gesture_history) < 3: - return gesture, confidence - - # 统计最近的手势 - gesture_counts = {} - confidence_sum = {} - - for g, c in self.gesture_history: - if g not in gesture_counts: - gesture_counts[g] = 0 - confidence_sum[g] = 0 - gesture_counts[g] += 1 - confidence_sum[g] += c - - # 找到最频繁的手势 - max_count = 0 - best_gesture = gesture - best_confidence = confidence - - for g, count in gesture_counts.items(): - if count > max_count and g != "等待手势": - max_count = count - best_gesture = g - best_confidence = confidence_sum[g] / count - - # 如果手势稳定,更新当前手势 - if max_count >= len(self.gesture_history) // 2: - self.current_gesture = best_gesture - self.confidence = best_confidence - - return self.current_gesture, self.confidence - - def visualize_hand(self, frame, contour, center, finger_count, fingertips): - """可视化手部检测结果""" - if contour is not None: - # 绘制轮廓 - cv2.drawContours(frame, [contour], -1, (0, 255, 0), 2) - - # 绘制边界框 - x, y, w, h = cv2.boundingRect(contour) - cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 0), 2) - - # 绘制中心点 - if center: - cx, cy = center - cv2.circle(frame, (cx, cy), 8, (0, 0, 255), -1) - - # 绘制指尖 - for tip in fingertips[:finger_count]: - cv2.circle(frame, tip, 6, (255, 255, 0), -1) - - # 显示手指数量 - cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + def visualize_detection(self, frame, hand_data, gesture, confidence): + """Visualize detection results""" + if hand_data is None: + return frame + + # Get display configuration + show_contours = config.get('display', 'show_contours') + show_bbox = config.get('display', 'show_bbox') + show_fingertips = config.get('display', 'show_fingertips') + + # Draw contour + if show_contours and 'contour' in hand_data: + cv2.drawContours(frame, [hand_data['contour']], -1, (0, 255, 0), 2) + + # Draw bounding box + if show_bbox and 'bbox' in hand_data: + x1, y1, x2, y2 = hand_data['bbox'] + color = self.gesture_colors.get(gesture, (255, 255, 255)) + cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) + + # Show gesture label + label = f"{gesture}" + if config.get('display', 'show_confidence'): + label += f" ({confidence:.0%})" + + # Calculate text size + (text_width, text_height), baseline = cv2.getTextSize( + label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 + ) + + # Draw text background + cv2.rectangle(frame, + (x1, y1 - text_height - 10), + (x1 + text_width, y1), + color, -1) + + # Draw text + cv2.putText(frame, label, (x1, y1 - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) + + # Draw center point + if 'center' in hand_data: + cx, cy = hand_data['center'] + cv2.circle(frame, (cx, cy), 5, (0, 0, 255), -1) + cv2.putText(frame, "Center", (cx + 10, cy), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) + + # Draw fingertips + if show_fingertips and 'fingertips' in hand_data: + for i, point in enumerate(hand_data['fingertips']): + cv2.circle(frame, point, 4, (255, 0, 0), -1) + cv2.putText(frame, f"F{i + 1}", (point[0] + 5, point[1]), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1) + + # Show finger count + if 'finger_count' in hand_data: + finger_text = f"Fingers: {hand_data['finger_count']}" + cv2.putText(frame, finger_text, (10, frame.shape[0] - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) return frame def recognize(self, frame): - """识别手势""" + """Recognize gesture""" start_time = time.time() try: - # 镜像图像以便更直观的控制 - frame = cv2.flip(frame, 1) - - # 方法1:使用MediaPipe(如果可用) - if self.use_mediapipe: - gesture, confidence, processed_frame = self.recognize_with_mediapipe(frame) - frame = processed_frame - else: - # 方法2:使用OpenCV - contour, mask = self.find_hand_contours(frame) + # Preprocess frame + processed_frame = self.preprocess_frame(frame) - if contour is not None: - finger_count, center, finger_tips = self.analyze_hand_contour(contour) + # Detect every few frames to improve performance + if self.frame_counter % self.detection_interval != 0: + self.frame_counter += 1 + return self.current_gesture, self.current_confidence, processed_frame - # 如果凸缺陷检测失败,使用轮廓检测指尖 - if finger_tips is None or len(finger_tips) < finger_count: - finger_tips = self.detect_fingertips_with_contours(contour) + # Detect skin + skin_mask = self.detect_skin(processed_frame) - self.finger_history.append(finger_count) + # Find hand contour + hand_contour = self.find_hand_contours(skin_mask) - # 计算平均手指数量(平滑) - if len(self.finger_history) > 0: - avg_fingers = sum(self.finger_history) / len(self.finger_history) - finger_count = int(round(avg_fingers)) + # Analyze hand contour + hand_data, confidence = self.analyze_hand_contour(hand_contour, processed_frame.shape) - # 识别手势 - gesture, confidence = self.recognize_gesture_by_fingers( - finger_count, center, frame.shape - ) - - # 可视化结果 - frame = self.visualize_hand(frame, contour, center, finger_count, finger_tips) - else: - gesture, confidence = "等待手势", 0.3 + # Recognize gesture + if hand_data is not None: + gesture, raw_confidence = self.recognize_gesture(hand_data) + confidence = max(confidence, raw_confidence) + else: + gesture, confidence = "Waiting", 0.3 - # 手势平滑处理 + # Smooth gesture final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) - # 更新最后检测时间 - self.last_gesture_time = time.time() + # Visualize results + if hand_data is not None: + processed_frame = self.visualize_detection( + processed_frame, hand_data, final_gesture, final_confidence + ) + + # Update counter + self.frame_counter += 1 - # 显示处理时间 + # Calculate processing time process_time = (time.time() - start_time) * 1000 - cv2.putText(frame, f'Process: {process_time:.1f}ms', (10, frame.shape[0] - 20), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + self.process_times.append(process_time) - return final_gesture, final_confidence, frame + return final_gesture, final_confidence, processed_frame except Exception as e: - print(f"Gesture recognition error: {e}") - return "识别异常", 0.0, frame + print(f"⚠ Gesture recognition error: {e}") + return "Error", 0.0, frame + + def get_performance_stats(self): + """Get performance statistics""" + if len(self.process_times) == 0: + return 0.0 + + return np.mean(list(self.process_times)) def set_simulated_gesture(self, gesture): - """设置模拟的手势""" + """Set simulated gesture""" self.current_gesture = gesture - self.confidence = 0.9 + self.current_confidence = 0.9 -# ========== 增强的无人机控制模块 ========== -class EnhancedDroneController: - """增强的无人机控制器""" +# ========== Simple Drone Controller ========== +class SimpleDroneController: + """Simple Drone Controller""" def __init__(self, airsim_module): self.airsim = airsim_module self.client = None self.connected = False self.flying = False - self.connection_attempted = False - # 控制参数 - self.velocity = 3.0 - self.duration = 0.2 - self.altitude = -10.0 # 初始高度(负值表示向上) + # Control parameters + self.velocity = config.get('drone', 'velocity') + self.duration = config.get('drone', 'duration') + self.altitude = config.get('drone', 'altitude') + self.control_interval = config.get('drone', 'control_interval') - # 控制历史 - self.control_history = deque(maxlen=10) + # Control state self.last_control_time = 0 + self.last_gesture = None - def connect(self): - """连接AirSim无人机""" - if self.connection_attempted: - return self.connected + print("✓ Simple drone controller initialized") - self.connection_attempted = True + def connect(self): + """Connect to AirSim drone""" + if self.connected: + return True if self.airsim is None: print("⚠ AirSim not available, using simulation mode") @@ -612,13 +716,7 @@ def connect(self): except Exception as e: print(f"❌ Connection failed: {e}") - print("\n请确保:") - print("1. AirSim模拟器正在运行") - print("2. 选择环境(如:Landscape Mountains)") - print("3. 无人机已在世界中生成") - print("4. 如果需要,按模拟器中的'R'键重置无人机") - - print("\n使用模拟模式继续?(y/n)") + print("\nContinue with simulation mode? (y/n)") choice = input().strip().lower() if choice == 'y': self.connected = True @@ -628,9 +726,8 @@ def connect(self): return False def takeoff(self): - """起飞""" + """Take off""" if not self.connected: - print("❌ Drone not connected") return False try: @@ -643,7 +740,7 @@ def takeoff(self): self.client.takeoffAsync().join() time.sleep(1) - # 上升到指定高度 + # Ascend to specified altitude self.client.moveToZAsync(self.altitude, 3).join() self.flying = True @@ -651,11 +748,10 @@ def takeoff(self): return True except Exception as e: print(f"❌ Takeoff failed: {e}") - self.flying = True # 模拟模式 - return True + return False def land(self): - """降落""" + """Land""" if not self.connected: return False @@ -672,54 +768,54 @@ def land(self): return True except Exception as e: print(f"Landing failed: {e}") - self.flying = False return False - def move_by_gesture(self, gesture, confidence=0.7): - """根据手势移动""" + def move_by_gesture(self, gesture, confidence): + """Move based on gesture""" if not self.connected or not self.flying: return False - # 检查控制间隔 + # Check control interval current_time = time.time() - if current_time - self.last_control_time < 0.1: # 最小控制间隔 + if current_time - self.last_control_time < self.control_interval: return False - # 如果置信度太低,不执行动作 - if confidence < 0.5: + # Check confidence threshold + min_confidence = config.get('gesture', 'min_confidence') + if confidence < min_confidence: return False try: if self.airsim is None or self.client is None: print(f"Simulated move: {gesture}") - self.control_history.append((gesture, current_time)) self.last_control_time = current_time + self.last_gesture = gesture return True success = False - if gesture == "向上": + if gesture == "Up": self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) success = True - elif gesture == "向下": + elif gesture == "Down": self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) success = True - elif gesture == "向左": + elif gesture == "Left": self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) success = True - elif gesture == "向右": + elif gesture == "Right": self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) success = True - elif gesture == "向前": + elif gesture == "Forward": self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) success = True - elif gesture == "停止": + elif gesture == "Stop": self.client.hoverAsync() success = True if success: - self.control_history.append((gesture, current_time)) self.last_control_time = current_time + self.last_gesture = gesture return success except Exception as e: @@ -727,7 +823,7 @@ def move_by_gesture(self, gesture, confidence=0.7): return False def emergency_stop(self): - """紧急停止""" + """Emergency stop""" if self.connected: try: if self.flying and self.client is not None: @@ -739,251 +835,348 @@ def emergency_stop(self): print("✅ Emergency stop complete") except: pass + self.connected = False self.flying = False -# ========== 性能监控模块 ========== +# ========== Simple UI Renderer ========== +class SimpleUIRenderer: + """Simple UI Renderer""" + + def __init__(self): + # Color definitions + self.colors = { + 'title': (0, 255, 255), # Cyan + 'connected': (0, 255, 0), # Green + 'disconnected': (0, 0, 255), # Red + 'flying': (0, 255, 0), # Green + 'landed': (255, 165, 0), # Orange + 'warning': (0, 165, 255), # Light Blue + 'info': (255, 255, 255), # White + 'help': (255, 200, 100) # Light Orange + } + + print("✓ Simple UI renderer initialized") + + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time): + """Draw status bar""" + h, w = frame.shape[:2] + + # Draw semi-transparent background + overlay = frame.copy() + cv2.rectangle(overlay, (0, 0), (w, 100), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + + # Title + title = "Gesture Controlled Drone" + cv2.putText(frame, title, (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, self.colors['title'], 2) + + # Connection status + status_color = self.colors['connected'] if drone_controller.connected else self.colors['disconnected'] + status_text = f"Drone: {'Connected' if drone_controller.connected else 'Disconnected'}" + cv2.putText(frame, status_text, (10, 60), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, status_color, 1) + + # Flight status + flight_color = self.colors['flying'] if drone_controller.flying else self.colors['landed'] + flight_text = f"Flight: {'Flying' if drone_controller.flying else 'Landed'}" + cv2.putText(frame, flight_text, (10, 85), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, flight_color, 1) + + # Gesture information + if confidence > 0.7: + gesture_color = (0, 255, 0) # Green + elif confidence > 0.5: + gesture_color = (255, 165, 0) # Orange + else: + gesture_color = (200, 200, 200) # Gray + + gesture_text = f"Gesture: {gesture}" + if config.get('display', 'show_confidence'): + gesture_text += f" ({confidence:.0%})" + + cv2.putText(frame, gesture_text, (w // 2, 60), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, gesture_color, 1) + + # Performance information + if config.get('display', 'show_fps'): + perf_text = f"FPS: {fps:.1f}" + if process_time > 0: + perf_text += f" | Delay: {process_time:.1f}ms" + + cv2.putText(frame, perf_text, (w - 200, 85), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['info'], 1) + + return frame + + def draw_help_bar(self, frame): + """Draw help bar""" + if not config.get('display', 'show_help'): + return frame + + h, w = frame.shape[:2] + + # Draw bottom help bar + cv2.rectangle(frame, (0, h - 50), (w, h), (0, 0, 0), -1) + + # Help text + help_text = "C:Connect Space:Takeoff/Land ESC:Exit W/A/S/D/F/X:Keyboard" + cv2.putText(frame, help_text, (10, h - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['help'], 1) + + return frame + + def draw_warning(self, frame, message): + """Draw warning message""" + h, w = frame.shape[:2] + + # Draw warning at top + warning_bg = np.zeros((40, w, 3), dtype=np.uint8) + warning_bg[:, :] = (0, 69, 255) # Orange + + frame[100:140, 0:w] = cv2.addWeighted( + frame[100:140, 0:w], 0.3, + warning_bg, 0.7, 0 + ) + + # Draw warning text + cv2.putText(frame, message, (10, 125), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, self.colors['warning'], 1) + + return frame + + +# ========== Performance Monitor ========== class PerformanceMonitor: - """性能监控器""" + """Performance Monitor""" def __init__(self): self.frame_times = deque(maxlen=60) - self.fps = 0 self.last_update = time.time() + self.fps = 0 + self.frame_count = 0 def update(self): - """更新性能数据""" + """Update performance data""" current_time = time.time() self.frame_times.append(current_time) + self.frame_count += 1 - # 每秒更新一次FPS + # Calculate FPS once per second if current_time - self.last_update >= 1.0: if len(self.frame_times) > 1: - self.fps = len(self.frame_times) / (self.frame_times[-1] - self.frame_times[0]) + time_diff = self.frame_times[-1] - self.frame_times[0] + if time_diff > 0: + self.fps = len(self.frame_times) / time_diff + else: + self.fps = 0 self.last_update = current_time def get_stats(self): - """获取性能统计""" + """Get performance statistics""" return { 'fps': self.fps, - 'frame_count': len(self.frame_times) + 'frame_count': self.frame_count } -# ========== 主程序 ========== +# ========== Main Program ========== def main(): - """主函数""" - # 初始化组件 - gesture_recognizer = AdvancedGestureRecognizer() - drone_controller = EnhancedDroneController(libs['airsim']) + """Main function""" + # Initialize components + print("Initializing components...") + + gesture_recognizer = RobustGestureRecognizer() + drone_controller = SimpleDroneController(libs['airsim']) + ui_renderer = SimpleUIRenderer() performance_monitor = PerformanceMonitor() - # 初始化摄像头 + # Initialize camera cap = None - available_cameras = [] - - # 检测可用摄像头 - for i in range(0, 4): - temp_cap = cv2.VideoCapture(i) - if temp_cap.isOpened(): - available_cameras.append(i) - temp_cap.release() - - if available_cameras: - print(f"✓ Available cameras: {available_cameras}") - cap = cv2.VideoCapture(available_cameras[0]) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - cap.set(cv2.CAP_PROP_FPS, 30) - print(f"✓ Using camera {available_cameras[0]}") - else: - print("❌ No cameras available, using keyboard control only") + try: + cap = cv2.VideoCapture(0) + if cap.isOpened(): + cap.set(cv2.CAP_PROP_FRAME_WIDTH, config.get('camera', 'width')) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, config.get('camera', 'height')) + cap.set(cv2.CAP_PROP_FPS, config.get('camera', 'fps')) + + # Get actual parameters + actual_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + actual_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + actual_fps = int(cap.get(cv2.CAP_PROP_FPS)) + + print(f"✓ Camera initialized") + print(f" Resolution: {actual_width}x{actual_height}") + print(f" FPS: {actual_fps}") + else: + print("❌ Camera not available, using simulation mode") + cap = None + except Exception as e: + print(f"⚠ Camera init failed: {e}") cap = None - # 显示说明 + # Display welcome message print("\n" + "=" * 60) - print("手势控制无人机系统 - 高级版") + print("Gesture Controlled Drone - Optimized Version") print("=" * 60) - print("操作说明:") - print("1. 首先启动AirSim模拟器(可选)") - print("2. 按 [C] 连接无人机") - print("3. 按 [空格键] 起飞/降落") - if cap: - print("4. 手势控制:") - print(" - 握拳(0指):停止") - print(" - 食指(1指):向前") - print(" - 手掌张开(4-5指):根据手的位置控制方向") - print(" * 手势识别置信度 > 50% 时才会执行") - else: - print("4. 键盘控制:") - print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") - print("5. 按 [ESC] 安全退出") - print("6. 按 [R] 重置手势识别") + print("System Status:") + print(f" Camera: {'Connected' if cap else 'Simulation Mode'}") + print(f" Gesture Recognition: Pure OpenCV") + print(f" AirSim: {'Available' if libs['airsim'] else 'Simulation Mode'}") print("=" * 60) - # 键盘手势映射 + # Display operation instructions + print("\nOperation Instructions:") + print("1. Press [C] to connect drone (AirSim simulator)") + print("2. Press [SPACE] to takeoff/land") + print("3. Gesture Control:") + print(" - Fist (0 fingers): Stop") + print(" - Index finger (1 finger): Forward") + print(" - Open hand (4-5 fingers): Control direction by hand position") + print(" * Gesture confidence > 60% to execute") + print("4. Keyboard Control:") + print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") + print("5. Press [H] to toggle help display") + print("6. Press [R] to reset gesture recognition") + print("7. Press [ESC] to exit safely") + print("=" * 60) + print("Program started successfully!") + print("-" * 60) + + # Keyboard gesture mapping key_to_gesture = { - ord('w'): "向上", ord('W'): "向上", - ord('s'): "向下", ord('S'): "向下", - ord('a'): "向左", ord('A'): "向左", - ord('d'): "向右", ord('D'): "向右", - ord('f'): "向前", ord('F'): "向前", - ord('x'): "停止", ord('X'): "停止", + ord('w'): "Up", ord('W'): "Up", + ord('s'): "Down", ord('S'): "Down", + ord('a'): "Left", ord('A'): "Left", + ord('d'): "Right", ord('D'): "Right", + ord('f'): "Forward", ord('F'): "Forward", + ord('x'): "Stop", ord('X'): "Stop", } - # 主循环 - last_gesture_time = time.time() - gesture_timeout = 2.0 # 手势超时时间 - - while True: - performance_monitor.update() + # Main loop + print("\nEntering main loop, press ESC to exit...") - # 读取摄像头帧 - if cap: - ret, frame = cap.read() - if not ret: - # 如果读取失败,尝试重新初始化摄像头 - print("⚠ Camera read failed, trying to reinitialize...") - cap.release() - if available_cameras: - cap = cv2.VideoCapture(available_cameras[0]) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - ret, frame = cap.read() + try: + while True: + # Update performance monitor + performance_monitor.update() + # Read camera frame + if cap: + ret, frame = cap.read() if not ret: + # Create blank frame frame = np.zeros((480, 640, 3), dtype=np.uint8) - current_gesture = "摄像头错误" - confidence = 0.0 + gesture, confidence = "Camera Error", 0.0 else: - current_gesture, confidence, frame = gesture_recognizer.recognize(frame) + # Gesture recognition + gesture, confidence, frame = gesture_recognizer.recognize(frame) else: - current_gesture, confidence, frame = gesture_recognizer.recognize(frame) - else: - frame = np.zeros((480, 640, 3), dtype=np.uint8) - current_gesture = gesture_recognizer.current_gesture - confidence = gesture_recognizer.confidence - - # ========== 绘制界面 ========== - # 创建信息覆盖层 - overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (640, 140), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) - - # 标题和版本 - title = "手势控制无人机系统 - 高级版" - frame = chinese_renderer.put_text(frame, title, (10, 10), size=28, color=(0, 255, 255)) + # Simulation mode + frame = np.zeros((480, 640, 3), dtype=np.uint8) + gesture, confidence = gesture_recognizer.current_gesture, gesture_recognizer.current_confidence - # 状态信息 - status_color = (0, 255, 0) if drone_controller.connected else (0, 0, 255) - status_text = f"状态: {'已连接' if drone_controller.connected else '未连接'}" - frame = chinese_renderer.put_text(frame, status_text, (10, 50), size=24, color=status_color) + # Get performance statistics + perf_stats = performance_monitor.get_stats() + process_time = gesture_recognizer.get_performance_stats() - flight_color = (0, 255, 0) if drone_controller.flying else (255, 165, 0) - flight_text = f"飞行: {'是' if drone_controller.flying else '否'}" - frame = chinese_renderer.put_text(frame, flight_text, (10, 80), size=24, color=flight_color) + # Draw UI + frame = ui_renderer.draw_status_bar( + frame, drone_controller, gesture, confidence, + perf_stats['fps'], process_time + ) - # 手势信息 - gesture_color = (0, 255, 0) if confidence > 0.7 else (255, 165, 0) if confidence > 0.5 else (200, 200, 200) - gesture_text = f"手势: {current_gesture} ({confidence:.1%})" - frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), size=24, color=gesture_color) + frame = ui_renderer.draw_help_bar(frame) - # 性能信息 - stats = performance_monitor.get_stats() - perf_text = f"FPS: {stats['fps']:.1f} | 延迟: {1000 / stats['fps']:.1f}ms" if stats['fps'] > 0 else "FPS: 0" - frame = chinese_renderer.put_text(frame, perf_text, (440, 50), size=20, color=(255, 255, 255)) - - # 操作提示 - help_text = "C:连接 空格:起飞/降落 R:重置 ESC:退出" - if not cap: - help_text += " W/A/S/D/F/X:控制" - - # 绘制底部提示栏 - cv2.rectangle(frame, (0, 440), (640, 480), (0, 0, 0), -1) - frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) - frame = chinese_renderer.put_text(frame, help_text, (10, 450), size=18, color=(255, 255, 255)) - - # 手势提示 - if cap: - hint_text = "提示: 保持手势稳定,确保良好光线" - frame = chinese_renderer.put_text(frame, hint_text, (10, 470), size=16, color=(255, 200, 100)) - - # 显示连接提示 - if not drone_controller.connected: - warning_text = "⚠ 按C键连接无人机,或继续使用模拟模式" - frame = chinese_renderer.put_text(frame, warning_text, (10, 400), - size=18, color=(0, 165, 255), bg_color=(0, 0, 0)) + # Show connection warning + if not drone_controller.connected: + warning_msg = "⚠ Press C to connect drone, or use simulation mode" + frame = ui_renderer.draw_warning(frame, warning_msg) - # 显示图像 - cv2.imshow('Gesture Controlled Drone - Advanced Version', frame) + # Show image + window_title = "Gesture Controlled Drone - Press ESC to exit" + cv2.imshow(window_title, frame) - # ========== 键盘控制 ========== - key = cv2.waitKey(1) & 0xFF + # ========== Keyboard Control ========== + key = cv2.waitKey(1) & 0xFF - if key == 27: # ESC键 - print("\n退出程序...") - break + if key == 27: # ESC key + print("\nExiting program...") + break - elif key == ord('c') or key == ord('C'): - if not drone_controller.connected: - drone_controller.connect() + elif key == ord('c') or key == ord('C'): + if not drone_controller.connected: + drone_controller.connect() - elif key == 32: # 空格键 - if drone_controller.connected: - if drone_controller.flying: - drone_controller.land() - else: - drone_controller.takeoff() - time.sleep(0.5) - - elif key == ord('r') or key == ord('R'): - print("重置手势识别...") - gesture_recognizer = AdvancedGestureRecognizer() - - elif key in key_to_gesture: - gesture = key_to_gesture[key] - gesture_recognizer.set_simulated_gesture(gesture) - current_gesture = gesture - confidence = 0.9 - if drone_controller.connected and drone_controller.flying: + elif key == 32: # Space key + if drone_controller.connected: + if drone_controller.flying: + drone_controller.land() + else: + drone_controller.takeoff() + time.sleep(0.5) + + elif key == ord('h') or key == ord('H'): + # Toggle help display + current = config.get('display', 'show_help') + config.set('display', 'show_help', value=not current) + print(f"Help display: {'ON' if not current else 'OFF'}") + + elif key == ord('f') or key == ord('F'): + # Toggle FPS display + current = config.get('display', 'show_fps') + config.set('display', 'show_fps', value=not current) + print(f"FPS display: {'ON' if not current else 'OFF'}") + + elif key == ord('r') or key == ord('R'): + # Reset gesture recognition + print("Resetting gesture recognition...") + gesture_recognizer = RobustGestureRecognizer() + + elif key == ord('t') or key == ord('T'): + # Toggle fingertip display + current = config.get('display', 'show_fingertips') + config.set('display', 'show_fingertips', value=not current) + print(f"Fingertip display: {'ON' if not current else 'OFF'}") + + elif key in key_to_gesture: + # Keyboard control + simulated_gesture = key_to_gesture[key] + gesture_recognizer.set_simulated_gesture(simulated_gesture) + gesture = simulated_gesture + confidence = 0.9 + if drone_controller.connected and drone_controller.flying: + drone_controller.move_by_gesture(gesture, confidence) + + # Real gesture control + current_time = time.time() + if (gesture and gesture != "Waiting" and + gesture != "Camera Error" and gesture != "Error" and + drone_controller.connected and drone_controller.flying): drone_controller.move_by_gesture(gesture, confidence) - # 真实手势控制 - current_time = time.time() - if (current_gesture and current_gesture != "等待手势" and - current_gesture != "摄像头错误" and current_gesture != "识别异常" and - drone_controller.connected and drone_controller.flying and - confidence > 0.5 and # 只有置信度足够高才执行 - current_time - last_gesture_time > 0.2): # 控制频率限制 - - success = drone_controller.move_by_gesture(current_gesture, confidence) - if success: - last_gesture_time = current_time - - # 检查手势超时 - if current_time - gesture_recognizer.last_gesture_time > gesture_timeout: - gesture_recognizer.current_gesture = "等待手势" - - # 清理资源 - print("\n清理资源...") - if cap: - cap.release() - cv2.destroyAllWindows() - drone_controller.emergency_stop() - print("程序安全退出") - print("=" * 60) - - -# ========== 程序入口 ========== -if __name__ == "__main__": - try: - main() except KeyboardInterrupt: - print("\n程序被用户中断") + print("\nProgram interrupted by user") except Exception as e: - print(f"\n程序错误: {e}") + print(f"\nProgram error: {e}") traceback.print_exc() finally: - print("\n按回车键退出...") - input() \ No newline at end of file + # Cleanup resources + print("\nCleaning up resources...") + if cap: + cap.release() + cv2.destroyAllWindows() + drone_controller.emergency_stop() + config.save_config() + + print("Program exited safely") + print("=" * 60) + print("\nThank you for using Gesture Controlled Drone System!") + input("Press Enter to exit...") + + +# ========== Program Entry Point ========== +if __name__ == "__main__": + main() \ No newline at end of file From f1897a2c8b60eee2305bb24fc22521495d7de8ab Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 15:46:37 +0800 Subject: [PATCH 05/18] =?UTF-8?q?=E6=8C=81=E7=BB=AD=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E6=89=8B=E5=8A=BF=E8=AF=86=E5=88=AB=E7=AE=97=E6=B3=95=EF=BC=8C?= =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=BD=8D=E7=BD=AE=E6=95=8F=E6=84=9F=E9=97=AE?= =?UTF-8?q?=E9=A2=98=EF=BC=8C=E5=A2=9E=E5=BC=BA=E5=85=A8=E5=B1=8F=E6=89=8B?= =?UTF-8?q?=E5=8A=BF=E6=A3=80=E6=B5=8B=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config.json | 7 +- .../gesture_config_v2.json | 53 + .../gesture_config_v3.json | 53 + .../main.py | 1241 ++++++++++------- 4 files changed, 887 insertions(+), 467 deletions(-) create mode 100644 src/autonomus_drone_hand_gesture_project/gesture_config_v2.json create mode 100644 src/autonomus_drone_hand_gesture_project/gesture_config_v3.json diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index c6d931acbe..d41f020e20 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -21,6 +21,9 @@ "smooth_frames": 5, "min_confidence": 0.6, "detection_interval": 2, + "hand_region_threshold": 0.2, + "face_exclusion_area": 8000, + "palm_aspect_ratio": 1.3, "single_finger_angle_threshold": 60 }, "drone": { @@ -35,7 +38,9 @@ "show_help": true, "show_contours": true, "show_bbox": true, - "show_fingertips": true + "show_fingertips": true, + "show_direction": true, + "show_region_grid": true }, "performance": { "target_fps": 30, diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json new file mode 100644 index 0000000000..b5479855de --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json @@ -0,0 +1,53 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "skin_lower_h": 0, + "skin_upper_h": 178, + "skin_lower_s": 0, + "skin_upper_s": 67, + "skin_lower_v": 36, + "skin_upper_v": 98, + "min_hand_area": 2000, + "max_hand_area": 30000, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.5, + "detection_interval": 1, + "hand_ratio_threshold": 1.5, + "contour_simplify_epsilon": 0.02, + "defect_distance_threshold": 20, + "palm_circle_radius_ratio": 0.3 + }, + "drone": { + "velocity": 2.5, + "duration": 0.3, + "altitude": -10.0, + "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": true, + "show_hand_direction": true, + "show_debug_info": false + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false + }, + "calibration": { + "auto_calibrate_skin": true, + "skin_calibration_frames": 30, + "hand_size_calibration": true + } +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json new file mode 100644 index 0000000000..7491f9cd30 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json @@ -0,0 +1,53 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "skin_lower_h": 0, + "skin_upper_h": 25, + "skin_lower_s": 30, + "skin_upper_s": 255, + "skin_lower_v": 60, + "skin_upper_v": 255, + "min_hand_area": 2000, + "max_hand_area": 30000, + "history_size": 15, + "smooth_frames": 10, + "min_confidence": 0.5, + "detection_interval": 1, + "gesture_lock_frames": 8, + "contour_simplify_epsilon": 0.02, + "defect_distance_threshold": 20, + "palm_circle_radius_ratio": 0.3 + }, + "drone": { + "velocity": 2.5, + "duration": 0.3, + "altitude": -10.0, + "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": false, + "show_hand_direction": false, + "show_debug_info": false + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false + }, + "calibration": { + "auto_calibrate_skin": true, + "skin_calibration_frames": 30, + "hand_size_calibration": true + } +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 4bd8903a38..f6f31be3fe 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- """ -Gesture Controlled AirSim Drone - Optimized Version -Pure OpenCV implementation with English interface -Author: xiaoshiyuan888 +手势控制AirSim无人机 - 手势识别优化版 +优化手势识别算法,解决位置敏感问题 +作者: xiaoshiyuan888 """ import sys @@ -18,55 +18,55 @@ from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Optimized Version") +print("Gesture Controlled Drone - Gesture Recognition Optimized") print("=" * 60) -# ========== Fix import path ========== +# ========== 修复导入路径 ========== current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -# ========== Core module imports ========== +# ========== 核心模块导入 ========== def safe_import(): - """Safely import all modules""" + """安全导入所有模块""" modules_status = {} try: from PIL import Image, ImageDraw, ImageFont modules_status['PIL'] = True - print("[PIL] ✓ Image processing library ready") + print("[PIL] ✓ 图像处理库就绪") except Exception as e: modules_status['PIL'] = False - print(f"[PIL] ✗ Import failed: {e}") + print(f"[PIL] ✗ 导入失败: {e}") return None, modules_status try: import cv2 import numpy as np modules_status['OpenCV'] = True - print("[OpenCV] ✓ Computer vision library ready") + print("[OpenCV] ✓ 计算机视觉库就绪") except Exception as e: modules_status['OpenCV'] = False - print(f"[OpenCV] ✗ Import failed: {e}") + print(f"[OpenCV] ✗ 导入失败: {e}") return None, modules_status airsim_module = None try: airsim_module = __import__('airsim') modules_status['AirSim'] = True - print(f"[AirSim] ✓ Successfully imported") + print(f"[AirSim] ✓ 成功导入") except ImportError: print("\n" + "!" * 60) - print("⚠ AirSim library NOT FOUND!") + print("⚠ AirSim库未找到!") print("!" * 60) - print("To install AirSim, run:") - print("1. First install: pip install msgpack-rpc-python") - print("2. Then install: pip install airsim") - print("\nOr from source:") + print("安装AirSim:") + print("1. 首先安装: pip install msgpack-rpc-python") + print("2. 然后安装: pip install airsim") + print("\n或从源码安装:") print(" pip install git+https://github.com/microsoft/AirSim.git") print("!" * 60) - print("\nContinue without AirSim? (y/n)") + print("\n无AirSim继续运行? (y/n)") choice = input().strip().lower() if choice != 'y': sys.exit(1) @@ -79,28 +79,28 @@ def safe_import(): }, modules_status -# Execute imports +# 执行导入 libs, status = safe_import() if not status.get('OpenCV', False) or not status.get('PIL', False): - print("\n❌ Core libraries missing, cannot start.") - input("Press Enter to exit...") + print("\n❌ 核心库缺失,无法启动。") + input("按回车键退出...") sys.exit(1) print("-" * 60) -print("✅ Environment check passed, initializing...") +print("✅ 环境检查通过,正在初始化...") print("-" * 60) -# Unpack libraries +# 解包库 cv2, np = libs['cv2'], libs['np'] Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -# ========== Configuration Manager ========== +# ========== 配置管理器 ========== class ConfigManager: - """Configuration Manager""" + """配置管理器""" def __init__(self): - self.config_file = os.path.join(current_dir, 'gesture_config.json') + self.config_file = os.path.join(current_dir, 'gesture_config_v2.json') self.default_config = { 'camera': { 'index': 0, @@ -109,14 +109,22 @@ def __init__(self): 'fps': 30 }, 'gesture': { - 'skin_lower': [0, 30, 60], - 'skin_upper': [25, 255, 255], - 'min_hand_area': 3000, + 'skin_lower_h': 0, + 'skin_upper_h': 25, + 'skin_lower_s': 30, + 'skin_upper_s': 255, + 'skin_lower_v': 60, + 'skin_upper_v': 255, + 'min_hand_area': 2000, + 'max_hand_area': 30000, 'history_size': 10, 'smooth_frames': 5, - 'min_confidence': 0.6, + 'min_confidence': 0.5, 'detection_interval': 1, - 'single_finger_angle_threshold': 60 # 新增:单个手指角度阈值 + 'hand_ratio_threshold': 1.5, # 手部长宽比阈值 + 'contour_simplify_epsilon': 0.02, + 'defect_distance_threshold': 20, # 凸缺陷距离阈值 + 'palm_circle_radius_ratio': 0.3, # 手掌半径与边界框的比例 }, 'drone': { 'velocity': 2.5, @@ -130,35 +138,46 @@ def __init__(self): 'show_help': True, 'show_contours': True, 'show_bbox': True, - 'show_fingertips': True # 新增:显示指尖 + 'show_fingertips': True, + 'show_palm_center': True, + 'show_hand_direction': True, + 'show_debug_info': False }, 'performance': { 'target_fps': 30, 'resize_factor': 1.0, 'enable_multiprocessing': False + }, + 'calibration': { + 'auto_calibrate_skin': True, + 'skin_calibration_frames': 30, + 'hand_size_calibration': True } } self.config = self.load_config() + self.skin_calibration_data = [] + self.hand_size_calibration_done = False + self.reference_hand_size = 0 def load_config(self): - """Load configuration""" + """加载配置""" if os.path.exists(self.config_file): try: with open(self.config_file, 'r', encoding='utf-8') as f: loaded_config = json.load(f) config = self.default_config.copy() self._merge_config(config, loaded_config) - print("✓ Configuration loaded from file") + print("✓ 从文件加载配置") return config except Exception as e: - print(f"⚠ Failed to load config: {e}, using defaults") + print(f"⚠ 加载配置失败: {e}, 使用默认配置") return self.default_config.copy() else: - print("✓ Using default configuration") + print("✓ 使用默认配置") return self.default_config.copy() def _merge_config(self, base, update): - """Recursively merge configurations""" + """递归合并配置""" for key, value in update.items(): if key in base and isinstance(base[key], dict) and isinstance(value, dict): self._merge_config(base[key], value) @@ -166,16 +185,16 @@ def _merge_config(self, base, update): base[key] = value def save_config(self): - """Save configuration""" + """保存配置""" try: with open(self.config_file, 'w', encoding='utf-8') as f: json.dump(self.config, f, indent=2, ensure_ascii=False) - print("✓ Configuration saved") + print("✓ 配置已保存") except Exception as e: - print(f"⚠ Failed to save config: {e}") + print(f"⚠ 保存配置失败: {e}") def get(self, *keys): - """Get configuration value""" + """获取配置值""" value = self.config for key in keys: if isinstance(value, dict) and key in value: @@ -185,7 +204,7 @@ def get(self, *keys): return value def set(self, *keys, value): - """Set configuration value""" + """设置配置值""" if len(keys) == 0: return @@ -198,208 +217,223 @@ def set(self, *keys, value): config[keys[-1]] = value self.save_config() + def calibrate_skin_color(self, frame, hand_mask): + """自动校准肤色范围""" + if not self.get('calibration', 'auto_calibrate_skin'): + return + + if len(self.skin_calibration_data) < self.get('calibration', 'skin_calibration_frames'): + # 转换到HSV + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + + # 获取肤色区域的HSV值 + skin_pixels = hsv[hand_mask > 0] + + if len(skin_pixels) > 100: # 确保有足够的像素 + self.skin_calibration_data.append(skin_pixels) + + if len(self.skin_calibration_data) == self.get('calibration', 'skin_calibration_frames'): + # 计算肤色范围 + all_skin_pixels = np.vstack(self.skin_calibration_data) + + h_min, h_max = np.percentile(all_skin_pixels[:, 0], [2, 98]) + s_min, s_max = np.percentile(all_skin_pixels[:, 1], [2, 98]) + v_min, v_max = np.percentile(all_skin_pixels[:, 2], [2, 98]) + + # 更新配置 + self.set('gesture', 'skin_lower_h', value=int(max(0, h_min - 5))) + self.set('gesture', 'skin_upper_h', value=int(min(180, h_max + 5))) + self.set('gesture', 'skin_lower_s', value=int(max(0, s_min - 10))) + self.set('gesture', 'skin_upper_s', value=int(min(255, s_max + 10))) + self.set('gesture', 'skin_lower_v', value=int(max(0, v_min - 10))) + self.set('gesture', 'skin_upper_v', value=int(min(255, v_max + 10))) + + print("✓ 肤色校准完成") + print(f" 肤色范围: H[{self.get('gesture', 'skin_lower_h')}-{self.get('gesture', 'skin_upper_h')}], " + f"S[{self.get('gesture', 'skin_lower_s')}-{self.get('gesture', 'skin_upper_s')}], " + f"V[{self.get('gesture', 'skin_lower_v')}-{self.get('gesture', 'skin_upper_v')}]") + + def calibrate_hand_size(self, hand_area): + """校准手部大小""" + if not self.get('calibration', 'hand_size_calibration') or self.hand_size_calibration_done: + return + + if hand_area > 0: + self.reference_hand_size = hand_area + self.hand_size_calibration_done = True + print(f"✓ 手部大小校准完成: {self.reference_hand_size:.0f} 像素") + config = ConfigManager() -# ========== Robust Gesture Recognizer ========== -class RobustGestureRecognizer: - """Robust Gesture Recognizer - Pure OpenCV Implementation""" +# ========== 改进的手势识别器 ========== +class ImprovedGestureRecognizer: + """改进的手势识别器 - 纯OpenCV实现""" def __init__(self): self.config = config.get('gesture') - # Gesture history and smoothing + # 手势历史和平滑 self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) self.confidence_history = deque(maxlen=self.history_size) - self.current_gesture = "Waiting" + self.current_gesture = "等待手势" self.current_confidence = 0.0 - # Hand tracking + # 手部跟踪和状态 self.last_hand_position = None self.hand_tracking = False self.track_window = None + self.hand_states = deque(maxlen=10) # 手部状态历史 - # Performance statistics + # 性能统计 self.process_times = deque(maxlen=30) self.frame_counter = 0 self.detection_interval = self.config['detection_interval'] - # Gesture color mapping + # 手势颜色映射 self.gesture_colors = { - "Stop": (0, 0, 255), # Red - "Forward": (0, 255, 0), # Green - "Up": (255, 255, 0), # Cyan - "Down": (255, 0, 255), # Purple - "Left": (255, 165, 0), # Orange - "Right": (0, 165, 255), # Light Blue - "Waiting": (200, 200, 200), # Gray - "Error": (255, 0, 0) # Blue + "停止": (0, 0, 255), # 红色 + "向前": (0, 255, 0), # 绿色 + "向上": (255, 255, 0), # 青色 + "向下": (255, 0, 255), # 紫色 + "向左": (255, 165, 0), # 橙色 + "向右": (0, 165, 255), # 浅蓝色 + "等待手势": (200, 200, 200), # 灰色 + "识别异常": (255, 0, 0), # 蓝色 + "悬停": (255, 255, 255) # 白色 } - # Skin detection parameters - self.skin_lower = np.array(self.config['skin_lower'], dtype=np.uint8) - self.skin_upper = np.array(self.config['skin_upper'], dtype=np.uint8) - - # Background subtractor + # 背景减除器 self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( history=100, varThreshold=25, detectShadows=True ) - # Morphological operation kernels + # 形态学操作核 self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - # Fingertip detection - self.prev_fingertips = None - self.fingertip_stability_threshold = 3 + print("✓ 改进的手势识别器已初始化 (纯OpenCV实现)") - print("✓ Robust gesture recognizer initialized (Pure OpenCV)") + def get_skin_mask(self, frame): + """获取肤色掩码""" + # 从配置获取肤色范围 + h_low = config.get('gesture', 'skin_lower_h') + h_high = config.get('gesture', 'skin_upper_h') + s_low = config.get('gesture', 'skin_lower_s') + s_high = config.get('gesture', 'skin_upper_s') + v_low = config.get('gesture', 'skin_lower_v') + v_high = config.get('gesture', 'skin_upper_v') - def preprocess_frame(self, frame): - """Preprocess frame""" - # Flip image horizontally for intuitive control - frame = cv2.flip(frame, 1) + # 转换为HSV颜色空间 + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - # Adjust size for performance - resize_factor = config.get('performance', 'resize_factor') - if resize_factor != 1.0: - new_width = int(frame.shape[1] * resize_factor) - new_height = int(frame.shape[0] * resize_factor) - frame = cv2.resize(frame, (new_width, new_height)) + # 定义肤色范围 + lower_skin = np.array([h_low, s_low, v_low], dtype=np.uint8) + upper_skin = np.array([h_high, s_high, v_high], dtype=np.uint8) - return frame + # 创建肤色掩码 + skin_mask = cv2.inRange(hsv, lower_skin, upper_skin) - def detect_skin(self, frame): - """Detect skin regions""" - # Convert to HSV color space - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + return skin_mask, hsv - # Skin mask - skin_mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) - - # Apply background subtraction + def enhance_skin_detection(self, frame, skin_mask): + """增强肤色检测""" + # 应用背景减除 fg_mask = self.bg_subtractor.apply(frame) - # Combine skin mask and foreground mask + # 结合肤色掩码和前景掩码 combined_mask = cv2.bitwise_and(skin_mask, fg_mask) - # Morphological operations + # 形态学操作 combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - # Gaussian blur + # 高斯模糊 combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) return combined_mask - def find_hand_contours(self, mask): - """Find hand contours""" - # Find contours + def find_best_hand_contour(self, mask, frame): + """找到最佳的手部轮廓""" + # 查找轮廓 contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: - return None + return None, 0.0 - # Find the largest contour (assumed to be hand) + # 按面积排序 contours = sorted(contours, key=cv2.contourArea, reverse=True) - for contour in contours: - area = cv2.contourArea(contour) + best_contour = None + best_score = 0.0 - # Filter out too small contours - if area < self.config['min_hand_area']: - continue + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') - # Calculate contour perimeter - perimeter = cv2.arcLength(contour, True) + for contour in contours[:3]: # 只检查前3个最大轮廓 + area = cv2.contourArea(contour) - # Filter out too simple contours - if perimeter < 100: + # 面积过滤 + if area < min_area or area > max_area: continue - return contour - - return None - - def detect_fingertips_advanced(self, contour): - """Advanced fingertip detection""" - # Simplify contour - epsilon = 0.01 * cv2.arcLength(contour, True) - approx = cv2.approxPolyDP(contour, epsilon, True) - - # Find convex hull - hull = cv2.convexHull(approx, returnPoints=False) + # 计算轮廓评分 + score = self.rate_contour(contour, frame.shape) - if hull is None or len(hull) < 3: - return [], 0 + if score > best_score: + best_score = score + best_contour = contour - # Find convexity defects - defects = cv2.convexityDefects(approx, hull) + return best_contour, best_score - if defects is None: - return [], 0 + def rate_contour(self, contour, frame_shape): + """评估轮廓作为手部的可能性""" + score = 0.0 - fingertips = [] - defects_points = [] - - for i in range(defects.shape[0]): - s, e, f, d = defects[i, 0] - start = tuple(approx[s][0]) - end = tuple(approx[e][0]) - far = tuple(approx[f][0]) - - defects_points.append((start, end, far)) - - # Calculate triangle sides - a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) - b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) - c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) - - # Calculate angle - if b * c != 0: - angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) - angle_degrees = math.degrees(angle) - - # If angle is less than 90 degrees, it might be a finger - if angle_degrees < 90: - # Check if this is a deep defect (far from hull) - if d > 1000: # Adjust this threshold based on your needs - # Both start and end points could be fingertips - for point in [start, end]: - if point not in fingertips: - fingertips.append(point) - - # If we found few fingertips, try alternative method - if len(fingertips) <= 2: - # Use convex hull points as fingertips - hull_points = cv2.convexHull(approx, returnPoints=True) - - # Find extreme points in hull - if len(hull_points) > 0: - hull_points = hull_points.reshape(-1, 2) - - # Sort by y-coordinate (top to bottom) - hull_points = hull_points[hull_points[:, 1].argsort()] - - # Take top points as fingertips - num_fingertips = min(5, len(hull_points)) - for i in range(num_fingertips): - point = tuple(hull_points[i]) - if point not in fingertips: - fingertips.append(point) + # 面积评分 + area = cv2.contourArea(contour) + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') + + if min_area < area < max_area: + # 面积在合理范围内 + area_ratio = min(area / max_area, 1.0) + score += area_ratio * 0.3 + + # 周长评分 + perimeter = cv2.arcLength(contour, True) + if perimeter > 100: # 最小周长 + score += 0.2 + + # 紧凑度评分 (周长^2 / 面积) + if area > 0: + compactness = perimeter ** 2 / area + # 手的紧凑度通常在14-20之间 + if 12 < compactness < 25: + compactness_score = 1.0 - abs(compactness - 18) / 6 + score += compactness_score * 0.3 + + # 长宽比评分 + x, y, w, h = cv2.boundingRect(contour) + if h > 0: + aspect_ratio = w / h + # 手的长宽比通常在0.5-2.0之间 + if 0.4 < aspect_ratio < 2.5: + aspect_score = 1.0 - abs(aspect_ratio - 1.0) / 1.5 + score += aspect_score * 0.2 - return fingertips, len(fingertips) + return score - def analyze_hand_contour(self, contour, frame_shape): - """Analyze hand contour""" + def analyze_hand_features(self, contour, frame_shape): + """分析手部特征""" if contour is None: return None, 0.0 - # Calculate contour area + # 计算轮廓面积 area = cv2.contourArea(contour) - # Calculate contour center + # 计算轮廓中心 M = cv2.moments(contour) if M["m00"] == 0: return None, 0.0 @@ -407,27 +441,41 @@ def analyze_hand_contour(self, contour, frame_shape): cx = int(M["m10"] / M["m00"]) cy = int(M["m01"] / M["m00"]) - # Calculate bounding box + # 计算边界框 x, y, w, h = cv2.boundingRect(contour) + bbox_area = w * h + + # 计算手掌中心(假设是轮廓的中心) + palm_center = (cx, cy) - # Advanced fingertip detection - fingertips, finger_count = self.detect_fingertips_advanced(contour) + # 计算手掌半径(假设为边界框宽度的30%) + palm_radius = int(w * config.get('gesture', 'palm_circle_radius_ratio')) - # Calculate hand position (normalized) + # 分析手指 + fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius) + + # 计算手部方向 + direction = self.calculate_hand_direction(contour, cx, cy) + + # 计算手部位置(归一化) h_img, w_img = frame_shape[:2] norm_x = cx / w_img norm_y = cy / h_img - # Calculate gesture confidence - confidence = self.calculate_confidence(area, finger_count, len(contour)) + # 计算手势置信度 + confidence = self.calculate_confidence(area, len(fingers), len(contour), bbox_area) - # Return result + # 返回结果 result = { 'contour': contour, 'center': (cx, cy), 'bbox': (x, y, x + w, y + h), - 'finger_count': finger_count, - 'fingertips': fingertips, + 'fingers': fingers, # 手指列表 + 'fingertips': fingertips, # 指尖点列表 + 'defects': defects, # 凸缺陷列表 + 'palm_center': palm_center, + 'palm_radius': palm_radius, + 'direction': direction, # 手部方向(角度) 'area': area, 'position': (norm_x, norm_y), 'confidence': confidence @@ -435,104 +483,241 @@ def analyze_hand_contour(self, contour, frame_shape): return result, confidence - def calculate_confidence(self, area, finger_count, contour_length): - """Calculate gesture confidence""" - confidence = 0.5 # Base confidence + def analyze_fingers(self, contour, palm_center, palm_radius): + """分析手指""" + # 简化轮廓 + epsilon = config.get('gesture', 'contour_simplify_epsilon') * cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon, True) - # Confidence based on area - if area > 5000: - confidence += 0.2 - elif area > 3000: - confidence += 0.1 + # 计算凸包 + hull = cv2.convexHull(approx, returnPoints=False) - # Confidence based on finger count + if hull is None or len(hull) < 3: + return [], [], [] + + # 计算凸缺陷 + defects = cv2.convexityDefects(approx, hull) + + fingers = [] + fingertips = [] + defect_points = [] + + if defects is not None: + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(approx[s][0]) + end = tuple(approx[e][0]) + far = tuple(approx[f][0]) + + # 计算距离 + start_dist = np.linalg.norm(np.array(start) - np.array(palm_center)) + end_dist = np.linalg.norm(np.array(end) - np.array(palm_center)) + far_dist = np.linalg.norm(np.array(far) - np.array(palm_center)) + + # 检查是否可能是手指 + if (start_dist > palm_radius * 1.2 and + end_dist > palm_radius * 1.2 and + d > config.get('gesture', 'defect_distance_threshold') * 256): + + # 检查角度 + a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + + if b * c != 0: + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + angle_degrees = math.degrees(angle) + + # 如果角度小于90度,可能是指缝 + if angle_degrees < 90: + # 添加手指 + finger = { + 'start': start, + 'end': end, + 'far': far, + 'depth': d, + 'angle': angle_degrees + } + fingers.append(finger) + + # 添加指尖(开始和结束点都可能是指尖) + for point in [start, end]: + if point not in fingertips: + # 检查点是否在手掌半径之外 + point_dist = np.linalg.norm(np.array(point) - np.array(palm_center)) + if point_dist > palm_radius * 1.5: + fingertips.append(point) + + defect_points.append((start, end, far, d)) + + # 如果没有检测到手指,尝试使用凸包点作为指尖 + if len(fingertips) == 0: + hull_points = cv2.convexHull(approx, returnPoints=True) + if len(hull_points) > 0: + hull_points = hull_points.reshape(-1, 2) + + # 过滤掉离手掌中心太近的点 + for point in hull_points: + point_tuple = tuple(point) + point_dist = np.linalg.norm(point - np.array(palm_center)) + if point_dist > palm_radius * 1.5 and point_tuple not in fingertips: + fingertips.append(point_tuple) + + # 限制指尖数量为5 + fingertips = fingertips[:5] + + return fingers, fingertips, defect_points + + def calculate_hand_direction(self, contour, cx, cy): + """计算手部方向""" + # 使用PCA(主成分分析)计算手部方向 + if len(contour) < 5: + return 0.0 + + # 将轮廓转换为点集 + points = contour.reshape(-1, 2).astype(np.float32) + + # 计算PCA + mean = np.empty((0)) + mean, eigenvectors, eigenvalues = cv2.PCACompute2(points, mean) + + # 计算方向角度(度) + direction = math.degrees(math.atan2(eigenvectors[0, 1], eigenvectors[0, 0])) + + return direction + + def calculate_confidence(self, area, finger_count, contour_length, bbox_area): + """计算手势置信度""" + confidence = 0.5 # 基础置信度 + + # 基于面积的置信度 + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') + + if min_area < area < max_area: + # 面积在合理范围内 + area_norm = (area - min_area) / (max_area - min_area) + confidence += area_norm * 0.2 + + # 基于手指数量的置信度 if 0 <= finger_count <= 5: confidence += 0.2 - # Confidence based on contour complexity + # 基于轮廓复杂度的置信度 if contour_length > 200: confidence += 0.1 + # 基于填充度的置信度(轮廓面积/边界框面积) + if bbox_area > 0: + fill_ratio = area / bbox_area + # 手的填充度通常在0.3-0.7之间 + if 0.2 < fill_ratio < 0.8: + fill_score = 1.0 - abs(fill_ratio - 0.5) / 0.3 + confidence += fill_score * 0.1 + return min(confidence, 1.0) - def recognize_gesture(self, hand_data): - """Recognize gesture based on hand data""" + def recognize_gesture_improved(self, hand_data): + """改进的手势识别逻辑""" if hand_data is None: - return "Waiting", 0.3 + return "等待手势", 0.3 - finger_count = hand_data['finger_count'] + finger_count = len(hand_data.get('fingers', [])) + fingertips = hand_data.get('fingertips', []) norm_x, norm_y = hand_data['position'] + direction = hand_data.get('direction', 0.0) confidence = hand_data['confidence'] - fingertips = hand_data.get('fingertips', []) - # Special case: single finger detection - if finger_count == 1: - # Additional check for single finger gesture - if len(fingertips) == 1: - # Check if the fingertip is above the hand center (typical for pointing) + # 根据手指数量分类 + if finger_count == 0: + # 握拳或没有手指 + if len(fingertips) == 0: + return "停止", confidence * 0.9 + else: + # 有指尖但不是手指,可能是部分握拳 + return "停止", confidence * 0.7 + + elif finger_count == 1: + # 单指手势 + if len(fingertips) >= 1: + # 检查指尖方向 cx, cy = hand_data['center'] fingertip = fingertips[0] - if fingertip[1] < cy - 20: # Fingertip is above hand center - return "Forward", confidence * 0.9 - else: - return "Stop", confidence * 0.7 - return "Forward", confidence * 0.8 + # 计算指尖相对于手中心的方向 + dx = fingertip[0] - cx + dy = fingertip[1] - cy - # No fingers (fist) - elif finger_count == 0: - return "Stop", confidence * 0.9 + # 判断指尖方向 + if abs(dx) > abs(dy): # 主要水平方向 + if dx > 0: + return "向右", confidence * 0.8 + else: + return "向左", confidence * 0.8 + else: # 主要垂直方向 + if dy < 0: # 指尖在手中心上方 + return "向上", confidence * 0.8 + else: + return "向前", confidence * 0.8 + return "向前", confidence * 0.7 - # Two fingers (V sign) elif finger_count == 2: - return "Forward", confidence * 0.7 + # 双指手势 + return "向前", confidence * 0.7 - # Three fingers elif finger_count == 3: - # Judge direction based on position - if norm_y < 0.4: - return "Up", confidence * 0.7 - elif norm_y > 0.6: - return "Down", confidence * 0.7 - else: - return "Stop", confidence * 0.6 + # 三指手势,根据手部方向判断 + if -45 <= direction <= 45: # 大致水平 + if direction > 0: + return "向右", confidence * 0.7 + else: + return "向左", confidence * 0.7 + else: # 大致垂直 + if direction > 0: + return "向下", confidence * 0.7 + else: + return "向上", confidence * 0.7 - # Four or five fingers (open hand) elif finger_count >= 4: - # Judge direction based on hand position - if norm_x < 0.3: - return "Left", confidence * 0.8 - elif norm_x > 0.7: - return "Right", confidence * 0.8 - elif norm_y < 0.4: - return "Up", confidence * 0.8 - elif norm_y > 0.6: - return "Down", confidence * 0.8 - else: - return "Stop", confidence * 0.7 - - return "Waiting", confidence * 0.5 + # 多指手势(手掌张开) + # 根据手部位置判断 + if norm_x < 0.4: # 左侧 + return "向左", confidence * 0.8 + elif norm_x > 0.6: # 右侧 + return "向右", confidence * 0.8 + elif norm_y < 0.4: # 上方 + return "向上", confidence * 0.8 + elif norm_y > 0.6: # 下方 + return "向下", confidence * 0.8 + else: # 中间 + # 根据手部方向判断 + if -45 <= direction <= 45: # 大致水平 + return "向前", confidence * 0.7 + else: # 大致垂直 + return "停止", confidence * 0.7 + + return "等待手势", confidence * 0.5 def smooth_gesture(self, new_gesture, new_confidence): - """Smooth gesture output""" - # Add to history + """平滑手势输出""" + # 添加到历史 self.gesture_history.append(new_gesture) self.confidence_history.append(new_confidence) - # If not enough history, return directly + # 如果历史记录不足,直接返回 if len(self.gesture_history) < 3: self.current_gesture = new_gesture self.current_confidence = new_confidence return new_gesture, new_confidence - # Count most common gesture + # 统计最常见的姿势 gesture_counter = Counter(self.gesture_history) most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] - # Calculate average confidence + # 计算平均置信度 avg_confidence = np.mean(list(self.confidence_history)) - # If the most common gesture appears enough times, use it + # 如果最常见的姿势出现次数足够多,使用它 smooth_frames = self.config['smooth_frames'] if most_common_count >= smooth_frames: self.current_gesture = most_common_gesture @@ -541,134 +726,175 @@ def smooth_gesture(self, new_gesture, new_confidence): return self.current_gesture, self.current_confidence def visualize_detection(self, frame, hand_data, gesture, confidence): - """Visualize detection results""" + """可视化检测结果""" if hand_data is None: return frame - # Get display configuration + # 获取显示配置 show_contours = config.get('display', 'show_contours') show_bbox = config.get('display', 'show_bbox') show_fingertips = config.get('display', 'show_fingertips') + show_palm_center = config.get('display', 'show_palm_center') + show_hand_direction = config.get('display', 'show_hand_direction') + show_debug_info = config.get('display', 'show_debug_info') - # Draw contour + # 绘制轮廓 if show_contours and 'contour' in hand_data: cv2.drawContours(frame, [hand_data['contour']], -1, (0, 255, 0), 2) - # Draw bounding box + # 绘制边界框 if show_bbox and 'bbox' in hand_data: x1, y1, x2, y2 = hand_data['bbox'] color = self.gesture_colors.get(gesture, (255, 255, 255)) cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) - # Show gesture label + # 显示手势标签 label = f"{gesture}" if config.get('display', 'show_confidence'): label += f" ({confidence:.0%})" - # Calculate text size + # 计算文本大小 (text_width, text_height), baseline = cv2.getTextSize( label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 ) - # Draw text background + # 绘制文本背景 cv2.rectangle(frame, (x1, y1 - text_height - 10), (x1 + text_width, y1), color, -1) - # Draw text + # 绘制文本 cv2.putText(frame, label, (x1, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) - # Draw center point - if 'center' in hand_data: - cx, cy = hand_data['center'] + # 绘制手掌中心 + if show_palm_center and 'palm_center' in hand_data: + cx, cy = hand_data['palm_center'] + palm_radius = hand_data.get('palm_radius', 20) cv2.circle(frame, (cx, cy), 5, (0, 0, 255), -1) - cv2.putText(frame, "Center", (cx + 10, cy), + cv2.circle(frame, (cx, cy), palm_radius, (0, 0, 255), 1) + cv2.putText(frame, "Palm", (cx + 10, cy), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) - # Draw fingertips + # 绘制指尖 if show_fingertips and 'fingertips' in hand_data: for i, point in enumerate(hand_data['fingertips']): cv2.circle(frame, point, 4, (255, 0, 0), -1) cv2.putText(frame, f"F{i + 1}", (point[0] + 5, point[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1) - # Show finger count - if 'finger_count' in hand_data: - finger_text = f"Fingers: {hand_data['finger_count']}" - cv2.putText(frame, finger_text, (10, frame.shape[0] - 10), + # 绘制手部方向 + if show_hand_direction and 'direction' in hand_data and 'center' in hand_data: + cx, cy = hand_data['center'] + direction = hand_data['direction'] + length = 50 + + # 计算方向向量 + dx = length * math.cos(math.radians(direction)) + dy = length * math.sin(math.radians(direction)) + + # 绘制方向线 + end_point = (int(cx + dx), int(cy + dy)) + cv2.arrowedLine(frame, (cx, cy), end_point, (255, 255, 0), 2) + + # 显示方向角度 + angle_text = f"Dir: {direction:.0f}°" + cv2.putText(frame, angle_text, (cx, cy - 20), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 1) + + # 显示调试信息 + if show_debug_info: + # 显示手指数量 + finger_count = len(hand_data.get('fingers', [])) + finger_text = f"Fingers: {finger_count}" + cv2.putText(frame, finger_text, (10, frame.shape[0] - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 显示手部位置 + pos_text = f"Pos: ({hand_data['position'][0]:.2f}, {hand_data['position'][1]:.2f})" + cv2.putText(frame, pos_text, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) return frame def recognize(self, frame): - """Recognize gesture""" + """识别手势""" start_time = time.time() try: - # Preprocess frame - processed_frame = self.preprocess_frame(frame) + # 预处理帧 + processed_frame = cv2.flip(frame, 1) - # Detect every few frames to improve performance + # 每隔几帧检测一次以提高性能 if self.frame_counter % self.detection_interval != 0: self.frame_counter += 1 return self.current_gesture, self.current_confidence, processed_frame - # Detect skin - skin_mask = self.detect_skin(processed_frame) + # 获取肤色掩码 + skin_mask, hsv = self.get_skin_mask(processed_frame) + + # 增强肤色检测 + enhanced_mask = self.enhance_skin_detection(processed_frame, skin_mask) - # Find hand contour - hand_contour = self.find_hand_contours(skin_mask) + # 找到最佳的手部轮廓 + hand_contour, contour_score = self.find_best_hand_contour(enhanced_mask, processed_frame) - # Analyze hand contour - hand_data, confidence = self.analyze_hand_contour(hand_contour, processed_frame.shape) + # 分析手部特征 + hand_data, confidence = self.analyze_hand_features(hand_contour, processed_frame.shape) - # Recognize gesture + # 识别手势 if hand_data is not None: - gesture, raw_confidence = self.recognize_gesture(hand_data) + # 校准肤色(如果需要) + config.calibrate_skin_color(processed_frame, enhanced_mask) + + # 校准手部大小(如果需要) + config.calibrate_hand_size(hand_data['area']) + + # 识别手势 + gesture, raw_confidence = self.recognize_gesture_improved(hand_data) confidence = max(confidence, raw_confidence) else: - gesture, confidence = "Waiting", 0.3 + gesture, confidence = "等待手势", 0.3 - # Smooth gesture + # 平滑手势 final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) - # Visualize results + # 可视化结果 if hand_data is not None: processed_frame = self.visualize_detection( processed_frame, hand_data, final_gesture, final_confidence ) - # Update counter + # 更新计数器 self.frame_counter += 1 - # Calculate processing time + # 计算处理时间 process_time = (time.time() - start_time) * 1000 self.process_times.append(process_time) return final_gesture, final_confidence, processed_frame except Exception as e: - print(f"⚠ Gesture recognition error: {e}") - return "Error", 0.0, frame + print(f"⚠ 手势识别错误: {e}") + return "识别异常", 0.0, frame def get_performance_stats(self): - """Get performance statistics""" + """获取性能统计""" if len(self.process_times) == 0: return 0.0 return np.mean(list(self.process_times)) def set_simulated_gesture(self, gesture): - """Set simulated gesture""" + """设置模拟的手势""" self.current_gesture = gesture self.current_confidence = 0.9 -# ========== Simple Drone Controller ========== +# ========== 简单的无人机控制器 ========== class SimpleDroneController: - """Simple Drone Controller""" + """简单的无人机控制器""" def __init__(self, airsim_module): self.airsim = airsim_module @@ -676,140 +902,143 @@ def __init__(self, airsim_module): self.connected = False self.flying = False - # Control parameters + # 控制参数 self.velocity = config.get('drone', 'velocity') self.duration = config.get('drone', 'duration') self.altitude = config.get('drone', 'altitude') self.control_interval = config.get('drone', 'control_interval') - # Control state + # 控制状态 self.last_control_time = 0 self.last_gesture = None - print("✓ Simple drone controller initialized") + print("✓ 简单的无人机控制器已初始化") def connect(self): - """Connect to AirSim drone""" + """连接AirSim无人机""" if self.connected: return True if self.airsim is None: - print("⚠ AirSim not available, using simulation mode") + print("⚠ AirSim不可用,使用模拟模式") self.connected = True return True - print("Connecting to AirSim...") + print("连接AirSim...") try: self.client = self.airsim.MultirotorClient() self.client.confirmConnection() - print("✅ Connected to AirSim!") + print("✅ 已连接AirSim!") self.client.enableApiControl(True) - print("✅ API control enabled") + print("✅ API控制已启用") self.client.armDisarm(True) - print("✅ Drone armed") + print("✅ 无人机已武装") self.connected = True return True except Exception as e: - print(f"❌ Connection failed: {e}") - print("\nContinue with simulation mode? (y/n)") + print(f"❌ 连接失败: {e}") + print("\n使用模拟模式继续? (y/n)") choice = input().strip().lower() if choice == 'y': self.connected = True - print("✅ Using simulation mode") + print("✅ 使用模拟模式") return True return False def takeoff(self): - """Take off""" + """起飞""" if not self.connected: return False try: if self.airsim is None or self.client is None: - print("✅ Simulated takeoff") + print("✅ 模拟起飞") self.flying = True return True - print("Taking off...") + print("起飞中...") self.client.takeoffAsync().join() time.sleep(1) - # Ascend to specified altitude + # 上升到指定高度 self.client.moveToZAsync(self.altitude, 3).join() self.flying = True - print("✅ Drone took off successfully") + print("✅ 无人机成功起飞") return True except Exception as e: - print(f"❌ Takeoff failed: {e}") + print(f"❌ 起飞失败: {e}") return False def land(self): - """Land""" + """降落""" if not self.connected: return False try: if self.airsim is None or self.client is None: - print("✅ Simulated landing") + print("✅ 模拟降落") self.flying = False return True - print("Landing...") + print("降落中...") self.client.landAsync().join() self.flying = False - print("✅ Drone landed") + print("✅ 无人机已降落") return True except Exception as e: - print(f"Landing failed: {e}") + print(f"降落失败: {e}") return False def move_by_gesture(self, gesture, confidence): - """Move based on gesture""" + """根据手势移动""" if not self.connected or not self.flying: return False - # Check control interval + # 检查控制间隔 current_time = time.time() if current_time - self.last_control_time < self.control_interval: return False - # Check confidence threshold + # 检查置信度阈值 min_confidence = config.get('gesture', 'min_confidence') if confidence < min_confidence: return False try: if self.airsim is None or self.client is None: - print(f"Simulated move: {gesture}") + print(f"模拟移动: {gesture}") self.last_control_time = current_time self.last_gesture = gesture return True success = False - if gesture == "Up": + if gesture == "向上": self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) success = True - elif gesture == "Down": + elif gesture == "向下": self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) success = True - elif gesture == "Left": + elif gesture == "向左": self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) success = True - elif gesture == "Right": + elif gesture == "向右": self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) success = True - elif gesture == "Forward": + elif gesture == "向前": self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) success = True - elif gesture == "Stop": + elif gesture == "停止": + self.client.hoverAsync() + success = True + elif gesture == "悬停": self.client.hoverAsync() success = True @@ -819,20 +1048,20 @@ def move_by_gesture(self, gesture, confidence): return success except Exception as e: - print(f"Control command failed: {e}") + print(f"控制命令失败: {e}") return False def emergency_stop(self): - """Emergency stop""" + """紧急停止""" if self.connected: try: if self.flying and self.client is not None: - print("Emergency landing...") + print("紧急降落...") self.land() if self.client is not None: self.client.armDisarm(False) self.client.enableApiControl(False) - print("✅ Emergency stop complete") + print("✅ 紧急停止完成") except: pass @@ -840,117 +1069,173 @@ def emergency_stop(self): self.flying = False -# ========== Simple UI Renderer ========== -class SimpleUIRenderer: - """Simple UI Renderer""" +# ========== 中文UI渲染器 ========== +class ChineseUIRenderer: + """中文UI渲染器""" def __init__(self): - # Color definitions + self.fonts = {} + self.load_fonts() + + # 颜色定义 self.colors = { - 'title': (0, 255, 255), # Cyan - 'connected': (0, 255, 0), # Green - 'disconnected': (0, 0, 255), # Red - 'flying': (0, 255, 0), # Green - 'landed': (255, 165, 0), # Orange - 'warning': (0, 165, 255), # Light Blue - 'info': (255, 255, 255), # White - 'help': (255, 200, 100) # Light Orange + 'title': (0, 255, 255), # 青色 + 'connected': (0, 255, 0), # 绿色 + 'disconnected': (0, 0, 255), # 红色 + 'flying': (0, 255, 0), # 绿色 + 'landed': (255, 165, 0), # 橙色 + 'warning': (0, 165, 255), # 浅蓝色 + 'info': (255, 255, 255), # 白色 + 'help': (255, 200, 100) # 浅橙色 } - print("✓ Simple UI renderer initialized") + print("✓ 中文UI渲染器已初始化") + + def load_fonts(self): + """加载字体""" + font_paths = [ + 'simhei.ttf', + 'C:/Windows/Fonts/simhei.ttf', + 'C:/Windows/Fonts/msyh.ttc', + '/System/Library/Fonts/PingFang.ttc', + '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' + ] + + for path in font_paths: + try: + self.fonts[14] = ImageFont.truetype(path, 14) + self.fonts[16] = ImageFont.truetype(path, 16) + self.fonts[18] = ImageFont.truetype(path, 18) + self.fonts[20] = ImageFont.truetype(path, 20) + self.fonts[24] = ImageFont.truetype(path, 24) + print(f"✓ 字体已加载: {path}") + return + except: + continue + + print("⚠ 未找到字体,使用默认") + + def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): + """在图像上绘制文本""" + try: + img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + pil_img = Image.fromarray(img_rgb) + draw = ImageDraw.Draw(pil_img) + + font = self.fonts.get(size, self.fonts.get(16)) + + # 绘制阴影 + shadow_color = (0, 0, 0) + shadow_pos = (pos[0] + 1, pos[1] + 1) + draw.text(shadow_pos, text, font=font, fill=shadow_color) + + # 绘制文字 + rgb_color = color[::-1] # BGR to RGB + draw.text(pos, text, font=font, fill=rgb_color) + + return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) + except: + # 备用方案:使用OpenCV绘制英文 + cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, + size / 25, color, 1) + return frame def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time): - """Draw status bar""" + """绘制状态栏""" h, w = frame.shape[:2] - # Draw semi-transparent background + # 绘制半透明背景 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (w, 100), (0, 0, 0), -1) + cv2.rectangle(overlay, (0, 0), (w, 120), (0, 0, 0), -1) frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) - # Title - title = "Gesture Controlled Drone" - cv2.putText(frame, title, (10, 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, self.colors['title'], 2) + # 标题 + title = "手势控制无人机系统 - 优化版" + frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) - # Connection status + # 连接状态 status_color = self.colors['connected'] if drone_controller.connected else self.colors['disconnected'] - status_text = f"Drone: {'Connected' if drone_controller.connected else 'Disconnected'}" - cv2.putText(frame, status_text, (10, 60), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, status_color, 1) + status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" + frame = self.draw_text(frame, status_text, (10, 40), size=16, color=status_color) - # Flight status + # 飞行状态 flight_color = self.colors['flying'] if drone_controller.flying else self.colors['landed'] - flight_text = f"Flight: {'Flying' if drone_controller.flying else 'Landed'}" - cv2.putText(frame, flight_text, (10, 85), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, flight_color, 1) + flight_text = f"飞行状态: {'飞行中' if drone_controller.flying else '已降落'}" + frame = self.draw_text(frame, flight_text, (10, 65), size=16, color=flight_color) - # Gesture information + # 手势信息 if confidence > 0.7: - gesture_color = (0, 255, 0) # Green + gesture_color = (0, 255, 0) # 绿色 elif confidence > 0.5: - gesture_color = (255, 165, 0) # Orange + gesture_color = (255, 165, 0) # 橙色 else: - gesture_color = (200, 200, 200) # Gray + gesture_color = (200, 200, 200) # 灰色 - gesture_text = f"Gesture: {gesture}" + gesture_text = f"当前手势: {gesture}" if config.get('display', 'show_confidence'): gesture_text += f" ({confidence:.0%})" - cv2.putText(frame, gesture_text, (w // 2, 60), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, gesture_color, 1) + frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) - # Performance information + # 性能信息 if config.get('display', 'show_fps'): - perf_text = f"FPS: {fps:.1f}" + perf_text = f"帧率: {fps:.1f}" if process_time > 0: - perf_text += f" | Delay: {process_time:.1f}ms" + perf_text += f" | 延迟: {process_time:.1f}ms" + + frame = self.draw_text(frame, perf_text, (w - 200, 65), size=14, color=self.colors['info']) - cv2.putText(frame, perf_text, (w - 200, 85), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['info'], 1) + # 控制提示 + control_text = "提示: 确保手部完全进入画面,光线充足" + frame = self.draw_text(frame, control_text, (10, 90), size=14, color=self.colors['info']) return frame def draw_help_bar(self, frame): - """Draw help bar""" + """绘制帮助栏""" if not config.get('display', 'show_help'): return frame h, w = frame.shape[:2] - # Draw bottom help bar - cv2.rectangle(frame, (0, h - 50), (w, h), (0, 0, 0), -1) + # 绘制底部帮助栏 + cv2.rectangle(frame, (0, h - 60), (w, h), (0, 0, 0), -1) + + # 帮助文本 + help_lines = [ + "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", + "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息" + ] - # Help text - help_text = "C:Connect Space:Takeoff/Land ESC:Exit W/A/S/D/F/X:Keyboard" - cv2.putText(frame, help_text, (10, h - 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['help'], 1) + for i, line in enumerate(help_lines): + y_pos = h - 45 + i * 20 + frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame def draw_warning(self, frame, message): - """Draw warning message""" + """绘制警告信息""" h, w = frame.shape[:2] - # Draw warning at top + # 在顶部绘制警告 warning_bg = np.zeros((40, w, 3), dtype=np.uint8) - warning_bg[:, :] = (0, 69, 255) # Orange + warning_bg[:, :] = (0, 69, 255) # 橙色 - frame[100:140, 0:w] = cv2.addWeighted( - frame[100:140, 0:w], 0.3, + frame[120:160, 0:w] = cv2.addWeighted( + frame[120:160, 0:w], 0.3, warning_bg, 0.7, 0 ) - # Draw warning text - cv2.putText(frame, message, (10, 125), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, self.colors['warning'], 1) + # 绘制警告文本 + frame = self.draw_text(frame, message, (10, 135), + size=16, color=self.colors['warning']) return frame -# ========== Performance Monitor ========== +# ========== 性能监控器 ========== class PerformanceMonitor: - """Performance Monitor""" + """性能监控器""" def __init__(self): self.frame_times = deque(maxlen=60) @@ -959,12 +1244,12 @@ def __init__(self): self.frame_count = 0 def update(self): - """Update performance data""" + """更新性能数据""" current_time = time.time() self.frame_times.append(current_time) self.frame_count += 1 - # Calculate FPS once per second + # 每秒计算一次FPS if current_time - self.last_update >= 1.0: if len(self.frame_times) > 1: time_diff = self.frame_times[-1] - self.frame_times[0] @@ -975,25 +1260,25 @@ def update(self): self.last_update = current_time def get_stats(self): - """Get performance statistics""" + """获取性能统计""" return { 'fps': self.fps, 'frame_count': self.frame_count } -# ========== Main Program ========== +# ========== 主程序 ========== def main(): - """Main function""" - # Initialize components - print("Initializing components...") + """主函数""" + # 初始化组件 + print("初始化组件...") - gesture_recognizer = RobustGestureRecognizer() + gesture_recognizer = ImprovedGestureRecognizer() drone_controller = SimpleDroneController(libs['airsim']) - ui_renderer = SimpleUIRenderer() + ui_renderer = ChineseUIRenderer() performance_monitor = PerformanceMonitor() - # Initialize camera + # 初始化摄像头 cap = None try: cap = cv2.VideoCapture(0) @@ -1002,87 +1287,111 @@ def main(): cap.set(cv2.CAP_PROP_FRAME_HEIGHT, config.get('camera', 'height')) cap.set(cv2.CAP_PROP_FPS, config.get('camera', 'fps')) - # Get actual parameters + # 获取实际参数 actual_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) actual_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) actual_fps = int(cap.get(cv2.CAP_PROP_FPS)) - print(f"✓ Camera initialized") - print(f" Resolution: {actual_width}x{actual_height}") - print(f" FPS: {actual_fps}") + print(f"✓ 摄像头已初始化") + print(f" 分辨率: {actual_width}x{actual_height}") + print(f" 帧率: {actual_fps}") else: - print("❌ Camera not available, using simulation mode") + print("❌ 摄像头不可用,使用模拟模式") cap = None except Exception as e: - print(f"⚠ Camera init failed: {e}") + print(f"⚠ 摄像头初始化失败: {e}") cap = None - # Display welcome message + # 显示欢迎信息 print("\n" + "=" * 60) - print("Gesture Controlled Drone - Optimized Version") + print("手势控制无人机系统 - 手势识别优化版") print("=" * 60) - print("System Status:") - print(f" Camera: {'Connected' if cap else 'Simulation Mode'}") - print(f" Gesture Recognition: Pure OpenCV") - print(f" AirSim: {'Available' if libs['airsim'] else 'Simulation Mode'}") + print("系统状态:") + print(f" 摄像头: {'已连接' if cap else '模拟模式'}") + print(f" 手势识别: 改进的OpenCV算法") + print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) - # Display operation instructions - print("\nOperation Instructions:") - print("1. Press [C] to connect drone (AirSim simulator)") - print("2. Press [SPACE] to takeoff/land") - print("3. Gesture Control:") - print(" - Fist (0 fingers): Stop") - print(" - Index finger (1 finger): Forward") - print(" - Open hand (4-5 fingers): Control direction by hand position") - print(" * Gesture confidence > 60% to execute") - print("4. Keyboard Control:") - print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") - print("5. Press [H] to toggle help display") - print("6. Press [R] to reset gesture recognition") - print("7. Press [ESC] to exit safely") + # 显示操作说明 + print("\n操作说明:") + print("1. 按 [C] 连接无人机 (AirSim模拟器)") + print("2. 按 [空格键] 起飞/降落") + print("3. 手势控制改进:") + print(" - 握拳或握紧: 停止") + print(" - 单指指向: 根据指尖方向判断上下左右") + print(" - 双指: 向前") + print(" - 手掌张开: 根据手的位置判断方向") + print(" * 手势识别置信度 > 50% 时才会执行") + print("4. 键盘控制:") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print("5. 调试功能:") + print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") + print("6. 按 [ESC] 安全退出") print("=" * 60) - print("Program started successfully!") + print("程序启动成功!") print("-" * 60) - # Keyboard gesture mapping + # 键盘手势映射 key_to_gesture = { - ord('w'): "Up", ord('W'): "Up", - ord('s'): "Down", ord('S'): "Down", - ord('a'): "Left", ord('A'): "Left", - ord('d'): "Right", ord('D'): "Right", - ord('f'): "Forward", ord('F'): "Forward", - ord('x'): "Stop", ord('X'): "Stop", + ord('w'): "向上", ord('W'): "向上", + ord('s'): "向下", ord('S'): "向下", + ord('a'): "向左", ord('A'): "向左", + ord('d'): "向右", ord('D'): "向右", + ord('f'): "向前", ord('F'): "向前", + ord('x'): "停止", ord('X'): "停止", } - # Main loop - print("\nEntering main loop, press ESC to exit...") + # 显示模式 + display_modes = ['normal', 'detailed', 'minimal'] + current_display_mode = 0 + + # 主循环 + print("\n进入主循环,按ESC退出...") try: while True: - # Update performance monitor + # 更新性能监控 performance_monitor.update() - # Read camera frame + # 读取摄像头帧 if cap: ret, frame = cap.read() if not ret: - # Create blank frame + # 创建空白帧 frame = np.zeros((480, 640, 3), dtype=np.uint8) - gesture, confidence = "Camera Error", 0.0 + gesture, confidence = "摄像头错误", 0.0 else: - # Gesture recognition + # 手势识别 gesture, confidence, frame = gesture_recognizer.recognize(frame) else: - # Simulation mode + # 模拟模式 frame = np.zeros((480, 640, 3), dtype=np.uint8) gesture, confidence = gesture_recognizer.current_gesture, gesture_recognizer.current_confidence - # Get performance statistics + # 获取性能统计 perf_stats = performance_monitor.get_stats() process_time = gesture_recognizer.get_performance_stats() - # Draw UI + # 根据显示模式调整显示选项 + if display_modes[current_display_mode] == 'normal': + config.set('display', 'show_contours', value=True) + config.set('display', 'show_bbox', value=True) + config.set('display', 'show_fingertips', value=True) + config.set('display', 'show_debug_info', value=False) + elif display_modes[current_display_mode] == 'detailed': + config.set('display', 'show_contours', value=True) + config.set('display', 'show_bbox', value=True) + config.set('display', 'show_fingertips', value=True) + config.set('display', 'show_palm_center', value=True) + config.set('display', 'show_hand_direction', value=True) + config.set('display', 'show_debug_info', value=True) + elif display_modes[current_display_mode] == 'minimal': + config.set('display', 'show_contours', value=False) + config.set('display', 'show_bbox', value=True) + config.set('display', 'show_fingertips', value=False) + config.set('display', 'show_debug_info', value=False) + + # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, perf_stats['fps'], process_time @@ -1090,27 +1399,26 @@ def main(): frame = ui_renderer.draw_help_bar(frame) - # Show connection warning + # 显示连接提示 if not drone_controller.connected: - warning_msg = "⚠ Press C to connect drone, or use simulation mode" + warning_msg = "⚠ 按C键连接无人机,或使用模拟模式" frame = ui_renderer.draw_warning(frame, warning_msg) - # Show image - window_title = "Gesture Controlled Drone - Press ESC to exit" - cv2.imshow(window_title, frame) + # 显示图像(窗口标题用英文) + cv2.imshow('Gesture Controlled Drone - Optimized', frame) - # ========== Keyboard Control ========== + # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF - if key == 27: # ESC key - print("\nExiting program...") + if key == 27: # ESC键 + print("\n退出程序...") break elif key == ord('c') or key == ord('C'): if not drone_controller.connected: drone_controller.connect() - elif key == 32: # Space key + elif key == 32: # 空格键 if drone_controller.connected: if drone_controller.flying: drone_controller.land() @@ -1119,30 +1427,31 @@ def main(): time.sleep(0.5) elif key == ord('h') or key == ord('H'): - # Toggle help display + # 切换帮助显示 current = config.get('display', 'show_help') config.set('display', 'show_help', value=not current) - print(f"Help display: {'ON' if not current else 'OFF'}") - - elif key == ord('f') or key == ord('F'): - # Toggle FPS display - current = config.get('display', 'show_fps') - config.set('display', 'show_fps', value=not current) - print(f"FPS display: {'ON' if not current else 'OFF'}") + print(f"帮助显示: {'开启' if not current else '关闭'}") elif key == ord('r') or key == ord('R'): - # Reset gesture recognition - print("Resetting gesture recognition...") - gesture_recognizer = RobustGestureRecognizer() + # 重置手势识别 + print("重置手势识别...") + gesture_recognizer = ImprovedGestureRecognizer() + print("✓ 手势识别已重置") elif key == ord('t') or key == ord('T'): - # Toggle fingertip display - current = config.get('display', 'show_fingertips') - config.set('display', 'show_fingertips', value=not current) - print(f"Fingertip display: {'ON' if not current else 'OFF'}") + # 切换显示模式 + current_display_mode = (current_display_mode + 1) % len(display_modes) + mode_name = display_modes[current_display_mode] + print(f"显示模式: {mode_name}") + + elif key == ord('d') or key == ord('D'): + # 切换调试信息 + current = config.get('display', 'show_debug_info') + config.set('display', 'show_debug_info', value=not current) + print(f"调试信息: {'开启' if not current else '关闭'}") elif key in key_to_gesture: - # Keyboard control + # 键盘控制 simulated_gesture = key_to_gesture[key] gesture_recognizer.set_simulated_gesture(simulated_gesture) gesture = simulated_gesture @@ -1150,33 +1459,33 @@ def main(): if drone_controller.connected and drone_controller.flying: drone_controller.move_by_gesture(gesture, confidence) - # Real gesture control + # 真实手势控制 current_time = time.time() - if (gesture and gesture != "Waiting" and - gesture != "Camera Error" and gesture != "Error" and + if (gesture and gesture != "等待手势" and + gesture != "摄像头错误" and gesture != "识别异常" and drone_controller.connected and drone_controller.flying): drone_controller.move_by_gesture(gesture, confidence) except KeyboardInterrupt: - print("\nProgram interrupted by user") + print("\n程序被用户中断") except Exception as e: - print(f"\nProgram error: {e}") + print(f"\n程序错误: {e}") traceback.print_exc() finally: - # Cleanup resources - print("\nCleaning up resources...") + # 清理资源 + print("\n清理资源...") if cap: cap.release() cv2.destroyAllWindows() drone_controller.emergency_stop() config.save_config() - print("Program exited safely") + print("程序安全退出") print("=" * 60) - print("\nThank you for using Gesture Controlled Drone System!") - input("Press Enter to exit...") + print("\n感谢使用手势控制无人机系统!") + input("按回车键退出...") -# ========== Program Entry Point ========== +# ========== 程序入口 ========== if __name__ == "__main__": main() \ No newline at end of file From 1c7a07d2a683967d9c6e7563bd8b1eb75a071e31 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 17:15:10 +0800 Subject: [PATCH 06/18] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E6=A0=87=E7=AD=BE=E6=98=BE=E7=A4=BA=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=B8=AD=E6=96=87=E5=AD=97=E4=BD=93=E7=BC=BA=E5=A4=B1=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=9A=84=E9=97=AE=E5=8F=B7=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config_v2.json | 6 +- .../main.py | 104 +++++++++--------- 2 files changed, 55 insertions(+), 55 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json index b5479855de..57205fa642 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json @@ -9,9 +9,9 @@ "skin_lower_h": 0, "skin_upper_h": 178, "skin_lower_s": 0, - "skin_upper_s": 67, - "skin_lower_v": 36, - "skin_upper_v": 98, + "skin_upper_s": 255, + "skin_lower_v": 0, + "skin_upper_v": 77, "min_hand_area": 2000, "max_hand_area": 30000, "history_size": 10, diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index f6f31be3fe..bfa1b35eef 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -278,7 +278,7 @@ def __init__(self): self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) self.confidence_history = deque(maxlen=self.history_size) - self.current_gesture = "等待手势" + self.current_gesture = "Waiting" self.current_confidence = 0.0 # 手部跟踪和状态 @@ -294,15 +294,15 @@ def __init__(self): # 手势颜色映射 self.gesture_colors = { - "停止": (0, 0, 255), # 红色 - "向前": (0, 255, 0), # 绿色 - "向上": (255, 255, 0), # 青色 - "向下": (255, 0, 255), # 紫色 - "向左": (255, 165, 0), # 橙色 - "向右": (0, 165, 255), # 浅蓝色 - "等待手势": (200, 200, 200), # 灰色 - "识别异常": (255, 0, 0), # 蓝色 - "悬停": (255, 255, 255) # 白色 + "Stop": (0, 0, 255), # 红色 + "Forward": (0, 255, 0), # 绿色 + "Up": (255, 255, 0), # 青色 + "Down": (255, 0, 255), # 紫色 + "Left": (255, 165, 0), # 橙色 + "Right": (0, 165, 255), # 浅蓝色 + "Waiting": (200, 200, 200), # 灰色 + "Error": (255, 0, 0), # 蓝色 + "Hover": (255, 255, 255) # 白色 } # 背景减除器 @@ -620,7 +620,7 @@ def calculate_confidence(self, area, finger_count, contour_length, bbox_area): def recognize_gesture_improved(self, hand_data): """改进的手势识别逻辑""" if hand_data is None: - return "等待手势", 0.3 + return "Waiting", 0.3 finger_count = len(hand_data.get('fingers', [])) fingertips = hand_data.get('fingertips', []) @@ -632,10 +632,10 @@ def recognize_gesture_improved(self, hand_data): if finger_count == 0: # 握拳或没有手指 if len(fingertips) == 0: - return "停止", confidence * 0.9 + return "Stop", confidence * 0.9 else: # 有指尖但不是手指,可能是部分握拳 - return "停止", confidence * 0.7 + return "Stop", confidence * 0.7 elif finger_count == 1: # 单指手势 @@ -651,52 +651,52 @@ def recognize_gesture_improved(self, hand_data): # 判断指尖方向 if abs(dx) > abs(dy): # 主要水平方向 if dx > 0: - return "向右", confidence * 0.8 + return "Right", confidence * 0.8 else: - return "向左", confidence * 0.8 + return "Left", confidence * 0.8 else: # 主要垂直方向 if dy < 0: # 指尖在手中心上方 - return "向上", confidence * 0.8 + return "Up", confidence * 0.8 else: - return "向前", confidence * 0.8 - return "向前", confidence * 0.7 + return "Forward", confidence * 0.8 + return "Forward", confidence * 0.7 elif finger_count == 2: # 双指手势 - return "向前", confidence * 0.7 + return "Forward", confidence * 0.7 elif finger_count == 3: # 三指手势,根据手部方向判断 if -45 <= direction <= 45: # 大致水平 if direction > 0: - return "向右", confidence * 0.7 + return "Right", confidence * 0.7 else: - return "向左", confidence * 0.7 + return "Left", confidence * 0.7 else: # 大致垂直 if direction > 0: - return "向下", confidence * 0.7 + return "Down", confidence * 0.7 else: - return "向上", confidence * 0.7 + return "Up", confidence * 0.7 elif finger_count >= 4: # 多指手势(手掌张开) # 根据手部位置判断 if norm_x < 0.4: # 左侧 - return "向左", confidence * 0.8 + return "Left", confidence * 0.8 elif norm_x > 0.6: # 右侧 - return "向右", confidence * 0.8 + return "Right", confidence * 0.8 elif norm_y < 0.4: # 上方 - return "向上", confidence * 0.8 + return "Up", confidence * 0.8 elif norm_y > 0.6: # 下方 - return "向下", confidence * 0.8 + return "Down", confidence * 0.8 else: # 中间 # 根据手部方向判断 if -45 <= direction <= 45: # 大致水平 - return "向前", confidence * 0.7 + return "Forward", confidence * 0.7 else: # 大致垂直 - return "停止", confidence * 0.7 + return "Stop", confidence * 0.7 - return "等待手势", confidence * 0.5 + return "Waiting", confidence * 0.5 def smooth_gesture(self, new_gesture, new_confidence): """平滑手势输出""" @@ -748,7 +748,7 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): color = self.gesture_colors.get(gesture, (255, 255, 255)) cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) - # 显示手势标签 + # 显示手势标签 - 使用英文避免字体问题 label = f"{gesture}" if config.get('display', 'show_confidence'): label += f" ({confidence:.0%})" @@ -855,7 +855,7 @@ def recognize(self, frame): gesture, raw_confidence = self.recognize_gesture_improved(hand_data) confidence = max(confidence, raw_confidence) else: - gesture, confidence = "等待手势", 0.3 + gesture, confidence = "Waiting", 0.3 # 平滑手势 final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) @@ -877,7 +877,7 @@ def recognize(self, frame): except Exception as e: print(f"⚠ 手势识别错误: {e}") - return "识别异常", 0.0, frame + return "Error", 0.0, frame def get_performance_stats(self): """获取性能统计""" @@ -1020,25 +1020,25 @@ def move_by_gesture(self, gesture, confidence): success = False - if gesture == "向上": + if gesture == "Up": self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) success = True - elif gesture == "向下": + elif gesture == "Down": self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) success = True - elif gesture == "向左": + elif gesture == "Left": self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) success = True - elif gesture == "向右": + elif gesture == "Right": self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) success = True - elif gesture == "向前": + elif gesture == "Forward": self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) success = True - elif gesture == "停止": + elif gesture == "Stop": self.client.hoverAsync() success = True - elif gesture == "悬停": + elif gesture == "Hover": self.client.hoverAsync() success = True @@ -1317,13 +1317,13 @@ def main(): print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") print("3. 手势控制改进:") - print(" - 握拳或握紧: 停止") - print(" - 单指指向: 根据指尖方向判断上下左右") - print(" - 双指: 向前") + print(" - 握拳或握紧: Stop") + print(" - 单指指向: 根据指尖方向判断Up/Down/Left/Right") + print(" - 双指: Forward") print(" - 手掌张开: 根据手的位置判断方向") print(" * 手势识别置信度 > 50% 时才会执行") print("4. 键盘控制:") - print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") print("5. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") print("6. 按 [ESC] 安全退出") @@ -1333,12 +1333,12 @@ def main(): # 键盘手势映射 key_to_gesture = { - ord('w'): "向上", ord('W'): "向上", - ord('s'): "向下", ord('S'): "向下", - ord('a'): "向左", ord('A'): "向左", - ord('d'): "向右", ord('D'): "向右", - ord('f'): "向前", ord('F'): "向前", - ord('x'): "停止", ord('X'): "停止", + ord('w'): "Up", ord('W'): "Up", + ord('s'): "Down", ord('S'): "Down", + ord('a'): "Left", ord('A'): "Left", + ord('d'): "Right", ord('D'): "Right", + ord('f'): "Forward", ord('F'): "Forward", + ord('x'): "Stop", ord('X'): "Stop", } # 显示模式 @@ -1461,8 +1461,8 @@ def main(): # 真实手势控制 current_time = time.time() - if (gesture and gesture != "等待手势" and - gesture != "摄像头错误" and gesture != "识别异常" and + if (gesture and gesture != "Waiting" and + gesture != "摄像头错误" and gesture != "Error" and drone_controller.connected and drone_controller.flying): drone_controller.move_by_gesture(gesture, confidence) From 36f5403394d01f3d3811cc6c2e3e994da5b6e6a7 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 21:57:39 +0800 Subject: [PATCH 07/18] =?UTF-8?q?=E6=96=B0=E5=A2=9E=EF=BC=9A=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E5=8F=8D=E9=A6=88=E7=B3=BB=E7=BB=9F=EF=BC=8C=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=A4=9A=E5=B9=B3=E5=8F=B0=E9=9F=B3=E9=A2=91=E6=92=AD?= =?UTF-8?q?=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config.json | 108 ++- .../gesture_config_v2.json | 20 +- .../gesture_config_v3.json | 83 ++- .../main.py | 650 +++++++++++++++++- 4 files changed, 800 insertions(+), 61 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index d41f020e20..f07c4b5817 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -1,11 +1,95 @@ { + "system": { + "version": "1.0", + "last_updated": "2025-12-19 20:51:44", + "debug_mode": false + }, "camera": { + "device_id": 0, + "resolution": { + "width": 640, + "height": 480 + }, + "fps": 30, + "flip_horizontal": true, + "use_virtual_camera": false, "index": 0, "width": 640, - "height": 480, - "fps": 30 + "height": 480 + }, + "gesture_detection": { + "skin_hsv_ranges": { + "h_low": 0, + "h_high": 25, + "s_low": 30, + "s_high": 255, + "v_low": 60, + "v_high": 255 + }, + "min_hand_area": 2000, + "max_hand_area": 30000, + "confidence_threshold": 0.5, + "smoothing_window": 5, + "detection_interval": 1 + }, + "gesture_mapping": { + "0": "stop", + "1": "forward", + "2": "backward", + "3": "up", + "4": "down", + "5": "left", + "6": "right", + "7": "hover" + }, + "drone_control": { + "base_speed": 2.0, + "altitude": -10.0, + "control_delay": 0.3, + "takeoff_height": 5.0, + "emergency_stop_enabled": true + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_gesture_info": true, + "show_control_help": true, + "show_detection_overlay": true, + "theme": "dark", + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": false, + "show_hand_direction": false, + "show_debug_info": false, + "show_direction": true, + "show_region_grid": true + }, + "performance": { + "target_fps": 30, + "enable_multithreading": false, + "enable_hardware_acceleration": true, + "resize_factor": 1.0, + "enable_multiprocessing": false }, "gesture": { + "skin_lower_h": 0, + "skin_upper_h": 25, + "skin_lower_s": 30, + "skin_upper_s": 255, + "skin_lower_v": 60, + "skin_upper_v": 255, + "min_hand_area": 3000, + "max_hand_area": 30000, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.6, + "detection_interval": 2, + "hand_ratio_threshold": 1.5, + "contour_simplify_epsilon": 0.02, + "defect_distance_threshold": 20, + "palm_circle_radius_ratio": 0.3, "skin_lower": [ 0, 30, @@ -16,11 +100,6 @@ 255, 255 ], - "min_hand_area": 3000, - "history_size": 10, - "smooth_frames": 5, - "min_confidence": 0.6, - "detection_interval": 2, "hand_region_threshold": 0.2, "face_exclusion_area": 8000, "palm_aspect_ratio": 1.3, @@ -31,20 +110,5 @@ "duration": 0.3, "altitude": -10.0, "control_interval": 0.3 - }, - "display": { - "show_fps": true, - "show_confidence": true, - "show_help": true, - "show_contours": true, - "show_bbox": true, - "show_fingertips": true, - "show_direction": true, - "show_region_grid": true - }, - "performance": { - "target_fps": 30, - "resize_factor": 1.0, - "enable_multiprocessing": false } } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json index 57205fa642..fca5800e11 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json @@ -7,11 +7,11 @@ }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 178, + "skin_upper_h": 175, "skin_lower_s": 0, - "skin_upper_s": 255, + "skin_upper_s": 57, "skin_lower_v": 0, - "skin_upper_v": 77, + "skin_upper_v": 53, "min_hand_area": 2000, "max_hand_area": 30000, "history_size": 10, @@ -21,7 +21,15 @@ "hand_ratio_threshold": 1.5, "contour_simplify_epsilon": 0.02, "defect_distance_threshold": 20, - "palm_circle_radius_ratio": 0.3 + "palm_circle_radius_ratio": 0.3, + "validation_buffer_size": 5, + "validation_threshold": 0.8, + "min_hold_frames": 8, + "false_trigger_cooldown": 0.5, + "min_gesture_duration": 0.3, + "stability_threshold": 3, + "position_change_threshold": 0.15, + "area_change_threshold": 0.3 }, "drone": { "velocity": 2.5, @@ -38,7 +46,9 @@ "show_fingertips": true, "show_palm_center": true, "show_hand_direction": true, - "show_debug_info": false + "show_debug_info": false, + "show_validation_info": true, + "show_stability_indicator": true }, "performance": { "target_fps": 30, diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json index 7491f9cd30..0f240d3668 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json @@ -3,31 +3,56 @@ "index": 0, "width": 640, "height": 480, - "fps": 30 + "fps": 30, + "flip_horizontal": true, + "auto_exposure": true }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 25, - "skin_lower_s": 30, - "skin_upper_s": 255, - "skin_lower_v": 60, - "skin_upper_v": 255, + "skin_upper_h": 176, + "skin_lower_s": 0, + "skin_upper_s": 72, + "skin_lower_v": 1, + "skin_upper_v": 96, "min_hand_area": 2000, "max_hand_area": 30000, "history_size": 15, "smooth_frames": 10, "min_confidence": 0.5, "detection_interval": 1, - "gesture_lock_frames": 8, + "hand_ratio_threshold": 1.5, "contour_simplify_epsilon": 0.02, "defect_distance_threshold": 20, - "palm_circle_radius_ratio": 0.3 + "palm_circle_radius_ratio": 0.3, + "validation_buffer_size": 8, + "validation_threshold": 0.75, + "min_hold_frames": 10, + "false_trigger_cooldown": 0.8, + "min_gesture_duration": 0.5, + "stability_threshold": 5, + "position_change_threshold": 0.1, + "area_change_threshold": 0.25, + "use_background_subtraction": true, + "bg_history": 50, + "bg_var_threshold": 16, + "morph_close_iterations": 2, + "morph_open_iterations": 2, + "use_roi": true, + "roi_margin": 50, + "finger_min_length_ratio": 0.15, + "finger_max_length_ratio": 0.8, + "enable_multi_hand": false, + "hand_tracking_smoothness": 0.7, + "gesture_lock_frames": 8 }, "drone": { "velocity": 2.5, "duration": 0.3, "altitude": -10.0, - "control_interval": 0.3 + "control_interval": 0.3, + "angular_velocity": 45.0, + "smooth_movement": true, + "emergency_landing_timeout": 10.0 }, "display": { "show_fps": true, @@ -36,18 +61,48 @@ "show_contours": true, "show_bbox": true, "show_fingertips": true, - "show_palm_center": false, - "show_hand_direction": false, - "show_debug_info": false + "show_palm_center": true, + "show_hand_direction": true, + "show_debug_info": false, + "show_speech_status": true, + "show_validation_info": true, + "show_stability_indicator": true, + "use_english_text": true, + "show_skin_mask": false, + "show_trajectory": false, + "show_gesture_history": false, + "ui_theme": "dark", + "font_size": 16 }, "performance": { "target_fps": 30, "resize_factor": 1.0, - "enable_multiprocessing": false + "enable_multiprocessing": false, + "optimize_for_low_end": false, + "max_frame_skip": 2 }, "calibration": { "auto_calibrate_skin": true, "skin_calibration_frames": 30, - "hand_size_calibration": true + "hand_size_calibration": true, + "calibration_done": true, + "environment_adaptation": true, + "adaptive_threshold": true + }, + "speech": { + "enabled": true, + "volume": 1.0, + "rate": 150, + "announce_gestures": true, + "announce_connections": true, + "announce_flight_events": true, + "min_gesture_confidence": 0.7 + }, + "advanced": { + "use_ml_backup": false, + "enable_gesture_learning": false, + "save_gesture_samples": false, + "log_level": "INFO", + "auto_save_config": true } } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index bfa1b35eef..4e33bea6dd 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 手势识别优化版 -优化手势识别算法,解决位置敏感问题 +手势控制AirSim无人机 - 手势识别优化版(添加语音反馈) +优化手势识别算法,解决位置敏感问题,添加语音反馈功能 作者: xiaoshiyuan888 """ @@ -11,6 +11,8 @@ import traceback import json import math +import threading +import tempfile from datetime import datetime from PIL import Image, ImageDraw, ImageFont import cv2 @@ -19,6 +21,7 @@ print("=" * 60) print("Gesture Controlled Drone - Gesture Recognition Optimized") +print("现在添加了语音反馈功能!") print("=" * 60) # ========== 修复导入路径 ========== @@ -71,11 +74,71 @@ def safe_import(): if choice != 'y': sys.exit(1) + # 尝试导入语音合成库 + speech_module = None + try: + # 尝试导入pyttsx3(离线TTS) + import pyttsx3 + speech_module = pyttsx3 + modules_status['Speech'] = True + print("[Speech] ✓ pyttsx3语音库就绪 (离线)") + except ImportError: + print("\n" + "!" * 60) + print("⚠ pyttsx3语音库未找到!") + print("!" * 60) + print("安装语音库 (使用清华大学镜像源):") + print("1. 安装离线语音库: pip install pyttsx3 -i https://pypi.tuna.tsinghua.edu.cn/simple") + print("2. 或者安装在线语音库: pip install gtts pygame -i https://pypi.tuna.tsinghua.edu.cn/simple") + print("!" * 60) + + # 尝试其他语音库 + try: + # 尝试使用gTTS(需要网络) + from gtts import gTTS + speech_module = {'gTTS': gTTS, 'type': 'gtts'} + modules_status['Speech'] = True + print("[Speech] ✓ gTTS语音库就绪 (需要网络连接)") + + # 尝试导入音频播放库 + try: + import pygame + pygame.mixer.init() + speech_module['pygame'] = pygame + print("[Speech] ✓ pygame音频播放库就绪") + except ImportError: + # 尝试其他播放方式 + try: + import pydub + from pydub import AudioSegment + from pydub.playback import play + speech_module['pydub'] = pydub + speech_module['AudioSegment'] = AudioSegment + speech_module['play'] = play + print("[Speech] ✓ pydub音频播放库就绪") + except ImportError: + # 最后尝试使用系统命令 + if os.name == 'nt': # Windows + speech_module['play_method'] = 'windows' + print("[Speech] ✓ 使用Windows系统命令播放音频") + elif os.name == 'posix': # Linux/Mac + speech_module['play_method'] = 'posix' + print("[Speech] ✓ 使用系统命令播放音频") + else: + print("[Speech] ✗ 所有音频播放库导入失败,语音功能将不可用") + speech_module = None + modules_status['Speech'] = False + + except ImportError: + print("[Speech] ✗ 所有语音库导入失败,语音功能将不可用") + speech_module = None + modules_status['Speech'] = False + return { 'cv2': cv2, 'np': np, 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, - 'airsim': airsim_module + 'airsim': airsim_module, + 'speech': speech_module }, modules_status @@ -95,12 +158,324 @@ def safe_import(): Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] +# ========== 语音反馈管理器 ========== +class SpeechFeedbackManager: + """语音反馈管理器""" + + def __init__(self, speech_lib): + self.speech_lib = speech_lib + self.enabled = True + self.volume = 1.0 + self.rate = 150 + self.voice_id = None + self.last_speech_time = {} + self.min_interval = 2.0 # 相同语音的最小间隔(秒) + + # 语音队列,避免语音重叠 + self.speech_queue = [] + self.is_speaking = False + self.queue_thread = None + + # 音频播放方法 + self.audio_method = None + + # 语音消息映射 + self.messages = { + # 连接相关 + 'connecting': "正在连接无人机,请稍候", + 'connected': "无人机连接成功", + 'connection_failed': "无人机连接失败,进入模拟模式", + + # 飞行相关 + 'taking_off': "无人机正在起飞", + 'takeoff_success': "起飞成功", + 'takeoff_failed': "起飞失败", + 'landing': "无人机正在降落", + 'land_success': "降落成功", + 'emergency_stop': "紧急停止,无人机已降落", + + # 手势相关 + 'gesture_detected': "手势识别就绪", + 'gesture_stop': "停止", + 'gesture_up': "向上", + 'gesture_down': "向下", + 'gesture_left': "向左", + 'gesture_right': "向右", + 'gesture_forward': "向前", + 'gesture_waiting': "等待手势", + 'gesture_error': "手势识别错误", + + # 系统相关 + 'program_start': "手势控制无人机系统已启动", + 'program_exit': "程序退出,感谢使用", + 'camera_error': "摄像头错误,请检查连接", + 'low_confidence': "置信度过低,请调整手势", + + # 模式相关 + 'simulation_mode': "进入模拟模式", + 'debug_mode_on': "调试模式已开启", + 'debug_mode_off': "调试模式已关闭", + 'display_mode_changed': "显示模式已切换", + 'help_toggled': "帮助信息已切换", + } + + # 初始化语音引擎 + self.init_speech_engine() + + def init_speech_engine(self): + """初始化语音引擎""" + if self.speech_lib is None: + print("⚠ 语音库未找到,语音功能禁用") + self.enabled = False + return + + try: + if hasattr(self.speech_lib, 'init'): # pyttsx3 + self.engine = self.speech_lib.init() + self.audio_method = 'pyttsx3' + + # 设置语音参数 + voices = self.engine.getProperty('voices') + + # 尝试寻找中文语音 + for voice in voices: + # 检查语音名称是否包含中文相关标识 + if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower(): + self.engine.setProperty('voice', voice.id) + self.voice_id = voice.id + print(f"[Speech] 使用中文语音: {voice.name}") + break + + # 如果没找到中文语音,使用第一个可用语音 + if self.voice_id is None and len(voices) > 0: + self.engine.setProperty('voice', voices[0].id) + print(f"[Speech] 使用默认语音: {voices[0].name}") + + # 设置语速和音量 + self.engine.setProperty('rate', self.rate) + self.engine.setProperty('volume', self.volume) + + print("✅ 语音引擎初始化成功 (pyttsx3)") + + elif isinstance(self.speech_lib, dict) and self.speech_lib.get('type') == 'gtts': + print("✅ 语音引擎初始化成功 (gTTS,需要网络连接)") + self.audio_method = 'gtts' + + # 确定播放方法 + if 'pygame' in self.speech_lib: + self.audio_method = 'gtts_pygame' + print("✅ 使用pygame播放音频") + elif 'pydub' in self.speech_lib: + self.audio_method = 'gtts_pydub' + print("✅ 使用pydub播放音频") + elif 'play_method' in self.speech_lib: + self.audio_method = f"gtts_{self.speech_lib['play_method']}" + print(f"✅ 使用系统命令播放音频") + else: + self.audio_method = 'gtts_system' + print("✅ 使用默认系统播放器") + + else: + print("⚠ 未知语音库类型,语音功能可能不正常") + self.enabled = False + + except Exception as e: + print(f"⚠ 语音引擎初始化失败: {e}") + self.enabled = False + + def play_audio_file(self, audio_file): + """播放音频文件(根据可用库选择方法)""" + try: + if self.audio_method == 'gtts_pygame' and 'pygame' in self.speech_lib: + pygame = self.speech_lib['pygame'] + pygame.mixer.music.load(audio_file) + pygame.mixer.music.play() + + # 等待播放完成 + while pygame.mixer.music.get_busy(): + pygame.time.Clock().tick(10) + + elif self.audio_method == 'gtts_pydub' and 'pydub' in self.speech_lib: + AudioSegment = self.speech_lib['AudioSegment'] + play = self.speech_lib['play'] + + audio = AudioSegment.from_mp3(audio_file) + play(audio) + + elif self.audio_method == 'gtts_windows': + # Windows系统命令 + os.startfile(audio_file) + # 等待播放完成(简单等待) + time.sleep(2) + + elif self.audio_method == 'gtts_posix': + # Linux/Mac系统命令 + import subprocess + if sys.platform == 'darwin': # macOS + subprocess.call(['afplay', audio_file]) + else: # Linux + subprocess.call(['xdg-open', audio_file]) + + else: + # 通用方法:使用系统默认播放器 + import subprocess + if sys.platform == 'win32': + os.startfile(audio_file) + elif sys.platform == 'darwin': + subprocess.call(['open', audio_file]) + else: + subprocess.call(['xdg-open', audio_file]) + + return True + + except Exception as e: + print(f"⚠ 音频播放失败: {e}") + return False + + def speak(self, message_key, force=False): + """播放语音""" + if not self.enabled: + return + + # 检查是否在最小间隔内 + current_time = time.time() + if not force and message_key in self.last_speech_time: + if current_time - self.last_speech_time[message_key] < self.min_interval: + return + + # 获取消息文本 + if message_key in self.messages: + text = self.messages[message_key] + else: + text = message_key # 直接使用传入的文本 + + # 添加到语音队列 + self.speech_queue.append(text) + + # 如果没有在播放,启动播放线程 + if not self.is_speaking and self.queue_thread is None: + self.queue_thread = threading.Thread(target=self._process_speech_queue) + self.queue_thread.daemon = True + self.queue_thread.start() + + # 更新时间戳 + self.last_speech_time[message_key] = current_time + + def _process_speech_queue(self): + """处理语音队列""" + while self.speech_queue and self.enabled: + self.is_speaking = True + + text = self.speech_queue.pop(0) + + try: + if self.audio_method == 'pyttsx3': + # 清除之前的语音 + self.engine.stop() + # 播报新语音 + self.engine.say(text) + self.engine.runAndWait() + + elif self.audio_method.startswith('gtts'): + # gTTS需要网络连接 + tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') + + # 保存临时文件 + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: + temp_file = f.name + tts.save(temp_file) + + # 播放音频 + self.play_audio_file(temp_file) + + # 删除临时文件 + try: + os.unlink(temp_file) + except: + pass # 忽略删除错误 + + except Exception as e: + print(f"⚠ 语音播放失败: {e}") + + time.sleep(0.1) # 避免过快 + + self.is_speaking = False + self.queue_thread = None + + def speak_direct(self, text): + """直接播放文本(不通过消息映射)""" + if not self.enabled: + return + + # 在新线程中播放 + thread = threading.Thread(target=self._speak_thread, args=(text,)) + thread.daemon = True + thread.start() + + def _speak_thread(self, text): + """语音播放线程""" + try: + if self.audio_method == 'pyttsx3': + self.engine.say(text) + self.engine.runAndWait() + + elif self.audio_method.startswith('gtts'): + tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') + + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: + temp_file = f.name + tts.save(temp_file) + + self.play_audio_file(temp_file) + + try: + os.unlink(temp_file) + except: + pass + + except Exception as e: + print(f"⚠ 直接语音播放失败: {e}") + + def stop(self): + """停止所有语音""" + if hasattr(self, 'engine'): + self.engine.stop() + + self.speech_queue.clear() + self.is_speaking = False + + def set_enabled(self, enabled): + """启用/禁用语音""" + self.enabled = enabled + if not enabled: + self.stop() + + def toggle_enabled(self): + """切换语音启用状态""" + self.enabled = not self.enabled + status = "启用" if self.enabled else "禁用" + self.speak_direct(f"语音反馈已{status}") + return self.enabled + + def get_status(self): + """获取语音状态""" + return { + 'enabled': self.enabled, + 'engine': 'pyttsx3' if self.audio_method == 'pyttsx3' else + 'gTTS' if self.audio_method.startswith('gtts') else + 'None', + 'queue_size': len(self.speech_queue), + 'is_speaking': self.is_speaking, + 'audio_method': self.audio_method + } + + # ========== 配置管理器 ========== class ConfigManager: """配置管理器""" def __init__(self): - self.config_file = os.path.join(current_dir, 'gesture_config_v2.json') + self.config_file = os.path.join(current_dir, 'gesture_config_v3.json') # 更新版本号 self.default_config = { 'camera': { 'index': 0, @@ -141,7 +516,8 @@ def __init__(self): 'show_fingertips': True, 'show_palm_center': True, 'show_hand_direction': True, - 'show_debug_info': False + 'show_debug_info': False, + 'show_speech_status': True, # 新增:显示语音状态 }, 'performance': { 'target_fps': 30, @@ -152,6 +528,15 @@ def __init__(self): 'auto_calibrate_skin': True, 'skin_calibration_frames': 30, 'hand_size_calibration': True + }, + 'speech': { # 新增:语音配置 + 'enabled': True, + 'volume': 1.0, + 'rate': 150, + 'announce_gestures': True, # 是否播报手势 + 'announce_connections': True, # 是否播报连接状态 + 'announce_flight_events': True, # 是否播报飞行事件 + 'min_gesture_confidence': 0.7, # 播报手势的最小置信度 } } self.config = self.load_config() @@ -271,8 +656,9 @@ def calibrate_hand_size(self, hand_area): class ImprovedGestureRecognizer: """改进的手势识别器 - 纯OpenCV实现""" - def __init__(self): + def __init__(self, speech_manager=None): self.config = config.get('gesture') + self.speech_manager = speech_manager # 手势历史和平滑 self.history_size = self.config['history_size'] @@ -281,6 +667,11 @@ def __init__(self): self.current_gesture = "Waiting" self.current_confidence = 0.0 + # 记录上次播报的手势,避免重复播报 + self.last_announced_gesture = None + self.last_announced_time = 0 + self.gesture_announce_interval = 3.0 # 手势播报最小间隔 + # 手部跟踪和状态 self.last_hand_position = None self.hand_tracking = False @@ -305,6 +696,18 @@ def __init__(self): "Hover": (255, 255, 255) # 白色 } + # 手势到语音的映射 + self.gesture_speech_map = { + "Stop": "gesture_stop", + "Forward": "gesture_forward", + "Up": "gesture_up", + "Down": "gesture_down", + "Left": "gesture_left", + "Right": "gesture_right", + "Waiting": "gesture_waiting", + "Error": "gesture_error", + } + # 背景减除器 self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( history=100, varThreshold=25, detectShadows=True @@ -315,6 +718,10 @@ def __init__(self): print("✓ 改进的手势识别器已初始化 (纯OpenCV实现)") + # 语音提示初始化完成 + if self.speech_manager: + self.speech_manager.speak('gesture_detected') + def get_skin_mask(self, frame): """获取肤色掩码""" # 从配置获取肤色范围 @@ -860,6 +1267,22 @@ def recognize(self, frame): # 平滑手势 final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) + # 手势语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gestures') and + final_confidence >= config.get('speech', 'min_gesture_confidence')): + + current_time = time.time() + if (final_gesture != self.last_announced_gesture or + current_time - self.last_announced_time > self.gesture_announce_interval): + + if final_gesture in self.gesture_speech_map: + self.speech_manager.speak(self.gesture_speech_map[final_gesture]) + + self.last_announced_gesture = final_gesture + self.last_announced_time = current_time + # 可视化结果 if hand_data is not None: processed_frame = self.visualize_detection( @@ -891,16 +1314,26 @@ def set_simulated_gesture(self, gesture): self.current_gesture = gesture self.current_confidence = 0.9 + # 模拟手势也触发语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gestures') and + gesture in self.gesture_speech_map): + self.speech_manager.speak(self.gesture_speech_map[gesture]) + self.last_announced_gesture = gesture + self.last_announced_time = time.time() + # ========== 简单的无人机控制器 ========== class SimpleDroneController: """简单的无人机控制器""" - def __init__(self, airsim_module): + def __init__(self, airsim_module, speech_manager=None): self.airsim = airsim_module self.client = None self.connected = False self.flying = False + self.speech_manager = speech_manager # 控制参数 self.velocity = config.get('drone', 'velocity') @@ -912,6 +1345,11 @@ def __init__(self, airsim_module): self.last_control_time = 0 self.last_gesture = None + # 上次语音提示状态 + self.last_connection_announced = False + self.last_takeoff_announced = False + self.last_land_announced = False + print("✓ 简单的无人机控制器已初始化") def connect(self): @@ -919,8 +1357,21 @@ def connect(self): if self.connected: return True + # 语音提示:正在连接 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('connecting') + if self.airsim is None: print("⚠ AirSim不可用,使用模拟模式") + + # 语音提示:模拟模式 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('simulation_mode') + self.connected = True return True @@ -931,6 +1382,12 @@ def connect(self): self.client.confirmConnection() print("✅ 已连接AirSim!") + # 语音提示:连接成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('connected') + self.client.enableApiControl(True) print("✅ API控制已启用") @@ -942,11 +1399,25 @@ def connect(self): except Exception as e: print(f"❌ 连接失败: {e}") + + # 语音提示:连接失败 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('connection_failed') + print("\n使用模拟模式继续? (y/n)") choice = input().strip().lower() if choice == 'y': self.connected = True print("✅ 使用模拟模式") + + # 语音提示:模拟模式 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('simulation_mode') + return True return False @@ -956,10 +1427,26 @@ def takeoff(self): if not self.connected: return False + # 语音提示:正在起飞 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events') and + not self.last_takeoff_announced): + self.speech_manager.speak('taking_off') + self.last_takeoff_announced = True + self.last_land_announced = False + try: if self.airsim is None or self.client is None: print("✅ 模拟起飞") self.flying = True + + # 语音提示:起飞成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('takeoff_success') + return True print("起飞中...") @@ -971,9 +1458,23 @@ def takeoff(self): self.flying = True print("✅ 无人机成功起飞") + + # 语音提示:起飞成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('takeoff_success') + return True except Exception as e: print(f"❌ 起飞失败: {e}") + + # 语音提示:起飞失败 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('takeoff_failed') + return False def land(self): @@ -981,16 +1482,39 @@ def land(self): if not self.connected: return False + # 语音提示:正在降落 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events') and + not self.last_land_announced): + self.speech_manager.speak('landing') + self.last_land_announced = True + self.last_takeoff_announced = False + try: if self.airsim is None or self.client is None: print("✅ 模拟降落") self.flying = False + + # 语音提示:降落成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('land_success') + return True print("降落中...") self.client.landAsync().join() self.flying = False print("✅ 无人机已降落") + + # 语音提示:降落成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('land_success') + return True except Exception as e: print(f"降落失败: {e}") @@ -1009,6 +1533,12 @@ def move_by_gesture(self, gesture, confidence): # 检查置信度阈值 min_confidence = config.get('gesture', 'min_confidence') if confidence < min_confidence: + # 低置信度语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gestures') and + confidence < min_confidence * 0.8): # 如果置信度特别低 + self.speech_manager.speak('low_confidence') return False try: @@ -1057,6 +1587,13 @@ def emergency_stop(self): try: if self.flying and self.client is not None: print("紧急降落...") + + # 语音提示:紧急停止 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('emergency_stop') + self.land() if self.client is not None: self.client.armDisarm(False) @@ -1073,8 +1610,9 @@ def emergency_stop(self): class ChineseUIRenderer: """中文UI渲染器""" - def __init__(self): + def __init__(self, speech_manager=None): self.fonts = {} + self.speech_manager = speech_manager self.load_fonts() # 颜色定义 @@ -1086,7 +1624,9 @@ def __init__(self): 'landed': (255, 165, 0), # 橙色 'warning': (0, 165, 255), # 浅蓝色 'info': (255, 255, 255), # 白色 - 'help': (255, 200, 100) # 浅橙色 + 'help': (255, 200, 100), # 浅橙色 + 'speech_enabled': (0, 255, 0), # 绿色 + 'speech_disabled': (255, 0, 0), # 红色 } print("✓ 中文UI渲染器已初始化") @@ -1150,7 +1690,7 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 优化版" + title = "手势控制无人机系统 - 优化版 (带语音反馈)" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -1177,6 +1717,13 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) + # 语音状态 + if config.get('display', 'show_speech_status') and self.speech_manager: + speech_status = self.speech_manager.get_status() + speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] + speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" + frame = self.draw_text(frame, speech_text, (w // 2, 65), size=16, color=speech_color) + # 性能信息 if config.get('display', 'show_fps'): perf_text = f"帧率: {fps:.1f}" @@ -1199,16 +1746,17 @@ def draw_help_bar(self, frame): h, w = frame.shape[:2] # 绘制底部帮助栏 - cv2.rectangle(frame, (0, h - 60), (w, h), (0, 0, 0), -1) + cv2.rectangle(frame, (0, h - 80), (w, h), (0, 0, 0), -1) # 帮助文本 help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", - "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息" + "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", + "V:切换语音反馈 M:测试语音" ] for i, line in enumerate(help_lines): - y_pos = h - 45 + i * 20 + y_pos = h - 65 + i * 20 frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame @@ -1270,12 +1818,20 @@ def get_stats(self): # ========== 主程序 ========== def main(): """主函数""" + # 初始化语音管理器 + print("初始化语音反馈系统...") + speech_manager = SpeechFeedbackManager(libs['speech']) + + # 程序启动语音提示 + if speech_manager.enabled: + speech_manager.speak('program_start', force=True) + # 初始化组件 print("初始化组件...") - gesture_recognizer = ImprovedGestureRecognizer() - drone_controller = SimpleDroneController(libs['airsim']) - ui_renderer = ChineseUIRenderer() + gesture_recognizer = ImprovedGestureRecognizer(speech_manager) + drone_controller = SimpleDroneController(libs['airsim'], speech_manager) + ui_renderer = ChineseUIRenderer(speech_manager) performance_monitor = PerformanceMonitor() # 初始化摄像头 @@ -1297,9 +1853,19 @@ def main(): print(f" 帧率: {actual_fps}") else: print("❌ 摄像头不可用,使用模拟模式") + + # 摄像头错误语音提示 + if speech_manager.enabled: + speech_manager.speak('camera_error') + cap = None except Exception as e: print(f"⚠ 摄像头初始化失败: {e}") + + # 摄像头错误语音提示 + if speech_manager.enabled: + speech_manager.speak('camera_error') + cap = None # 显示欢迎信息 @@ -1309,6 +1875,7 @@ def main(): print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 改进的OpenCV算法") + print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) @@ -1326,7 +1893,9 @@ def main(): print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") print("5. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") - print("6. 按 [ESC] 安全退出") + print("6. 语音控制:") + print(" [V]切换语音反馈 [M]测试语音") + print("7. 按 [ESC] 安全退出") print("=" * 60) print("程序启动成功!") print("-" * 60) @@ -1405,7 +1974,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Optimized', frame) + cv2.imshow('Gesture Controlled Drone - Optimized with Speech', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -1432,23 +2001,58 @@ def main(): config.set('display', 'show_help', value=not current) print(f"帮助显示: {'开启' if not current else '关闭'}") + # 语音提示 + if speech_manager.enabled: + speech_manager.speak('help_toggled') + elif key == ord('r') or key == ord('R'): # 重置手势识别 print("重置手势识别...") - gesture_recognizer = ImprovedGestureRecognizer() + gesture_recognizer = ImprovedGestureRecognizer(speech_manager) print("✓ 手势识别已重置") + # 语音提示 + if speech_manager.enabled: + speech_manager.speak_direct("手势识别已重置") + elif key == ord('t') or key == ord('T'): # 切换显示模式 current_display_mode = (current_display_mode + 1) % len(display_modes) mode_name = display_modes[current_display_mode] print(f"显示模式: {mode_name}") + # 语音提示 + if speech_manager.enabled: + speech_manager.speak('display_mode_changed') + elif key == ord('d') or key == ord('D'): # 切换调试信息 current = config.get('display', 'show_debug_info') config.set('display', 'show_debug_info', value=not current) - print(f"调试信息: {'开启' if not current else '关闭'}") + status = '开启' if not current else '关闭' + print(f"调试信息: {status}") + + # 语音提示 + if speech_manager.enabled: + if not current: + speech_manager.speak('debug_mode_on') + else: + speech_manager.speak('debug_mode_off') + + elif key == ord('v') or key == ord('V'): + # 切换语音反馈 + new_status = speech_manager.toggle_enabled() + status = '启用' if new_status else '禁用' + print(f"语音反馈: {status}") + config.set('speech', 'enabled', value=new_status) + + elif key == ord('m') or key == ord('M'): + # 测试语音 + if speech_manager.enabled: + print("测试语音...") + speech_manager.speak_direct("语音反馈测试,这是一条测试消息") + else: + print("语音反馈已禁用,按V键启用") elif key in key_to_gesture: # 键盘控制 @@ -1477,6 +2081,12 @@ def main(): if cap: cap.release() cv2.destroyAllWindows() + + # 程序退出语音提示 + if speech_manager.enabled: + speech_manager.speak('program_exit', force=True) + time.sleep(1) # 确保语音播报完成 + drone_controller.emergency_stop() config.save_config() From fb866db00dc95facc02a5c036b92a0527e8c3592 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sat, 20 Dec 2025 11:09:56 +0800 Subject: [PATCH 08/18] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E5=B9=B3=E6=BB=91=E5=A4=84=E7=90=86=EF=BC=8C=E6=89=A9=E5=B1=95?= =?UTF-8?q?=E8=AF=AD=E9=9F=B3=E5=8F=8D=E9=A6=88=E7=B3=BB=E7=BB=9F=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=89=8B=E5=8A=BF=E7=8A=B6=E6=80=81=E8=BF=BD?= =?UTF-8?q?=E8=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 8 +- .../gesture_config.json | 168 +++-- .../main.py | 660 ++++++++++++------ 3 files changed, 555 insertions(+), 281 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index 0ad6fac5ef..b1c539fd44 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -10,4 +10,10 @@ __pycache__/ .idea/ # 忽略日志文件夹(临时日志文件,无需提交到仓库) -logs/ \ No newline at end of file +logs/ + +# 忽略手势识别配置文件(保留本地,不提交到仓库) +gesture_config.json +gesture_config (1).json +gesture_config_v2.json +gesture_config_v3.json \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index f07c4b5817..c60f9cec3d 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -1,85 +1,24 @@ { - "system": { - "version": "1.0", - "last_updated": "2025-12-19 20:51:44", - "debug_mode": false - }, "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30, "device_id": 0, "resolution": { "width": 640, "height": 480 }, - "fps": 30, "flip_horizontal": true, - "use_virtual_camera": false, - "index": 0, - "width": 640, - "height": 480 - }, - "gesture_detection": { - "skin_hsv_ranges": { - "h_low": 0, - "h_high": 25, - "s_low": 30, - "s_high": 255, - "v_low": 60, - "v_high": 255 - }, - "min_hand_area": 2000, - "max_hand_area": 30000, - "confidence_threshold": 0.5, - "smoothing_window": 5, - "detection_interval": 1 - }, - "gesture_mapping": { - "0": "stop", - "1": "forward", - "2": "backward", - "3": "up", - "4": "down", - "5": "left", - "6": "right", - "7": "hover" - }, - "drone_control": { - "base_speed": 2.0, - "altitude": -10.0, - "control_delay": 0.3, - "takeoff_height": 5.0, - "emergency_stop_enabled": true - }, - "display": { - "show_fps": true, - "show_confidence": true, - "show_gesture_info": true, - "show_control_help": true, - "show_detection_overlay": true, - "theme": "dark", - "show_help": true, - "show_contours": true, - "show_bbox": true, - "show_fingertips": true, - "show_palm_center": false, - "show_hand_direction": false, - "show_debug_info": false, - "show_direction": true, - "show_region_grid": true - }, - "performance": { - "target_fps": 30, - "enable_multithreading": false, - "enable_hardware_acceleration": true, - "resize_factor": 1.0, - "enable_multiprocessing": false + "use_virtual_camera": false }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 25, - "skin_lower_s": 30, - "skin_upper_s": 255, - "skin_lower_v": 60, - "skin_upper_v": 255, + "skin_upper_h": 178, + "skin_lower_s": 0, + "skin_upper_s": 71, + "skin_lower_v": 32, + "skin_upper_v": 85, "min_hand_area": 3000, "max_hand_area": 30000, "history_size": 10, @@ -90,6 +29,10 @@ "contour_simplify_epsilon": 0.02, "defect_distance_threshold": 20, "palm_circle_radius_ratio": 0.3, + "gesture_stability_threshold": 5, + "transition_threshold": 0.3, + "position_stability_weight": 0.4, + "gesture_cooldown": 0.5, "skin_lower": [ 0, 30, @@ -110,5 +53,88 @@ "duration": 0.3, "altitude": -10.0, "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": true, + "show_hand_direction": true, + "show_debug_info": false, + "show_speech_status": true, + "show_gesture_history": true, + "show_stability_indicator": true, + "show_gesture_info": true, + "show_control_help": true, + "show_detection_overlay": true, + "theme": "dark", + "show_direction": true, + "show_region_grid": true + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false, + "enable_multithreading": false, + "enable_hardware_acceleration": true + }, + "calibration": { + "auto_calibrate_skin": true, + "skin_calibration_frames": 30, + "hand_size_calibration": true + }, + "speech": { + "enabled": true, + "volume": 1.0, + "rate": 150, + "announce_gestures": true, + "announce_connections": true, + "announce_flight_events": true, + "announce_gesture_changes": true, + "announce_hand_status": true, + "announce_performance": true, + "min_gesture_confidence": 0.7, + "gesture_start_threshold": 3, + "gesture_end_threshold": 10 + }, + "system": { + "version": "1.0", + "last_updated": "2025-12-19 20:51:44", + "debug_mode": false + }, + "gesture_detection": { + "skin_hsv_ranges": { + "h_low": 0, + "h_high": 25, + "s_low": 30, + "s_high": 255, + "v_low": 60, + "v_high": 255 + }, + "min_hand_area": 2000, + "max_hand_area": 30000, + "confidence_threshold": 0.5, + "smoothing_window": 5, + "detection_interval": 1 + }, + "gesture_mapping": { + "0": "stop", + "1": "forward", + "2": "backward", + "3": "up", + "4": "down", + "5": "left", + "6": "right", + "7": "hover" + }, + "drone_control": { + "base_speed": 2.0, + "altitude": -10.0, + "control_delay": 0.3, + "takeoff_height": 5.0, + "emergency_stop_enabled": true } } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 4e33bea6dd..fe271047a5 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 手势识别优化版(添加语音反馈) -优化手势识别算法,解决位置敏感问题,添加语音反馈功能 +手势控制AirSim无人机 - 手势识别优化版(增强语音反馈和手势平滑) +优化手势识别算法,增强语音反馈,改进手势平滑处理 作者: xiaoshiyuan888 """ @@ -20,8 +20,8 @@ from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Gesture Recognition Optimized") -print("现在添加了语音反馈功能!") +print("Gesture Controlled Drone - Enhanced Gesture Recognition") +print("增强语音反馈和手势平滑处理!") print("=" * 60) # ========== 修复导入路径 ========== @@ -158,9 +158,9 @@ def safe_import(): Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -# ========== 语音反馈管理器 ========== -class SpeechFeedbackManager: - """语音反馈管理器""" +# ========== 增强语音反馈管理器 ========== +class EnhancedSpeechFeedbackManager: + """增强的语音反馈管理器""" def __init__(self, speech_lib): self.speech_lib = speech_lib @@ -169,7 +169,7 @@ def __init__(self, speech_lib): self.rate = 150 self.voice_id = None self.last_speech_time = {} - self.min_interval = 2.0 # 相同语音的最小间隔(秒) + self.min_interval = 1.5 # 缩短相同语音的最小间隔(秒) # 语音队列,避免语音重叠 self.speech_queue = [] @@ -179,7 +179,11 @@ def __init__(self, speech_lib): # 音频播放方法 self.audio_method = None - # 语音消息映射 + # 新增:手势状态追踪 + self.last_gesture_state = "none" # 记录上次手势状态 + self.gesture_active_time = 0 # 手势持续活跃时间 + + # 增强的语音消息映射 self.messages = { # 连接相关 'connecting': "正在连接无人机,请稍候", @@ -193,9 +197,12 @@ def __init__(self, speech_lib): 'landing': "无人机正在降落", 'land_success': "降落成功", 'emergency_stop': "紧急停止,无人机已降落", + 'hovering': "无人机悬停中", - # 手势相关 - 'gesture_detected': "手势识别就绪", + # 手势相关 - 增强 + 'gesture_detected': "手势识别就绪,请开始手势", + 'gesture_start': "开始识别手势", + 'gesture_end': "手势识别结束", 'gesture_stop': "停止", 'gesture_up': "向上", 'gesture_down': "向下", @@ -204,12 +211,17 @@ def __init__(self, speech_lib): 'gesture_forward': "向前", 'gesture_waiting': "等待手势", 'gesture_error': "手势识别错误", + 'gesture_stable': "手势稳定", + 'gesture_change': "手势变化", + 'gesture_low_confidence': "手势识别置信度低", + 'gesture_good_confidence': "手势识别置信度高", # 系统相关 'program_start': "手势控制无人机系统已启动", 'program_exit': "程序退出,感谢使用", 'camera_error': "摄像头错误,请检查连接", - 'low_confidence': "置信度过低,请调整手势", + 'camera_ready': "摄像头就绪", + 'system_ready': "系统准备就绪", # 模式相关 'simulation_mode': "进入模拟模式", @@ -217,6 +229,17 @@ def __init__(self, speech_lib): 'debug_mode_off': "调试模式已关闭", 'display_mode_changed': "显示模式已切换", 'help_toggled': "帮助信息已切换", + + # 新增:性能相关 + 'performance_good': "系统运行流畅", + 'performance_warning': "系统性能警告", + + # 新增:手势指导 + 'move_closer': "请将手靠近摄像头", + 'move_away': "请将手移远一些", + 'good_position': "手部位置良好", + 'hand_detected': "手部已检测到", + 'hand_lost': "手部丢失,请重新放置", } # 初始化语音引擎 @@ -240,7 +263,7 @@ def init_speech_engine(self): # 尝试寻找中文语音 for voice in voices: # 检查语音名称是否包含中文相关标识 - if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower(): + if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower() or 'zh_CN' in voice.name.lower(): self.engine.setProperty('voice', voice.id) self.voice_id = voice.id print(f"[Speech] 使用中文语音: {voice.name}") @@ -306,7 +329,7 @@ def play_audio_file(self, audio_file): # Windows系统命令 os.startfile(audio_file) # 等待播放完成(简单等待) - time.sleep(2) + time.sleep(1.5) elif self.audio_method == 'gtts_posix': # Linux/Mac系统命令 @@ -332,7 +355,7 @@ def play_audio_file(self, audio_file): print(f"⚠ 音频播放失败: {e}") return False - def speak(self, message_key, force=False): + def speak(self, message_key, force=False, immediate=False): """播放语音""" if not self.enabled: return @@ -349,14 +372,18 @@ def speak(self, message_key, force=False): else: text = message_key # 直接使用传入的文本 - # 添加到语音队列 - self.speech_queue.append(text) + # 如果是立即播放,直接在新线程中播放 + if immediate: + self.speak_direct(text) + else: + # 添加到语音队列 + self.speech_queue.append(text) - # 如果没有在播放,启动播放线程 - if not self.is_speaking and self.queue_thread is None: - self.queue_thread = threading.Thread(target=self._process_speech_queue) - self.queue_thread.daemon = True - self.queue_thread.start() + # 如果没有在播放,启动播放线程 + if not self.is_speaking and self.queue_thread is None: + self.queue_thread = threading.Thread(target=self._process_speech_queue) + self.queue_thread.daemon = True + self.queue_thread.start() # 更新时间戳 self.last_speech_time[message_key] = current_time @@ -397,7 +424,7 @@ def _process_speech_queue(self): except Exception as e: print(f"⚠ 语音播放失败: {e}") - time.sleep(0.1) # 避免过快 + time.sleep(0.05) # 减少等待时间,避免卡顿 self.is_speaking = False self.queue_thread = None @@ -475,7 +502,7 @@ class ConfigManager: """配置管理器""" def __init__(self): - self.config_file = os.path.join(current_dir, 'gesture_config_v3.json') # 更新版本号 + self.config_file = os.path.join(current_dir, 'gesture_config.json') self.default_config = { 'camera': { 'index': 0, @@ -492,14 +519,18 @@ def __init__(self): 'skin_upper_v': 255, 'min_hand_area': 2000, 'max_hand_area': 30000, - 'history_size': 10, - 'smooth_frames': 5, - 'min_confidence': 0.5, + 'history_size': 15, # 增加历史记录大小 + 'smooth_frames': 7, # 增加平滑帧数 + 'min_confidence': 0.6, # 提高最小置信度阈值 'detection_interval': 1, - 'hand_ratio_threshold': 1.5, # 手部长宽比阈值 + 'hand_ratio_threshold': 1.5, 'contour_simplify_epsilon': 0.02, - 'defect_distance_threshold': 20, # 凸缺陷距离阈值 - 'palm_circle_radius_ratio': 0.3, # 手掌半径与边界框的比例 + 'defect_distance_threshold': 20, + 'palm_circle_radius_ratio': 0.3, + 'gesture_stability_threshold': 5, # 新增:手势稳定性阈值 + 'transition_threshold': 0.3, # 新增:手势转换阈值 + 'position_stability_weight': 0.4, # 新增:位置稳定性权重 + 'gesture_cooldown': 0.5, # 新增:手势冷却时间 }, 'drone': { 'velocity': 2.5, @@ -517,7 +548,9 @@ def __init__(self): 'show_palm_center': True, 'show_hand_direction': True, 'show_debug_info': False, - 'show_speech_status': True, # 新增:显示语音状态 + 'show_speech_status': True, + 'show_gesture_history': True, # 新增:显示手势历史 + 'show_stability_indicator': True, # 新增:显示稳定性指示器 }, 'performance': { 'target_fps': 30, @@ -529,14 +562,19 @@ def __init__(self): 'skin_calibration_frames': 30, 'hand_size_calibration': True }, - 'speech': { # 新增:语音配置 + 'speech': { 'enabled': True, 'volume': 1.0, 'rate': 150, - 'announce_gestures': True, # 是否播报手势 - 'announce_connections': True, # 是否播报连接状态 - 'announce_flight_events': True, # 是否播报飞行事件 - 'min_gesture_confidence': 0.7, # 播报手势的最小置信度 + 'announce_gestures': True, + 'announce_connections': True, + 'announce_flight_events': True, + 'announce_gesture_changes': True, # 新增:播报手势变化 + 'announce_hand_status': True, # 新增:播报手部状态 + 'announce_performance': True, # 新增:播报性能状态 + 'min_gesture_confidence': 0.7, + 'gesture_start_threshold': 3, # 新增:手势开始识别阈值 + 'gesture_end_threshold': 10, # 新增:手势结束识别阈值 } } self.config = self.load_config() @@ -652,36 +690,48 @@ def calibrate_hand_size(self, hand_area): config = ConfigManager() -# ========== 改进的手势识别器 ========== -class ImprovedGestureRecognizer: - """改进的手势识别器 - 纯OpenCV实现""" +# ========== 改进的手势识别器(增强平滑处理) ========== +class EnhancedGestureRecognizer: + """增强的手势识别器 - 改进平滑处理和稳定性""" def __init__(self, speech_manager=None): self.config = config.get('gesture') self.speech_manager = speech_manager - # 手势历史和平滑 + # 增强的手势历史和平滑 self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) self.confidence_history = deque(maxlen=self.history_size) + self.position_history = deque(maxlen=self.history_size) # 新增:位置历史 self.current_gesture = "Waiting" self.current_confidence = 0.0 - # 记录上次播报的手势,避免重复播报 + # 新增:手势状态追踪 + self.gesture_state = "none" # none, starting, active, ending + self.gesture_stability_counter = 0 + self.last_stable_gesture = "Waiting" + self.gesture_active_frames = 0 + self.last_gesture_change_time = 0 + + # 记录上次播报的手势 self.last_announced_gesture = None self.last_announced_time = 0 - self.gesture_announce_interval = 3.0 # 手势播报最小间隔 + self.last_hand_status_time = 0 + self.gesture_announce_interval = 2.0 # 缩短手势播报间隔 # 手部跟踪和状态 self.last_hand_position = None self.hand_tracking = False self.track_window = None - self.hand_states = deque(maxlen=10) # 手部状态历史 + self.hand_states = deque(maxlen=15) + self.hand_detected_frames = 0 + self.hand_lost_frames = 0 # 性能统计 self.process_times = deque(maxlen=30) self.frame_counter = 0 self.detection_interval = self.config['detection_interval'] + self.last_performance_report = 0 # 手势颜色映射 self.gesture_colors = { @@ -708,6 +758,14 @@ def __init__(self, speech_manager=None): "Error": "gesture_error", } + # 手势状态颜色 + self.state_colors = { + "none": (100, 100, 100), # 灰色 + "starting": (255, 165, 0), # 橙色 + "active": (0, 255, 0), # 绿色 + "ending": (255, 0, 0), # 红色 + } + # 背景减除器 self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( history=100, varThreshold=25, detectShadows=True @@ -716,15 +774,15 @@ def __init__(self, speech_manager=None): # 形态学操作核 self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - print("✓ 改进的手势识别器已初始化 (纯OpenCV实现)") + # 新增:性能监控 + self.avg_process_time = 0 + self.frame_rate = 0 + self.last_fps_check = time.time() - # 语音提示初始化完成 - if self.speech_manager: - self.speech_manager.speak('gesture_detected') + print("✓ 增强的手势识别器已初始化 (改进平滑处理)") def get_skin_mask(self, frame): """获取肤色掩码""" - # 从配置获取肤色范围 h_low = config.get('gesture', 'skin_lower_h') h_high = config.get('gesture', 'skin_upper_h') s_low = config.get('gesture', 'skin_lower_s') @@ -732,60 +790,42 @@ def get_skin_mask(self, frame): v_low = config.get('gesture', 'skin_lower_v') v_high = config.get('gesture', 'skin_upper_v') - # 转换为HSV颜色空间 hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - - # 定义肤色范围 lower_skin = np.array([h_low, s_low, v_low], dtype=np.uint8) upper_skin = np.array([h_high, s_high, v_high], dtype=np.uint8) - - # 创建肤色掩码 skin_mask = cv2.inRange(hsv, lower_skin, upper_skin) return skin_mask, hsv def enhance_skin_detection(self, frame, skin_mask): """增强肤色检测""" - # 应用背景减除 fg_mask = self.bg_subtractor.apply(frame) - - # 结合肤色掩码和前景掩码 combined_mask = cv2.bitwise_and(skin_mask, fg_mask) - - # 形态学操作 combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - - # 高斯模糊 combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) return combined_mask def find_best_hand_contour(self, mask, frame): """找到最佳的手部轮廓""" - # 查找轮廓 contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: return None, 0.0 - # 按面积排序 contours = sorted(contours, key=cv2.contourArea, reverse=True) - best_contour = None best_score = 0.0 - min_area = config.get('gesture', 'min_hand_area') max_area = config.get('gesture', 'max_hand_area') - for contour in contours[:3]: # 只检查前3个最大轮廓 + for contour in contours[:3]: area = cv2.contourArea(contour) - # 面积过滤 if area < min_area or area > max_area: continue - # 计算轮廓评分 score = self.rate_contour(contour, frame.shape) if score > best_score: @@ -797,35 +837,27 @@ def find_best_hand_contour(self, mask, frame): def rate_contour(self, contour, frame_shape): """评估轮廓作为手部的可能性""" score = 0.0 - - # 面积评分 area = cv2.contourArea(contour) min_area = config.get('gesture', 'min_hand_area') max_area = config.get('gesture', 'max_hand_area') if min_area < area < max_area: - # 面积在合理范围内 area_ratio = min(area / max_area, 1.0) score += area_ratio * 0.3 - # 周长评分 perimeter = cv2.arcLength(contour, True) - if perimeter > 100: # 最小周长 + if perimeter > 100: score += 0.2 - # 紧凑度评分 (周长^2 / 面积) if area > 0: compactness = perimeter ** 2 / area - # 手的紧凑度通常在14-20之间 if 12 < compactness < 25: compactness_score = 1.0 - abs(compactness - 18) / 6 score += compactness_score * 0.3 - # 长宽比评分 x, y, w, h = cv2.boundingRect(contour) if h > 0: aspect_ratio = w / h - # 手的长宽比通常在0.5-2.0之间 if 0.4 < aspect_ratio < 2.5: aspect_score = 1.0 - abs(aspect_ratio - 1.0) / 1.5 score += aspect_score * 0.2 @@ -837,54 +869,37 @@ def analyze_hand_features(self, contour, frame_shape): if contour is None: return None, 0.0 - # 计算轮廓面积 area = cv2.contourArea(contour) - - # 计算轮廓中心 M = cv2.moments(contour) if M["m00"] == 0: return None, 0.0 cx = int(M["m10"] / M["m00"]) cy = int(M["m01"] / M["m00"]) - - # 计算边界框 x, y, w, h = cv2.boundingRect(contour) bbox_area = w * h - - # 计算手掌中心(假设是轮廓的中心) palm_center = (cx, cy) - - # 计算手掌半径(假设为边界框宽度的30%) palm_radius = int(w * config.get('gesture', 'palm_circle_radius_ratio')) - - # 分析手指 fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius) - - # 计算手部方向 direction = self.calculate_hand_direction(contour, cx, cy) - - # 计算手部位置(归一化) h_img, w_img = frame_shape[:2] norm_x = cx / w_img norm_y = cy / h_img - - # 计算手势置信度 confidence = self.calculate_confidence(area, len(fingers), len(contour), bbox_area) - # 返回结果 result = { 'contour': contour, 'center': (cx, cy), 'bbox': (x, y, x + w, y + h), - 'fingers': fingers, # 手指列表 - 'fingertips': fingertips, # 指尖点列表 - 'defects': defects, # 凸缺陷列表 + 'fingers': fingers, + 'fingertips': fingertips, + 'defects': defects, 'palm_center': palm_center, 'palm_radius': palm_radius, - 'direction': direction, # 手部方向(角度) + 'direction': direction, 'area': area, 'position': (norm_x, norm_y), + 'bbox_size': (w, h), 'confidence': confidence } @@ -892,19 +907,14 @@ def analyze_hand_features(self, contour, frame_shape): def analyze_fingers(self, contour, palm_center, palm_radius): """分析手指""" - # 简化轮廓 epsilon = config.get('gesture', 'contour_simplify_epsilon') * cv2.arcLength(contour, True) approx = cv2.approxPolyDP(contour, epsilon, True) - - # 计算凸包 hull = cv2.convexHull(approx, returnPoints=False) if hull is None or len(hull) < 3: return [], [], [] - # 计算凸缺陷 defects = cv2.convexityDefects(approx, hull) - fingers = [] fingertips = [] defect_points = [] @@ -916,17 +926,14 @@ def analyze_fingers(self, contour, palm_center, palm_radius): end = tuple(approx[e][0]) far = tuple(approx[f][0]) - # 计算距离 start_dist = np.linalg.norm(np.array(start) - np.array(palm_center)) end_dist = np.linalg.norm(np.array(end) - np.array(palm_center)) far_dist = np.linalg.norm(np.array(far) - np.array(palm_center)) - # 检查是否可能是手指 if (start_dist > palm_radius * 1.2 and end_dist > palm_radius * 1.2 and d > config.get('gesture', 'defect_distance_threshold') * 256): - # 检查角度 a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) @@ -935,9 +942,7 @@ def analyze_fingers(self, contour, palm_center, palm_radius): angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) angle_degrees = math.degrees(angle) - # 如果角度小于90度,可能是指缝 if angle_degrees < 90: - # 添加手指 finger = { 'start': start, 'end': end, @@ -947,77 +952,59 @@ def analyze_fingers(self, contour, palm_center, palm_radius): } fingers.append(finger) - # 添加指尖(开始和结束点都可能是指尖) for point in [start, end]: if point not in fingertips: - # 检查点是否在手掌半径之外 point_dist = np.linalg.norm(np.array(point) - np.array(palm_center)) if point_dist > palm_radius * 1.5: fingertips.append(point) defect_points.append((start, end, far, d)) - # 如果没有检测到手指,尝试使用凸包点作为指尖 if len(fingertips) == 0: hull_points = cv2.convexHull(approx, returnPoints=True) if len(hull_points) > 0: hull_points = hull_points.reshape(-1, 2) - # 过滤掉离手掌中心太近的点 for point in hull_points: point_tuple = tuple(point) point_dist = np.linalg.norm(point - np.array(palm_center)) if point_dist > palm_radius * 1.5 and point_tuple not in fingertips: fingertips.append(point_tuple) - # 限制指尖数量为5 fingertips = fingertips[:5] return fingers, fingertips, defect_points def calculate_hand_direction(self, contour, cx, cy): """计算手部方向""" - # 使用PCA(主成分分析)计算手部方向 if len(contour) < 5: return 0.0 - # 将轮廓转换为点集 points = contour.reshape(-1, 2).astype(np.float32) - - # 计算PCA mean = np.empty((0)) mean, eigenvectors, eigenvalues = cv2.PCACompute2(points, mean) - - # 计算方向角度(度) direction = math.degrees(math.atan2(eigenvectors[0, 1], eigenvectors[0, 0])) return direction def calculate_confidence(self, area, finger_count, contour_length, bbox_area): """计算手势置信度""" - confidence = 0.5 # 基础置信度 - - # 基于面积的置信度 + confidence = 0.5 min_area = config.get('gesture', 'min_hand_area') max_area = config.get('gesture', 'max_hand_area') if min_area < area < max_area: - # 面积在合理范围内 area_norm = (area - min_area) / (max_area - min_area) confidence += area_norm * 0.2 - # 基于手指数量的置信度 if 0 <= finger_count <= 5: confidence += 0.2 - # 基于轮廓复杂度的置信度 if contour_length > 200: confidence += 0.1 - # 基于填充度的置信度(轮廓面积/边界框面积) if bbox_area > 0: fill_ratio = area / bbox_area - # 手的填充度通常在0.3-0.7之间 if 0.2 < fill_ratio < 0.8: fill_score = 1.0 - abs(fill_ratio - 0.5) / 0.3 confidence += fill_score * 0.1 @@ -1034,116 +1021,219 @@ def recognize_gesture_improved(self, hand_data): norm_x, norm_y = hand_data['position'] direction = hand_data.get('direction', 0.0) confidence = hand_data['confidence'] + w, h = hand_data['bbox_size'] + aspect_ratio = w / h if h > 0 else 1.0 # 根据手指数量分类 if finger_count == 0: - # 握拳或没有手指 if len(fingertips) == 0: return "Stop", confidence * 0.9 else: - # 有指尖但不是手指,可能是部分握拳 return "Stop", confidence * 0.7 elif finger_count == 1: - # 单指手势 if len(fingertips) >= 1: - # 检查指尖方向 cx, cy = hand_data['center'] fingertip = fingertips[0] - - # 计算指尖相对于手中心的方向 dx = fingertip[0] - cx dy = fingertip[1] - cy - # 判断指尖方向 - if abs(dx) > abs(dy): # 主要水平方向 + if abs(dx) > abs(dy): if dx > 0: return "Right", confidence * 0.8 else: return "Left", confidence * 0.8 - else: # 主要垂直方向 - if dy < 0: # 指尖在手中心上方 + else: + if dy < 0: return "Up", confidence * 0.8 else: return "Forward", confidence * 0.8 return "Forward", confidence * 0.7 elif finger_count == 2: - # 双指手势 return "Forward", confidence * 0.7 elif finger_count == 3: - # 三指手势,根据手部方向判断 - if -45 <= direction <= 45: # 大致水平 + if -45 <= direction <= 45: if direction > 0: return "Right", confidence * 0.7 else: return "Left", confidence * 0.7 - else: # 大致垂直 + else: if direction > 0: return "Down", confidence * 0.7 else: return "Up", confidence * 0.7 elif finger_count >= 4: - # 多指手势(手掌张开) - # 根据手部位置判断 - if norm_x < 0.4: # 左侧 + if norm_x < 0.4: return "Left", confidence * 0.8 - elif norm_x > 0.6: # 右侧 + elif norm_x > 0.6: return "Right", confidence * 0.8 - elif norm_y < 0.4: # 上方 + elif norm_y < 0.4: return "Up", confidence * 0.8 - elif norm_y > 0.6: # 下方 + elif norm_y > 0.6: return "Down", confidence * 0.8 - else: # 中间 - # 根据手部方向判断 - if -45 <= direction <= 45: # 大致水平 + else: + if -45 <= direction <= 45: return "Forward", confidence * 0.7 - else: # 大致垂直 + else: return "Stop", confidence * 0.7 return "Waiting", confidence * 0.5 - def smooth_gesture(self, new_gesture, new_confidence): - """平滑手势输出""" + def smooth_gesture_enhanced(self, new_gesture, new_confidence, hand_data): + """增强的手势平滑处理""" + current_time = time.time() + + # 检查手势冷却时间 + if current_time - self.last_gesture_change_time < config.get('gesture', 'gesture_cooldown'): + return self.current_gesture, self.current_confidence + # 添加到历史 self.gesture_history.append(new_gesture) self.confidence_history.append(new_confidence) - # 如果历史记录不足,直接返回 - if len(self.gesture_history) < 3: - self.current_gesture = new_gesture - self.current_confidence = new_confidence - return new_gesture, new_confidence + if hand_data is not None: + self.position_history.append(hand_data['position']) + + # 计算手势稳定性 + if len(self.gesture_history) >= 3: + # 检查最近N个手势是否一致 + recent_gestures = list(self.gesture_history)[-3:] + gesture_counter = Counter(recent_gestures) + most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] + + # 计算位置稳定性 + position_stability = 1.0 + if len(self.position_history) >= 2 and hand_data is not None: + current_pos = hand_data['position'] + prev_pos = self.position_history[-2] if len(self.position_history) >= 2 else current_pos + position_diff = math.sqrt((current_pos[0] - prev_pos[0]) ** 2 + (current_pos[1] - prev_pos[1]) ** 2) + position_stability = max(0, 1.0 - position_diff * 5.0) + + # 增强的稳定性检查 + stability_threshold = config.get('gesture', 'gesture_stability_threshold') + transition_threshold = config.get('gesture', 'transition_threshold') + position_weight = config.get('gesture', 'position_stability_weight') + + # 计算综合稳定性得分 + gesture_stability = most_common_count / 3.0 + overall_stability = gesture_stability * (1.0 - position_weight) + position_stability * position_weight + + # 手势状态转换逻辑 + if overall_stability >= transition_threshold: + if most_common_gesture != self.last_stable_gesture: + self.gesture_stability_counter += 1 + else: + self.gesture_stability_counter = max(0, self.gesture_stability_counter - 1) + + if self.gesture_stability_counter >= stability_threshold: + # 手势稳定,更新当前手势 + self.current_gesture = most_common_gesture + self.current_confidence = np.mean(list(self.confidence_history)[-3:]) + self.last_stable_gesture = most_common_gesture + self.gesture_stability_counter = 0 + self.last_gesture_change_time = current_time + else: + # 手势不稳定,重置计数器 + self.gesture_stability_counter = max(0, self.gesture_stability_counter - 2) + + return self.current_gesture, self.current_confidence + + def update_gesture_state(self, hand_data, gesture, confidence): + """更新手势状态""" + current_time = time.time() - # 统计最常见的姿势 - gesture_counter = Counter(self.gesture_history) - most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] + if hand_data is None: + # 手部丢失 + self.hand_lost_frames += 1 + self.hand_detected_frames = max(0, self.hand_detected_frames - 1) - # 计算平均置信度 - avg_confidence = np.mean(list(self.confidence_history)) + if self.hand_lost_frames > 10 and self.gesture_state != "none": + self.gesture_state = "none" + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_hand_status') and + current_time - self.last_hand_status_time > 3.0): + self.speech_manager.speak('hand_lost', immediate=True) + self.last_hand_status_time = current_time + return - # 如果最常见的姿势出现次数足够多,使用它 - smooth_frames = self.config['smooth_frames'] - if most_common_count >= smooth_frames: - self.current_gesture = most_common_gesture - self.current_confidence = avg_confidence + # 手部检测到 + self.hand_detected_frames += 1 + self.hand_lost_frames = 0 - return self.current_gesture, self.current_confidence + # 检查手部大小和位置 + hand_area = hand_data['area'] + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') + + # 提供手部位置反馈 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_hand_status') and + current_time - self.last_hand_status_time > 5.0): + + if hand_area < min_area * 0.8: + self.speech_manager.speak('move_closer', immediate=True) + self.last_hand_status_time = current_time + elif hand_area > max_area * 1.2: + self.speech_manager.speak('move_away', immediate=True) + self.last_hand_status_time = current_time + elif self.hand_detected_frames == 5: # 首次稳定检测 + self.speech_manager.speak('hand_detected', immediate=True) + self.last_hand_status_time = current_time + + # 手势状态转换 + if self.gesture_state == "none" and gesture != "Waiting" and confidence > 0.6: + # 手势开始 + self.gesture_state = "starting" + self.gesture_active_frames = 0 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gesture_changes')): + self.speech_manager.speak('gesture_start', immediate=True) + + elif self.gesture_state == "starting": + self.gesture_active_frames += 1 + if self.gesture_active_frames >= config.get('speech', 'gesture_start_threshold'): + self.gesture_state = "active" + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gesture_changes')): + self.speech_manager.speak('gesture_stable', immediate=True) + + elif self.gesture_state == "active": + if gesture == "Waiting" or confidence < 0.5: + self.gesture_active_frames = max(0, self.gesture_active_frames - 2) + if self.gesture_active_frames <= 0: + self.gesture_state = "ending" + else: + self.gesture_active_frames = min(20, self.gesture_active_frames + 1) + + elif self.gesture_state == "ending": + self.gesture_active_frames -= 1 + if self.gesture_active_frames <= 0: + self.gesture_state = "none" + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gesture_changes')): + self.speech_manager.speak('gesture_end', immediate=True) def visualize_detection(self, frame, hand_data, gesture, confidence): """可视化检测结果""" if hand_data is None: return frame - # 获取显示配置 show_contours = config.get('display', 'show_contours') show_bbox = config.get('display', 'show_bbox') show_fingertips = config.get('display', 'show_fingertips') show_palm_center = config.get('display', 'show_palm_center') show_hand_direction = config.get('display', 'show_hand_direction') show_debug_info = config.get('display', 'show_debug_info') + show_gesture_history = config.get('display', 'show_gesture_history') + show_stability_indicator = config.get('display', 'show_stability_indicator') # 绘制轮廓 if show_contours and 'contour' in hand_data: @@ -1153,13 +1243,24 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): if show_bbox and 'bbox' in hand_data: x1, y1, x2, y2 = hand_data['bbox'] color = self.gesture_colors.get(gesture, (255, 255, 255)) + + # 根据手势状态调整边界框颜色 + state_color = self.state_colors.get(self.gesture_state, color) + if self.gesture_state != "none": + color = state_color + cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) - # 显示手势标签 - 使用英文避免字体问题 + # 显示手势标签 label = f"{gesture}" if config.get('display', 'show_confidence'): label += f" ({confidence:.0%})" + # 显示手势状态 + if self.gesture_state != "none": + state_text = {"starting": "开始", "active": "活跃", "ending": "结束"} + label += f" [{state_text.get(self.gesture_state, '')}]" + # 计算文本大小 (text_width, text_height), baseline = cv2.getTextSize( label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 @@ -1197,19 +1298,65 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): direction = hand_data['direction'] length = 50 - # 计算方向向量 dx = length * math.cos(math.radians(direction)) dy = length * math.sin(math.radians(direction)) - # 绘制方向线 end_point = (int(cx + dx), int(cy + dy)) cv2.arrowedLine(frame, (cx, cy), end_point, (255, 255, 0), 2) - # 显示方向角度 angle_text = f"Dir: {direction:.0f}°" cv2.putText(frame, angle_text, (cx, cy - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 1) + # 绘制稳定性指示器 + if show_stability_indicator: + h, w = frame.shape[:2] + indicator_x = w - 100 + indicator_y = 30 + + # 绘制稳定性背景 + cv2.rectangle(frame, (indicator_x, indicator_y), + (indicator_x + 80, indicator_y + 15), (50, 50, 50), -1) + + # 计算稳定性指示条长度 + stability_level = min(1.0, + self.gesture_stability_counter / config.get('gesture', 'gesture_stability_threshold')) + bar_length = int(70 * stability_level) + + # 根据稳定性级别选择颜色 + if stability_level > 0.7: + bar_color = (0, 255, 0) # 绿色 + elif stability_level > 0.4: + bar_color = (255, 165, 0) # 橙色 + else: + bar_color = (255, 0, 0) # 红色 + + # 绘制稳定性指示条 + cv2.rectangle(frame, (indicator_x + 5, indicator_y + 5), + (indicator_x + 5 + bar_length, indicator_y + 10), bar_color, -1) + + # 绘制稳定性文本 + cv2.putText(frame, "稳定度", (indicator_x, indicator_y - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 绘制手势历史(如果启用) + if show_gesture_history and len(self.gesture_history) > 0: + h, w = frame.shape[:2] + history_y = h - 50 + + # 绘制历史背景 + cv2.rectangle(frame, (10, history_y - 20), (200, history_y + 10), (0, 0, 0), -1) + cv2.putText(frame, "手势历史:", (15, history_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 显示最近几个手势 + recent_gestures = list(self.gesture_history)[-5:] if len(self.gesture_history) >= 5 else list( + self.gesture_history) + for i, gest in enumerate(recent_gestures): + color = self.gesture_colors.get(gest, (255, 255, 255)) + cv2.putText(frame, gest[0], (85 + i * 20, history_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + # 显示调试信息 if show_debug_info: # 显示手指数量 @@ -1223,6 +1370,16 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): cv2.putText(frame, pos_text, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + # 显示手势状态 + state_text = f"State: {self.gesture_state}" + cv2.putText(frame, state_text, (150, frame.shape[0] - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 显示稳定性计数器 + stability_text = f"Stability: {self.gesture_stability_counter}" + cv2.putText(frame, stability_text, (150, frame.shape[0] - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + return frame def recognize(self, frame): @@ -1261,20 +1418,36 @@ def recognize(self, frame): # 识别手势 gesture, raw_confidence = self.recognize_gesture_improved(hand_data) confidence = max(confidence, raw_confidence) + + # 更新手势状态 + self.update_gesture_state(hand_data, gesture, confidence) + + # 增强的手势平滑 + final_gesture, final_confidence = self.smooth_gesture_enhanced(gesture, confidence, hand_data) else: gesture, confidence = "Waiting", 0.3 - - # 平滑手势 - final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) + self.update_gesture_state(None, gesture, confidence) + final_gesture, final_confidence = gesture, confidence # 手势语音提示 if (self.speech_manager and config.get('speech', 'enabled') and - config.get('speech', 'announce_gestures') and - final_confidence >= config.get('speech', 'min_gesture_confidence')): + config.get('speech', 'announce_gestures')): current_time = time.time() - if (final_gesture != self.last_announced_gesture or + + # 根据置信度提供反馈 + if confidence >= 0.8 and current_time - self.last_hand_status_time > 5.0: + self.speech_manager.speak('gesture_good_confidence', immediate=True) + self.last_hand_status_time = current_time + elif confidence < 0.5 and current_time - self.last_hand_status_time > 5.0: + self.speech_manager.speak('gesture_low_confidence', immediate=True) + self.last_hand_status_time = current_time + + # 手势变化播报 + if (final_gesture != self.last_announced_gesture and + final_gesture not in ["Waiting", "Error"] and + final_confidence >= config.get('speech', 'min_gesture_confidence') and current_time - self.last_announced_time > self.gesture_announce_interval): if final_gesture in self.gesture_speech_map: @@ -1283,6 +1456,20 @@ def recognize(self, frame): self.last_announced_gesture = final_gesture self.last_announced_time = current_time + # 性能报告 + current_time = time.time() + if current_time - self.last_performance_report > 30.0: # 每30秒报告一次 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_performance')): + + if self.avg_process_time < 20: # 处理时间小于20ms + self.speech_manager.speak('performance_good', immediate=True) + elif self.avg_process_time > 50: # 处理时间大于50ms + self.speech_manager.speak('performance_warning', immediate=True) + + self.last_performance_report = current_time + # 可视化结果 if hand_data is not None: processed_frame = self.visualize_detection( @@ -1296,6 +1483,17 @@ def recognize(self, frame): process_time = (time.time() - start_time) * 1000 self.process_times.append(process_time) + # 更新平均处理时间 + if len(self.process_times) > 0: + self.avg_process_time = np.mean(list(self.process_times)) + + # 更新帧率 + current_time = time.time() + if current_time - self.last_fps_check >= 1.0: + self.frame_rate = self.frame_counter + self.frame_counter = 0 + self.last_fps_check = current_time + return final_gesture, final_confidence, processed_frame except Exception as e: @@ -1305,9 +1503,9 @@ def recognize(self, frame): def get_performance_stats(self): """获取性能统计""" if len(self.process_times) == 0: - return 0.0 + return 0.0, self.frame_rate - return np.mean(list(self.process_times)) + return np.mean(list(self.process_times)), self.frame_rate def set_simulated_gesture(self, gesture): """设置模拟的手势""" @@ -1538,7 +1736,7 @@ def move_by_gesture(self, gesture, confidence): config.get('speech', 'enabled') and config.get('speech', 'announce_gestures') and confidence < min_confidence * 0.8): # 如果置信度特别低 - self.speech_manager.speak('low_confidence') + self.speech_manager.speak('gesture_low_confidence') return False try: @@ -1568,9 +1766,18 @@ def move_by_gesture(self, gesture, confidence): elif gesture == "Stop": self.client.hoverAsync() success = True + # 悬停语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('hovering') elif gesture == "Hover": self.client.hoverAsync() success = True + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('hovering') if success: self.last_control_time = current_time @@ -1627,6 +1834,9 @@ def __init__(self, speech_manager=None): 'help': (255, 200, 100), # 浅橙色 'speech_enabled': (0, 255, 0), # 绿色 'speech_disabled': (255, 0, 0), # 红色 + 'performance_good': (0, 255, 0), # 绿色 + 'performance_warning': (255, 165, 0), # 橙色 + 'performance_bad': (255, 0, 0), # 红色 } print("✓ 中文UI渲染器已初始化") @@ -1690,7 +1900,7 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 优化版 (带语音反馈)" + title = "手势控制无人机系统 - 增强版" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -1730,10 +1940,20 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro if process_time > 0: perf_text += f" | 延迟: {process_time:.1f}ms" - frame = self.draw_text(frame, perf_text, (w - 200, 65), size=14, color=self.colors['info']) + # 根据处理时间选择颜色 + if process_time < 20: + perf_color = self.colors['performance_good'] + elif process_time < 50: + perf_color = self.colors['performance_warning'] + else: + perf_color = self.colors['performance_bad'] + else: + perf_color = self.colors['info'] + + frame = self.draw_text(frame, perf_text, (w - 200, 65), size=14, color=perf_color) # 控制提示 - control_text = "提示: 确保手部完全进入画面,光线充足" + control_text = "提示: 确保手部完全进入画面,保持稳定手势" frame = self.draw_text(frame, control_text, (10, 90), size=14, color=self.colors['info']) return frame @@ -1752,7 +1972,7 @@ def draw_help_bar(self, frame): help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音" + "V:切换语音反馈 M:测试语音 P:性能报告" ] for i, line in enumerate(help_lines): @@ -1820,16 +2040,17 @@ def main(): """主函数""" # 初始化语音管理器 print("初始化语音反馈系统...") - speech_manager = SpeechFeedbackManager(libs['speech']) + speech_manager = EnhancedSpeechFeedbackManager(libs['speech']) # 程序启动语音提示 if speech_manager.enabled: - speech_manager.speak('program_start', force=True) + speech_manager.speak('program_start', force=True, immediate=True) + speech_manager.speak('system_ready', immediate=True) # 初始化组件 print("初始化组件...") - gesture_recognizer = ImprovedGestureRecognizer(speech_manager) + gesture_recognizer = EnhancedGestureRecognizer(speech_manager) drone_controller = SimpleDroneController(libs['airsim'], speech_manager) ui_renderer = ChineseUIRenderer(speech_manager) performance_monitor = PerformanceMonitor() @@ -1851,12 +2072,16 @@ def main(): print(f"✓ 摄像头已初始化") print(f" 分辨率: {actual_width}x{actual_height}") print(f" 帧率: {actual_fps}") + + # 摄像头就绪语音提示 + if speech_manager.enabled: + speech_manager.speak('camera_ready', immediate=True) else: print("❌ 摄像头不可用,使用模拟模式") # 摄像头错误语音提示 if speech_manager.enabled: - speech_manager.speak('camera_error') + speech_manager.speak('camera_error', immediate=True) cap = None except Exception as e: @@ -1864,18 +2089,19 @@ def main(): # 摄像头错误语音提示 if speech_manager.enabled: - speech_manager.speak('camera_error') + speech_manager.speak('camera_error', immediate=True) cap = None # 显示欢迎信息 print("\n" + "=" * 60) - print("手势控制无人机系统 - 手势识别优化版") + print("手势控制无人机系统 - 增强版") print("=" * 60) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") - print(f" 手势识别: 改进的OpenCV算法") + print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") + print(f" 手势状态: 支持开始/活跃/结束状态追踪") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) @@ -1884,17 +2110,16 @@ def main(): print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") print("3. 手势控制改进:") - print(" - 握拳或握紧: Stop") - print(" - 单指指向: 根据指尖方向判断Up/Down/Left/Right") - print(" - 双指: Forward") - print(" - 手掌张开: 根据手的位置判断方向") - print(" * 手势识别置信度 > 50% 时才会执行") + print(" - 系统会自动检测手势开始、稳定和结束状态") + print(" - 手势稳定性越高,识别越准确") + print(" - 手部距离摄像头适中时效果最佳") + print(" * 手势识别置信度 > 60% 时才会执行") print("4. 键盘控制:") print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") print("5. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") print("6. 语音控制:") - print(" [V]切换语音反馈 [M]测试语音") + print(" [V]切换语音反馈 [M]测试语音 [P]性能报告") print("7. 按 [ESC] 安全退出") print("=" * 60) print("程序启动成功!") @@ -1939,13 +2164,15 @@ def main(): # 获取性能统计 perf_stats = performance_monitor.get_stats() - process_time = gesture_recognizer.get_performance_stats() + process_time, frame_rate = gesture_recognizer.get_performance_stats() # 根据显示模式调整显示选项 if display_modes[current_display_mode] == 'normal': config.set('display', 'show_contours', value=True) config.set('display', 'show_bbox', value=True) config.set('display', 'show_fingertips', value=True) + config.set('display', 'show_gesture_history', value=True) + config.set('display', 'show_stability_indicator', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -1953,11 +2180,15 @@ def main(): config.set('display', 'show_fingertips', value=True) config.set('display', 'show_palm_center', value=True) config.set('display', 'show_hand_direction', value=True) + config.set('display', 'show_gesture_history', value=True) + config.set('display', 'show_stability_indicator', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) config.set('display', 'show_bbox', value=True) config.set('display', 'show_fingertips', value=False) + config.set('display', 'show_gesture_history', value=False) + config.set('display', 'show_stability_indicator', value=False) config.set('display', 'show_debug_info', value=False) # 绘制UI @@ -1974,7 +2205,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Optimized with Speech', frame) + cv2.imshow('Gesture Controlled Drone - Enhanced', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -2003,12 +2234,12 @@ def main(): # 语音提示 if speech_manager.enabled: - speech_manager.speak('help_toggled') + speech_manager.speak('help_toggled', immediate=True) elif key == ord('r') or key == ord('R'): # 重置手势识别 print("重置手势识别...") - gesture_recognizer = ImprovedGestureRecognizer(speech_manager) + gesture_recognizer = EnhancedGestureRecognizer(speech_manager) print("✓ 手势识别已重置") # 语音提示 @@ -2023,7 +2254,7 @@ def main(): # 语音提示 if speech_manager.enabled: - speech_manager.speak('display_mode_changed') + speech_manager.speak('display_mode_changed', immediate=True) elif key == ord('d') or key == ord('D'): # 切换调试信息 @@ -2035,9 +2266,9 @@ def main(): # 语音提示 if speech_manager.enabled: if not current: - speech_manager.speak('debug_mode_on') + speech_manager.speak('debug_mode_on', immediate=True) else: - speech_manager.speak('debug_mode_off') + speech_manager.speak('debug_mode_off', immediate=True) elif key == ord('v') or key == ord('V'): # 切换语音反馈 @@ -2050,10 +2281,21 @@ def main(): # 测试语音 if speech_manager.enabled: print("测试语音...") - speech_manager.speak_direct("语音反馈测试,这是一条测试消息") + speech_manager.speak_direct("语音反馈测试,系统运行正常") else: print("语音反馈已禁用,按V键启用") + elif key == ord('p') or key == ord('P'): + # 性能报告 + if speech_manager.enabled: + print("生成性能报告...") + if process_time < 20: + speech_manager.speak_direct("系统性能优秀,运行流畅") + elif process_time < 50: + speech_manager.speak_direct("系统性能良好") + else: + speech_manager.speak_direct("系统性能警告,请检查") + elif key in key_to_gesture: # 键盘控制 simulated_gesture = key_to_gesture[key] @@ -2084,7 +2326,7 @@ def main(): # 程序退出语音提示 if speech_manager.enabled: - speech_manager.speak('program_exit', force=True) + speech_manager.speak('program_exit', force=True, immediate=True) time.sleep(1) # 确保语音播报完成 drone_controller.emergency_stop() From d831140f4fa47e437dc68e18637947b379c81eb0 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sat, 20 Dec 2025 20:30:10 +0800 Subject: [PATCH 09/18] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=88=91=E7=9A=84?= =?UTF-8?q?=E9=A1=B9=E7=9B=AEREADME.md=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../README.md | 40 +++++++++++++++++++ .../gesture_config.json | 10 +++++ 2 files changed, 50 insertions(+) diff --git a/src/autonomus_drone_hand_gesture_project/README.md b/src/autonomus_drone_hand_gesture_project/README.md index e69de29bb2..0b5c35f915 100644 --- a/src/autonomus_drone_hand_gesture_project/README.md +++ b/src/autonomus_drone_hand_gesture_project/README.md @@ -0,0 +1,40 @@ +# 手势控制AirSim无人机系统 + +[项目简介] +基于计算机视觉和语音反馈技术的手势控制无人机系统。使用纯OpenCV实现手部检测和手势识别,结合语音反馈系统,提供直观的无人机控制体验。系统支持AirSim仿真环境和真实无人机控制,具备手势平滑处理、状态追踪和多模式显示功能。 + +[核心功能] + +1. 手势识别与控制 + - 纯OpenCV实现:不依赖深度学习框架,使用肤色检测、轮廓分析和凸缺陷检测识别手势 + - 多种手势支持:支持停止、前进、后退、上升、下降、左转、右转等多种控制手势 + - 实时手势追踪:跟踪手部位置和状态,提供连续控制体验 + +2. 智能语音反馈系统 + - 多引擎支持:兼容pyttsx3(离线)和gTTS(在线)语音引擎 + - 丰富语音提示:提供连接状态、飞行状态、手势识别、系统状态等多种语音反馈 + - 智能语音队列:避免语音重叠,支持优先级播放和即时播放 + +3. 增强手势平滑处理 + - 手势状态追踪:实现手势状态机(none/starting/active/ending),跟踪手势生命周期 + - 稳定性检测:使用手势稳定性计数器和位置稳定性权重,减少误判 + - 冷却时间机制:防止手势切换过于频繁,提高控制精度 + +4. 多平台兼容性 + - 跨平台音频播放:支持pygame、pydub和系统默认播放器 + - 自动库检测:自动检测可用库并选择最优方案 + - 详细错误处理:完善的错误处理和降级方案 + +[环境依赖] +在运行代码之前,请确保已安装以下Python库: + +```bash +# 基础依赖 +pip install opencv-python pillow numpy + +# AirSim无人机控制(可选) +pip install airsim + +# 语音反馈系统(选择安装) +pip install pyttsx3 # 离线语音(推荐) +pip install gtts pygame # 在线语音(需要网络) \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index c60f9cec3d..145a26faac 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -67,6 +67,9 @@ "show_speech_status": true, "show_gesture_history": true, "show_stability_indicator": true, + "show_trajectory": true, + "show_trajectory_heatmap": false, + "trajectory_max_points": 100, "show_gesture_info": true, "show_control_help": true, "show_detection_overlay": true, @@ -96,10 +99,17 @@ "announce_gesture_changes": true, "announce_hand_status": true, "announce_performance": true, + "announce_recording": true, "min_gesture_confidence": 0.7, "gesture_start_threshold": 3, "gesture_end_threshold": 10 }, + "recording": { + "auto_save": false, + "default_filename": "gesture_recording.pkl", + "playback_speed": 1.0, + "loop_playback": false + }, "system": { "version": "1.0", "last_updated": "2025-12-19 20:51:44", From 15eb615ac71b54721929b2b980b5891c2094f45f Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sun, 21 Dec 2025 09:07:37 +0800 Subject: [PATCH 10/18] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E8=BD=A8=E8=BF=B9=E8=AE=B0=E5=BD=95=E4=B8=8E=E5=9B=9E=E6=94=BE?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=EF=BC=8C=E4=BC=98=E5=8C=96=E8=AF=AD=E9=9F=B3?= =?UTF-8?q?=E5=8F=8D=E9=A6=88=E7=B3=BB=E7=BB=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config.json | 152 +++-- .../main.py | 554 +++++++++++++++++- .../trajectory_20251221_085734.pkl | Bin 0 -> 5540 bytes .../trajectories/trajectory_temp.pkl | Bin 0 -> 4719 bytes 4 files changed, 633 insertions(+), 73 deletions(-) create mode 100644 src/autonomus_drone_hand_gesture_project/trajectories/trajectory_20251221_085734.pkl create mode 100644 src/autonomus_drone_hand_gesture_project/trajectories/trajectory_temp.pkl diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index 145a26faac..1ebcdda50e 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -4,21 +4,25 @@ "width": 640, "height": 480, "fps": 30, + "auto_exposure": true, + "brightness": 0, + "contrast": 1.0, "device_id": 0, + "exposure": 0, + "flip_horizontal": true, "resolution": { - "width": 640, - "height": 480 + "height": 480, + "width": 640 }, - "flip_horizontal": true, "use_virtual_camera": false }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 178, + "skin_upper_h": 125, "skin_lower_s": 0, - "skin_upper_s": 71, - "skin_lower_v": 32, - "skin_upper_v": 85, + "skin_upper_s": 31, + "skin_lower_v": 0, + "skin_upper_v": 74, "min_hand_area": 3000, "max_hand_area": 30000, "history_size": 10, @@ -33,6 +37,17 @@ "transition_threshold": 0.3, "position_stability_weight": 0.4, "gesture_cooldown": 0.5, + "adaptive_learning_rate": 0.1, + "enable_background_subtraction": true, + "enable_dynamic_threshold": true, + "enable_morphological_ops": true, + "face_exclusion_area": 8000, + "hand_region_threshold": 0.2, + "max_fingertips": 5, + "min_fingertip_distance": 10, + "noise_reduction": 0.5, + "palm_aspect_ratio": 1.3, + "single_finger_angle_threshold": 60, "skin_lower": [ 0, 30, @@ -42,17 +57,18 @@ 25, 255, 255 - ], - "hand_region_threshold": 0.2, - "face_exclusion_area": 8000, - "palm_aspect_ratio": 1.3, - "single_finger_angle_threshold": 60 + ] }, "drone": { "velocity": 2.5, "duration": 0.3, "altitude": -10.0, - "control_interval": 0.3 + "control_interval": 0.3, + "acceleration_limit": 2.0, + "auto_takeoff_delay": 2.0, + "emergency_land_delay": 3.0, + "emergency_land_timeout": 5.0, + "smooth_movement": true }, "display": { "show_fps": true, @@ -68,26 +84,43 @@ "show_gesture_history": true, "show_stability_indicator": true, "show_trajectory": true, - "show_trajectory_heatmap": false, - "trajectory_max_points": 100, - "show_gesture_info": true, + "show_recording_status": true, + "font_size_large": 20, + "font_size_medium": 16, + "font_size_small": 14, + "show_calibration_info": true, "show_control_help": true, + "show_control_hints": true, "show_detection_overlay": true, - "theme": "dark", "show_direction": true, - "show_region_grid": true + "show_gesture_info": true, + "show_performance_stats": true, + "show_region_grid": true, + "show_trajectory_heatmap": false, + "theme": "dark", + "trajectory_max_points": 100, + "ui_opacity": 0.7 }, "performance": { "target_fps": 30, "resize_factor": 1.0, "enable_multiprocessing": false, + "enable_gpu_acceleration": false, + "enable_hardware_acceleration": true, "enable_multithreading": false, - "enable_hardware_acceleration": true + "max_queue_size": 10, + "memory_limit_mb": 500, + "process_time_critical": 100.0, + "process_time_warning": 50.0 }, "calibration": { "auto_calibrate_skin": true, "skin_calibration_frames": 30, - "hand_size_calibration": true + "hand_size_calibration": true, + "calibration_interval": 300, + "calibration_timeout": 10.0, + "enable_adaptive_calibration": true, + "min_calibration_samples": 50 }, "speech": { "enabled": true, @@ -99,36 +132,50 @@ "announce_gesture_changes": true, "announce_hand_status": true, "announce_performance": true, - "announce_recording": true, + "announce_recording_events": true, "min_gesture_confidence": 0.7, "gesture_start_threshold": 3, - "gesture_end_threshold": 10 + "gesture_end_threshold": 10, + "announce_gesture_directions": true, + "announce_recording": true, + "enable_short_feedback": true, + "max_queue_size": 5, + "speech_cooldown": 1.0, + "speech_language": "zh-cn" }, "recording": { + "auto_save_interval": 5, + "max_trajectory_points": 1000, + "show_trajectory": true, + "trajectory_thickness": 2, + "trajectory_max_length": 100, + "default_save_dir": "trajectories", "auto_save": false, "default_filename": "gesture_recording.pkl", - "playback_speed": 1.0, - "loop_playback": false + "loop_playback": false, + "playback_speed": 1.0 }, - "system": { - "version": "1.0", - "last_updated": "2025-12-19 20:51:44", - "debug_mode": false + "drone_control": { + "altitude": -10.0, + "base_speed": 2.0, + "control_delay": 0.3, + "emergency_stop_enabled": true, + "takeoff_height": 5.0 }, "gesture_detection": { + "confidence_threshold": 0.5, + "detection_interval": 1, + "max_hand_area": 30000, + "min_hand_area": 2000, "skin_hsv_ranges": { - "h_low": 0, "h_high": 25, - "s_low": 30, + "h_low": 0, "s_high": 255, - "v_low": 60, - "v_high": 255 + "s_low": 30, + "v_high": 255, + "v_low": 60 }, - "min_hand_area": 2000, - "max_hand_area": 30000, - "confidence_threshold": 0.5, - "smoothing_window": 5, - "detection_interval": 1 + "smoothing_window": 5 }, "gesture_mapping": { "0": "stop", @@ -140,11 +187,30 @@ "6": "right", "7": "hover" }, - "drone_control": { - "base_speed": 2.0, - "altitude": -10.0, - "control_delay": 0.3, - "takeoff_height": 5.0, - "emergency_stop_enabled": true + "keyboard": { + "connect_key": "c", + "emergency_key": "q", + "enable_keyboard_control": true, + "key_repeat_delay": 0.2, + "land_key": "l", + "takeoff_key": " " + }, + "system": { + "debug_mode": false, + "last_updated": "2025-12-19 20:51:44", + "version": "1.0" + }, + "training": { + "auto_save_training": true, + "enable_training_mode": false, + "samples_per_gesture": 50, + "training_gestures": [ + "Stop", + "Forward", + "Up", + "Down", + "Left", + "Right" + ] } } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index fe271047a5..c7a24d5374 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 手势识别优化版(增强语音反馈和手势平滑) -优化手势识别算法,增强语音反馈,改进手势平滑处理 +手势控制AirSim无人机 - 手势轨迹记录与回放优化版 +修复手势语音播报问题,确保所有手势语音正常 作者: xiaoshiyuan888 """ @@ -13,6 +13,7 @@ import math import threading import tempfile +import pickle # 新增:用于数据序列化 from datetime import datetime from PIL import Image, ImageDraw, ImageFont import cv2 @@ -20,8 +21,8 @@ from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Enhanced Gesture Recognition") -print("增强语音反馈和手势平滑处理!") +print("Gesture Controlled Drone - Enhanced with Trajectory Recording") +print("手势轨迹记录与回放优化版!") print("=" * 60) # ========== 修复导入路径 ========== @@ -209,12 +210,14 @@ def __init__(self, speech_lib): 'gesture_left': "向左", 'gesture_right': "向右", 'gesture_forward': "向前", + 'gesture_backward': "向后", # 新增:向后手势 'gesture_waiting': "等待手势", 'gesture_error': "手势识别错误", 'gesture_stable': "手势稳定", 'gesture_change': "手势变化", 'gesture_low_confidence': "手势识别置信度低", 'gesture_good_confidence': "手势识别置信度高", + 'gesture_hover': "悬停", # 新增:悬停手势 # 系统相关 'program_start': "手势控制无人机系统已启动", @@ -240,6 +243,19 @@ def __init__(self, speech_lib): 'good_position': "手部位置良好", 'hand_detected': "手部已检测到", 'hand_lost': "手部丢失,请重新放置", + + # 新增:录制相关 + 'recording_start': "开始录制手势轨迹", + 'recording_stop': "停止录制", + 'recording_saved': "轨迹已保存", + 'recording_loaded': "轨迹已加载", + 'recording_playback_start': "开始回放手势轨迹", + 'recording_playback_stop': "回放结束", + 'recording_cleared': "轨迹已清除", + 'recording_paused': "回放已暂停", + 'recording_resumed': "回放继续", + 'recording_not_found': "未找到轨迹数据", + 'recording_frame_count': "轨迹帧数", } # 初始化语音引擎 @@ -497,6 +513,341 @@ def get_status(self): } +# ========== 手势轨迹记录器 ========== +class GestureTrajectoryRecorder: + """手势轨迹记录器 - 记录、保存、加载和回放手势轨迹""" + + def __init__(self, speech_manager=None): + self.speech_manager = speech_manager + self.trajectory_data = [] + self.is_recording = False + self.is_playing = False + self.playback_index = 0 + self.playback_paused = False + self.max_trajectory_points = 1000 # 最大轨迹点数 + self.recording_start_time = 0 + self.last_save_time = 0 + self.save_interval = 5 # 自动保存间隔(秒) + + # 轨迹文件路径 + self.trajectory_dir = os.path.join(current_dir, 'trajectories') + if not os.path.exists(self.trajectory_dir): + os.makedirs(self.trajectory_dir) + + # 默认轨迹文件名 + self.default_filename = os.path.join(self.trajectory_dir, + f'trajectory_{datetime.now().strftime("%Y%m%d_%H%M%S")}.pkl') + + # 轨迹可视化设置 + self.trajectory_colors = [ + (255, 0, 0), # 红色 - 起点 + (0, 255, 0), # 绿色 - 正常点 + (0, 0, 255), # 蓝色 - 终点 + (255, 255, 0), # 青色 - 特殊点 + (255, 0, 255) # 紫色 - 特殊点 + ] + + self.show_trajectory = True + self.trajectory_thickness = 2 + self.trajectory_max_length = 100 # 显示的最大轨迹长度 + + print("✓ 手势轨迹记录器已初始化") + + def start_recording(self): + """开始录制手势轨迹""" + if self.is_recording: + return False + + self.trajectory_data = [] + self.is_recording = True + self.recording_start_time = time.time() + self.last_save_time = time.time() + + print("🎬 开始录制手势轨迹") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_start', immediate=True) + + return True + + def stop_recording(self): + """停止录制手势轨迹""" + if not self.is_recording: + return False + + self.is_recording = False + recording_duration = time.time() - self.recording_start_time + + print(f"⏹️ 停止录制手势轨迹") + print(f" 录制时长: {recording_duration:.1f}秒") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_stop', immediate=True) + if len(self.trajectory_data) > 0: + self.speech_manager.speak_direct(f"录制了{len(self.trajectory_data)}个轨迹点") + + return True + + def add_trajectory_point(self, hand_data, gesture, confidence, frame_shape): + """添加轨迹点""" + if not self.is_recording or len(self.trajectory_data) >= self.max_trajectory_points: + return False + + if hand_data is None: + return False + + # 创建轨迹点数据 + trajectory_point = { + 'timestamp': time.time(), + 'hand_position': hand_data['position'] if 'position' in hand_data else (0.5, 0.5), + 'hand_center': hand_data['center'] if 'center' in hand_data else (0, 0), + 'gesture': gesture, + 'confidence': confidence, + 'fingertips': hand_data.get('fingertips', []), + 'frame_shape': frame_shape + } + + self.trajectory_data.append(trajectory_point) + + # 自动保存检查 + current_time = time.time() + if current_time - self.last_save_time >= self.save_interval and len(self.trajectory_data) > 10: + self.auto_save() + self.last_save_time = current_time + + return True + + def auto_save(self): + """自动保存轨迹(临时文件)""" + if len(self.trajectory_data) == 0: + return + + temp_file = os.path.join(self.trajectory_dir, 'trajectory_temp.pkl') + try: + with open(temp_file, 'wb') as f: + pickle.dump(self.trajectory_data, f) + print(f"💾 自动保存轨迹到临时文件 ({len(self.trajectory_data)}个点)") + except Exception as e: + print(f"⚠ 自动保存轨迹失败: {e}") + + def save_trajectory(self, filename=None): + """保存轨迹到文件""" + if len(self.trajectory_data) == 0: + print("⚠ 没有轨迹数据可保存") + return False + + if filename is None: + filename = self.default_filename + + try: + with open(filename, 'wb') as f: + pickle.dump(self.trajectory_data, f) + + print(f"💾 轨迹已保存到: {filename}") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_saved', immediate=True) + self.speech_manager.speak_direct(f"保存了{len(self.trajectory_data)}个轨迹点") + + return True + except Exception as e: + print(f"❌ 保存轨迹失败: {e}") + return False + + def load_trajectory(self, filename): + """从文件加载轨迹""" + try: + if not os.path.exists(filename): + print(f"❌ 轨迹文件不存在: {filename}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_not_found', immediate=True) + + return False + + with open(filename, 'rb') as f: + self.trajectory_data = pickle.load(f) + + print(f"📂 轨迹已从文件加载: {filename}") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_loaded', immediate=True) + self.speech_manager.speak_direct(f"加载了{len(self.trajectory_data)}个轨迹点") + + return True + except Exception as e: + print(f"❌ 加载轨迹失败: {e}") + return False + + def start_playback(self): + """开始回放轨迹""" + if len(self.trajectory_data) == 0: + print("⚠ 没有轨迹数据可回放") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_not_found', immediate=True) + + return False + + self.is_playing = True + self.playback_index = 0 + self.playback_paused = False + + print(f"▶️ 开始回放手势轨迹") + print(f" 总帧数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_playback_start', immediate=True) + + return True + + def stop_playback(self): + """停止回放轨迹""" + if not self.is_playing: + return False + + self.is_playing = False + self.playback_paused = False + + print("⏹️ 停止回放手势轨迹") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_playback_stop', immediate=True) + + return True + + def pause_playback(self): + """暂停/继续回放""" + self.playback_paused = not self.playback_paused + + status = "暂停" if self.playback_paused else "继续" + print(f"⏸️ 回放已{status}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + if self.playback_paused: + self.speech_manager.speak('recording_paused', immediate=True) + else: + self.speech_manager.speak('recording_resumed', immediate=True) + + return self.playback_paused + + def get_next_playback_point(self): + """获取下一个回放点""" + if not self.is_playing or self.playback_paused or len(self.trajectory_data) == 0: + return None + + if self.playback_index >= len(self.trajectory_data): + self.stop_playback() + return None + + point = self.trajectory_data[self.playback_index] + self.playback_index += 1 + + # 检查是否到达末尾 + if self.playback_index >= len(self.trajectory_data): + self.stop_playback() + + return point + + def clear_trajectory(self): + """清除轨迹数据""" + self.trajectory_data = [] + self.is_recording = False + self.is_playing = False + self.playback_index = 0 + + print("🗑️ 轨迹数据已清除") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_cleared', immediate=True) + + return True + + def draw_trajectory(self, frame): + """在帧上绘制轨迹""" + if not self.show_trajectory or len(self.trajectory_data) == 0: + return frame + + h, w = frame.shape[:2] + + # 限制显示的轨迹点数 + display_points = min(len(self.trajectory_data), self.trajectory_max_length) + start_idx = max(0, len(self.trajectory_data) - display_points) + + # 绘制轨迹线 + for i in range(start_idx, len(self.trajectory_data) - 1): + point1 = self.trajectory_data[i] + point2 = self.trajectory_data[i + 1] + + # 获取手部中心位置(转换为图像坐标) + if 'hand_center' in point1 and 'hand_center' in point2: + x1, y1 = point1['hand_center'] + x2, y2 = point2['hand_center'] + + # 确保坐标在图像范围内 + x1 = max(0, min(w - 1, x1)) + y1 = max(0, min(h - 1, y1)) + x2 = max(0, min(w - 1, x2)) + y2 = max(0, min(h - 1, y2)) + + # 根据索引计算颜色(渐变色) + color_idx = int((i - start_idx) / display_points * (len(self.trajectory_colors) - 1)) + color = self.trajectory_colors[color_idx] + + # 绘制线条 + cv2.line(frame, (int(x1), int(y1)), (int(x2), int(y2)), + color, self.trajectory_thickness) + + # 绘制当前点(如果正在录制或回放) + if self.is_recording or self.is_playing: + current_idx = len(self.trajectory_data) - 1 if self.is_recording else self.playback_index - 1 + if 0 <= current_idx < len(self.trajectory_data): + point = self.trajectory_data[current_idx] + if 'hand_center' in point: + x, y = point['hand_center'] + x = max(0, min(w - 1, x)) + y = max(0, min(h - 1, y)) + + # 绘制当前点 + cv2.circle(frame, (int(x), int(y)), 8, (0, 255, 255), -1) + cv2.circle(frame, (int(x), int(y)), 8, (0, 0, 0), 2) + + return frame + + def get_status(self): + """获取录制状态""" + return { + 'is_recording': self.is_recording, + 'is_playing': self.is_playing, + 'playback_paused': self.playback_paused, + 'trajectory_points': len(self.trajectory_data), + 'playback_index': self.playback_index, + 'playback_total': len(self.trajectory_data), + 'recording_duration': time.time() - self.recording_start_time if self.is_recording else 0 + } + + def list_saved_trajectories(self): + """列出保存的轨迹文件""" + try: + files = [f for f in os.listdir(self.trajectory_dir) if f.endswith('.pkl')] + return sorted(files, reverse=True) + except: + return [] + + # ========== 配置管理器 ========== class ConfigManager: """配置管理器""" @@ -551,6 +902,8 @@ def __init__(self): 'show_speech_status': True, 'show_gesture_history': True, # 新增:显示手势历史 'show_stability_indicator': True, # 新增:显示稳定性指示器 + 'show_trajectory': True, # 新增:显示轨迹 + 'show_recording_status': True, # 新增:显示录制状态 }, 'performance': { 'target_fps': 30, @@ -572,9 +925,18 @@ def __init__(self): 'announce_gesture_changes': True, # 新增:播报手势变化 'announce_hand_status': True, # 新增:播报手部状态 'announce_performance': True, # 新增:播报性能状态 + 'announce_recording_events': True, # 新增:播报录制事件 'min_gesture_confidence': 0.7, 'gesture_start_threshold': 3, # 新增:手势开始识别阈值 'gesture_end_threshold': 10, # 新增:手势结束识别阈值 + }, + 'recording': { # 新增:录制配置 + 'auto_save_interval': 5, # 自动保存间隔(秒) + 'max_trajectory_points': 1000, # 最大轨迹点数 + 'show_trajectory': True, # 显示轨迹 + 'trajectory_thickness': 2, # 轨迹线粗细 + 'trajectory_max_length': 100, # 显示的最大轨迹长度 + 'default_save_dir': 'trajectories', # 默认保存目录 } } self.config = self.load_config() @@ -746,7 +1108,7 @@ def __init__(self, speech_manager=None): "Hover": (255, 255, 255) # 白色 } - # 手势到语音的映射 + # 手势到语音的映射 - 修复:确保所有手势都有对应语音 self.gesture_speech_map = { "Stop": "gesture_stop", "Forward": "gesture_forward", @@ -756,6 +1118,7 @@ def __init__(self, speech_manager=None): "Right": "gesture_right", "Waiting": "gesture_waiting", "Error": "gesture_error", + "Hover": "gesture_hover", # 新增悬停手势语音 } # 手势状态颜色 @@ -779,6 +1142,9 @@ def __init__(self, speech_manager=None): self.frame_rate = 0 self.last_fps_check = time.time() + # 新增:存储手部数据用于轨迹记录 + self.last_hand_data = None + print("✓ 增强的手势识别器已初始化 (改进平滑处理)") def get_skin_mask(self, frame): @@ -1407,6 +1773,9 @@ def recognize(self, frame): # 分析手部特征 hand_data, confidence = self.analyze_hand_features(hand_contour, processed_frame.shape) + # 保存手部数据用于轨迹记录 + self.last_hand_data = hand_data + # 识别手势 if hand_data is not None: # 校准肤色(如果需要) @@ -1429,7 +1798,8 @@ def recognize(self, frame): self.update_gesture_state(None, gesture, confidence) final_gesture, final_confidence = gesture, confidence - # 手势语音提示 + # ========== 修复手势语音播报 ========== + # 确保所有手势都有语音反馈 if (self.speech_manager and config.get('speech', 'enabled') and config.get('speech', 'announce_gestures')): @@ -1444,14 +1814,20 @@ def recognize(self, frame): self.speech_manager.speak('gesture_low_confidence', immediate=True) self.last_hand_status_time = current_time - # 手势变化播报 - if (final_gesture != self.last_announced_gesture and - final_gesture not in ["Waiting", "Error"] and + # 手势语音播报 - 修复:确保所有手势都能播报 + if (final_gesture != "Waiting" and + final_gesture != "Error" and + final_gesture != "摄像头错误" and final_confidence >= config.get('speech', 'min_gesture_confidence') and current_time - self.last_announced_time > self.gesture_announce_interval): + # 检查手势是否在语音映射中 if final_gesture in self.gesture_speech_map: - self.speech_manager.speak(self.gesture_speech_map[final_gesture]) + speech_key = self.gesture_speech_map[final_gesture] + self.speech_manager.speak(speech_key) + else: + # 如果手势不在映射中,直接播报手势名称 + self.speech_manager.speak_direct(f"手势{final_gesture}") self.last_announced_gesture = final_gesture self.last_announced_time = current_time @@ -1512,12 +1888,17 @@ def set_simulated_gesture(self, gesture): self.current_gesture = gesture self.current_confidence = 0.9 - # 模拟手势也触发语音提示 + # 模拟手势也触发语音提示 - 修复:确保所有模拟手势都有语音 if (self.speech_manager and config.get('speech', 'enabled') and - config.get('speech', 'announce_gestures') and - gesture in self.gesture_speech_map): - self.speech_manager.speak(self.gesture_speech_map[gesture]) + config.get('speech', 'announce_gestures')): + + if gesture in self.gesture_speech_map: + self.speech_manager.speak(self.gesture_speech_map[gesture]) + else: + # 如果手势不在映射中,直接播报手势名称 + self.speech_manager.speak_direct(f"手势{gesture}") + self.last_announced_gesture = gesture self.last_announced_time = time.time() @@ -1837,6 +2218,9 @@ def __init__(self, speech_manager=None): 'performance_good': (0, 255, 0), # 绿色 'performance_warning': (255, 165, 0), # 橙色 'performance_bad': (255, 0, 0), # 红色 + 'recording': (255, 50, 50), # 亮红色 + 'playback': (50, 50, 255), # 亮蓝色 + 'paused': (255, 255, 0), # 黄色 } print("✓ 中文UI渲染器已初始化") @@ -1890,7 +2274,8 @@ def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): size / 25, color, 1) return frame - def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time): + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, + trajectory_recorder=None): """绘制状态栏""" h, w = frame.shape[:2] @@ -1900,7 +2285,7 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 增强版" + title = "手势控制无人机系统 - 轨迹记录版" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -1927,12 +2312,32 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) + # 录制/回放状态 + if trajectory_recorder and config.get('display', 'show_recording_status'): + recorder_status = trajectory_recorder.get_status() + + if recorder_status['is_recording']: + status_color = self.colors['recording'] + status_text = f"录制中: {recorder_status['trajectory_points']}点" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=16, color=status_color) + elif recorder_status['is_playing']: + if recorder_status['playback_paused']: + status_color = self.colors['paused'] + status_text = f"回放暂停: {recorder_status['playback_index']}/{recorder_status['playback_total']}" + else: + status_color = self.colors['playback'] + status_text = f"回放中: {recorder_status['playback_index']}/{recorder_status['playback_total']}" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=16, color=status_color) + else: + status_text = f"轨迹点: {recorder_status['trajectory_points']}" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=self.colors['info']) + # 语音状态 if config.get('display', 'show_speech_status') and self.speech_manager: speech_status = self.speech_manager.get_status() speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" - frame = self.draw_text(frame, speech_text, (w // 2, 65), size=16, color=speech_color) + frame = self.draw_text(frame, speech_text, (w // 2, 90), size=14, color=speech_color) # 性能信息 if config.get('display', 'show_fps'): @@ -1966,17 +2371,18 @@ def draw_help_bar(self, frame): h, w = frame.shape[:2] # 绘制底部帮助栏 - cv2.rectangle(frame, (0, h - 80), (w, h), (0, 0, 0), -1) + cv2.rectangle(frame, (0, h - 100), (w, h), (0, 0, 0), -1) # 帮助文本 help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音 P:性能报告" + "V:切换语音反馈 M:测试语音 P:性能报告", + "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" ] for i, line in enumerate(help_lines): - y_pos = h - 65 + i * 20 + y_pos = h - 85 + i * 20 frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame @@ -2055,6 +2461,10 @@ def main(): ui_renderer = ChineseUIRenderer(speech_manager) performance_monitor = PerformanceMonitor() + # 新增:手势轨迹记录器 + print("初始化手势轨迹记录器...") + trajectory_recorder = GestureTrajectoryRecorder(speech_manager) + # 初始化摄像头 cap = None try: @@ -2095,12 +2505,13 @@ def main(): # 显示欢迎信息 print("\n" + "=" * 60) - print("手势控制无人机系统 - 增强版") + print("手势控制无人机系统 - 轨迹记录版") print("=" * 60) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") + print(f" 轨迹记录: 支持录制/回放功能") print(f" 手势状态: 支持开始/活跃/结束状态追踪") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) @@ -2109,18 +2520,23 @@ def main(): print("\n操作说明:") print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") - print("3. 手势控制改进:") - print(" - 系统会自动检测手势开始、稳定和结束状态") + print("3. 手势控制:") + print(" - 手势识别后会有语音提示: 向上、向下、向左、向右、向前、停止") print(" - 手势稳定性越高,识别越准确") print(" - 手部距离摄像头适中时效果最佳") print(" * 手势识别置信度 > 60% 时才会执行") - print("4. 键盘控制:") - print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") - print("5. 调试功能:") + print("4. 轨迹记录功能:") + print(" [1]开始录制 [2]停止录制 [3]保存轨迹 [4]回放轨迹 [5]清除轨迹 [6]暂停/继续") + print(" - 录制时系统会记录手部位置和手势") + print(" - 回放时可以查看录制的轨迹") + print(" - 轨迹数据自动保存到 trajectories/ 目录") + print("5. 键盘控制:") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print("6. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") - print("6. 语音控制:") + print("7. 语音控制:") print(" [V]切换语音反馈 [M]测试语音 [P]性能报告") - print("7. 按 [ESC] 安全退出") + print("8. 按 [ESC] 安全退出") print("=" * 60) print("程序启动成功!") print("-" * 60) @@ -2133,6 +2549,7 @@ def main(): ord('d'): "Right", ord('D'): "Right", ord('f'): "Forward", ord('F'): "Forward", ord('x'): "Stop", ord('X'): "Stop", + ord('h'): "Hover", ord('H'): "Hover", } # 显示模式 @@ -2173,6 +2590,8 @@ def main(): config.set('display', 'show_fingertips', value=True) config.set('display', 'show_gesture_history', value=True) config.set('display', 'show_stability_indicator', value=True) + config.set('display', 'show_trajectory', value=True) + config.set('display', 'show_recording_status', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -2182,6 +2601,8 @@ def main(): config.set('display', 'show_hand_direction', value=True) config.set('display', 'show_gesture_history', value=True) config.set('display', 'show_stability_indicator', value=True) + config.set('display', 'show_trajectory', value=True) + config.set('display', 'show_recording_status', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) @@ -2189,12 +2610,18 @@ def main(): config.set('display', 'show_fingertips', value=False) config.set('display', 'show_gesture_history', value=False) config.set('display', 'show_stability_indicator', value=False) + config.set('display', 'show_trajectory', value=True) + config.set('display', 'show_recording_status', value=True) config.set('display', 'show_debug_info', value=False) + # 绘制轨迹(如果启用) + if config.get('display', 'show_trajectory'): + frame = trajectory_recorder.draw_trajectory(frame) + # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, - perf_stats['fps'], process_time + perf_stats['fps'], process_time, trajectory_recorder ) frame = ui_renderer.draw_help_bar(frame) @@ -2205,7 +2632,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Enhanced', frame) + cv2.imshow('Gesture Controlled Drone - Trajectory Recording', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -2296,6 +2723,49 @@ def main(): else: speech_manager.speak_direct("系统性能警告,请检查") + # 新增:轨迹记录控制 + elif key == ord('1'): + # 开始录制轨迹 + if trajectory_recorder.start_recording(): + print("✅ 开始录制手势轨迹") + else: + print("⚠ 已经在录制中") + + elif key == ord('2'): + # 停止录制轨迹 + if trajectory_recorder.stop_recording(): + print("✅ 停止录制手势轨迹") + else: + print("⚠ 当前没有在录制") + + elif key == ord('3'): + # 保存轨迹 + if trajectory_recorder.save_trajectory(): + print("✅ 轨迹保存成功") + else: + print("⚠ 没有轨迹数据可保存") + + elif key == ord('4'): + # 回放轨迹 + if trajectory_recorder.start_playback(): + print("✅ 开始回放手势轨迹") + else: + print("⚠ 没有轨迹数据可回放") + + elif key == ord('5'): + # 清除轨迹 + if trajectory_recorder.clear_trajectory(): + print("✅ 轨迹数据已清除") + else: + print("⚠ 清除轨迹失败") + + elif key == ord('6'): + # 暂停/继续回放 + if trajectory_recorder.pause_playback(): + print("✅ 切换回放暂停状态") + else: + print("⚠ 当前没有在回放") + elif key in key_to_gesture: # 键盘控制 simulated_gesture = key_to_gesture[key] @@ -2312,6 +2782,24 @@ def main(): drone_controller.connected and drone_controller.flying): drone_controller.move_by_gesture(gesture, confidence) + # 处理轨迹记录 + if cap and ret: + # 如果正在录制,添加轨迹点 + if trajectory_recorder.is_recording: + # 获取手势识别的手部数据 + if hasattr(gesture_recognizer, 'last_hand_data'): + hand_data = gesture_recognizer.last_hand_data + trajectory_recorder.add_trajectory_point( + hand_data, gesture, confidence, frame.shape + ) + + # 如果正在回放,获取回放点 + if trajectory_recorder.is_playing and not trajectory_recorder.playback_paused: + playback_point = trajectory_recorder.get_next_playback_point() + if playback_point: + # 这里可以添加回放点的可视化或处理 + pass + except KeyboardInterrupt: print("\n程序被用户中断") except Exception as e: @@ -2330,6 +2818,12 @@ def main(): time.sleep(1) # 确保语音播报完成 drone_controller.emergency_stop() + + # 自动保存轨迹 + if trajectory_recorder and len(trajectory_recorder.trajectory_data) > 0: + print("自动保存轨迹数据...") + trajectory_recorder.save_trajectory() + config.save_config() print("程序安全退出") diff --git a/src/autonomus_drone_hand_gesture_project/trajectories/trajectory_20251221_085734.pkl b/src/autonomus_drone_hand_gesture_project/trajectories/trajectory_20251221_085734.pkl new file mode 100644 index 0000000000000000000000000000000000000000..2e15536f58fda247e254762d50bee730511e0186 GIT binary patch literal 5540 zcmchbYitx%6vsQ;F0>RSXm}KqheD{dHiTyb$bu+~U8jIh1VgDyyJdFU-IncEK!a32 zloD1{7*sGsi6$UaX%`rk+5z1fH>B2_0)BGYB(w6^?S-fZhxunoOOP|nf=uRgW=mWcc37zw4}@% z@|Fa2r{fqu$LY9Kg>7S{9!_S*_e!`_dC;SW6BdR_O7(D`{E|SS zx4;v~hp_>Ha)0UCjQo}hNbPR9?2uzd;w?XDAQsscZ5w&iWo!Bt+Skg#5( z0iGXWV98|HDBJdc>_s3u$GD-kSR%W|zCV-6el#k(?K-v_O!jB$Bi6Ei5oHw^vbfH6 zG4Jo5e@V4Dqwp7+h@Wql3`EzW*uU;9hiX4>FA)fL9?q7MRi+q zpCTa~b2NoCT_4Z@&mX0b@_T52=Nl-b{7xF+`PV6=d<_lod~XWnRa`SQ@kQ_@48FWz z@@2Ekm$^UN(3f^9ht_*o0|9pRu%c#I`|>3T;kqu+^pWnj#QZrUFP+b<3nxJLhoJi- zlkS5u-PvbrneIZ?;OR5USglvoG;7^wL|r%Oen-q-?t$+44GEz86zD!}(%m4_z3AW= z)ZIb4n~YUM{aSHd2M2FnHCjYjLntJ>zoG#?9kVH<{4pBf`G+Z_{9zj4`ELzuK2Rs- z4;p#tbcVO3A?MkY=sTko&VdGZw7ZU|>DJTp7aU=Ft*xiqlFln)evXls>e#H0)Z%@K zaHMAUC1Dq{*VhL_>vvwwmffS6ezsh|deOQU+dihhUY*=cKey>`QQotLJh$vNdi5HR zcQ=0T91iUu!w*d&yiTZI*(TT64(+v#ZnxclR0p27jnT`DGcBzMJ=)h*hPvs!21*u zz$y`7ypfjz^#5@>0OX)lyKgae2EgMON&p7m7$g96T*P*X01B-Eib)X%$fA${%0+;2 zMqUaq0BE02tZEg;Yd^ zbmp+eoV=>2erqSEiB9DjH{#ZAp#dJEKWuFzryb(?la0L8=`l-E!D+lOIK34+!)YBu zNvVvz7-3G+ORJOrV1S*98n6a@nG{I?W{3c$Qj!9sx19rk_c*|Q5dil}GJtX4=P-c2 zY>|mAI}G5mqOR%@fR}hAGqBLeO993=KLY@tqg1;OU(2Q$IF6xY1~MY?T23 z(+o68XCOs%c!eR5Pf5I(f2Di=T^itrV~GeQxSIxeekp}yP%rWNxSU4Mml`y+WjPr1 z9UKV;HDYI!=y?n!2FrGb{z1**FaV@xlOz z6JP>Nz)%AC)cPq1*xt@&9so@n$CHX$W?jOo$#SkY_DIqUrHcTjoud?B&C^`~FpL8X z7Xe1d0U~p%zF;#%FU%J4h1uU4V7(}?6XsVa!EX5nh?f92(Ev9fk3#vDCNlTbF#w#v z0Vj%plWqa1PI?0YKjvIcN5r&cbhz#XMX^$5UwYuf4OT0|;p&D{2;Xrebr6|1rUlf^ zK5EW-NJFr|AV@m%xX}V%hfFx)#cqn-I9t1RCl#8x*cpmXQKPr6q|axb#WBJ z6;1Zx_FgECfSAYsL85tCbw`%nf%X=mRJ*SjyV#`p{1{q4o{=X96(GQsaP$q+WZH~$ zDK7lJ61!0BBU$Wm)(8mWpAs`+;K_L~lN8}sTJ+F!R9(+ECtWf3we+81-ea~SR NlkIyRLo>f8;~#jwO6>pu literal 0 HcmV?d00001 diff --git a/src/autonomus_drone_hand_gesture_project/trajectories/trajectory_temp.pkl b/src/autonomus_drone_hand_gesture_project/trajectories/trajectory_temp.pkl new file mode 100644 index 0000000000000000000000000000000000000000..03d88104f34a56b85d410659b9b7749aaba9edb9 GIT binary patch literal 4719 zcmbW4Yitx%6vsQ;9cU?rpy91NjD$*SLqLH5vLMQ0*J)HJN+Ob_-7>qd+p@bA&>(y& zC5)(WtAZiXXaWLC3qDA&P!TWzYJ#W;$kV<=0zP7*(TL~VyQg>VGB*O@tvGLoo( zT)#wZyk^(_bbTn(@~nCJAWiY69ClIdtoip^(sZ@Z7< zi{3AvV&CEM3_lX``{9N=XZTBuOjyUlPC->N9FHtW*F48_S>f2!|rM4+4=SWd!RkY9_c;3`LEMZ#Qpb52SlP?tv6Fx$09n+1Aj)rmC@iz zkp!}fb<=Nb>W{9RJdUF^82Diyj!!|L89HJW8t})>w!BL^gvv`464o1(5cS~_ODDV0 zXxIj_=YZ^T`9gcHMs}mQYa*BZSVngHH5}Ku?9Z%6y=DKZ%BqyIgw77N9?!17qSgc1 z_4R7~K3Ugd7wo(O*ipcaxv&@OuqS-860zI&jKdKObkg2no7Qf_3VU-OBOwBFB!$$j zk0>GP2PvfbHcE(kD}_|wLJ3iSlR~ODQbN>wQm9|Wjg#_T245WTvg`&`o)L7 zbkK5WyPFRXU|RbPtHj%vFG+~db(Y#ky5CXjr(|8*&(gC;K=;R>`xBS$y*k}xC!4tL zDn8)XCzr9+Vpv7qx=*OOZqogpTEBD)x*u=#gYILX`?yPYt4{Zvy`xZfC+WT*cMa=n zBXkWMym{BStjZcpA=&*EB}6-BP)PMdlo0htD5Uy+N{ITm5?dsi)%spp*S6DXC_>JY zgED7E8{7i}-e`snSjFD$`5R`qj`z2s<~H2QzImmXDWidt<#Y z^gi<%H*CF!>*w1A>=$j@aqQswTdeU}`h`vZsPbNv^1`~;u${|6-d#f8P*vVAy}YKJ zqcc%n`@D?3=nM>$XYDYoWm)<%hjJS!5_(ebZ!un^ zzqpNSz?<@Z?}q?gs2{&6y6`wU-pT>48dlmH;4M-l1n!}b0G{zF3uuC@>lR2;!37Hd zrlC|bHXSb#;2{jX7f4f|iya(5=mr3>QaHfRhPBum;B8VQ06wIU0G6r%V`W_n(D$b) z08oxn&DdPLNPwp>)Bp^;K0pEJJcr{v0aSSdEFnb#pp-%as8s>R$hsC_@E>ge5JRbE zY%yLW0KQHKFmxS!0f6x>eCflU>H@fKSpO3ss{`Y10boSC5VATWq^q0{?&KB2ihDab zS#@f@e37tr10_U={_wq#oDQh>kC%0=)1#g)1gG)A;PghkNT(Twnx!)O+-~kP9a>%D z4-VL6SP5^yS4oiuV5$n>T1r}g;`UPj@PPo>r2^o$k`7?Z_vIX*H{WEsw;c{}$*`8* z5so-Z7heE?&rzxw!(-Xifx{STIxz7(KaM(oQDeon`j4T&=yML0@p<)%l_2sAqSpNJC02nF&hN%F<^#HqP)_=h} oLLE Date: Sun, 21 Dec 2025 12:15:47 +0800 Subject: [PATCH 11/18] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E9=80=89=E6=8B=A9=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config.json | 27 +- .../main.py | 534 +++++++++++++----- 2 files changed, 410 insertions(+), 151 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index 1ebcdda50e..40656680c9 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -18,25 +18,25 @@ }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 125, + "skin_upper_h": 165, "skin_lower_s": 0, - "skin_upper_s": 31, + "skin_upper_s": 36, "skin_lower_v": 0, - "skin_upper_v": 74, + "skin_upper_v": 69, "min_hand_area": 3000, "max_hand_area": 30000, - "history_size": 10, - "smooth_frames": 5, - "min_confidence": 0.6, - "detection_interval": 2, "hand_ratio_threshold": 1.5, - "contour_simplify_epsilon": 0.02, "defect_distance_threshold": 20, "palm_circle_radius_ratio": 0.3, - "gesture_stability_threshold": 5, "transition_threshold": 0.3, "position_stability_weight": 0.4, "gesture_cooldown": 0.5, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.6, + "detection_interval": 2, + "contour_simplify_epsilon": 0.02, + "gesture_stability_threshold": 5, "adaptive_learning_rate": 0.1, "enable_background_subtraction": true, "enable_dynamic_threshold": true, @@ -85,6 +85,7 @@ "show_stability_indicator": true, "show_trajectory": true, "show_recording_status": true, + "show_performance_mode": true, "font_size_large": 20, "font_size_medium": 16, "font_size_small": 14, @@ -105,6 +106,13 @@ "target_fps": 30, "resize_factor": 1.0, "enable_multiprocessing": false, + "mode": "accurate", + "current_mode_index": 2, + "modes": [ + "fast", + "balanced", + "accurate" + ], "enable_gpu_acceleration": false, "enable_hardware_acceleration": true, "enable_multithreading": false, @@ -133,6 +141,7 @@ "announce_hand_status": true, "announce_performance": true, "announce_recording_events": true, + "announce_performance_mode": true, "min_gesture_confidence": 0.7, "gesture_start_threshold": 3, "gesture_end_threshold": 10, diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index c7a24d5374..3076402e7a 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 手势轨迹记录与回放优化版 -修复手势语音播报问题,确保所有手势语音正常 +手势控制AirSim无人机 - 性能模式选择优化版 +新增fast(最快)、balanced(平衡)、accurate(最准)三种性能模式 作者: xiaoshiyuan888 """ @@ -21,8 +21,8 @@ from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Enhanced with Trajectory Recording") -print("手势轨迹记录与回放优化版!") +print("Gesture Controlled Drone - Performance Mode Selection") +print("性能模式选择优化版!") print("=" * 60) # ========== 修复导入路径 ========== @@ -210,14 +210,14 @@ def __init__(self, speech_lib): 'gesture_left': "向左", 'gesture_right': "向右", 'gesture_forward': "向前", - 'gesture_backward': "向后", # 新增:向后手势 + 'gesture_backward': "向后", 'gesture_waiting': "等待手势", 'gesture_error': "手势识别错误", 'gesture_stable': "手势稳定", 'gesture_change': "手势变化", 'gesture_low_confidence': "手势识别置信度低", 'gesture_good_confidence': "手势识别置信度高", - 'gesture_hover': "悬停", # 新增:悬停手势 + 'gesture_hover': "悬停", # 系统相关 'program_start': "手势控制无人机系统已启动", @@ -232,6 +232,9 @@ def __init__(self, speech_lib): 'debug_mode_off': "调试模式已关闭", 'display_mode_changed': "显示模式已切换", 'help_toggled': "帮助信息已切换", + 'performance_mode_fast': "切换到最快性能模式", + 'performance_mode_balanced': "切换到平衡性能模式", + 'performance_mode_accurate': "切换到最准确性能模式", # 新增:性能相关 'performance_good': "系统运行流畅", @@ -854,6 +857,53 @@ class ConfigManager: def __init__(self): self.config_file = os.path.join(current_dir, 'gesture_config.json') + + # 性能模式配置 + self.performance_modes = { + 'fast': { + 'name': '最快', + 'description': '性能优先,降低识别精度换取更高帧率', + 'detection_interval': 2, # 每2帧检测一次 + 'smooth_frames': 3, # 平滑帧数 + 'min_confidence': 0.5, # 最小置信度阈值 + 'resize_factor': 0.5, # 图像缩放因子 + 'skin_detection_enabled': True, + 'background_subtraction_enabled': False, + 'contour_simplify_epsilon': 0.03, + 'history_size': 10, + 'gesture_stability_threshold': 3, + 'color': (0, 255, 0), # 绿色 + }, + 'balanced': { + 'name': '平衡', + 'description': '平衡性能与精度,适用于大多数场景', + 'detection_interval': 1, # 每帧检测 + 'smooth_frames': 5, # 平滑帧数 + 'min_confidence': 0.6, # 最小置信度阈值 + 'resize_factor': 0.75, # 图像缩放因子 + 'skin_detection_enabled': True, + 'background_subtraction_enabled': True, + 'contour_simplify_epsilon': 0.02, + 'history_size': 15, + 'gesture_stability_threshold': 5, + 'color': (255, 165, 0), # 橙色 + }, + 'accurate': { + 'name': '最准', + 'description': '精度优先,提供最准确的手势识别', + 'detection_interval': 1, # 每帧检测 + 'smooth_frames': 7, # 平滑帧数 + 'min_confidence': 0.7, # 最小置信度阈值 + 'resize_factor': 1.0, # 不缩放 + 'skin_detection_enabled': True, + 'background_subtraction_enabled': True, + 'contour_simplify_epsilon': 0.01, + 'history_size': 20, + 'gesture_stability_threshold': 7, + 'color': (255, 0, 0), # 红色 + } + } + self.default_config = { 'camera': { 'index': 0, @@ -870,18 +920,12 @@ def __init__(self): 'skin_upper_v': 255, 'min_hand_area': 2000, 'max_hand_area': 30000, - 'history_size': 15, # 增加历史记录大小 - 'smooth_frames': 7, # 增加平滑帧数 - 'min_confidence': 0.6, # 提高最小置信度阈值 - 'detection_interval': 1, 'hand_ratio_threshold': 1.5, - 'contour_simplify_epsilon': 0.02, 'defect_distance_threshold': 20, 'palm_circle_radius_ratio': 0.3, - 'gesture_stability_threshold': 5, # 新增:手势稳定性阈值 - 'transition_threshold': 0.3, # 新增:手势转换阈值 - 'position_stability_weight': 0.4, # 新增:位置稳定性权重 - 'gesture_cooldown': 0.5, # 新增:手势冷却时间 + 'transition_threshold': 0.3, + 'position_stability_weight': 0.4, + 'gesture_cooldown': 0.5, }, 'drone': { 'velocity': 2.5, @@ -900,15 +944,19 @@ def __init__(self): 'show_hand_direction': True, 'show_debug_info': False, 'show_speech_status': True, - 'show_gesture_history': True, # 新增:显示手势历史 - 'show_stability_indicator': True, # 新增:显示稳定性指示器 - 'show_trajectory': True, # 新增:显示轨迹 - 'show_recording_status': True, # 新增:显示录制状态 + 'show_gesture_history': True, + 'show_stability_indicator': True, + 'show_trajectory': True, + 'show_recording_status': True, + 'show_performance_mode': True, # 新增:显示性能模式 }, 'performance': { 'target_fps': 30, 'resize_factor': 1.0, - 'enable_multiprocessing': False + 'enable_multiprocessing': False, + 'mode': 'balanced', # 新增:性能模式选择 + 'current_mode_index': 1, # 当前模式索引 + 'modes': ['fast', 'balanced', 'accurate'], # 可用模式列表 }, 'calibration': { 'auto_calibrate_skin': True, @@ -922,21 +970,22 @@ def __init__(self): 'announce_gestures': True, 'announce_connections': True, 'announce_flight_events': True, - 'announce_gesture_changes': True, # 新增:播报手势变化 - 'announce_hand_status': True, # 新增:播报手部状态 - 'announce_performance': True, # 新增:播报性能状态 - 'announce_recording_events': True, # 新增:播报录制事件 + 'announce_gesture_changes': True, + 'announce_hand_status': True, + 'announce_performance': True, + 'announce_recording_events': True, + 'announce_performance_mode': True, # 新增:播报性能模式切换 'min_gesture_confidence': 0.7, - 'gesture_start_threshold': 3, # 新增:手势开始识别阈值 - 'gesture_end_threshold': 10, # 新增:手势结束识别阈值 + 'gesture_start_threshold': 3, + 'gesture_end_threshold': 10, }, - 'recording': { # 新增:录制配置 - 'auto_save_interval': 5, # 自动保存间隔(秒) - 'max_trajectory_points': 1000, # 最大轨迹点数 - 'show_trajectory': True, # 显示轨迹 - 'trajectory_thickness': 2, # 轨迹线粗细 - 'trajectory_max_length': 100, # 显示的最大轨迹长度 - 'default_save_dir': 'trajectories', # 默认保存目录 + 'recording': { + 'auto_save_interval': 5, + 'max_trajectory_points': 1000, + 'show_trajectory': True, + 'trajectory_thickness': 2, + 'trajectory_max_length': 100, + 'default_save_dir': 'trajectories', } } self.config = self.load_config() @@ -1002,6 +1051,56 @@ def set(self, *keys, value): config[keys[-1]] = value self.save_config() + def get_performance_mode_config(self, mode=None): + """获取性能模式配置""" + if mode is None: + mode = self.get('performance', 'mode') + + if mode in self.performance_modes: + return self.performance_modes[mode] + else: + return self.performance_modes['balanced'] + + def get_current_performance_mode(self): + """获取当前性能模式""" + mode = self.get('performance', 'mode') + if mode in self.performance_modes: + return mode + return 'balanced' + + def set_performance_mode(self, mode): + """设置性能模式""" + if mode in self.performance_modes: + self.set('performance', 'mode', value=mode) + + # 更新当前模式索引 + modes = self.get('performance', 'modes') + if modes and mode in modes: + index = modes.index(mode) + self.set('performance', 'current_mode_index', value=index) + + print(f"✓ 性能模式设置为: {self.performance_modes[mode]['name']}") + return True + return False + + def cycle_performance_mode(self): + """循环切换性能模式""" + modes = self.get('performance', 'modes') + if not modes: + modes = ['fast', 'balanced', 'accurate'] + + current_index = self.get('performance', 'current_mode_index') + if current_index is None: + current_index = 0 + + # 计算下一个模式索引 + next_index = (current_index + 1) % len(modes) + next_mode = modes[next_index] + + # 设置新模式 + self.set('performance', 'current_mode_index', value=next_index) + return self.set_performance_mode(next_mode) + def calibrate_skin_color(self, frame, hand_mask): """自动校准肤色范围""" if not self.get('calibration', 'auto_calibrate_skin'): @@ -1052,24 +1151,33 @@ def calibrate_hand_size(self, hand_area): config = ConfigManager() -# ========== 改进的手势识别器(增强平滑处理) ========== +# ========== 改进的手势识别器(支持性能模式) ========== class EnhancedGestureRecognizer: - """增强的手势识别器 - 改进平滑处理和稳定性""" + """增强的手势识别器 - 支持性能模式选择""" def __init__(self, speech_manager=None): - self.config = config.get('gesture') self.speech_manager = speech_manager + # 加载性能模式配置 + self.performance_mode = config.get_current_performance_mode() + self.mode_config = config.get_performance_mode_config(self.performance_mode) + + # 根据性能模式初始化参数 + self.history_size = self.mode_config['history_size'] + self.detection_interval = self.mode_config['detection_interval'] + self.smooth_frames = self.mode_config['smooth_frames'] + self.min_confidence = self.mode_config['min_confidence'] + self.resize_factor = self.mode_config['resize_factor'] + # 增强的手势历史和平滑 - self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) self.confidence_history = deque(maxlen=self.history_size) - self.position_history = deque(maxlen=self.history_size) # 新增:位置历史 + self.position_history = deque(maxlen=self.history_size) self.current_gesture = "Waiting" self.current_confidence = 0.0 # 新增:手势状态追踪 - self.gesture_state = "none" # none, starting, active, ending + self.gesture_state = "none" self.gesture_stability_counter = 0 self.last_stable_gesture = "Waiting" self.gesture_active_frames = 0 @@ -1079,7 +1187,7 @@ def __init__(self, speech_manager=None): self.last_announced_gesture = None self.last_announced_time = 0 self.last_hand_status_time = 0 - self.gesture_announce_interval = 2.0 # 缩短手势播报间隔 + self.gesture_announce_interval = 2.0 # 手部跟踪和状态 self.last_hand_position = None @@ -1092,23 +1200,22 @@ def __init__(self, speech_manager=None): # 性能统计 self.process_times = deque(maxlen=30) self.frame_counter = 0 - self.detection_interval = self.config['detection_interval'] self.last_performance_report = 0 # 手势颜色映射 self.gesture_colors = { - "Stop": (0, 0, 255), # 红色 - "Forward": (0, 255, 0), # 绿色 - "Up": (255, 255, 0), # 青色 - "Down": (255, 0, 255), # 紫色 - "Left": (255, 165, 0), # 橙色 - "Right": (0, 165, 255), # 浅蓝色 - "Waiting": (200, 200, 200), # 灰色 - "Error": (255, 0, 0), # 蓝色 - "Hover": (255, 255, 255) # 白色 + "Stop": (0, 0, 255), + "Forward": (0, 255, 0), + "Up": (255, 255, 0), + "Down": (255, 0, 255), + "Left": (255, 165, 0), + "Right": (0, 165, 255), + "Waiting": (200, 200, 200), + "Error": (255, 0, 0), + "Hover": (255, 255, 255) } - # 手势到语音的映射 - 修复:确保所有手势都有对应语音 + # 手势到语音的映射 self.gesture_speech_map = { "Stop": "gesture_stop", "Forward": "gesture_forward", @@ -1118,34 +1225,71 @@ def __init__(self, speech_manager=None): "Right": "gesture_right", "Waiting": "gesture_waiting", "Error": "gesture_error", - "Hover": "gesture_hover", # 新增悬停手势语音 + "Hover": "gesture_hover", } # 手势状态颜色 self.state_colors = { - "none": (100, 100, 100), # 灰色 - "starting": (255, 165, 0), # 橙色 - "active": (0, 255, 0), # 绿色 - "ending": (255, 0, 0), # 红色 + "none": (100, 100, 100), + "starting": (255, 165, 0), + "active": (0, 255, 0), + "ending": (255, 0, 0), } - # 背景减除器 - self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( - history=100, varThreshold=25, detectShadows=True - ) + # 根据性能模式初始化背景减除器 + self.bg_subtractor = None + if self.mode_config['background_subtraction_enabled']: + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( + history=100, varThreshold=25, detectShadows=True + ) # 形态学操作核 self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - # 新增:性能监控 + # 性能监控 self.avg_process_time = 0 self.frame_rate = 0 self.last_fps_check = time.time() - # 新增:存储手部数据用于轨迹记录 + # 存储手部数据用于轨迹记录 self.last_hand_data = None - print("✓ 增强的手势识别器已初始化 (改进平滑处理)") + # 性能模式信息 + self.performance_mode_color = self.mode_config['color'] + self.performance_mode_name = self.mode_config['name'] + + print(f"✓ 增强的手势识别器已初始化 - 性能模式: {self.performance_mode_name}") + + def set_performance_mode(self, mode): + """设置性能模式""" + self.performance_mode = mode + self.mode_config = config.get_performance_mode_config(mode) + + # 更新参数 + self.history_size = self.mode_config['history_size'] + self.detection_interval = self.mode_config['detection_interval'] + self.smooth_frames = self.mode_config['smooth_frames'] + self.min_confidence = self.mode_config['min_confidence'] + self.resize_factor = self.mode_config['resize_factor'] + + # 更新背景减除器 + if self.mode_config['background_subtraction_enabled'] and self.bg_subtractor is None: + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( + history=100, varThreshold=25, detectShadows=True + ) + elif not self.mode_config['background_subtraction_enabled']: + self.bg_subtractor = None + + # 更新队列大小 + self.gesture_history = deque(maxlen=self.history_size) + self.confidence_history = deque(maxlen=self.history_size) + self.position_history = deque(maxlen=self.history_size) + + # 更新显示信息 + self.performance_mode_color = self.mode_config['color'] + self.performance_mode_name = self.mode_config['name'] + + print(f"✓ 切换到性能模式: {self.performance_mode_name}") def get_skin_mask(self, frame): """获取肤色掩码""" @@ -1165,14 +1309,36 @@ def get_skin_mask(self, frame): def enhance_skin_detection(self, frame, skin_mask): """增强肤色检测""" - fg_mask = self.bg_subtractor.apply(frame) - combined_mask = cv2.bitwise_and(skin_mask, fg_mask) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) + if not self.mode_config['skin_detection_enabled']: + return skin_mask + + if self.bg_subtractor is not None: + fg_mask = self.bg_subtractor.apply(frame) + combined_mask = cv2.bitwise_and(skin_mask, fg_mask) + else: + combined_mask = skin_mask + + # 根据性能模式决定形态学操作次数 + if self.performance_mode == 'accurate': + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) + combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) + elif self.performance_mode == 'balanced': + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=1) + combined_mask = cv2.GaussianBlur(combined_mask, (3, 3), 0) + # fast模式不进行额外的形态学操作 return combined_mask + def preprocess_frame(self, frame): + """预处理帧(根据性能模式调整大小)""" + if self.resize_factor != 1.0: + new_width = int(frame.shape[1] * self.resize_factor) + new_height = int(frame.shape[0] * self.resize_factor) + resized_frame = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_AREA) + return resized_frame + return frame + def find_best_hand_contour(self, mask, frame): """找到最佳的手部轮廓""" contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) @@ -1183,10 +1349,13 @@ def find_best_hand_contour(self, mask, frame): contours = sorted(contours, key=cv2.contourArea, reverse=True) best_contour = None best_score = 0.0 - min_area = config.get('gesture', 'min_hand_area') - max_area = config.get('gesture', 'max_hand_area') + min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) + + # 根据性能模式调整检查的轮廓数量 + max_contours = 3 if self.performance_mode != 'fast' else 1 - for contour in contours[:3]: + for contour in contours[:max_contours]: area = cv2.contourArea(contour) if area < min_area or area > max_area: @@ -1204,8 +1373,8 @@ def rate_contour(self, contour, frame_shape): """评估轮廓作为手部的可能性""" score = 0.0 area = cv2.contourArea(contour) - min_area = config.get('gesture', 'min_hand_area') - max_area = config.get('gesture', 'max_hand_area') + min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) if min_area < area < max_area: area_ratio = min(area / max_area, 1.0) @@ -1246,7 +1415,11 @@ def analyze_hand_features(self, contour, frame_shape): bbox_area = w * h palm_center = (cx, cy) palm_radius = int(w * config.get('gesture', 'palm_circle_radius_ratio')) - fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius) + + # 根据性能模式调整轮廓简化程度 + epsilon = self.mode_config['contour_simplify_epsilon'] * cv2.arcLength(contour, True) + fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius, epsilon) + direction = self.calculate_hand_direction(contour, cx, cy) h_img, w_img = frame_shape[:2] norm_x = cx / w_img @@ -1271,9 +1444,8 @@ def analyze_hand_features(self, contour, frame_shape): return result, confidence - def analyze_fingers(self, contour, palm_center, palm_radius): + def analyze_fingers(self, contour, palm_center, palm_radius, epsilon): """分析手指""" - epsilon = config.get('gesture', 'contour_simplify_epsilon') * cv2.arcLength(contour, True) approx = cv2.approxPolyDP(contour, epsilon, True) hull = cv2.convexHull(approx, returnPoints=False) @@ -1356,8 +1528,8 @@ def calculate_hand_direction(self, contour, cx, cy): def calculate_confidence(self, area, finger_count, contour_length, bbox_area): """计算手势置信度""" confidence = 0.5 - min_area = config.get('gesture', 'min_hand_area') - max_area = config.get('gesture', 'max_hand_area') + min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) if min_area < area < max_area: area_norm = (area - min_area) / (max_area - min_area) @@ -1479,7 +1651,7 @@ def smooth_gesture_enhanced(self, new_gesture, new_confidence, hand_data): position_stability = max(0, 1.0 - position_diff * 5.0) # 增强的稳定性检查 - stability_threshold = config.get('gesture', 'gesture_stability_threshold') + stability_threshold = self.mode_config['gesture_stability_threshold'] transition_threshold = config.get('gesture', 'transition_threshold') position_weight = config.get('gesture', 'position_stability_weight') @@ -1532,8 +1704,8 @@ def update_gesture_state(self, hand_data, gesture, confidence): # 检查手部大小和位置 hand_area = hand_data['area'] - min_area = config.get('gesture', 'min_hand_area') - max_area = config.get('gesture', 'max_hand_area') + min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) # 提供手部位置反馈 if (self.speech_manager and @@ -1686,7 +1858,7 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): # 计算稳定性指示条长度 stability_level = min(1.0, - self.gesture_stability_counter / config.get('gesture', 'gesture_stability_threshold')) + self.gesture_stability_counter / self.mode_config['gesture_stability_threshold']) bar_length = int(70 * stability_level) # 根据稳定性级别选择颜色 @@ -1756,10 +1928,15 @@ def recognize(self, frame): # 预处理帧 processed_frame = cv2.flip(frame, 1) + # 根据性能模式调整图像大小 + original_frame = processed_frame.copy() + if self.resize_factor != 1.0: + processed_frame = self.preprocess_frame(processed_frame) + # 每隔几帧检测一次以提高性能 if self.frame_counter % self.detection_interval != 0: self.frame_counter += 1 - return self.current_gesture, self.current_confidence, processed_frame + return self.current_gesture, self.current_confidence, original_frame # 获取肤色掩码 skin_mask, hsv = self.get_skin_mask(processed_frame) @@ -1798,8 +1975,7 @@ def recognize(self, frame): self.update_gesture_state(None, gesture, confidence) final_gesture, final_confidence = gesture, confidence - # ========== 修复手势语音播报 ========== - # 确保所有手势都有语音反馈 + # 手势语音播报 if (self.speech_manager and config.get('speech', 'enabled') and config.get('speech', 'announce_gestures')): @@ -1814,19 +1990,17 @@ def recognize(self, frame): self.speech_manager.speak('gesture_low_confidence', immediate=True) self.last_hand_status_time = current_time - # 手势语音播报 - 修复:确保所有手势都能播报 + # 手势语音播报 if (final_gesture != "Waiting" and final_gesture != "Error" and final_gesture != "摄像头错误" and - final_confidence >= config.get('speech', 'min_gesture_confidence') and + final_confidence >= self.min_confidence and current_time - self.last_announced_time > self.gesture_announce_interval): - # 检查手势是否在语音映射中 if final_gesture in self.gesture_speech_map: speech_key = self.gesture_speech_map[final_gesture] self.speech_manager.speak(speech_key) else: - # 如果手势不在映射中,直接播报手势名称 self.speech_manager.speak_direct(f"手势{final_gesture}") self.last_announced_gesture = final_gesture @@ -1834,22 +2008,44 @@ def recognize(self, frame): # 性能报告 current_time = time.time() - if current_time - self.last_performance_report > 30.0: # 每30秒报告一次 + if current_time - self.last_performance_report > 30.0: if (self.speech_manager and config.get('speech', 'enabled') and config.get('speech', 'announce_performance')): - if self.avg_process_time < 20: # 处理时间小于20ms + if self.avg_process_time < 20: self.speech_manager.speak('performance_good', immediate=True) - elif self.avg_process_time > 50: # 处理时间大于50ms + elif self.avg_process_time > 50: self.speech_manager.speak('performance_warning', immediate=True) self.last_performance_report = current_time - # 可视化结果 + # 可视化结果(在原图上) if hand_data is not None: - processed_frame = self.visualize_detection( - processed_frame, hand_data, final_gesture, final_confidence + # 需要将坐标转换回原始图像大小 + if self.resize_factor != 1.0: + # 缩放坐标 + scale_factor = 1.0 / self.resize_factor + if 'center' in hand_data: + hand_data['center'] = (int(hand_data['center'][0] * scale_factor), + int(hand_data['center'][1] * scale_factor)) + if 'bbox' in hand_data: + x1, y1, x2, y2 = hand_data['bbox'] + hand_data['bbox'] = (int(x1 * scale_factor), int(y1 * scale_factor), + int(x2 * scale_factor), int(y2 * scale_factor)) + if 'fingertips' in hand_data: + hand_data['fingertips'] = [(int(x * scale_factor), int(y * scale_factor)) + for (x, y) in hand_data['fingertips']] + if 'palm_center' in hand_data: + hand_data['palm_center'] = (int(hand_data['palm_center'][0] * scale_factor), + int(hand_data['palm_center'][1] * scale_factor)) + + original_frame = self.visualize_detection( + original_frame, hand_data, final_gesture, final_confidence + ) + else: + original_frame = self.visualize_detection( + original_frame, None, final_gesture, final_confidence ) # 更新计数器 @@ -1870,7 +2066,7 @@ def recognize(self, frame): self.frame_counter = 0 self.last_fps_check = current_time - return final_gesture, final_confidence, processed_frame + return final_gesture, final_confidence, original_frame except Exception as e: print(f"⚠ 手势识别错误: {e}") @@ -1883,12 +2079,24 @@ def get_performance_stats(self): return np.mean(list(self.process_times)), self.frame_rate + def get_performance_mode_info(self): + """获取性能模式信息""" + return { + 'name': self.performance_mode_name, + 'mode': self.performance_mode, + 'color': self.performance_mode_color, + 'detection_interval': self.detection_interval, + 'resize_factor': self.resize_factor, + 'smooth_frames': self.smooth_frames, + 'min_confidence': self.min_confidence + } + def set_simulated_gesture(self, gesture): """设置模拟的手势""" self.current_gesture = gesture self.current_confidence = 0.9 - # 模拟手势也触发语音提示 - 修复:确保所有模拟手势都有语音 + # 模拟手势也触发语音提示 if (self.speech_manager and config.get('speech', 'enabled') and config.get('speech', 'announce_gestures')): @@ -1896,7 +2104,6 @@ def set_simulated_gesture(self, gesture): if gesture in self.gesture_speech_map: self.speech_manager.speak(self.gesture_speech_map[gesture]) else: - # 如果手势不在映射中,直接播报手势名称 self.speech_manager.speak_direct(f"手势{gesture}") self.last_announced_gesture = gesture @@ -2116,7 +2323,7 @@ def move_by_gesture(self, gesture, confidence): if (self.speech_manager and config.get('speech', 'enabled') and config.get('speech', 'announce_gestures') and - confidence < min_confidence * 0.8): # 如果置信度特别低 + confidence < min_confidence * 0.8): self.speech_manager.speak('gesture_low_confidence') return False @@ -2205,22 +2412,25 @@ def __init__(self, speech_manager=None): # 颜色定义 self.colors = { - 'title': (0, 255, 255), # 青色 - 'connected': (0, 255, 0), # 绿色 - 'disconnected': (0, 0, 255), # 红色 - 'flying': (0, 255, 0), # 绿色 - 'landed': (255, 165, 0), # 橙色 - 'warning': (0, 165, 255), # 浅蓝色 - 'info': (255, 255, 255), # 白色 - 'help': (255, 200, 100), # 浅橙色 - 'speech_enabled': (0, 255, 0), # 绿色 - 'speech_disabled': (255, 0, 0), # 红色 - 'performance_good': (0, 255, 0), # 绿色 - 'performance_warning': (255, 165, 0), # 橙色 - 'performance_bad': (255, 0, 0), # 红色 - 'recording': (255, 50, 50), # 亮红色 - 'playback': (50, 50, 255), # 亮蓝色 - 'paused': (255, 255, 0), # 黄色 + 'title': (0, 255, 255), + 'connected': (0, 255, 0), + 'disconnected': (0, 0, 255), + 'flying': (0, 255, 0), + 'landed': (255, 165, 0), + 'warning': (0, 165, 255), + 'info': (255, 255, 255), + 'help': (255, 200, 100), + 'speech_enabled': (0, 255, 0), + 'speech_disabled': (255, 0, 0), + 'performance_good': (0, 255, 0), + 'performance_warning': (255, 165, 0), + 'performance_bad': (255, 0, 0), + 'recording': (255, 50, 50), + 'playback': (50, 50, 255), + 'paused': (255, 255, 0), + 'performance_fast': (0, 255, 0), # 绿色 + 'performance_balanced': (255, 165, 0), # 橙色 + 'performance_accurate': (255, 0, 0), # 红色 } print("✓ 中文UI渲染器已初始化") @@ -2274,8 +2484,8 @@ def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): size / 25, color, 1) return frame - def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, - trajectory_recorder=None): + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, trajectory_recorder=None, + gesture_recognizer=None): """绘制状态栏""" h, w = frame.shape[:2] @@ -2285,7 +2495,7 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 轨迹记录版" + title = "手势控制无人机系统 - 性能模式版" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -2300,11 +2510,11 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro # 手势信息 if confidence > 0.7: - gesture_color = (0, 255, 0) # 绿色 + gesture_color = (0, 255, 0) elif confidence > 0.5: - gesture_color = (255, 165, 0) # 橙色 + gesture_color = (255, 165, 0) else: - gesture_color = (200, 200, 200) # 灰色 + gesture_color = (200, 200, 200) gesture_text = f"当前手势: {gesture}" if config.get('display', 'show_confidence'): @@ -2312,14 +2522,29 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) + # 性能模式显示 + if gesture_recognizer and config.get('display', 'show_performance_mode'): + mode_info = gesture_recognizer.get_performance_mode_info() + + # 根据模式选择颜色 + if mode_info['mode'] == 'fast': + mode_color = self.colors['performance_fast'] + elif mode_info['mode'] == 'balanced': + mode_color = self.colors['performance_balanced'] + else: # accurate + mode_color = self.colors['performance_accurate'] + + mode_text = f"性能模式: {mode_info['name']}" + frame = self.draw_text(frame, mode_text, (w // 2, 65), size=14, color=mode_color) + # 录制/回放状态 - if trajectory_recorder and config.get('display', 'show_recording_status'): + elif trajectory_recorder and config.get('display', 'show_recording_status'): recorder_status = trajectory_recorder.get_status() if recorder_status['is_recording']: status_color = self.colors['recording'] status_text = f"录制中: {recorder_status['trajectory_points']}点" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=16, color=status_color) + frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) elif recorder_status['is_playing']: if recorder_status['playback_paused']: status_color = self.colors['paused'] @@ -2327,17 +2552,17 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro else: status_color = self.colors['playback'] status_text = f"回放中: {recorder_status['playback_index']}/{recorder_status['playback_total']}" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=16, color=status_color) + frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) else: status_text = f"轨迹点: {recorder_status['trajectory_points']}" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=self.colors['info']) + frame = self.draw_text(frame, status_text, (w // 2, 65), size=12, color=self.colors['info']) # 语音状态 if config.get('display', 'show_speech_status') and self.speech_manager: speech_status = self.speech_manager.get_status() speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" - frame = self.draw_text(frame, speech_text, (w // 2, 90), size=14, color=speech_color) + frame = self.draw_text(frame, speech_text, (w // 2, 90), size=12, color=speech_color) # 性能信息 if config.get('display', 'show_fps'): @@ -2355,11 +2580,11 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro else: perf_color = self.colors['info'] - frame = self.draw_text(frame, perf_text, (w - 200, 65), size=14, color=perf_color) + frame = self.draw_text(frame, perf_text, (w - 200, 65), size=12, color=perf_color) # 控制提示 control_text = "提示: 确保手部完全进入画面,保持稳定手势" - frame = self.draw_text(frame, control_text, (10, 90), size=14, color=self.colors['info']) + frame = self.draw_text(frame, control_text, (10, 90), size=12, color=self.colors['info']) return frame @@ -2377,7 +2602,7 @@ def draw_help_bar(self, frame): help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音 P:性能报告", + "V:切换语音反馈 M:测试语音 P:性能报告 O:切换性能模式", "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" ] @@ -2393,7 +2618,7 @@ def draw_warning(self, frame, message): # 在顶部绘制警告 warning_bg = np.zeros((40, w, 3), dtype=np.uint8) - warning_bg[:, :] = (0, 69, 255) # 橙色 + warning_bg[:, :] = (0, 69, 255) frame[120:160, 0:w] = cv2.addWeighted( frame[120:160, 0:w], 0.3, @@ -2461,7 +2686,7 @@ def main(): ui_renderer = ChineseUIRenderer(speech_manager) performance_monitor = PerformanceMonitor() - # 新增:手势轨迹记录器 + # 手势轨迹记录器 print("初始化手势轨迹记录器...") trajectory_recorder = GestureTrajectoryRecorder(speech_manager) @@ -2505,14 +2730,14 @@ def main(): # 显示欢迎信息 print("\n" + "=" * 60) - print("手势控制无人机系统 - 轨迹记录版") + print("手势控制无人机系统 - 性能模式版") print("=" * 60) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") + print(f" 性能模式: {gesture_recognizer.performance_mode_name}") print(f" 轨迹记录: 支持录制/回放功能") - print(f" 手势状态: 支持开始/活跃/结束状态追踪") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) @@ -2520,23 +2745,28 @@ def main(): print("\n操作说明:") print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") - print("3. 手势控制:") + print("3. 性能模式选择:") + print(" - 按 [O] 键循环切换性能模式: 最快(fast) → 平衡(balanced) → 最准(accurate)") + print(" - 最快模式: 性能优先,帧率最高,识别精度较低") + print(" - 平衡模式: 性能与精度平衡,适用于大多数场景") + print(" - 最准模式: 精度优先,提供最准确的手势识别") + print("4. 手势控制:") print(" - 手势识别后会有语音提示: 向上、向下、向左、向右、向前、停止") print(" - 手势稳定性越高,识别越准确") print(" - 手部距离摄像头适中时效果最佳") print(" * 手势识别置信度 > 60% 时才会执行") - print("4. 轨迹记录功能:") + print("5. 轨迹记录功能:") print(" [1]开始录制 [2]停止录制 [3]保存轨迹 [4]回放轨迹 [5]清除轨迹 [6]暂停/继续") print(" - 录制时系统会记录手部位置和手势") print(" - 回放时可以查看录制的轨迹") print(" - 轨迹数据自动保存到 trajectories/ 目录") - print("5. 键盘控制:") + print("6. 键盘控制:") print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") - print("6. 调试功能:") + print("7. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") - print("7. 语音控制:") + print("8. 语音控制:") print(" [V]切换语音反馈 [M]测试语音 [P]性能报告") - print("8. 按 [ESC] 安全退出") + print("9. 按 [ESC] 安全退出") print("=" * 60) print("程序启动成功!") print("-" * 60) @@ -2592,6 +2822,7 @@ def main(): config.set('display', 'show_stability_indicator', value=True) config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) + config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -2603,6 +2834,7 @@ def main(): config.set('display', 'show_stability_indicator', value=True) config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) + config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) @@ -2612,6 +2844,7 @@ def main(): config.set('display', 'show_stability_indicator', value=False) config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) + config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_debug_info', value=False) # 绘制轨迹(如果启用) @@ -2621,7 +2854,7 @@ def main(): # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, - perf_stats['fps'], process_time, trajectory_recorder + perf_stats['fps'], process_time, trajectory_recorder, gesture_recognizer ) frame = ui_renderer.draw_help_bar(frame) @@ -2632,7 +2865,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Trajectory Recording', frame) + cv2.imshow('Gesture Controlled Drone - Performance Mode', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -2723,6 +2956,23 @@ def main(): else: speech_manager.speak_direct("系统性能警告,请检查") + elif key == ord('o') or key == ord('O'): + # 切换性能模式 + if config.cycle_performance_mode(): + new_mode = config.get_current_performance_mode() + gesture_recognizer.set_performance_mode(new_mode) + + # 语音提示 + if speech_manager.enabled: + if new_mode == 'fast': + speech_manager.speak('performance_mode_fast', immediate=True) + elif new_mode == 'balanced': + speech_manager.speak('performance_mode_balanced', immediate=True) + else: # accurate + speech_manager.speak('performance_mode_accurate', immediate=True) + + print(f"✓ 已切换到性能模式: {gesture_recognizer.performance_mode_name}") + # 新增:轨迹记录控制 elif key == ord('1'): # 开始录制轨迹 @@ -2815,7 +3065,7 @@ def main(): # 程序退出语音提示 if speech_manager.enabled: speech_manager.speak('program_exit', force=True, immediate=True) - time.sleep(1) # 确保语音播报完成 + time.sleep(1) drone_controller.emergency_stop() From 40d100ca2593624379cb7392211bb053cdf921c5 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sun, 21 Dec 2025 13:14:53 +0800 Subject: [PATCH 12/18] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E9=80=89=E6=8B=A9=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 2 + .../gesture_config.json | 12 +- .../main.py | 845 ++++++++++++++++-- 3 files changed, 769 insertions(+), 90 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index b1c539fd44..9b0346af15 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -12,6 +12,8 @@ __pycache__/ # 忽略日志文件夹(临时日志文件,无需提交到仓库) logs/ +trajectories/ + # 忽略手势识别配置文件(保留本地,不提交到仓库) gesture_config.json gesture_config (1).json diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index 40656680c9..3cdf48a5cb 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -18,11 +18,11 @@ }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 165, + "skin_upper_h": 176, "skin_lower_s": 0, - "skin_upper_s": 36, + "skin_upper_s": 116, "skin_lower_v": 0, - "skin_upper_v": 69, + "skin_upper_v": 92, "min_hand_area": 3000, "max_hand_area": 30000, "hand_ratio_threshold": 1.5, @@ -86,6 +86,8 @@ "show_trajectory": true, "show_recording_status": true, "show_performance_mode": true, + "show_performance_stats": true, + "show_system_resources": true, "font_size_large": 20, "font_size_medium": 16, "font_size_small": 14, @@ -95,7 +97,6 @@ "show_detection_overlay": true, "show_direction": true, "show_gesture_info": true, - "show_performance_stats": true, "show_region_grid": true, "show_trajectory_heatmap": false, "theme": "dark", @@ -113,6 +114,8 @@ "balanced", "accurate" ], + "auto_report_interval": 60, + "enable_performance_monitor": true, "enable_gpu_acceleration": false, "enable_hardware_acceleration": true, "enable_multithreading": false, @@ -142,6 +145,7 @@ "announce_performance": true, "announce_recording_events": true, "announce_performance_mode": true, + "announce_performance_events": true, "min_gesture_confidence": 0.7, "gesture_start_threshold": 3, "gesture_end_threshold": 10, diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 3076402e7a..8bda7d3989 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 性能模式选择优化版 -新增fast(最快)、balanced(平衡)、accurate(最准)三种性能模式 +手势控制AirSim无人机 - 详细性能统计优化版 +新增完整的性能监控、统计和报告功能 作者: xiaoshiyuan888 """ @@ -13,16 +13,18 @@ import math import threading import tempfile -import pickle # 新增:用于数据序列化 +import pickle from datetime import datetime from PIL import Image, ImageDraw, ImageFont import cv2 import numpy as np from collections import deque, Counter +import psutil # 新增:用于系统资源监控 +import csv # 新增:用于性能日志导出 print("=" * 60) -print("Gesture Controlled Drone - Performance Mode Selection") -print("性能模式选择优化版!") +print("Gesture Controlled Drone - Performance Statistics Enhanced") +print("详细性能统计优化版!") print("=" * 60) # ========== 修复导入路径 ========== @@ -54,6 +56,15 @@ def safe_import(): print(f"[OpenCV] ✗ 导入失败: {e}") return None, modules_status + try: + import psutil + modules_status['psutil'] = True + print("[psutil] ✓ 系统资源监控库就绪") + except Exception as e: + modules_status['psutil'] = False + print(f"[psutil] ✗ 导入失败: {e}") + # 继续运行,psutil不是必需的 + airsim_module = None try: airsim_module = __import__('airsim') @@ -139,7 +150,8 @@ def safe_import(): 'np': np, 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, 'airsim': airsim_module, - 'speech': speech_module + 'speech': speech_module, + 'psutil': psutil if 'psutil' in modules_status and modules_status['psutil'] else None }, modules_status @@ -159,6 +171,497 @@ def safe_import(): Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] +# ========== 性能分析器类 ========== +class PerformanceAnalyzer: + """性能分析器 - 监控和报告系统性能""" + + def __init__(self, speech_manager=None): + self.speech_manager = speech_manager + self.start_time = time.time() + self.session_start_time = time.time() + + # 帧率统计 + self.frame_times = deque(maxlen=300) # 保留最近300帧的时间戳 + self.frame_count = 0 + self.fps_history = deque(maxlen=100) # FPS历史记录 + + # 手势识别性能 + self.gesture_recognition_times = deque(maxlen=100) + self.avg_recognition_time = 0 + self.max_recognition_time = 0 + + # 系统资源监控 + self.cpu_usage_history = deque(maxlen=100) + self.memory_usage_history = deque(maxlen=100) + + # 性能事件记录 + self.performance_events = [] + self.performance_snapshots = [] + + # 手势统计 + self.gesture_counts = {} + self.gesture_confidence_sum = {} + self.gesture_confidence_count = {} + + # 错误统计 + self.error_count = 0 + self.warning_count = 0 + + # 无人机控制统计 + self.drone_commands = 0 + self.successful_commands = 0 + self.failed_commands = 0 + + # 轨迹记录统计 + self.recording_sessions = 0 + self.total_trajectory_points = 0 + + # 性能日志 + self.performance_log = [] + self.log_file = os.path.join(current_dir, 'performance_log.csv') + + # 性能阈值 + self.performance_thresholds = { + 'fps_warning': 15, + 'fps_critical': 5, + 'cpu_warning': 80, + 'cpu_critical': 90, + 'memory_warning': 85, + 'memory_critical': 95, + 'recognition_warning': 50, # ms + 'recognition_critical': 100 # ms + } + + # 性能状态 + self.performance_status = "良好" + self.last_performance_report = 0 + self.auto_report_interval = 60 # 自动报告间隔(秒) + + print("✓ 性能分析器已初始化") + + def update_frame(self): + """更新帧统计""" + current_time = time.time() + self.frame_times.append(current_time) + self.frame_count += 1 + + # 计算当前FPS + if len(self.frame_times) > 1: + time_span = self.frame_times[-1] - self.frame_times[0] + if time_span > 0: + current_fps = (len(self.frame_times) - 1) / time_span + self.fps_history.append(current_fps) + + def update_gesture_recognition_time(self, recognition_time_ms): + """更新手势识别时间""" + self.gesture_recognition_times.append(recognition_time_ms) + + # 更新平均识别时间 + if len(self.gesture_recognition_times) > 0: + self.avg_recognition_time = np.mean(list(self.gesture_recognition_times)) + self.max_recognition_time = max(self.max_recognition_time, recognition_time_ms) + + def update_system_resources(self): + """更新系统资源使用情况""" + try: + if libs['psutil']: + cpu_percent = libs['psutil'].cpu_percent(interval=0.1) + memory_percent = libs['psutil'].virtual_memory().percent + + self.cpu_usage_history.append(cpu_percent) + self.memory_usage_history.append(memory_percent) + + # 检查性能问题 + self.check_performance_issues(cpu_percent, memory_percent) + except: + pass # psutil不可用时跳过 + + def check_performance_issues(self, cpu_percent, memory_percent): + """检查性能问题""" + issues = [] + + # 检查FPS + if len(self.fps_history) > 0: + avg_fps = np.mean(list(self.fps_history[-10:])) if len(self.fps_history) >= 10 else self.fps_history[-1] + + if avg_fps < self.performance_thresholds['fps_critical']: + issues.append(("严重", f"帧率过低: {avg_fps:.1f} FPS")) + self.performance_status = "严重" + elif avg_fps < self.performance_thresholds['fps_warning']: + issues.append(("警告", f"帧率较低: {avg_fps:.1f} FPS")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查CPU使用率 + if cpu_percent > self.performance_thresholds['cpu_critical']: + issues.append(("严重", f"CPU使用率过高: {cpu_percent:.1f}%")) + self.performance_status = "严重" + elif cpu_percent > self.performance_thresholds['cpu_warning']: + issues.append(("警告", f"CPU使用率较高: {cpu_percent:.1f}%")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查内存使用率 + if memory_percent > self.performance_thresholds['memory_critical']: + issues.append(("严重", f"内存使用率过高: {memory_percent:.1f}%")) + self.performance_status = "严重" + elif memory_percent > self.performance_thresholds['memory_warning']: + issues.append(("警告", f"内存使用率较高: {memory_percent:.1f}%")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查手势识别时间 + if self.avg_recognition_time > self.performance_thresholds['recognition_critical']: + issues.append(("严重", f"手势识别时间过长: {self.avg_recognition_time:.1f}ms")) + self.performance_status = "严重" + elif self.avg_recognition_time > self.performance_thresholds['recognition_warning']: + issues.append(("警告", f"手势识别时间较长: {self.avg_recognition_time:.1f}ms")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 记录性能事件 + if issues: + for level, message in issues: + self.add_performance_event(level, message) + + # 语音提示(仅在状态变化时) + if (self.speech_manager and + self.speech_manager.enabled and + level == "严重"): + current_time = time.time() + if current_time - self.last_performance_report > 10: # 避免频繁提示 + self.speech_manager.speak_direct(f"性能{level}: {message}") + self.last_performance_report = current_time + + def add_performance_event(self, level, message): + """添加性能事件""" + event = { + 'timestamp': time.time(), + 'level': level, + 'message': message, + 'session_time': time.time() - self.session_start_time + } + self.performance_events.append(event) + + # 记录到日志 + self.log_performance_event(event) + + if level == "警告": + self.warning_count += 1 + elif level == "严重": + self.error_count += 1 + + def log_performance_event(self, event): + """记录性能事件到日志""" + log_entry = { + 'timestamp': datetime.fromtimestamp(event['timestamp']).strftime('%Y-%m-%d %H:%M:%S'), + 'session_time': f"{event['session_time']:.1f}", + 'level': event['level'], + 'message': event['message'] + } + self.performance_log.append(log_entry) + + def record_gesture(self, gesture, confidence): + """记录手势统计""" + if gesture not in self.gesture_counts: + self.gesture_counts[gesture] = 0 + self.gesture_confidence_sum[gesture] = 0 + self.gesture_confidence_count[gesture] = 0 + + self.gesture_counts[gesture] += 1 + self.gesture_confidence_sum[gesture] += confidence + self.gesture_confidence_count[gesture] += 1 + + def record_drone_command(self, success=True): + """记录无人机命令""" + self.drone_commands += 1 + if success: + self.successful_commands += 1 + else: + self.failed_commands += 1 + + def record_recording_session(self, points_count=0): + """记录录制会话""" + self.recording_sessions += 1 + self.total_trajectory_points += points_count + + def take_snapshot(self, label=""): + """拍摄性能快照""" + snapshot = { + 'timestamp': time.time(), + 'label': label, + 'fps': self.get_current_fps(), + 'avg_fps': self.get_average_fps(), + 'avg_recognition_time': self.avg_recognition_time, + 'max_recognition_time': self.max_recognition_time, + 'cpu_usage': self.get_current_cpu_usage(), + 'memory_usage': self.get_current_memory_usage(), + 'gesture_counts': dict(self.gesture_counts), + 'performance_status': self.performance_status, + 'frame_count': self.frame_count, + 'session_duration': time.time() - self.session_start_time + } + self.performance_snapshots.append(snapshot) + + print(f"📸 性能快照已保存: {label}") + return snapshot + + def get_current_fps(self): + """获取当前FPS""" + if len(self.fps_history) > 0: + return self.fps_history[-1] + return 0 + + def get_average_fps(self): + """获取平均FPS""" + if len(self.fps_history) > 0: + return np.mean(list(self.fps_history)) + return 0 + + def get_current_cpu_usage(self): + """获取当前CPU使用率""" + if len(self.cpu_usage_history) > 0: + return self.cpu_usage_history[-1] + return 0 + + def get_current_memory_usage(self): + """获取当前内存使用率""" + if len(self.memory_usage_history) > 0: + return self.memory_usage_history[-1] + return 0 + + def generate_report(self, detailed=True): + """生成性能报告""" + report_time = time.time() + session_duration = report_time - self.session_start_time + + # 基础报告 + report = { + '生成时间': datetime.fromtimestamp(report_time).strftime('%Y-%m-%d %H:%M:%S'), + '会话时长': f"{session_duration:.1f}秒", + '总帧数': self.frame_count, + '平均FPS': f"{self.get_average_fps():.1f}", + '当前FPS': f"{self.get_current_fps():.1f}", + '平均手势识别时间': f"{self.avg_recognition_time:.1f}ms", + '最大手势识别时间': f"{self.max_recognition_time:.1f}ms", + '当前CPU使用率': f"{self.get_current_cpu_usage():.1f}%", + '当前内存使用率': f"{self.get_current_memory_usage():.1f}%", + '性能状态': self.performance_status, + '警告数量': self.warning_count, + '错误数量': self.error_count, + '无人机命令': { + '总数': self.drone_commands, + '成功': self.successful_commands, + '失败': self.failed_commands, + '成功率': f"{(self.successful_commands / self.drone_commands * 100 if self.drone_commands > 0 else 0):.1f}%" + }, + '录制统计': { + '会话数': self.recording_sessions, + '总轨迹点数': self.total_trajectory_points + } + } + + # 详细报告 + if detailed: + # 手势统计 + gesture_stats = {} + for gesture in self.gesture_counts: + count = self.gesture_counts[gesture] + if gesture in self.gesture_confidence_count and self.gesture_confidence_count[gesture] > 0: + avg_confidence = self.gesture_confidence_sum[gesture] / self.gesture_confidence_count[gesture] + else: + avg_confidence = 0 + + gesture_stats[gesture] = { + '次数': count, + '占比': f"{(count / self.frame_count * 100 if self.frame_count > 0 else 0):.1f}%", + '平均置信度': f"{avg_confidence:.1%}" + } + + report['手势统计'] = gesture_stats + + # 性能事件 + if self.performance_events: + recent_events = list(self.performance_events)[-10:] # 最近10个事件 + report['最近性能事件'] = [ + { + '时间': datetime.fromtimestamp(e['timestamp']).strftime('%H:%M:%S'), + '级别': e['level'], + '消息': e['message'] + } + for e in recent_events + ] + + # 性能快照 + if self.performance_snapshots: + report['性能快照数'] = len(self.performance_snapshots) + + # 系统建议 + suggestions = self.generate_suggestions() + if suggestions: + report['优化建议'] = suggestions + + return report + + def generate_suggestions(self): + """生成优化建议""" + suggestions = [] + + # 检查FPS + avg_fps = self.get_average_fps() + if avg_fps < self.performance_thresholds['fps_warning']: + suggestions.append(f"帧率较低({avg_fps:.1f}FPS),建议切换到'最快'性能模式") + + # 检查CPU + cpu_usage = self.get_current_cpu_usage() + if cpu_usage > self.performance_thresholds['cpu_warning']: + suggestions.append(f"CPU使用率较高({cpu_usage:.1f}%),请关闭其他占用CPU的程序") + + # 检查内存 + memory_usage = self.get_current_memory_usage() + if memory_usage > self.performance_thresholds['memory_warning']: + suggestions.append(f"内存使用率较高({memory_usage:.1f}%),请关闭不必要的程序") + + # 检查识别时间 + if self.avg_recognition_time > self.performance_thresholds['recognition_warning']: + suggestions.append(f"手势识别时间较长({self.avg_recognition_time:.1f}ms),建议调整摄像头位置或光线") + + return suggestions + + def print_report(self, detailed=True): + """打印性能报告""" + report = self.generate_report(detailed) + + print("\n" + "=" * 80) + print("📊 性能分析报告") + print("=" * 80) + + # 基础信息 + print(f"生成时间: {report['生成时间']}") + print(f"会话时长: {report['会话时长']}") + print(f"总帧数: {report['总帧数']}") + print(f"平均FPS: {report['平均FPS']}") + print(f"当前FPS: {report['当前FPS']}") + print(f"平均手势识别时间: {report['平均手势识别时间']}") + print(f"最大手势识别时间: {report['最大手势识别时间']}") + print(f"当前CPU使用率: {report['当前CPU使用率']}") + print(f"当前内存使用率: {report['当前内存使用率']}") + print(f"性能状态: {report['性能状态']}") + + # 无人机命令统计 + cmd_stats = report['无人机命令'] + print(f"\n无人机命令统计:") + print(f" 总数: {cmd_stats['总数']}") + print(f" 成功: {cmd_stats['成功']}") + print(f" 失败: {cmd_stats['失败']}") + print(f" 成功率: {cmd_stats['成功率']}") + + # 录制统计 + rec_stats = report['录制统计'] + print(f"\n录制统计:") + print(f" 会话数: {rec_stats['会话数']}") + print(f" 总轨迹点数: {rec_stats['总轨迹点数']}") + + # 详细报告 + if detailed and '手势统计' in report: + print(f"\n手势统计:") + for gesture, stats in report['手势统计'].items(): + print(f" {gesture}: {stats['次数']}次 ({stats['占比']}), 平均置信度: {stats['平均置信度']}") + + # 性能事件 + if detailed and '最近性能事件' in report and report['最近性能事件']: + print(f"\n最近性能事件:") + for event in report['最近性能事件']: + print(f" [{event['时间']}] {event['级别']}: {event['消息']}") + + # 优化建议 + if detailed and '优化建议' in report and report['优化建议']: + print(f"\n优化建议:") + for i, suggestion in enumerate(report['优化建议'], 1): + print(f" {i}. {suggestion}") + + print("=" * 80) + + # 语音播报摘要 + if self.speech_manager and self.speech_manager.enabled: + summary = (f"性能报告: 平均帧率{report['平均FPS']},识别时间{report['平均手势识别时间']}," + f"性能状态{report['性能状态']},无人机命令成功率{cmd_stats['成功率']}") + self.speech_manager.speak_direct(summary) + + def export_log(self, filename=None): + """导出性能日志""" + if filename is None: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = os.path.join(current_dir, f'performance_log_{timestamp}.csv') + + try: + with open(filename, 'w', newline='', encoding='utf-8') as f: + if self.performance_log: + fieldnames = self.performance_log[0].keys() + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(self.performance_log) + + print(f"📤 性能日志已导出到: {filename}") + return True + except Exception as e: + print(f"❌ 导出性能日志失败: {e}") + return False + + def auto_report(self): + """自动性能报告(定期执行)""" + current_time = time.time() + if current_time - self.last_performance_report > self.auto_report_interval: + # 生成简要报告 + report = self.generate_report(detailed=False) + + # 检查是否需要报告 + if (self.performance_status == "严重" or + self.warning_count > 5 or + self.error_count > 0): + + print(f"⚠ 自动性能检查: {report['性能状态']}, FPS: {report['当前FPS']}, " + f"CPU: {report['当前CPU使用率']}, 内存: {report['当前内存使用率']}") + + # 语音提示 + if (self.speech_manager and + self.speech_manager.enabled and + self.performance_status == "严重"): + self.speech_manager.speak_direct(f"系统性能{self.performance_status},建议检查") + + self.last_performance_report = current_time + + def reset_session(self): + """重置会话统计""" + self.session_start_time = time.time() + self.performance_events = [] + self.performance_snapshots = [] + self.gesture_counts = {} + self.gesture_confidence_sum = {} + self.gesture_confidence_count = {} + self.error_count = 0 + self.warning_count = 0 + self.drone_commands = 0 + self.successful_commands = 0 + self.failed_commands = 0 + self.performance_status = "良好" + + print("✓ 性能统计会话已重置") + + def get_stats_summary(self): + """获取统计摘要""" + return { + 'fps': self.get_current_fps(), + 'avg_fps': self.get_average_fps(), + 'recognition_time': self.avg_recognition_time, + 'cpu_usage': self.get_current_cpu_usage(), + 'memory_usage': self.get_current_memory_usage(), + 'performance_status': self.performance_status, + 'gesture_count': sum(self.gesture_counts.values()), + 'unique_gestures': len(self.gesture_counts) + } + + # ========== 增强语音反馈管理器 ========== class EnhancedSpeechFeedbackManager: """增强的语音反馈管理器""" @@ -239,6 +742,10 @@ def __init__(self, speech_lib): # 新增:性能相关 'performance_good': "系统运行流畅", 'performance_warning': "系统性能警告", + 'performance_critical': "系统性能严重警告", + 'performance_report': "性能报告生成完成", + 'performance_snapshot': "性能快照已保存", + 'performance_log_exported': "性能日志已导出", # 新增:手势指导 'move_closer': "请将手靠近摄像头", @@ -949,6 +1456,8 @@ def __init__(self): 'show_trajectory': True, 'show_recording_status': True, 'show_performance_mode': True, # 新增:显示性能模式 + 'show_performance_stats': True, # 新增:显示性能统计 + 'show_system_resources': True, # 新增:显示系统资源 }, 'performance': { 'target_fps': 30, @@ -957,6 +1466,8 @@ def __init__(self): 'mode': 'balanced', # 新增:性能模式选择 'current_mode_index': 1, # 当前模式索引 'modes': ['fast', 'balanced', 'accurate'], # 可用模式列表 + 'auto_report_interval': 60, # 自动报告间隔 + 'enable_performance_monitor': True, # 启用性能监控 }, 'calibration': { 'auto_calibrate_skin': True, @@ -975,6 +1486,7 @@ def __init__(self): 'announce_performance': True, 'announce_recording_events': True, 'announce_performance_mode': True, # 新增:播报性能模式切换 + 'announce_performance_events': True, # 新增:播报性能事件 'min_gesture_confidence': 0.7, 'gesture_start_threshold': 3, 'gesture_end_threshold': 10, @@ -2401,7 +2913,7 @@ def emergency_stop(self): self.flying = False -# ========== 中文UI渲染器 ========== +# ========== 中文UI渲染器(增强版) ========== class ChineseUIRenderer: """中文UI渲染器""" @@ -2431,6 +2943,12 @@ def __init__(self, speech_manager=None): 'performance_fast': (0, 255, 0), # 绿色 'performance_balanced': (255, 165, 0), # 橙色 'performance_accurate': (255, 0, 0), # 红色 + 'cpu_good': (0, 255, 0), # CPU使用率颜色 + 'cpu_warning': (255, 165, 0), + 'cpu_critical': (255, 0, 0), + 'memory_good': (0, 255, 0), # 内存使用率颜色 + 'memory_warning': (255, 165, 0), + 'memory_critical': (255, 0, 0), } print("✓ 中文UI渲染器已初始化") @@ -2484,18 +3002,18 @@ def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): size / 25, color, 1) return frame - def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, trajectory_recorder=None, - gesture_recognizer=None): - """绘制状态栏""" + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, + trajectory_recorder=None, gesture_recognizer=None, performance_analyzer=None): + """绘制状态栏(增强版)""" h, w = frame.shape[:2] # 绘制半透明背景 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (w, 120), (0, 0, 0), -1) + cv2.rectangle(overlay, (0, 0), (w, 130), (0, 0, 0), -1) frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 性能模式版" + title = "手势控制无人机系统 - 性能统计版" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -2557,12 +3075,52 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro status_text = f"轨迹点: {recorder_status['trajectory_points']}" frame = self.draw_text(frame, status_text, (w // 2, 65), size=12, color=self.colors['info']) + # 性能统计显示 + if performance_analyzer and config.get('display', 'show_performance_stats'): + perf_stats = performance_analyzer.get_stats_summary() + + # 性能状态颜色 + if perf_stats['performance_status'] == "良好": + perf_color = self.colors['performance_good'] + elif perf_stats['performance_status'] == "警告": + perf_color = self.colors['performance_warning'] + else: + perf_color = self.colors['performance_bad'] + + perf_text = f"性能: {perf_stats['performance_status']} ({perf_stats['fps']:.0f}FPS)" + frame = self.draw_text(frame, perf_text, (w // 2, 90), size=12, color=perf_color) + + # 系统资源显示 + if performance_analyzer and config.get('display', 'show_system_resources'): + cpu_usage = performance_analyzer.get_current_cpu_usage() + memory_usage = performance_analyzer.get_current_memory_usage() + + # CPU颜色 + if cpu_usage < 70: + cpu_color = self.colors['cpu_good'] + elif cpu_usage < 85: + cpu_color = self.colors['cpu_warning'] + else: + cpu_color = self.colors['cpu_critical'] + + # 内存颜色 + if memory_usage < 70: + mem_color = self.colors['memory_good'] + elif memory_usage < 85: + mem_color = self.colors['memory_warning'] + else: + mem_color = self.colors['memory_critical'] + + # 绘制系统资源信息 + sys_text = f"CPU:{cpu_usage:.0f}% 内存:{memory_usage:.0f}%" + frame = self.draw_text(frame, sys_text, (w // 2, 110), size=11, color=self.colors['info']) + # 语音状态 if config.get('display', 'show_speech_status') and self.speech_manager: speech_status = self.speech_manager.get_status() speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" - frame = self.draw_text(frame, speech_text, (w // 2, 90), size=12, color=speech_color) + frame = self.draw_text(frame, speech_text, (w - 200, 90), size=12, color=speech_color) # 性能信息 if config.get('display', 'show_fps'): @@ -2586,6 +3144,11 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro control_text = "提示: 确保手部完全进入画面,保持稳定手势" frame = self.draw_text(frame, control_text, (10, 90), size=12, color=self.colors['info']) + # 性能快捷键提示 + if performance_analyzer: + perf_tip = "P:报告 L:快照 K:导出" + frame = self.draw_text(frame, perf_tip, (10, 110), size=11, color=self.colors['help']) + return frame def draw_help_bar(self, frame): @@ -2596,18 +3159,19 @@ def draw_help_bar(self, frame): h, w = frame.shape[:2] # 绘制底部帮助栏 - cv2.rectangle(frame, (0, h - 100), (w, h), (0, 0, 0), -1) + cv2.rectangle(frame, (0, h - 120), (w, h), (0, 0, 0), -1) - # 帮助文本 + # 帮助文本(增强版,包含性能统计快捷键) help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音 P:性能报告 O:切换性能模式", + "V:切换语音反馈 M:测试语音 P:性能报告 L:性能快照 K:导出日志", + "O:切换性能模式 P:详细报告 R:重置统计", "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" ] for i, line in enumerate(help_lines): - y_pos = h - 85 + i * 20 + y_pos = h - 105 + i * 20 frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame @@ -2620,50 +3184,74 @@ def draw_warning(self, frame, message): warning_bg = np.zeros((40, w, 3), dtype=np.uint8) warning_bg[:, :] = (0, 69, 255) - frame[120:160, 0:w] = cv2.addWeighted( - frame[120:160, 0:w], 0.3, + frame[130:170, 0:w] = cv2.addWeighted( + frame[130:170, 0:w], 0.3, warning_bg, 0.7, 0 ) # 绘制警告文本 - frame = self.draw_text(frame, message, (10, 135), + frame = self.draw_text(frame, message, (10, 145), size=16, color=self.colors['warning']) return frame + def draw_performance_chart(self, frame, performance_analyzer): + """绘制性能图表""" + if not performance_analyzer: + return frame -# ========== 性能监控器 ========== -class PerformanceMonitor: - """性能监控器""" - - def __init__(self): - self.frame_times = deque(maxlen=60) - self.last_update = time.time() - self.fps = 0 - self.frame_count = 0 - - def update(self): - """更新性能数据""" - current_time = time.time() - self.frame_times.append(current_time) - self.frame_count += 1 + h, w = frame.shape[:2] - # 每秒计算一次FPS - if current_time - self.last_update >= 1.0: - if len(self.frame_times) > 1: - time_diff = self.frame_times[-1] - self.frame_times[0] - if time_diff > 0: - self.fps = len(self.frame_times) / time_diff - else: - self.fps = 0 - self.last_update = current_time + # 绘制图表区域背景 + chart_x = w - 250 + chart_y = 130 + chart_width = 240 + chart_height = 150 + + cv2.rectangle(frame, (chart_x, chart_y), + (chart_x + chart_width, chart_y + chart_height), + (30, 30, 30), -1) + cv2.rectangle(frame, (chart_x, chart_y), + (chart_x + chart_width, chart_y + chart_height), + (100, 100, 100), 1) + + # 绘制标题 + frame = self.draw_text(frame, "性能监控", (chart_x + 10, chart_y + 20), + size=14, color=self.colors['title']) + + # 绘制FPS图表 + if len(performance_analyzer.fps_history) > 1: + fps_values = list(performance_analyzer.fps_history)[-20:] # 最近20个值 + + # 计算最大值和最小值 + if fps_values: + max_fps = max(fps_values) + min_fps = min(fps_values) + if max_fps == min_fps: + max_fps = min_fps + 1 + + # 绘制FPS曲线 + points = [] + for i, fps in enumerate(fps_values): + x = chart_x + 10 + i * (chart_width - 20) / len(fps_values) + y = chart_y + chart_height - 30 - (fps - min_fps) * (chart_height - 50) / (max_fps - min_fps) + points.append((int(x), int(y))) + + # 连接点形成曲线 + for i in range(len(points) - 1): + cv2.line(frame, points[i], points[i + 1], (0, 255, 0), 2) + + # 显示当前FPS值 + current_fps = fps_values[-1] if fps_values else 0 + fps_color = self.colors['performance_good'] if current_fps > 20 else \ + self.colors['performance_warning'] if current_fps > 10 else \ + self.colors['performance_bad'] + + frame = self.draw_text(frame, f"FPS: {current_fps:.1f}", + (chart_x + 120, chart_y + 40), + size=12, color=fps_color) - def get_stats(self): - """获取性能统计""" - return { - 'fps': self.fps, - 'frame_count': self.frame_count - } + return frame # ========== 主程序 ========== @@ -2684,7 +3272,10 @@ def main(): gesture_recognizer = EnhancedGestureRecognizer(speech_manager) drone_controller = SimpleDroneController(libs['airsim'], speech_manager) ui_renderer = ChineseUIRenderer(speech_manager) - performance_monitor = PerformanceMonitor() + + # 初始化性能分析器 + print("初始化性能分析器...") + performance_analyzer = PerformanceAnalyzer(speech_manager) # 手势轨迹记录器 print("初始化手势轨迹记录器...") @@ -2729,47 +3320,48 @@ def main(): cap = None # 显示欢迎信息 - print("\n" + "=" * 60) - print("手势控制无人机系统 - 性能模式版") - print("=" * 60) + print("\n" + "=" * 80) + print("手势控制无人机系统 - 详细性能统计版") + print("=" * 80) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") print(f" 性能模式: {gesture_recognizer.performance_mode_name}") + print(f" 性能监控: {'已启用' if config.get('performance', 'enable_performance_monitor') else '已禁用'}") print(f" 轨迹记录: 支持录制/回放功能") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") - print("=" * 60) + print("=" * 80) # 显示操作说明 print("\n操作说明:") print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") - print("3. 性能模式选择:") + print("3. 性能统计功能:") + print(" - 按 [P] 键显示详细性能报告") + print(" - 按 [L] 键记录性能快照") + print(" - 按 [K] 键导出性能日志") + print(" - 按 [R] 键重置性能统计") + print(" - 系统自动监控: FPS, CPU, 内存, 识别时间等") + print("4. 性能模式选择:") print(" - 按 [O] 键循环切换性能模式: 最快(fast) → 平衡(balanced) → 最准(accurate)") - print(" - 最快模式: 性能优先,帧率最高,识别精度较低") - print(" - 平衡模式: 性能与精度平衡,适用于大多数场景") - print(" - 最准模式: 精度优先,提供最准确的手势识别") - print("4. 手势控制:") + print("5. 手势控制:") print(" - 手势识别后会有语音提示: 向上、向下、向左、向右、向前、停止") print(" - 手势稳定性越高,识别越准确") print(" - 手部距离摄像头适中时效果最佳") print(" * 手势识别置信度 > 60% 时才会执行") - print("5. 轨迹记录功能:") + print("6. 轨迹记录功能:") print(" [1]开始录制 [2]停止录制 [3]保存轨迹 [4]回放轨迹 [5]清除轨迹 [6]暂停/继续") - print(" - 录制时系统会记录手部位置和手势") - print(" - 回放时可以查看录制的轨迹") - print(" - 轨迹数据自动保存到 trajectories/ 目录") - print("6. 键盘控制:") + print("7. 键盘控制:") print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") - print("7. 调试功能:") + print("8. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") - print("8. 语音控制:") - print(" [V]切换语音反馈 [M]测试语音 [P]性能报告") - print("9. 按 [ESC] 安全退出") - print("=" * 60) + print("9. 语音控制:") + print(" [V]切换语音反馈 [M]测试语音") + print("10. 按 [ESC] 安全退出") + print("=" * 80) print("程序启动成功!") - print("-" * 60) + print("-" * 80) # 键盘手势映射 key_to_gesture = { @@ -2792,7 +3384,9 @@ def main(): try: while True: # 更新性能监控 - performance_monitor.update() + performance_analyzer.update_frame() + performance_analyzer.update_system_resources() + performance_analyzer.auto_report() # 读取摄像头帧 if cap: @@ -2804,15 +3398,21 @@ def main(): else: # 手势识别 gesture, confidence, frame = gesture_recognizer.recognize(frame) + + # 记录手势统计 + if gesture and gesture != "Waiting" and gesture != "摄像头错误": + performance_analyzer.record_gesture(gesture, confidence) else: # 模拟模式 frame = np.zeros((480, 640, 3), dtype=np.uint8) gesture, confidence = gesture_recognizer.current_gesture, gesture_recognizer.current_confidence # 获取性能统计 - perf_stats = performance_monitor.get_stats() process_time, frame_rate = gesture_recognizer.get_performance_stats() + # 更新手势识别时间 + performance_analyzer.update_gesture_recognition_time(process_time) + # 根据显示模式调整显示选项 if display_modes[current_display_mode] == 'normal': config.set('display', 'show_contours', value=True) @@ -2823,6 +3423,8 @@ def main(): config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) config.set('display', 'show_performance_mode', value=True) + config.set('display', 'show_performance_stats', value=True) + config.set('display', 'show_system_resources', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -2835,6 +3437,8 @@ def main(): config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) config.set('display', 'show_performance_mode', value=True) + config.set('display', 'show_performance_stats', value=True) + config.set('display', 'show_system_resources', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) @@ -2845,6 +3449,8 @@ def main(): config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) config.set('display', 'show_performance_mode', value=True) + config.set('display', 'show_performance_stats', value=True) + config.set('display', 'show_system_resources', value=True) config.set('display', 'show_debug_info', value=False) # 绘制轨迹(如果启用) @@ -2854,9 +3460,14 @@ def main(): # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, - perf_stats['fps'], process_time, trajectory_recorder, gesture_recognizer + performance_analyzer.get_current_fps(), process_time, + trajectory_recorder, gesture_recognizer, performance_analyzer ) + # 绘制性能图表(如果启用详细模式) + if display_modes[current_display_mode] == 'detailed': + frame = ui_renderer.draw_performance_chart(frame, performance_analyzer) + frame = ui_renderer.draw_help_bar(frame) # 显示连接提示 @@ -2864,8 +3475,13 @@ def main(): warning_msg = "⚠ 按C键连接无人机,或使用模拟模式" frame = ui_renderer.draw_warning(frame, warning_msg) + # 显示性能警告 + if performance_analyzer.performance_status != "良好": + warning_msg = f"⚠ 性能状态: {performance_analyzer.performance_status}" + frame = ui_renderer.draw_warning(frame, warning_msg) + # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Performance Mode', frame) + cv2.imshow('Gesture Controlled Drone - Performance Statistics', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -2947,14 +3563,42 @@ def main(): elif key == ord('p') or key == ord('P'): # 性能报告 + if key == ord('p'): # 小写p - 简要报告 + print("生成简要性能报告...") + performance_analyzer.print_report(detailed=False) + else: # 大写P - 详细报告 + print("生成详细性能报告...") + performance_analyzer.print_report(detailed=True) + + # 语音提示 if speech_manager.enabled: - print("生成性能报告...") - if process_time < 20: - speech_manager.speak_direct("系统性能优秀,运行流畅") - elif process_time < 50: - speech_manager.speak_direct("系统性能良好") - else: - speech_manager.speak_direct("系统性能警告,请检查") + speech_manager.speak('performance_report', immediate=True) + + elif key == ord('l') or key == ord('L'): + # 性能快照 + snapshot_label = f"快照_{datetime.now().strftime('%H:%M:%S')}" + snapshot = performance_analyzer.take_snapshot(snapshot_label) + + print(f"📸 性能快照已保存: {snapshot_label}") + print(f" 当前FPS: {snapshot['fps']:.1f}") + print(f" 平均FPS: {snapshot['avg_fps']:.1f}") + print(f" CPU使用率: {snapshot['cpu_usage']:.1f}%") + print(f" 内存使用率: {snapshot['memory_usage']:.1f}%") + + # 语音提示 + if speech_manager.enabled: + speech_manager.speak('performance_snapshot', immediate=True) + + elif key == ord('k') or key == ord('K'): + # 导出性能日志 + if performance_analyzer.export_log(): + print("✅ 性能日志导出成功") + + # 语音提示 + if speech_manager.enabled: + speech_manager.speak('performance_log_exported', immediate=True) + else: + print("❌ 性能日志导出失败") elif key == ord('o') or key == ord('O'): # 切换性能模式 @@ -2973,11 +3617,22 @@ def main(): print(f"✓ 已切换到性能模式: {gesture_recognizer.performance_mode_name}") + # 新增:重置性能统计 + elif key == ord('R') and chr(key).isupper(): # 大写R + print("重置性能统计...") + performance_analyzer.reset_session() + print("✓ 性能统计已重置") + + # 语音提示 + if speech_manager.enabled: + speech_manager.speak_direct("性能统计已重置") + # 新增:轨迹记录控制 elif key == ord('1'): # 开始录制轨迹 if trajectory_recorder.start_recording(): print("✅ 开始录制手势轨迹") + performance_analyzer.record_recording_session() else: print("⚠ 已经在录制中") @@ -2985,6 +3640,7 @@ def main(): # 停止录制轨迹 if trajectory_recorder.stop_recording(): print("✅ 停止录制手势轨迹") + performance_analyzer.record_recording_session(len(trajectory_recorder.trajectory_data)) else: print("⚠ 当前没有在录制") @@ -3022,15 +3678,21 @@ def main(): gesture_recognizer.set_simulated_gesture(simulated_gesture) gesture = simulated_gesture confidence = 0.9 + + # 记录手势统计 + performance_analyzer.record_gesture(gesture, confidence) + if drone_controller.connected and drone_controller.flying: - drone_controller.move_by_gesture(gesture, confidence) + success = drone_controller.move_by_gesture(gesture, confidence) + performance_analyzer.record_drone_command(success) # 真实手势控制 current_time = time.time() if (gesture and gesture != "Waiting" and gesture != "摄像头错误" and gesture != "Error" and drone_controller.connected and drone_controller.flying): - drone_controller.move_by_gesture(gesture, confidence) + success = drone_controller.move_by_gesture(gesture, confidence) + performance_analyzer.record_drone_command(success) # 处理轨迹记录 if cap and ret: @@ -3062,6 +3724,17 @@ def main(): cap.release() cv2.destroyAllWindows() + # 生成最终性能报告 + print("\n" + "=" * 80) + print("📊 最终性能总结") + print("=" * 80) + performance_analyzer.print_report(detailed=True) + + # 导出最终性能日志 + performance_analyzer.export_log() + + print("=" * 80) + # 程序退出语音提示 if speech_manager.enabled: speech_manager.speak('program_exit', force=True, immediate=True) @@ -3077,11 +3750,11 @@ def main(): config.save_config() print("程序安全退出") - print("=" * 60) + print("=" * 80) print("\n感谢使用手势控制无人机系统!") input("按回车键退出...") # ========== 程序入口 ========== if __name__ == "__main__": - main() \ No newline at end of file + main() From 729daf90b3a7af04d6c1cb2da821b3bd0305d8c9 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sun, 21 Dec 2025 13:23:16 +0800 Subject: [PATCH 13/18] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E9=80=89=E6=8B=A9=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main.py | 845 ++---------------- 1 file changed, 86 insertions(+), 759 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 8bda7d3989..3076402e7a 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 详细性能统计优化版 -新增完整的性能监控、统计和报告功能 +手势控制AirSim无人机 - 性能模式选择优化版 +新增fast(最快)、balanced(平衡)、accurate(最准)三种性能模式 作者: xiaoshiyuan888 """ @@ -13,18 +13,16 @@ import math import threading import tempfile -import pickle +import pickle # 新增:用于数据序列化 from datetime import datetime from PIL import Image, ImageDraw, ImageFont import cv2 import numpy as np from collections import deque, Counter -import psutil # 新增:用于系统资源监控 -import csv # 新增:用于性能日志导出 print("=" * 60) -print("Gesture Controlled Drone - Performance Statistics Enhanced") -print("详细性能统计优化版!") +print("Gesture Controlled Drone - Performance Mode Selection") +print("性能模式选择优化版!") print("=" * 60) # ========== 修复导入路径 ========== @@ -56,15 +54,6 @@ def safe_import(): print(f"[OpenCV] ✗ 导入失败: {e}") return None, modules_status - try: - import psutil - modules_status['psutil'] = True - print("[psutil] ✓ 系统资源监控库就绪") - except Exception as e: - modules_status['psutil'] = False - print(f"[psutil] ✗ 导入失败: {e}") - # 继续运行,psutil不是必需的 - airsim_module = None try: airsim_module = __import__('airsim') @@ -150,8 +139,7 @@ def safe_import(): 'np': np, 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, 'airsim': airsim_module, - 'speech': speech_module, - 'psutil': psutil if 'psutil' in modules_status and modules_status['psutil'] else None + 'speech': speech_module }, modules_status @@ -171,497 +159,6 @@ def safe_import(): Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -# ========== 性能分析器类 ========== -class PerformanceAnalyzer: - """性能分析器 - 监控和报告系统性能""" - - def __init__(self, speech_manager=None): - self.speech_manager = speech_manager - self.start_time = time.time() - self.session_start_time = time.time() - - # 帧率统计 - self.frame_times = deque(maxlen=300) # 保留最近300帧的时间戳 - self.frame_count = 0 - self.fps_history = deque(maxlen=100) # FPS历史记录 - - # 手势识别性能 - self.gesture_recognition_times = deque(maxlen=100) - self.avg_recognition_time = 0 - self.max_recognition_time = 0 - - # 系统资源监控 - self.cpu_usage_history = deque(maxlen=100) - self.memory_usage_history = deque(maxlen=100) - - # 性能事件记录 - self.performance_events = [] - self.performance_snapshots = [] - - # 手势统计 - self.gesture_counts = {} - self.gesture_confidence_sum = {} - self.gesture_confidence_count = {} - - # 错误统计 - self.error_count = 0 - self.warning_count = 0 - - # 无人机控制统计 - self.drone_commands = 0 - self.successful_commands = 0 - self.failed_commands = 0 - - # 轨迹记录统计 - self.recording_sessions = 0 - self.total_trajectory_points = 0 - - # 性能日志 - self.performance_log = [] - self.log_file = os.path.join(current_dir, 'performance_log.csv') - - # 性能阈值 - self.performance_thresholds = { - 'fps_warning': 15, - 'fps_critical': 5, - 'cpu_warning': 80, - 'cpu_critical': 90, - 'memory_warning': 85, - 'memory_critical': 95, - 'recognition_warning': 50, # ms - 'recognition_critical': 100 # ms - } - - # 性能状态 - self.performance_status = "良好" - self.last_performance_report = 0 - self.auto_report_interval = 60 # 自动报告间隔(秒) - - print("✓ 性能分析器已初始化") - - def update_frame(self): - """更新帧统计""" - current_time = time.time() - self.frame_times.append(current_time) - self.frame_count += 1 - - # 计算当前FPS - if len(self.frame_times) > 1: - time_span = self.frame_times[-1] - self.frame_times[0] - if time_span > 0: - current_fps = (len(self.frame_times) - 1) / time_span - self.fps_history.append(current_fps) - - def update_gesture_recognition_time(self, recognition_time_ms): - """更新手势识别时间""" - self.gesture_recognition_times.append(recognition_time_ms) - - # 更新平均识别时间 - if len(self.gesture_recognition_times) > 0: - self.avg_recognition_time = np.mean(list(self.gesture_recognition_times)) - self.max_recognition_time = max(self.max_recognition_time, recognition_time_ms) - - def update_system_resources(self): - """更新系统资源使用情况""" - try: - if libs['psutil']: - cpu_percent = libs['psutil'].cpu_percent(interval=0.1) - memory_percent = libs['psutil'].virtual_memory().percent - - self.cpu_usage_history.append(cpu_percent) - self.memory_usage_history.append(memory_percent) - - # 检查性能问题 - self.check_performance_issues(cpu_percent, memory_percent) - except: - pass # psutil不可用时跳过 - - def check_performance_issues(self, cpu_percent, memory_percent): - """检查性能问题""" - issues = [] - - # 检查FPS - if len(self.fps_history) > 0: - avg_fps = np.mean(list(self.fps_history[-10:])) if len(self.fps_history) >= 10 else self.fps_history[-1] - - if avg_fps < self.performance_thresholds['fps_critical']: - issues.append(("严重", f"帧率过低: {avg_fps:.1f} FPS")) - self.performance_status = "严重" - elif avg_fps < self.performance_thresholds['fps_warning']: - issues.append(("警告", f"帧率较低: {avg_fps:.1f} FPS")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 检查CPU使用率 - if cpu_percent > self.performance_thresholds['cpu_critical']: - issues.append(("严重", f"CPU使用率过高: {cpu_percent:.1f}%")) - self.performance_status = "严重" - elif cpu_percent > self.performance_thresholds['cpu_warning']: - issues.append(("警告", f"CPU使用率较高: {cpu_percent:.1f}%")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 检查内存使用率 - if memory_percent > self.performance_thresholds['memory_critical']: - issues.append(("严重", f"内存使用率过高: {memory_percent:.1f}%")) - self.performance_status = "严重" - elif memory_percent > self.performance_thresholds['memory_warning']: - issues.append(("警告", f"内存使用率较高: {memory_percent:.1f}%")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 检查手势识别时间 - if self.avg_recognition_time > self.performance_thresholds['recognition_critical']: - issues.append(("严重", f"手势识别时间过长: {self.avg_recognition_time:.1f}ms")) - self.performance_status = "严重" - elif self.avg_recognition_time > self.performance_thresholds['recognition_warning']: - issues.append(("警告", f"手势识别时间较长: {self.avg_recognition_time:.1f}ms")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 记录性能事件 - if issues: - for level, message in issues: - self.add_performance_event(level, message) - - # 语音提示(仅在状态变化时) - if (self.speech_manager and - self.speech_manager.enabled and - level == "严重"): - current_time = time.time() - if current_time - self.last_performance_report > 10: # 避免频繁提示 - self.speech_manager.speak_direct(f"性能{level}: {message}") - self.last_performance_report = current_time - - def add_performance_event(self, level, message): - """添加性能事件""" - event = { - 'timestamp': time.time(), - 'level': level, - 'message': message, - 'session_time': time.time() - self.session_start_time - } - self.performance_events.append(event) - - # 记录到日志 - self.log_performance_event(event) - - if level == "警告": - self.warning_count += 1 - elif level == "严重": - self.error_count += 1 - - def log_performance_event(self, event): - """记录性能事件到日志""" - log_entry = { - 'timestamp': datetime.fromtimestamp(event['timestamp']).strftime('%Y-%m-%d %H:%M:%S'), - 'session_time': f"{event['session_time']:.1f}", - 'level': event['level'], - 'message': event['message'] - } - self.performance_log.append(log_entry) - - def record_gesture(self, gesture, confidence): - """记录手势统计""" - if gesture not in self.gesture_counts: - self.gesture_counts[gesture] = 0 - self.gesture_confidence_sum[gesture] = 0 - self.gesture_confidence_count[gesture] = 0 - - self.gesture_counts[gesture] += 1 - self.gesture_confidence_sum[gesture] += confidence - self.gesture_confidence_count[gesture] += 1 - - def record_drone_command(self, success=True): - """记录无人机命令""" - self.drone_commands += 1 - if success: - self.successful_commands += 1 - else: - self.failed_commands += 1 - - def record_recording_session(self, points_count=0): - """记录录制会话""" - self.recording_sessions += 1 - self.total_trajectory_points += points_count - - def take_snapshot(self, label=""): - """拍摄性能快照""" - snapshot = { - 'timestamp': time.time(), - 'label': label, - 'fps': self.get_current_fps(), - 'avg_fps': self.get_average_fps(), - 'avg_recognition_time': self.avg_recognition_time, - 'max_recognition_time': self.max_recognition_time, - 'cpu_usage': self.get_current_cpu_usage(), - 'memory_usage': self.get_current_memory_usage(), - 'gesture_counts': dict(self.gesture_counts), - 'performance_status': self.performance_status, - 'frame_count': self.frame_count, - 'session_duration': time.time() - self.session_start_time - } - self.performance_snapshots.append(snapshot) - - print(f"📸 性能快照已保存: {label}") - return snapshot - - def get_current_fps(self): - """获取当前FPS""" - if len(self.fps_history) > 0: - return self.fps_history[-1] - return 0 - - def get_average_fps(self): - """获取平均FPS""" - if len(self.fps_history) > 0: - return np.mean(list(self.fps_history)) - return 0 - - def get_current_cpu_usage(self): - """获取当前CPU使用率""" - if len(self.cpu_usage_history) > 0: - return self.cpu_usage_history[-1] - return 0 - - def get_current_memory_usage(self): - """获取当前内存使用率""" - if len(self.memory_usage_history) > 0: - return self.memory_usage_history[-1] - return 0 - - def generate_report(self, detailed=True): - """生成性能报告""" - report_time = time.time() - session_duration = report_time - self.session_start_time - - # 基础报告 - report = { - '生成时间': datetime.fromtimestamp(report_time).strftime('%Y-%m-%d %H:%M:%S'), - '会话时长': f"{session_duration:.1f}秒", - '总帧数': self.frame_count, - '平均FPS': f"{self.get_average_fps():.1f}", - '当前FPS': f"{self.get_current_fps():.1f}", - '平均手势识别时间': f"{self.avg_recognition_time:.1f}ms", - '最大手势识别时间': f"{self.max_recognition_time:.1f}ms", - '当前CPU使用率': f"{self.get_current_cpu_usage():.1f}%", - '当前内存使用率': f"{self.get_current_memory_usage():.1f}%", - '性能状态': self.performance_status, - '警告数量': self.warning_count, - '错误数量': self.error_count, - '无人机命令': { - '总数': self.drone_commands, - '成功': self.successful_commands, - '失败': self.failed_commands, - '成功率': f"{(self.successful_commands / self.drone_commands * 100 if self.drone_commands > 0 else 0):.1f}%" - }, - '录制统计': { - '会话数': self.recording_sessions, - '总轨迹点数': self.total_trajectory_points - } - } - - # 详细报告 - if detailed: - # 手势统计 - gesture_stats = {} - for gesture in self.gesture_counts: - count = self.gesture_counts[gesture] - if gesture in self.gesture_confidence_count and self.gesture_confidence_count[gesture] > 0: - avg_confidence = self.gesture_confidence_sum[gesture] / self.gesture_confidence_count[gesture] - else: - avg_confidence = 0 - - gesture_stats[gesture] = { - '次数': count, - '占比': f"{(count / self.frame_count * 100 if self.frame_count > 0 else 0):.1f}%", - '平均置信度': f"{avg_confidence:.1%}" - } - - report['手势统计'] = gesture_stats - - # 性能事件 - if self.performance_events: - recent_events = list(self.performance_events)[-10:] # 最近10个事件 - report['最近性能事件'] = [ - { - '时间': datetime.fromtimestamp(e['timestamp']).strftime('%H:%M:%S'), - '级别': e['level'], - '消息': e['message'] - } - for e in recent_events - ] - - # 性能快照 - if self.performance_snapshots: - report['性能快照数'] = len(self.performance_snapshots) - - # 系统建议 - suggestions = self.generate_suggestions() - if suggestions: - report['优化建议'] = suggestions - - return report - - def generate_suggestions(self): - """生成优化建议""" - suggestions = [] - - # 检查FPS - avg_fps = self.get_average_fps() - if avg_fps < self.performance_thresholds['fps_warning']: - suggestions.append(f"帧率较低({avg_fps:.1f}FPS),建议切换到'最快'性能模式") - - # 检查CPU - cpu_usage = self.get_current_cpu_usage() - if cpu_usage > self.performance_thresholds['cpu_warning']: - suggestions.append(f"CPU使用率较高({cpu_usage:.1f}%),请关闭其他占用CPU的程序") - - # 检查内存 - memory_usage = self.get_current_memory_usage() - if memory_usage > self.performance_thresholds['memory_warning']: - suggestions.append(f"内存使用率较高({memory_usage:.1f}%),请关闭不必要的程序") - - # 检查识别时间 - if self.avg_recognition_time > self.performance_thresholds['recognition_warning']: - suggestions.append(f"手势识别时间较长({self.avg_recognition_time:.1f}ms),建议调整摄像头位置或光线") - - return suggestions - - def print_report(self, detailed=True): - """打印性能报告""" - report = self.generate_report(detailed) - - print("\n" + "=" * 80) - print("📊 性能分析报告") - print("=" * 80) - - # 基础信息 - print(f"生成时间: {report['生成时间']}") - print(f"会话时长: {report['会话时长']}") - print(f"总帧数: {report['总帧数']}") - print(f"平均FPS: {report['平均FPS']}") - print(f"当前FPS: {report['当前FPS']}") - print(f"平均手势识别时间: {report['平均手势识别时间']}") - print(f"最大手势识别时间: {report['最大手势识别时间']}") - print(f"当前CPU使用率: {report['当前CPU使用率']}") - print(f"当前内存使用率: {report['当前内存使用率']}") - print(f"性能状态: {report['性能状态']}") - - # 无人机命令统计 - cmd_stats = report['无人机命令'] - print(f"\n无人机命令统计:") - print(f" 总数: {cmd_stats['总数']}") - print(f" 成功: {cmd_stats['成功']}") - print(f" 失败: {cmd_stats['失败']}") - print(f" 成功率: {cmd_stats['成功率']}") - - # 录制统计 - rec_stats = report['录制统计'] - print(f"\n录制统计:") - print(f" 会话数: {rec_stats['会话数']}") - print(f" 总轨迹点数: {rec_stats['总轨迹点数']}") - - # 详细报告 - if detailed and '手势统计' in report: - print(f"\n手势统计:") - for gesture, stats in report['手势统计'].items(): - print(f" {gesture}: {stats['次数']}次 ({stats['占比']}), 平均置信度: {stats['平均置信度']}") - - # 性能事件 - if detailed and '最近性能事件' in report and report['最近性能事件']: - print(f"\n最近性能事件:") - for event in report['最近性能事件']: - print(f" [{event['时间']}] {event['级别']}: {event['消息']}") - - # 优化建议 - if detailed and '优化建议' in report and report['优化建议']: - print(f"\n优化建议:") - for i, suggestion in enumerate(report['优化建议'], 1): - print(f" {i}. {suggestion}") - - print("=" * 80) - - # 语音播报摘要 - if self.speech_manager and self.speech_manager.enabled: - summary = (f"性能报告: 平均帧率{report['平均FPS']},识别时间{report['平均手势识别时间']}," - f"性能状态{report['性能状态']},无人机命令成功率{cmd_stats['成功率']}") - self.speech_manager.speak_direct(summary) - - def export_log(self, filename=None): - """导出性能日志""" - if filename is None: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - filename = os.path.join(current_dir, f'performance_log_{timestamp}.csv') - - try: - with open(filename, 'w', newline='', encoding='utf-8') as f: - if self.performance_log: - fieldnames = self.performance_log[0].keys() - writer = csv.DictWriter(f, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(self.performance_log) - - print(f"📤 性能日志已导出到: {filename}") - return True - except Exception as e: - print(f"❌ 导出性能日志失败: {e}") - return False - - def auto_report(self): - """自动性能报告(定期执行)""" - current_time = time.time() - if current_time - self.last_performance_report > self.auto_report_interval: - # 生成简要报告 - report = self.generate_report(detailed=False) - - # 检查是否需要报告 - if (self.performance_status == "严重" or - self.warning_count > 5 or - self.error_count > 0): - - print(f"⚠ 自动性能检查: {report['性能状态']}, FPS: {report['当前FPS']}, " - f"CPU: {report['当前CPU使用率']}, 内存: {report['当前内存使用率']}") - - # 语音提示 - if (self.speech_manager and - self.speech_manager.enabled and - self.performance_status == "严重"): - self.speech_manager.speak_direct(f"系统性能{self.performance_status},建议检查") - - self.last_performance_report = current_time - - def reset_session(self): - """重置会话统计""" - self.session_start_time = time.time() - self.performance_events = [] - self.performance_snapshots = [] - self.gesture_counts = {} - self.gesture_confidence_sum = {} - self.gesture_confidence_count = {} - self.error_count = 0 - self.warning_count = 0 - self.drone_commands = 0 - self.successful_commands = 0 - self.failed_commands = 0 - self.performance_status = "良好" - - print("✓ 性能统计会话已重置") - - def get_stats_summary(self): - """获取统计摘要""" - return { - 'fps': self.get_current_fps(), - 'avg_fps': self.get_average_fps(), - 'recognition_time': self.avg_recognition_time, - 'cpu_usage': self.get_current_cpu_usage(), - 'memory_usage': self.get_current_memory_usage(), - 'performance_status': self.performance_status, - 'gesture_count': sum(self.gesture_counts.values()), - 'unique_gestures': len(self.gesture_counts) - } - - # ========== 增强语音反馈管理器 ========== class EnhancedSpeechFeedbackManager: """增强的语音反馈管理器""" @@ -742,10 +239,6 @@ def __init__(self, speech_lib): # 新增:性能相关 'performance_good': "系统运行流畅", 'performance_warning': "系统性能警告", - 'performance_critical': "系统性能严重警告", - 'performance_report': "性能报告生成完成", - 'performance_snapshot': "性能快照已保存", - 'performance_log_exported': "性能日志已导出", # 新增:手势指导 'move_closer': "请将手靠近摄像头", @@ -1456,8 +949,6 @@ def __init__(self): 'show_trajectory': True, 'show_recording_status': True, 'show_performance_mode': True, # 新增:显示性能模式 - 'show_performance_stats': True, # 新增:显示性能统计 - 'show_system_resources': True, # 新增:显示系统资源 }, 'performance': { 'target_fps': 30, @@ -1466,8 +957,6 @@ def __init__(self): 'mode': 'balanced', # 新增:性能模式选择 'current_mode_index': 1, # 当前模式索引 'modes': ['fast', 'balanced', 'accurate'], # 可用模式列表 - 'auto_report_interval': 60, # 自动报告间隔 - 'enable_performance_monitor': True, # 启用性能监控 }, 'calibration': { 'auto_calibrate_skin': True, @@ -1486,7 +975,6 @@ def __init__(self): 'announce_performance': True, 'announce_recording_events': True, 'announce_performance_mode': True, # 新增:播报性能模式切换 - 'announce_performance_events': True, # 新增:播报性能事件 'min_gesture_confidence': 0.7, 'gesture_start_threshold': 3, 'gesture_end_threshold': 10, @@ -2913,7 +2401,7 @@ def emergency_stop(self): self.flying = False -# ========== 中文UI渲染器(增强版) ========== +# ========== 中文UI渲染器 ========== class ChineseUIRenderer: """中文UI渲染器""" @@ -2943,12 +2431,6 @@ def __init__(self, speech_manager=None): 'performance_fast': (0, 255, 0), # 绿色 'performance_balanced': (255, 165, 0), # 橙色 'performance_accurate': (255, 0, 0), # 红色 - 'cpu_good': (0, 255, 0), # CPU使用率颜色 - 'cpu_warning': (255, 165, 0), - 'cpu_critical': (255, 0, 0), - 'memory_good': (0, 255, 0), # 内存使用率颜色 - 'memory_warning': (255, 165, 0), - 'memory_critical': (255, 0, 0), } print("✓ 中文UI渲染器已初始化") @@ -3002,18 +2484,18 @@ def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): size / 25, color, 1) return frame - def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, - trajectory_recorder=None, gesture_recognizer=None, performance_analyzer=None): - """绘制状态栏(增强版)""" + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, trajectory_recorder=None, + gesture_recognizer=None): + """绘制状态栏""" h, w = frame.shape[:2] # 绘制半透明背景 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (w, 130), (0, 0, 0), -1) + cv2.rectangle(overlay, (0, 0), (w, 120), (0, 0, 0), -1) frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 性能统计版" + title = "手势控制无人机系统 - 性能模式版" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -3075,52 +2557,12 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro status_text = f"轨迹点: {recorder_status['trajectory_points']}" frame = self.draw_text(frame, status_text, (w // 2, 65), size=12, color=self.colors['info']) - # 性能统计显示 - if performance_analyzer and config.get('display', 'show_performance_stats'): - perf_stats = performance_analyzer.get_stats_summary() - - # 性能状态颜色 - if perf_stats['performance_status'] == "良好": - perf_color = self.colors['performance_good'] - elif perf_stats['performance_status'] == "警告": - perf_color = self.colors['performance_warning'] - else: - perf_color = self.colors['performance_bad'] - - perf_text = f"性能: {perf_stats['performance_status']} ({perf_stats['fps']:.0f}FPS)" - frame = self.draw_text(frame, perf_text, (w // 2, 90), size=12, color=perf_color) - - # 系统资源显示 - if performance_analyzer and config.get('display', 'show_system_resources'): - cpu_usage = performance_analyzer.get_current_cpu_usage() - memory_usage = performance_analyzer.get_current_memory_usage() - - # CPU颜色 - if cpu_usage < 70: - cpu_color = self.colors['cpu_good'] - elif cpu_usage < 85: - cpu_color = self.colors['cpu_warning'] - else: - cpu_color = self.colors['cpu_critical'] - - # 内存颜色 - if memory_usage < 70: - mem_color = self.colors['memory_good'] - elif memory_usage < 85: - mem_color = self.colors['memory_warning'] - else: - mem_color = self.colors['memory_critical'] - - # 绘制系统资源信息 - sys_text = f"CPU:{cpu_usage:.0f}% 内存:{memory_usage:.0f}%" - frame = self.draw_text(frame, sys_text, (w // 2, 110), size=11, color=self.colors['info']) - # 语音状态 if config.get('display', 'show_speech_status') and self.speech_manager: speech_status = self.speech_manager.get_status() speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" - frame = self.draw_text(frame, speech_text, (w - 200, 90), size=12, color=speech_color) + frame = self.draw_text(frame, speech_text, (w // 2, 90), size=12, color=speech_color) # 性能信息 if config.get('display', 'show_fps'): @@ -3144,11 +2586,6 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro control_text = "提示: 确保手部完全进入画面,保持稳定手势" frame = self.draw_text(frame, control_text, (10, 90), size=12, color=self.colors['info']) - # 性能快捷键提示 - if performance_analyzer: - perf_tip = "P:报告 L:快照 K:导出" - frame = self.draw_text(frame, perf_tip, (10, 110), size=11, color=self.colors['help']) - return frame def draw_help_bar(self, frame): @@ -3159,19 +2596,18 @@ def draw_help_bar(self, frame): h, w = frame.shape[:2] # 绘制底部帮助栏 - cv2.rectangle(frame, (0, h - 120), (w, h), (0, 0, 0), -1) + cv2.rectangle(frame, (0, h - 100), (w, h), (0, 0, 0), -1) - # 帮助文本(增强版,包含性能统计快捷键) + # 帮助文本 help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音 P:性能报告 L:性能快照 K:导出日志", - "O:切换性能模式 P:详细报告 R:重置统计", + "V:切换语音反馈 M:测试语音 P:性能报告 O:切换性能模式", "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" ] for i, line in enumerate(help_lines): - y_pos = h - 105 + i * 20 + y_pos = h - 85 + i * 20 frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame @@ -3184,74 +2620,50 @@ def draw_warning(self, frame, message): warning_bg = np.zeros((40, w, 3), dtype=np.uint8) warning_bg[:, :] = (0, 69, 255) - frame[130:170, 0:w] = cv2.addWeighted( - frame[130:170, 0:w], 0.3, + frame[120:160, 0:w] = cv2.addWeighted( + frame[120:160, 0:w], 0.3, warning_bg, 0.7, 0 ) # 绘制警告文本 - frame = self.draw_text(frame, message, (10, 145), + frame = self.draw_text(frame, message, (10, 135), size=16, color=self.colors['warning']) return frame - def draw_performance_chart(self, frame, performance_analyzer): - """绘制性能图表""" - if not performance_analyzer: - return frame - h, w = frame.shape[:2] +# ========== 性能监控器 ========== +class PerformanceMonitor: + """性能监控器""" - # 绘制图表区域背景 - chart_x = w - 250 - chart_y = 130 - chart_width = 240 - chart_height = 150 - - cv2.rectangle(frame, (chart_x, chart_y), - (chart_x + chart_width, chart_y + chart_height), - (30, 30, 30), -1) - cv2.rectangle(frame, (chart_x, chart_y), - (chart_x + chart_width, chart_y + chart_height), - (100, 100, 100), 1) - - # 绘制标题 - frame = self.draw_text(frame, "性能监控", (chart_x + 10, chart_y + 20), - size=14, color=self.colors['title']) - - # 绘制FPS图表 - if len(performance_analyzer.fps_history) > 1: - fps_values = list(performance_analyzer.fps_history)[-20:] # 最近20个值 - - # 计算最大值和最小值 - if fps_values: - max_fps = max(fps_values) - min_fps = min(fps_values) - if max_fps == min_fps: - max_fps = min_fps + 1 - - # 绘制FPS曲线 - points = [] - for i, fps in enumerate(fps_values): - x = chart_x + 10 + i * (chart_width - 20) / len(fps_values) - y = chart_y + chart_height - 30 - (fps - min_fps) * (chart_height - 50) / (max_fps - min_fps) - points.append((int(x), int(y))) - - # 连接点形成曲线 - for i in range(len(points) - 1): - cv2.line(frame, points[i], points[i + 1], (0, 255, 0), 2) - - # 显示当前FPS值 - current_fps = fps_values[-1] if fps_values else 0 - fps_color = self.colors['performance_good'] if current_fps > 20 else \ - self.colors['performance_warning'] if current_fps > 10 else \ - self.colors['performance_bad'] - - frame = self.draw_text(frame, f"FPS: {current_fps:.1f}", - (chart_x + 120, chart_y + 40), - size=12, color=fps_color) + def __init__(self): + self.frame_times = deque(maxlen=60) + self.last_update = time.time() + self.fps = 0 + self.frame_count = 0 - return frame + def update(self): + """更新性能数据""" + current_time = time.time() + self.frame_times.append(current_time) + self.frame_count += 1 + + # 每秒计算一次FPS + if current_time - self.last_update >= 1.0: + if len(self.frame_times) > 1: + time_diff = self.frame_times[-1] - self.frame_times[0] + if time_diff > 0: + self.fps = len(self.frame_times) / time_diff + else: + self.fps = 0 + self.last_update = current_time + + def get_stats(self): + """获取性能统计""" + return { + 'fps': self.fps, + 'frame_count': self.frame_count + } # ========== 主程序 ========== @@ -3272,10 +2684,7 @@ def main(): gesture_recognizer = EnhancedGestureRecognizer(speech_manager) drone_controller = SimpleDroneController(libs['airsim'], speech_manager) ui_renderer = ChineseUIRenderer(speech_manager) - - # 初始化性能分析器 - print("初始化性能分析器...") - performance_analyzer = PerformanceAnalyzer(speech_manager) + performance_monitor = PerformanceMonitor() # 手势轨迹记录器 print("初始化手势轨迹记录器...") @@ -3320,48 +2729,47 @@ def main(): cap = None # 显示欢迎信息 - print("\n" + "=" * 80) - print("手势控制无人机系统 - 详细性能统计版") - print("=" * 80) + print("\n" + "=" * 60) + print("手势控制无人机系统 - 性能模式版") + print("=" * 60) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") print(f" 性能模式: {gesture_recognizer.performance_mode_name}") - print(f" 性能监控: {'已启用' if config.get('performance', 'enable_performance_monitor') else '已禁用'}") print(f" 轨迹记录: 支持录制/回放功能") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") - print("=" * 80) + print("=" * 60) # 显示操作说明 print("\n操作说明:") print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") - print("3. 性能统计功能:") - print(" - 按 [P] 键显示详细性能报告") - print(" - 按 [L] 键记录性能快照") - print(" - 按 [K] 键导出性能日志") - print(" - 按 [R] 键重置性能统计") - print(" - 系统自动监控: FPS, CPU, 内存, 识别时间等") - print("4. 性能模式选择:") + print("3. 性能模式选择:") print(" - 按 [O] 键循环切换性能模式: 最快(fast) → 平衡(balanced) → 最准(accurate)") - print("5. 手势控制:") + print(" - 最快模式: 性能优先,帧率最高,识别精度较低") + print(" - 平衡模式: 性能与精度平衡,适用于大多数场景") + print(" - 最准模式: 精度优先,提供最准确的手势识别") + print("4. 手势控制:") print(" - 手势识别后会有语音提示: 向上、向下、向左、向右、向前、停止") print(" - 手势稳定性越高,识别越准确") print(" - 手部距离摄像头适中时效果最佳") print(" * 手势识别置信度 > 60% 时才会执行") - print("6. 轨迹记录功能:") + print("5. 轨迹记录功能:") print(" [1]开始录制 [2]停止录制 [3]保存轨迹 [4]回放轨迹 [5]清除轨迹 [6]暂停/继续") - print("7. 键盘控制:") + print(" - 录制时系统会记录手部位置和手势") + print(" - 回放时可以查看录制的轨迹") + print(" - 轨迹数据自动保存到 trajectories/ 目录") + print("6. 键盘控制:") print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") - print("8. 调试功能:") + print("7. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") - print("9. 语音控制:") - print(" [V]切换语音反馈 [M]测试语音") - print("10. 按 [ESC] 安全退出") - print("=" * 80) + print("8. 语音控制:") + print(" [V]切换语音反馈 [M]测试语音 [P]性能报告") + print("9. 按 [ESC] 安全退出") + print("=" * 60) print("程序启动成功!") - print("-" * 80) + print("-" * 60) # 键盘手势映射 key_to_gesture = { @@ -3384,9 +2792,7 @@ def main(): try: while True: # 更新性能监控 - performance_analyzer.update_frame() - performance_analyzer.update_system_resources() - performance_analyzer.auto_report() + performance_monitor.update() # 读取摄像头帧 if cap: @@ -3398,21 +2804,15 @@ def main(): else: # 手势识别 gesture, confidence, frame = gesture_recognizer.recognize(frame) - - # 记录手势统计 - if gesture and gesture != "Waiting" and gesture != "摄像头错误": - performance_analyzer.record_gesture(gesture, confidence) else: # 模拟模式 frame = np.zeros((480, 640, 3), dtype=np.uint8) gesture, confidence = gesture_recognizer.current_gesture, gesture_recognizer.current_confidence # 获取性能统计 + perf_stats = performance_monitor.get_stats() process_time, frame_rate = gesture_recognizer.get_performance_stats() - # 更新手势识别时间 - performance_analyzer.update_gesture_recognition_time(process_time) - # 根据显示模式调整显示选项 if display_modes[current_display_mode] == 'normal': config.set('display', 'show_contours', value=True) @@ -3423,8 +2823,6 @@ def main(): config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) config.set('display', 'show_performance_mode', value=True) - config.set('display', 'show_performance_stats', value=True) - config.set('display', 'show_system_resources', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -3437,8 +2835,6 @@ def main(): config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) config.set('display', 'show_performance_mode', value=True) - config.set('display', 'show_performance_stats', value=True) - config.set('display', 'show_system_resources', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) @@ -3449,8 +2845,6 @@ def main(): config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) config.set('display', 'show_performance_mode', value=True) - config.set('display', 'show_performance_stats', value=True) - config.set('display', 'show_system_resources', value=True) config.set('display', 'show_debug_info', value=False) # 绘制轨迹(如果启用) @@ -3460,14 +2854,9 @@ def main(): # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, - performance_analyzer.get_current_fps(), process_time, - trajectory_recorder, gesture_recognizer, performance_analyzer + perf_stats['fps'], process_time, trajectory_recorder, gesture_recognizer ) - # 绘制性能图表(如果启用详细模式) - if display_modes[current_display_mode] == 'detailed': - frame = ui_renderer.draw_performance_chart(frame, performance_analyzer) - frame = ui_renderer.draw_help_bar(frame) # 显示连接提示 @@ -3475,13 +2864,8 @@ def main(): warning_msg = "⚠ 按C键连接无人机,或使用模拟模式" frame = ui_renderer.draw_warning(frame, warning_msg) - # 显示性能警告 - if performance_analyzer.performance_status != "良好": - warning_msg = f"⚠ 性能状态: {performance_analyzer.performance_status}" - frame = ui_renderer.draw_warning(frame, warning_msg) - # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Performance Statistics', frame) + cv2.imshow('Gesture Controlled Drone - Performance Mode', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -3563,42 +2947,14 @@ def main(): elif key == ord('p') or key == ord('P'): # 性能报告 - if key == ord('p'): # 小写p - 简要报告 - print("生成简要性能报告...") - performance_analyzer.print_report(detailed=False) - else: # 大写P - 详细报告 - print("生成详细性能报告...") - performance_analyzer.print_report(detailed=True) - - # 语音提示 if speech_manager.enabled: - speech_manager.speak('performance_report', immediate=True) - - elif key == ord('l') or key == ord('L'): - # 性能快照 - snapshot_label = f"快照_{datetime.now().strftime('%H:%M:%S')}" - snapshot = performance_analyzer.take_snapshot(snapshot_label) - - print(f"📸 性能快照已保存: {snapshot_label}") - print(f" 当前FPS: {snapshot['fps']:.1f}") - print(f" 平均FPS: {snapshot['avg_fps']:.1f}") - print(f" CPU使用率: {snapshot['cpu_usage']:.1f}%") - print(f" 内存使用率: {snapshot['memory_usage']:.1f}%") - - # 语音提示 - if speech_manager.enabled: - speech_manager.speak('performance_snapshot', immediate=True) - - elif key == ord('k') or key == ord('K'): - # 导出性能日志 - if performance_analyzer.export_log(): - print("✅ 性能日志导出成功") - - # 语音提示 - if speech_manager.enabled: - speech_manager.speak('performance_log_exported', immediate=True) - else: - print("❌ 性能日志导出失败") + print("生成性能报告...") + if process_time < 20: + speech_manager.speak_direct("系统性能优秀,运行流畅") + elif process_time < 50: + speech_manager.speak_direct("系统性能良好") + else: + speech_manager.speak_direct("系统性能警告,请检查") elif key == ord('o') or key == ord('O'): # 切换性能模式 @@ -3617,22 +2973,11 @@ def main(): print(f"✓ 已切换到性能模式: {gesture_recognizer.performance_mode_name}") - # 新增:重置性能统计 - elif key == ord('R') and chr(key).isupper(): # 大写R - print("重置性能统计...") - performance_analyzer.reset_session() - print("✓ 性能统计已重置") - - # 语音提示 - if speech_manager.enabled: - speech_manager.speak_direct("性能统计已重置") - # 新增:轨迹记录控制 elif key == ord('1'): # 开始录制轨迹 if trajectory_recorder.start_recording(): print("✅ 开始录制手势轨迹") - performance_analyzer.record_recording_session() else: print("⚠ 已经在录制中") @@ -3640,7 +2985,6 @@ def main(): # 停止录制轨迹 if trajectory_recorder.stop_recording(): print("✅ 停止录制手势轨迹") - performance_analyzer.record_recording_session(len(trajectory_recorder.trajectory_data)) else: print("⚠ 当前没有在录制") @@ -3678,21 +3022,15 @@ def main(): gesture_recognizer.set_simulated_gesture(simulated_gesture) gesture = simulated_gesture confidence = 0.9 - - # 记录手势统计 - performance_analyzer.record_gesture(gesture, confidence) - if drone_controller.connected and drone_controller.flying: - success = drone_controller.move_by_gesture(gesture, confidence) - performance_analyzer.record_drone_command(success) + drone_controller.move_by_gesture(gesture, confidence) # 真实手势控制 current_time = time.time() if (gesture and gesture != "Waiting" and gesture != "摄像头错误" and gesture != "Error" and drone_controller.connected and drone_controller.flying): - success = drone_controller.move_by_gesture(gesture, confidence) - performance_analyzer.record_drone_command(success) + drone_controller.move_by_gesture(gesture, confidence) # 处理轨迹记录 if cap and ret: @@ -3724,17 +3062,6 @@ def main(): cap.release() cv2.destroyAllWindows() - # 生成最终性能报告 - print("\n" + "=" * 80) - print("📊 最终性能总结") - print("=" * 80) - performance_analyzer.print_report(detailed=True) - - # 导出最终性能日志 - performance_analyzer.export_log() - - print("=" * 80) - # 程序退出语音提示 if speech_manager.enabled: speech_manager.speak('program_exit', force=True, immediate=True) @@ -3750,11 +3077,11 @@ def main(): config.save_config() print("程序安全退出") - print("=" * 80) + print("=" * 60) print("\n感谢使用手势控制无人机系统!") input("按回车键退出...") # ========== 程序入口 ========== if __name__ == "__main__": - main() + main() \ No newline at end of file From a73dbea5e9cfdd1a0e943b23d9555d1234b7bba6 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sun, 21 Dec 2025 14:20:50 +0800 Subject: [PATCH 14/18] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AE=8C=E6=95=B4?= =?UTF-8?q?=E7=9A=84=E6=80=A7=E8=83=BD=E7=BB=9F=E8=AE=A1=E4=B8=8E=E5=88=86?= =?UTF-8?q?=E6=9E=90=E7=B3=BB=E7=BB=9F=EF=BC=8C=E5=AE=9E=E7=8E=B0=E5=AE=9E?= =?UTF-8?q?=E6=97=B6=E6=80=A7=E8=83=BD=E7=9B=91=E6=8E=A7=E3=80=81=E8=AF=A6?= =?UTF-8?q?=E7=BB=86=E6=8A=A5=E5=91=8A=E5=92=8C=E7=B3=BB=E7=BB=9F=E8=B5=84?= =?UTF-8?q?=E6=BA=90=E7=AE=A1=E7=90=86=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main.py | 843 ++++++++++++++++-- 1 file changed, 758 insertions(+), 85 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 3076402e7a..7e4ae44fbf 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 性能模式选择优化版 -新增fast(最快)、balanced(平衡)、accurate(最准)三种性能模式 +手势控制AirSim无人机 - 详细性能统计优化版 +新增完整的性能监控、统计和报告功能 作者: xiaoshiyuan888 """ @@ -13,16 +13,18 @@ import math import threading import tempfile -import pickle # 新增:用于数据序列化 +import pickle from datetime import datetime from PIL import Image, ImageDraw, ImageFont import cv2 import numpy as np from collections import deque, Counter +import psutil # 新增:用于系统资源监控 +import csv # 新增:用于性能日志导出 print("=" * 60) -print("Gesture Controlled Drone - Performance Mode Selection") -print("性能模式选择优化版!") +print("Gesture Controlled Drone - Performance Statistics Enhanced") +print("详细性能统计优化版!") print("=" * 60) # ========== 修复导入路径 ========== @@ -54,6 +56,15 @@ def safe_import(): print(f"[OpenCV] ✗ 导入失败: {e}") return None, modules_status + try: + import psutil + modules_status['psutil'] = True + print("[psutil] ✓ 系统资源监控库就绪") + except Exception as e: + modules_status['psutil'] = False + print(f"[psutil] ✗ 导入失败: {e}") + # 继续运行,psutil不是必需的 + airsim_module = None try: airsim_module = __import__('airsim') @@ -139,7 +150,8 @@ def safe_import(): 'np': np, 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, 'airsim': airsim_module, - 'speech': speech_module + 'speech': speech_module, + 'psutil': psutil if 'psutil' in modules_status and modules_status['psutil'] else None }, modules_status @@ -159,6 +171,497 @@ def safe_import(): Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] +# ========== 性能分析器类 ========== +class PerformanceAnalyzer: + """性能分析器 - 监控和报告系统性能""" + + def __init__(self, speech_manager=None): + self.speech_manager = speech_manager + self.start_time = time.time() + self.session_start_time = time.time() + + # 帧率统计 + self.frame_times = deque(maxlen=300) # 保留最近300帧的时间戳 + self.frame_count = 0 + self.fps_history = deque(maxlen=100) # FPS历史记录 + + # 手势识别性能 + self.gesture_recognition_times = deque(maxlen=100) + self.avg_recognition_time = 0 + self.max_recognition_time = 0 + + # 系统资源监控 + self.cpu_usage_history = deque(maxlen=100) + self.memory_usage_history = deque(maxlen=100) + + # 性能事件记录 + self.performance_events = [] + self.performance_snapshots = [] + + # 手势统计 + self.gesture_counts = {} + self.gesture_confidence_sum = {} + self.gesture_confidence_count = {} + + # 错误统计 + self.error_count = 0 + self.warning_count = 0 + + # 无人机控制统计 + self.drone_commands = 0 + self.successful_commands = 0 + self.failed_commands = 0 + + # 轨迹记录统计 + self.recording_sessions = 0 + self.total_trajectory_points = 0 + + # 性能日志 + self.performance_log = [] + self.log_file = os.path.join(current_dir, 'performance_log.csv') + + # 性能阈值 + self.performance_thresholds = { + 'fps_warning': 15, + 'fps_critical': 5, + 'cpu_warning': 80, + 'cpu_critical': 90, + 'memory_warning': 85, + 'memory_critical': 95, + 'recognition_warning': 50, # ms + 'recognition_critical': 100 # ms + } + + # 性能状态 + self.performance_status = "良好" + self.last_performance_report = 0 + self.auto_report_interval = 60 # 自动报告间隔(秒) + + print("✓ 性能分析器已初始化") + + def update_frame(self): + """更新帧统计""" + current_time = time.time() + self.frame_times.append(current_time) + self.frame_count += 1 + + # 计算当前FPS + if len(self.frame_times) > 1: + time_span = self.frame_times[-1] - self.frame_times[0] + if time_span > 0: + current_fps = (len(self.frame_times) - 1) / time_span + self.fps_history.append(current_fps) + + def update_gesture_recognition_time(self, recognition_time_ms): + """更新手势识别时间""" + self.gesture_recognition_times.append(recognition_time_ms) + + # 更新平均识别时间 + if len(self.gesture_recognition_times) > 0: + self.avg_recognition_time = np.mean(list(self.gesture_recognition_times)) + self.max_recognition_time = max(self.max_recognition_time, recognition_time_ms) + + def update_system_resources(self): + """更新系统资源使用情况""" + try: + if libs['psutil']: + cpu_percent = libs['psutil'].cpu_percent(interval=0.1) + memory_percent = libs['psutil'].virtual_memory().percent + + self.cpu_usage_history.append(cpu_percent) + self.memory_usage_history.append(memory_percent) + + # 检查性能问题 + self.check_performance_issues(cpu_percent, memory_percent) + except: + pass # psutil不可用时跳过 + + def check_performance_issues(self, cpu_percent, memory_percent): + """检查性能问题""" + issues = [] + + # 检查FPS + if len(self.fps_history) > 0: + avg_fps = np.mean(list(self.fps_history[-10:])) if len(self.fps_history) >= 10 else self.fps_history[-1] + + if avg_fps < self.performance_thresholds['fps_critical']: + issues.append(("严重", f"帧率过低: {avg_fps:.1f} FPS")) + self.performance_status = "严重" + elif avg_fps < self.performance_thresholds['fps_warning']: + issues.append(("警告", f"帧率较低: {avg_fps:.1f} FPS")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查CPU使用率 + if cpu_percent > self.performance_thresholds['cpu_critical']: + issues.append(("严重", f"CPU使用率过高: {cpu_percent:.1f}%")) + self.performance_status = "严重" + elif cpu_percent > self.performance_thresholds['cpu_warning']: + issues.append(("警告", f"CPU使用率较高: {cpu_percent:.1f}%")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查内存使用率 + if memory_percent > self.performance_thresholds['memory_critical']: + issues.append(("严重", f"内存使用率过高: {memory_percent:.1f}%")) + self.performance_status = "严重" + elif memory_percent > self.performance_thresholds['memory_warning']: + issues.append(("警告", f"内存使用率较高: {memory_percent:.1f}%")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查手势识别时间 + if self.avg_recognition_time > self.performance_thresholds['recognition_critical']: + issues.append(("严重", f"手势识别时间过长: {self.avg_recognition_time:.1f}ms")) + self.performance_status = "严重" + elif self.avg_recognition_time > self.performance_thresholds['recognition_warning']: + issues.append(("警告", f"手势识别时间较长: {self.avg_recognition_time:.1f}ms")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 记录性能事件 + if issues: + for level, message in issues: + self.add_performance_event(level, message) + + # 语音提示(仅在状态变化时) + if (self.speech_manager and + self.speech_manager.enabled and + level == "严重"): + current_time = time.time() + if current_time - self.last_performance_report > 10: # 避免频繁提示 + self.speech_manager.speak_direct(f"性能{level}: {message}") + self.last_performance_report = current_time + + def add_performance_event(self, level, message): + """添加性能事件""" + event = { + 'timestamp': time.time(), + 'level': level, + 'message': message, + 'session_time': time.time() - self.session_start_time + } + self.performance_events.append(event) + + # 记录到日志 + self.log_performance_event(event) + + if level == "警告": + self.warning_count += 1 + elif level == "严重": + self.error_count += 1 + + def log_performance_event(self, event): + """记录性能事件到日志""" + log_entry = { + 'timestamp': datetime.fromtimestamp(event['timestamp']).strftime('%Y-%m-%d %H:%M:%S'), + 'session_time': f"{event['session_time']:.1f}", + 'level': event['level'], + 'message': event['message'] + } + self.performance_log.append(log_entry) + + def record_gesture(self, gesture, confidence): + """记录手势统计""" + if gesture not in self.gesture_counts: + self.gesture_counts[gesture] = 0 + self.gesture_confidence_sum[gesture] = 0 + self.gesture_confidence_count[gesture] = 0 + + self.gesture_counts[gesture] += 1 + self.gesture_confidence_sum[gesture] += confidence + self.gesture_confidence_count[gesture] += 1 + + def record_drone_command(self, success=True): + """记录无人机命令""" + self.drone_commands += 1 + if success: + self.successful_commands += 1 + else: + self.failed_commands += 1 + + def record_recording_session(self, points_count=0): + """记录录制会话""" + self.recording_sessions += 1 + self.total_trajectory_points += points_count + + def take_snapshot(self, label=""): + """拍摄性能快照""" + snapshot = { + 'timestamp': time.time(), + 'label': label, + 'fps': self.get_current_fps(), + 'avg_fps': self.get_average_fps(), + 'avg_recognition_time': self.avg_recognition_time, + 'max_recognition_time': self.max_recognition_time, + 'cpu_usage': self.get_current_cpu_usage(), + 'memory_usage': self.get_current_memory_usage(), + 'gesture_counts': dict(self.gesture_counts), + 'performance_status': self.performance_status, + 'frame_count': self.frame_count, + 'session_duration': time.time() - self.session_start_time + } + self.performance_snapshots.append(snapshot) + + print(f"📸 性能快照已保存: {label}") + return snapshot + + def get_current_fps(self): + """获取当前FPS""" + if len(self.fps_history) > 0: + return self.fps_history[-1] + return 0 + + def get_average_fps(self): + """获取平均FPS""" + if len(self.fps_history) > 0: + return np.mean(list(self.fps_history)) + return 0 + + def get_current_cpu_usage(self): + """获取当前CPU使用率""" + if len(self.cpu_usage_history) > 0: + return self.cpu_usage_history[-1] + return 0 + + def get_current_memory_usage(self): + """获取当前内存使用率""" + if len(self.memory_usage_history) > 0: + return self.memory_usage_history[-1] + return 0 + + def generate_report(self, detailed=True): + """生成性能报告""" + report_time = time.time() + session_duration = report_time - self.session_start_time + + # 基础报告 + report = { + '生成时间': datetime.fromtimestamp(report_time).strftime('%Y-%m-%d %H:%M:%S'), + '会话时长': f"{session_duration:.1f}秒", + '总帧数': self.frame_count, + '平均FPS': f"{self.get_average_fps():.1f}", + '当前FPS': f"{self.get_current_fps():.1f}", + '平均手势识别时间': f"{self.avg_recognition_time:.1f}ms", + '最大手势识别时间': f"{self.max_recognition_time:.1f}ms", + '当前CPU使用率': f"{self.get_current_cpu_usage():.1f}%", + '当前内存使用率': f"{self.get_current_memory_usage():.1f}%", + '性能状态': self.performance_status, + '警告数量': self.warning_count, + '错误数量': self.error_count, + '无人机命令': { + '总数': self.drone_commands, + '成功': self.successful_commands, + '失败': self.failed_commands, + '成功率': f"{(self.successful_commands / self.drone_commands * 100 if self.drone_commands > 0 else 0):.1f}%" + }, + '录制统计': { + '会话数': self.recording_sessions, + '总轨迹点数': self.total_trajectory_points + } + } + + # 详细报告 + if detailed: + # 手势统计 + gesture_stats = {} + for gesture in self.gesture_counts: + count = self.gesture_counts[gesture] + if gesture in self.gesture_confidence_count and self.gesture_confidence_count[gesture] > 0: + avg_confidence = self.gesture_confidence_sum[gesture] / self.gesture_confidence_count[gesture] + else: + avg_confidence = 0 + + gesture_stats[gesture] = { + '次数': count, + '占比': f"{(count / self.frame_count * 100 if self.frame_count > 0 else 0):.1f}%", + '平均置信度': f"{avg_confidence:.1%}" + } + + report['手势统计'] = gesture_stats + + # 性能事件 + if self.performance_events: + recent_events = list(self.performance_events)[-10:] # 最近10个事件 + report['最近性能事件'] = [ + { + '时间': datetime.fromtimestamp(e['timestamp']).strftime('%H:%M:%S'), + '级别': e['level'], + '消息': e['message'] + } + for e in recent_events + ] + + # 性能快照 + if self.performance_snapshots: + report['性能快照数'] = len(self.performance_snapshots) + + # 系统建议 + suggestions = self.generate_suggestions() + if suggestions: + report['优化建议'] = suggestions + + return report + + def generate_suggestions(self): + """生成优化建议""" + suggestions = [] + + # 检查FPS + avg_fps = self.get_average_fps() + if avg_fps < self.performance_thresholds['fps_warning']: + suggestions.append(f"帧率较低({avg_fps:.1f}FPS),建议切换到'最快'性能模式") + + # 检查CPU + cpu_usage = self.get_current_cpu_usage() + if cpu_usage > self.performance_thresholds['cpu_warning']: + suggestions.append(f"CPU使用率较高({cpu_usage:.1f}%),请关闭其他占用CPU的程序") + + # 检查内存 + memory_usage = self.get_current_memory_usage() + if memory_usage > self.performance_thresholds['memory_warning']: + suggestions.append(f"内存使用率较高({memory_usage:.1f}%),请关闭不必要的程序") + + # 检查识别时间 + if self.avg_recognition_time > self.performance_thresholds['recognition_warning']: + suggestions.append(f"手势识别时间较长({self.avg_recognition_time:.1f}ms),建议调整摄像头位置或光线") + + return suggestions + + def print_report(self, detailed=True): + """打印性能报告""" + report = self.generate_report(detailed) + + print("\n" + "=" * 80) + print("📊 性能分析报告") + print("=" * 80) + + # 基础信息 + print(f"生成时间: {report['生成时间']}") + print(f"会话时长: {report['会话时长']}") + print(f"总帧数: {report['总帧数']}") + print(f"平均FPS: {report['平均FPS']}") + print(f"当前FPS: {report['当前FPS']}") + print(f"平均手势识别时间: {report['平均手势识别时间']}") + print(f"最大手势识别时间: {report['最大手势识别时间']}") + print(f"当前CPU使用率: {report['当前CPU使用率']}") + print(f"当前内存使用率: {report['当前内存使用率']}") + print(f"性能状态: {report['性能状态']}") + + # 无人机命令统计 + cmd_stats = report['无人机命令'] + print(f"\n无人机命令统计:") + print(f" 总数: {cmd_stats['总数']}") + print(f" 成功: {cmd_stats['成功']}") + print(f" 失败: {cmd_stats['失败']}") + print(f" 成功率: {cmd_stats['成功率']}") + + # 录制统计 + rec_stats = report['录制统计'] + print(f"\n录制统计:") + print(f" 会话数: {rec_stats['会话数']}") + print(f" 总轨迹点数: {rec_stats['总轨迹点数']}") + + # 详细报告 + if detailed and '手势统计' in report: + print(f"\n手势统计:") + for gesture, stats in report['手势统计'].items(): + print(f" {gesture}: {stats['次数']}次 ({stats['占比']}), 平均置信度: {stats['平均置信度']}") + + # 性能事件 + if detailed and '最近性能事件' in report and report['最近性能事件']: + print(f"\n最近性能事件:") + for event in report['最近性能事件']: + print(f" [{event['时间']}] {event['级别']}: {event['消息']}") + + # 优化建议 + if detailed and '优化建议' in report and report['优化建议']: + print(f"\n优化建议:") + for i, suggestion in enumerate(report['优化建议'], 1): + print(f" {i}. {suggestion}") + + print("=" * 80) + + # 语音播报摘要 + if self.speech_manager and self.speech_manager.enabled: + summary = (f"性能报告: 平均帧率{report['平均FPS']},识别时间{report['平均手势识别时间']}," + f"性能状态{report['性能状态']},无人机命令成功率{cmd_stats['成功率']}") + self.speech_manager.speak_direct(summary) + + def export_log(self, filename=None): + """导出性能日志""" + if filename is None: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = os.path.join(current_dir, f'performance_log_{timestamp}.csv') + + try: + with open(filename, 'w', newline='', encoding='utf-8') as f: + if self.performance_log: + fieldnames = self.performance_log[0].keys() + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(self.performance_log) + + print(f"📤 性能日志已导出到: {filename}") + return True + except Exception as e: + print(f"❌ 导出性能日志失败: {e}") + return False + + def auto_report(self): + """自动性能报告(定期执行)""" + current_time = time.time() + if current_time - self.last_performance_report > self.auto_report_interval: + # 生成简要报告 + report = self.generate_report(detailed=False) + + # 检查是否需要报告 + if (self.performance_status == "严重" or + self.warning_count > 5 or + self.error_count > 0): + + print(f"⚠ 自动性能检查: {report['性能状态']}, FPS: {report['当前FPS']}, " + f"CPU: {report['当前CPU使用率']}, 内存: {report['当前内存使用率']}") + + # 语音提示 + if (self.speech_manager and + self.speech_manager.enabled and + self.performance_status == "严重"): + self.speech_manager.speak_direct(f"系统性能{self.performance_status},建议检查") + + self.last_performance_report = current_time + + def reset_session(self): + """重置会话统计""" + self.session_start_time = time.time() + self.performance_events = [] + self.performance_snapshots = [] + self.gesture_counts = {} + self.gesture_confidence_sum = {} + self.gesture_confidence_count = {} + self.error_count = 0 + self.warning_count = 0 + self.drone_commands = 0 + self.successful_commands = 0 + self.failed_commands = 0 + self.performance_status = "良好" + + print("✓ 性能统计会话已重置") + + def get_stats_summary(self): + """获取统计摘要""" + return { + 'fps': self.get_current_fps(), + 'avg_fps': self.get_average_fps(), + 'recognition_time': self.avg_recognition_time, + 'cpu_usage': self.get_current_cpu_usage(), + 'memory_usage': self.get_current_memory_usage(), + 'performance_status': self.performance_status, + 'gesture_count': sum(self.gesture_counts.values()), + 'unique_gestures': len(self.gesture_counts) + } + + # ========== 增强语音反馈管理器 ========== class EnhancedSpeechFeedbackManager: """增强的语音反馈管理器""" @@ -239,6 +742,10 @@ def __init__(self, speech_lib): # 新增:性能相关 'performance_good': "系统运行流畅", 'performance_warning': "系统性能警告", + 'performance_critical': "系统性能严重警告", + 'performance_report': "性能报告生成完成", + 'performance_snapshot': "性能快照已保存", + 'performance_log_exported': "性能日志已导出", # 新增:手势指导 'move_closer': "请将手靠近摄像头", @@ -949,6 +1456,8 @@ def __init__(self): 'show_trajectory': True, 'show_recording_status': True, 'show_performance_mode': True, # 新增:显示性能模式 + 'show_performance_stats': True, # 新增:显示性能统计 + 'show_system_resources': True, # 新增:显示系统资源 }, 'performance': { 'target_fps': 30, @@ -957,6 +1466,8 @@ def __init__(self): 'mode': 'balanced', # 新增:性能模式选择 'current_mode_index': 1, # 当前模式索引 'modes': ['fast', 'balanced', 'accurate'], # 可用模式列表 + 'auto_report_interval': 60, # 自动报告间隔 + 'enable_performance_monitor': True, # 启用性能监控 }, 'calibration': { 'auto_calibrate_skin': True, @@ -975,6 +1486,7 @@ def __init__(self): 'announce_performance': True, 'announce_recording_events': True, 'announce_performance_mode': True, # 新增:播报性能模式切换 + 'announce_performance_events': True, # 新增:播报性能事件 'min_gesture_confidence': 0.7, 'gesture_start_threshold': 3, 'gesture_end_threshold': 10, @@ -2401,7 +2913,7 @@ def emergency_stop(self): self.flying = False -# ========== 中文UI渲染器 ========== +# ========== 中文UI渲染器(增强版) ========== class ChineseUIRenderer: """中文UI渲染器""" @@ -2431,6 +2943,12 @@ def __init__(self, speech_manager=None): 'performance_fast': (0, 255, 0), # 绿色 'performance_balanced': (255, 165, 0), # 橙色 'performance_accurate': (255, 0, 0), # 红色 + 'cpu_good': (0, 255, 0), # CPU使用率颜色 + 'cpu_warning': (255, 165, 0), + 'cpu_critical': (255, 0, 0), + 'memory_good': (0, 255, 0), # 内存使用率颜色 + 'memory_warning': (255, 165, 0), + 'memory_critical': (255, 0, 0), } print("✓ 中文UI渲染器已初始化") @@ -2484,18 +3002,18 @@ def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): size / 25, color, 1) return frame - def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, trajectory_recorder=None, - gesture_recognizer=None): - """绘制状态栏""" + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, + trajectory_recorder=None, gesture_recognizer=None, performance_analyzer=None): + """绘制状态栏(增强版)""" h, w = frame.shape[:2] # 绘制半透明背景 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (w, 120), (0, 0, 0), -1) + cv2.rectangle(overlay, (0, 0), (w, 130), (0, 0, 0), -1) frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 性能模式版" + title = "手势控制无人机系统 - 性能统计版" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -2557,12 +3075,52 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro status_text = f"轨迹点: {recorder_status['trajectory_points']}" frame = self.draw_text(frame, status_text, (w // 2, 65), size=12, color=self.colors['info']) + # 性能统计显示 + if performance_analyzer and config.get('display', 'show_performance_stats'): + perf_stats = performance_analyzer.get_stats_summary() + + # 性能状态颜色 + if perf_stats['performance_status'] == "良好": + perf_color = self.colors['performance_good'] + elif perf_stats['performance_status'] == "警告": + perf_color = self.colors['performance_warning'] + else: + perf_color = self.colors['performance_bad'] + + perf_text = f"性能: {perf_stats['performance_status']} ({perf_stats['fps']:.0f}FPS)" + frame = self.draw_text(frame, perf_text, (w // 2, 90), size=12, color=perf_color) + + # 系统资源显示 + if performance_analyzer and config.get('display', 'show_system_resources'): + cpu_usage = performance_analyzer.get_current_cpu_usage() + memory_usage = performance_analyzer.get_current_memory_usage() + + # CPU颜色 + if cpu_usage < 70: + cpu_color = self.colors['cpu_good'] + elif cpu_usage < 85: + cpu_color = self.colors['cpu_warning'] + else: + cpu_color = self.colors['cpu_critical'] + + # 内存颜色 + if memory_usage < 70: + mem_color = self.colors['memory_good'] + elif memory_usage < 85: + mem_color = self.colors['memory_warning'] + else: + mem_color = self.colors['memory_critical'] + + # 绘制系统资源信息 + sys_text = f"CPU:{cpu_usage:.0f}% 内存:{memory_usage:.0f}%" + frame = self.draw_text(frame, sys_text, (w // 2, 110), size=11, color=self.colors['info']) + # 语音状态 if config.get('display', 'show_speech_status') and self.speech_manager: speech_status = self.speech_manager.get_status() speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" - frame = self.draw_text(frame, speech_text, (w // 2, 90), size=12, color=speech_color) + frame = self.draw_text(frame, speech_text, (w - 200, 90), size=12, color=speech_color) # 性能信息 if config.get('display', 'show_fps'): @@ -2586,6 +3144,11 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro control_text = "提示: 确保手部完全进入画面,保持稳定手势" frame = self.draw_text(frame, control_text, (10, 90), size=12, color=self.colors['info']) + # 性能快捷键提示 + if performance_analyzer: + perf_tip = "P:报告 L:快照 K:导出" + frame = self.draw_text(frame, perf_tip, (10, 110), size=11, color=self.colors['help']) + return frame def draw_help_bar(self, frame): @@ -2596,18 +3159,19 @@ def draw_help_bar(self, frame): h, w = frame.shape[:2] # 绘制底部帮助栏 - cv2.rectangle(frame, (0, h - 100), (w, h), (0, 0, 0), -1) + cv2.rectangle(frame, (0, h - 120), (w, h), (0, 0, 0), -1) - # 帮助文本 + # 帮助文本(增强版,包含性能统计快捷键) help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音 P:性能报告 O:切换性能模式", + "V:切换语音反馈 M:测试语音 P:性能报告 L:性能快照 K:导出日志", + "O:切换性能模式 P:详细报告 R:重置统计", "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" ] for i, line in enumerate(help_lines): - y_pos = h - 85 + i * 20 + y_pos = h - 105 + i * 20 frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame @@ -2620,50 +3184,74 @@ def draw_warning(self, frame, message): warning_bg = np.zeros((40, w, 3), dtype=np.uint8) warning_bg[:, :] = (0, 69, 255) - frame[120:160, 0:w] = cv2.addWeighted( - frame[120:160, 0:w], 0.3, + frame[130:170, 0:w] = cv2.addWeighted( + frame[130:170, 0:w], 0.3, warning_bg, 0.7, 0 ) # 绘制警告文本 - frame = self.draw_text(frame, message, (10, 135), + frame = self.draw_text(frame, message, (10, 145), size=16, color=self.colors['warning']) return frame + def draw_performance_chart(self, frame, performance_analyzer): + """绘制性能图表""" + if not performance_analyzer: + return frame -# ========== 性能监控器 ========== -class PerformanceMonitor: - """性能监控器""" - - def __init__(self): - self.frame_times = deque(maxlen=60) - self.last_update = time.time() - self.fps = 0 - self.frame_count = 0 - - def update(self): - """更新性能数据""" - current_time = time.time() - self.frame_times.append(current_time) - self.frame_count += 1 + h, w = frame.shape[:2] - # 每秒计算一次FPS - if current_time - self.last_update >= 1.0: - if len(self.frame_times) > 1: - time_diff = self.frame_times[-1] - self.frame_times[0] - if time_diff > 0: - self.fps = len(self.frame_times) / time_diff - else: - self.fps = 0 - self.last_update = current_time + # 绘制图表区域背景 + chart_x = w - 250 + chart_y = 130 + chart_width = 240 + chart_height = 150 + + cv2.rectangle(frame, (chart_x, chart_y), + (chart_x + chart_width, chart_y + chart_height), + (30, 30, 30), -1) + cv2.rectangle(frame, (chart_x, chart_y), + (chart_x + chart_width, chart_y + chart_height), + (100, 100, 100), 1) + + # 绘制标题 + frame = self.draw_text(frame, "性能监控", (chart_x + 10, chart_y + 20), + size=14, color=self.colors['title']) + + # 绘制FPS图表 + if len(performance_analyzer.fps_history) > 1: + fps_values = list(performance_analyzer.fps_history)[-20:] # 最近20个值 + + # 计算最大值和最小值 + if fps_values: + max_fps = max(fps_values) + min_fps = min(fps_values) + if max_fps == min_fps: + max_fps = min_fps + 1 + + # 绘制FPS曲线 + points = [] + for i, fps in enumerate(fps_values): + x = chart_x + 10 + i * (chart_width - 20) / len(fps_values) + y = chart_y + chart_height - 30 - (fps - min_fps) * (chart_height - 50) / (max_fps - min_fps) + points.append((int(x), int(y))) + + # 连接点形成曲线 + for i in range(len(points) - 1): + cv2.line(frame, points[i], points[i + 1], (0, 255, 0), 2) + + # 显示当前FPS值 + current_fps = fps_values[-1] if fps_values else 0 + fps_color = self.colors['performance_good'] if current_fps > 20 else \ + self.colors['performance_warning'] if current_fps > 10 else \ + self.colors['performance_bad'] + + frame = self.draw_text(frame, f"FPS: {current_fps:.1f}", + (chart_x + 120, chart_y + 40), + size=12, color=fps_color) - def get_stats(self): - """获取性能统计""" - return { - 'fps': self.fps, - 'frame_count': self.frame_count - } + return frame # ========== 主程序 ========== @@ -2684,7 +3272,10 @@ def main(): gesture_recognizer = EnhancedGestureRecognizer(speech_manager) drone_controller = SimpleDroneController(libs['airsim'], speech_manager) ui_renderer = ChineseUIRenderer(speech_manager) - performance_monitor = PerformanceMonitor() + + # 初始化性能分析器 + print("初始化性能分析器...") + performance_analyzer = PerformanceAnalyzer(speech_manager) # 手势轨迹记录器 print("初始化手势轨迹记录器...") @@ -2729,47 +3320,48 @@ def main(): cap = None # 显示欢迎信息 - print("\n" + "=" * 60) - print("手势控制无人机系统 - 性能模式版") - print("=" * 60) + print("\n" + "=" * 80) + print("手势控制无人机系统 - 详细性能统计版") + print("=" * 80) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") print(f" 性能模式: {gesture_recognizer.performance_mode_name}") + print(f" 性能监控: {'已启用' if config.get('performance', 'enable_performance_monitor') else '已禁用'}") print(f" 轨迹记录: 支持录制/回放功能") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") - print("=" * 60) + print("=" * 80) # 显示操作说明 print("\n操作说明:") print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") - print("3. 性能模式选择:") + print("3. 性能统计功能:") + print(" - 按 [P] 键显示详细性能报告") + print(" - 按 [L] 键记录性能快照") + print(" - 按 [K] 键导出性能日志") + print(" - 按 [R] 键重置性能统计") + print(" - 系统自动监控: FPS, CPU, 内存, 识别时间等") + print("4. 性能模式选择:") print(" - 按 [O] 键循环切换性能模式: 最快(fast) → 平衡(balanced) → 最准(accurate)") - print(" - 最快模式: 性能优先,帧率最高,识别精度较低") - print(" - 平衡模式: 性能与精度平衡,适用于大多数场景") - print(" - 最准模式: 精度优先,提供最准确的手势识别") - print("4. 手势控制:") + print("5. 手势控制:") print(" - 手势识别后会有语音提示: 向上、向下、向左、向右、向前、停止") print(" - 手势稳定性越高,识别越准确") print(" - 手部距离摄像头适中时效果最佳") print(" * 手势识别置信度 > 60% 时才会执行") - print("5. 轨迹记录功能:") + print("6. 轨迹记录功能:") print(" [1]开始录制 [2]停止录制 [3]保存轨迹 [4]回放轨迹 [5]清除轨迹 [6]暂停/继续") - print(" - 录制时系统会记录手部位置和手势") - print(" - 回放时可以查看录制的轨迹") - print(" - 轨迹数据自动保存到 trajectories/ 目录") - print("6. 键盘控制:") + print("7. 键盘控制:") print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") - print("7. 调试功能:") + print("8. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") - print("8. 语音控制:") - print(" [V]切换语音反馈 [M]测试语音 [P]性能报告") - print("9. 按 [ESC] 安全退出") - print("=" * 60) + print("9. 语音控制:") + print(" [V]切换语音反馈 [M]测试语音") + print("10. 按 [ESC] 安全退出") + print("=" * 80) print("程序启动成功!") - print("-" * 60) + print("-" * 80) # 键盘手势映射 key_to_gesture = { @@ -2792,7 +3384,9 @@ def main(): try: while True: # 更新性能监控 - performance_monitor.update() + performance_analyzer.update_frame() + performance_analyzer.update_system_resources() + performance_analyzer.auto_report() # 读取摄像头帧 if cap: @@ -2804,15 +3398,21 @@ def main(): else: # 手势识别 gesture, confidence, frame = gesture_recognizer.recognize(frame) + + # 记录手势统计 + if gesture and gesture != "Waiting" and gesture != "摄像头错误": + performance_analyzer.record_gesture(gesture, confidence) else: # 模拟模式 frame = np.zeros((480, 640, 3), dtype=np.uint8) gesture, confidence = gesture_recognizer.current_gesture, gesture_recognizer.current_confidence # 获取性能统计 - perf_stats = performance_monitor.get_stats() process_time, frame_rate = gesture_recognizer.get_performance_stats() + # 更新手势识别时间 + performance_analyzer.update_gesture_recognition_time(process_time) + # 根据显示模式调整显示选项 if display_modes[current_display_mode] == 'normal': config.set('display', 'show_contours', value=True) @@ -2823,6 +3423,8 @@ def main(): config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) config.set('display', 'show_performance_mode', value=True) + config.set('display', 'show_performance_stats', value=True) + config.set('display', 'show_system_resources', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -2835,6 +3437,8 @@ def main(): config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) config.set('display', 'show_performance_mode', value=True) + config.set('display', 'show_performance_stats', value=True) + config.set('display', 'show_system_resources', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) @@ -2845,6 +3449,8 @@ def main(): config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) config.set('display', 'show_performance_mode', value=True) + config.set('display', 'show_performance_stats', value=True) + config.set('display', 'show_system_resources', value=True) config.set('display', 'show_debug_info', value=False) # 绘制轨迹(如果启用) @@ -2854,9 +3460,14 @@ def main(): # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, - perf_stats['fps'], process_time, trajectory_recorder, gesture_recognizer + performance_analyzer.get_current_fps(), process_time, + trajectory_recorder, gesture_recognizer, performance_analyzer ) + # 绘制性能图表(如果启用详细模式) + if display_modes[current_display_mode] == 'detailed': + frame = ui_renderer.draw_performance_chart(frame, performance_analyzer) + frame = ui_renderer.draw_help_bar(frame) # 显示连接提示 @@ -2864,8 +3475,13 @@ def main(): warning_msg = "⚠ 按C键连接无人机,或使用模拟模式" frame = ui_renderer.draw_warning(frame, warning_msg) + # 显示性能警告 + if performance_analyzer.performance_status != "良好": + warning_msg = f"⚠ 性能状态: {performance_analyzer.performance_status}" + frame = ui_renderer.draw_warning(frame, warning_msg) + # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Performance Mode', frame) + cv2.imshow('Gesture Controlled Drone - Performance Statistics', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -2947,14 +3563,42 @@ def main(): elif key == ord('p') or key == ord('P'): # 性能报告 + if key == ord('p'): # 小写p - 简要报告 + print("生成简要性能报告...") + performance_analyzer.print_report(detailed=False) + else: # 大写P - 详细报告 + print("生成详细性能报告...") + performance_analyzer.print_report(detailed=True) + + # 语音提示 if speech_manager.enabled: - print("生成性能报告...") - if process_time < 20: - speech_manager.speak_direct("系统性能优秀,运行流畅") - elif process_time < 50: - speech_manager.speak_direct("系统性能良好") - else: - speech_manager.speak_direct("系统性能警告,请检查") + speech_manager.speak('performance_report', immediate=True) + + elif key == ord('l') or key == ord('L'): + # 性能快照 + snapshot_label = f"快照_{datetime.now().strftime('%H:%M:%S')}" + snapshot = performance_analyzer.take_snapshot(snapshot_label) + + print(f"📸 性能快照已保存: {snapshot_label}") + print(f" 当前FPS: {snapshot['fps']:.1f}") + print(f" 平均FPS: {snapshot['avg_fps']:.1f}") + print(f" CPU使用率: {snapshot['cpu_usage']:.1f}%") + print(f" 内存使用率: {snapshot['memory_usage']:.1f}%") + + # 语音提示 + if speech_manager.enabled: + speech_manager.speak('performance_snapshot', immediate=True) + + elif key == ord('k') or key == ord('K'): + # 导出性能日志 + if performance_analyzer.export_log(): + print("✅ 性能日志导出成功") + + # 语音提示 + if speech_manager.enabled: + speech_manager.speak('performance_log_exported', immediate=True) + else: + print("❌ 性能日志导出失败") elif key == ord('o') or key == ord('O'): # 切换性能模式 @@ -2973,11 +3617,22 @@ def main(): print(f"✓ 已切换到性能模式: {gesture_recognizer.performance_mode_name}") + # 新增:重置性能统计 + elif key == ord('R') and chr(key).isupper(): # 大写R + print("重置性能统计...") + performance_analyzer.reset_session() + print("✓ 性能统计已重置") + + # 语音提示 + if speech_manager.enabled: + speech_manager.speak_direct("性能统计已重置") + # 新增:轨迹记录控制 elif key == ord('1'): # 开始录制轨迹 if trajectory_recorder.start_recording(): print("✅ 开始录制手势轨迹") + performance_analyzer.record_recording_session() else: print("⚠ 已经在录制中") @@ -2985,6 +3640,7 @@ def main(): # 停止录制轨迹 if trajectory_recorder.stop_recording(): print("✅ 停止录制手势轨迹") + performance_analyzer.record_recording_session(len(trajectory_recorder.trajectory_data)) else: print("⚠ 当前没有在录制") @@ -3022,15 +3678,21 @@ def main(): gesture_recognizer.set_simulated_gesture(simulated_gesture) gesture = simulated_gesture confidence = 0.9 + + # 记录手势统计 + performance_analyzer.record_gesture(gesture, confidence) + if drone_controller.connected and drone_controller.flying: - drone_controller.move_by_gesture(gesture, confidence) + success = drone_controller.move_by_gesture(gesture, confidence) + performance_analyzer.record_drone_command(success) # 真实手势控制 current_time = time.time() if (gesture and gesture != "Waiting" and gesture != "摄像头错误" and gesture != "Error" and drone_controller.connected and drone_controller.flying): - drone_controller.move_by_gesture(gesture, confidence) + success = drone_controller.move_by_gesture(gesture, confidence) + performance_analyzer.record_drone_command(success) # 处理轨迹记录 if cap and ret: @@ -3062,6 +3724,17 @@ def main(): cap.release() cv2.destroyAllWindows() + # 生成最终性能报告 + print("\n" + "=" * 80) + print("📊 最终性能总结") + print("=" * 80) + performance_analyzer.print_report(detailed=True) + + # 导出最终性能日志 + performance_analyzer.export_log() + + print("=" * 80) + # 程序退出语音提示 if speech_manager.enabled: speech_manager.speak('program_exit', force=True, immediate=True) @@ -3077,7 +3750,7 @@ def main(): config.save_config() print("程序安全退出") - print("=" * 60) + print("=" * 80) print("\n感谢使用手势控制无人机系统!") input("按回车键退出...") From 0de8fdf91e96b8f36900d57813df91312a4b453a Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sun, 21 Dec 2025 17:18:03 +0800 Subject: [PATCH 15/18] =?UTF-8?q?=E5=B0=86=E6=89=8B=E5=8A=BF=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E6=97=A0=E4=BA=BA=E6=9C=BA=E7=B3=BB=E7=BB=9F=E6=8B=86?= =?UTF-8?q?=E5=88=86=E4=B8=BA=E6=A8=A1=E5=9D=97=E5=8C=96=E6=9E=B6=E6=9E=84?= =?UTF-8?q?=EF=BC=8C=E4=BF=9D=E6=8C=81=E5=8A=9F=E8=83=BD=E5=AE=8C=E6=95=B4?= =?UTF-8?q?=E6=80=A7=EF=BC=8C=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81=E7=BB=B4?= =?UTF-8?q?=E6=8A=A4=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config_manager.py | 313 ++ .../drone_controller.py | 282 ++ .../gesture_config.json | 6 +- .../gesture_recognizer.py | 952 +++++ .../main.py | 3359 +---------------- .../performance_analyzer.py | 506 +++ .../speech_manager.py | 361 ++ .../trajectory_recorder.py | 347 ++ .../ui_renderer.py | 350 ++ 9 files changed, 3240 insertions(+), 3236 deletions(-) create mode 100644 src/autonomus_drone_hand_gesture_project/config_manager.py create mode 100644 src/autonomus_drone_hand_gesture_project/drone_controller.py create mode 100644 src/autonomus_drone_hand_gesture_project/gesture_recognizer.py create mode 100644 src/autonomus_drone_hand_gesture_project/performance_analyzer.py create mode 100644 src/autonomus_drone_hand_gesture_project/speech_manager.py create mode 100644 src/autonomus_drone_hand_gesture_project/trajectory_recorder.py create mode 100644 src/autonomus_drone_hand_gesture_project/ui_renderer.py diff --git a/src/autonomus_drone_hand_gesture_project/config_manager.py b/src/autonomus_drone_hand_gesture_project/config_manager.py new file mode 100644 index 0000000000..bfd2c46f73 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/config_manager.py @@ -0,0 +1,313 @@ +""" +配置管理器模块 +负责加载、保存和管理系统配置 +作者: xiaoshiyuan888 +""" + +import os +import json +import numpy as np +import cv2 + + +class ConfigManager: + """配置管理器""" + + def __init__(self): + import sys + current_dir = os.path.dirname(os.path.abspath(__file__)) + self.config_file = os.path.join(current_dir, 'gesture_config.json') + + # 性能模式配置 + self.performance_modes = { + 'fast': { + 'name': '最快', + 'description': '性能优先,降低识别精度换取更高帧率', + 'detection_interval': 2, + 'smooth_frames': 3, + 'min_confidence': 0.5, + 'resize_factor': 0.5, + 'skin_detection_enabled': True, + 'background_subtraction_enabled': False, + 'contour_simplify_epsilon': 0.03, + 'history_size': 10, + 'gesture_stability_threshold': 3, + 'color': (0, 255, 0), + }, + 'balanced': { + 'name': '平衡', + 'description': '平衡性能与精度,适用于大多数场景', + 'detection_interval': 1, + 'smooth_frames': 5, + 'min_confidence': 0.6, + 'resize_factor': 0.75, + 'skin_detection_enabled': True, + 'background_subtraction_enabled': True, + 'contour_simplify_epsilon': 0.02, + 'history_size': 15, + 'gesture_stability_threshold': 5, + 'color': (255, 165, 0), + }, + 'accurate': { + 'name': '最准', + 'description': '精度优先,提供最准确的手势识别', + 'detection_interval': 1, + 'smooth_frames': 7, + 'min_confidence': 0.7, + 'resize_factor': 1.0, + 'skin_detection_enabled': True, + 'background_subtraction_enabled': True, + 'contour_simplify_epsilon': 0.01, + 'history_size': 20, + 'gesture_stability_threshold': 7, + 'color': (255, 0, 0), + } + } + + self.default_config = { + 'camera': { + 'index': 0, + 'width': 640, + 'height': 480, + 'fps': 30 + }, + 'gesture': { + 'skin_lower_h': 0, + 'skin_upper_h': 25, + 'skin_lower_s': 30, + 'skin_upper_s': 255, + 'skin_lower_v': 60, + 'skin_upper_v': 255, + 'min_hand_area': 2000, + 'max_hand_area': 30000, + 'hand_ratio_threshold': 1.5, + 'defect_distance_threshold': 20, + 'palm_circle_radius_ratio': 0.3, + 'transition_threshold': 0.3, + 'position_stability_weight': 0.4, + 'gesture_cooldown': 0.5, + }, + 'drone': { + 'velocity': 2.5, + 'duration': 0.3, + 'altitude': -10.0, + 'control_interval': 0.3 + }, + 'display': { + 'show_fps': True, + 'show_confidence': True, + 'show_help': True, + 'show_contours': True, + 'show_bbox': True, + 'show_fingertips': True, + 'show_palm_center': True, + 'show_hand_direction': True, + 'show_debug_info': False, + 'show_speech_status': True, + 'show_gesture_history': True, + 'show_stability_indicator': True, + 'show_trajectory': True, + 'show_recording_status': True, + 'show_performance_mode': True, + 'show_performance_stats': True, + 'show_system_resources': True, + }, + 'performance': { + 'target_fps': 30, + 'resize_factor': 1.0, + 'enable_multiprocessing': False, + 'mode': 'balanced', + 'current_mode_index': 1, + 'modes': ['fast', 'balanced', 'accurate'], + 'auto_report_interval': 60, + 'enable_performance_monitor': True, + }, + 'calibration': { + 'auto_calibrate_skin': True, + 'skin_calibration_frames': 30, + 'hand_size_calibration': True + }, + 'speech': { + 'enabled': True, + 'volume': 1.0, + 'rate': 150, + 'announce_gestures': True, + 'announce_connections': True, + 'announce_flight_events': True, + 'announce_gesture_changes': True, + 'announce_hand_status': True, + 'announce_performance': True, + 'announce_recording_events': True, + 'announce_performance_mode': True, + 'announce_performance_events': True, + 'min_gesture_confidence': 0.7, + 'gesture_start_threshold': 3, + 'gesture_end_threshold': 10, + }, + 'recording': { + 'auto_save_interval': 5, + 'max_trajectory_points': 1000, + 'show_trajectory': True, + 'trajectory_thickness': 2, + 'trajectory_max_length': 100, + 'default_save_dir': 'trajectories', + } + } + self.config = self.load_config() + self.skin_calibration_data = [] + self.hand_size_calibration_done = False + self.reference_hand_size = 0 + + def load_config(self): + """加载配置""" + if os.path.exists(self.config_file): + try: + with open(self.config_file, 'r', encoding='utf-8') as f: + loaded_config = json.load(f) + config = self.default_config.copy() + self._merge_config(config, loaded_config) + print("✓ 从文件加载配置") + return config + except Exception as e: + print(f"⚠ 加载配置失败: {e}, 使用默认配置") + return self.default_config.copy() + else: + print("✓ 使用默认配置") + return self.default_config.copy() + + def _merge_config(self, base, update): + """递归合并配置""" + for key, value in update.items(): + if key in base and isinstance(base[key], dict) and isinstance(value, dict): + self._merge_config(base[key], value) + else: + base[key] = value + + def save_config(self): + """保存配置""" + try: + with open(self.config_file, 'w', encoding='utf-8') as f: + json.dump(self.config, f, indent=2, ensure_ascii=False) + print("✓ 配置已保存") + except Exception as e: + print(f"⚠ 保存配置失败: {e}") + + def get(self, *keys): + """获取配置值""" + value = self.config + for key in keys: + if isinstance(value, dict) and key in value: + value = value[key] + else: + return None + return value + + def set(self, *keys, value): + """设置配置值""" + if len(keys) == 0: + return + + config = self.config + for key in keys[:-1]: + if key not in config: + config[key] = {} + config = config[key] + + config[keys[-1]] = value + self.save_config() + + def get_performance_mode_config(self, mode=None): + """获取性能模式配置""" + if mode is None: + mode = self.get('performance', 'mode') + + if mode in self.performance_modes: + return self.performance_modes[mode] + else: + return self.performance_modes['balanced'] + + def get_current_performance_mode(self): + """获取当前性能模式""" + mode = self.get('performance', 'mode') + if mode in self.performance_modes: + return mode + return 'balanced' + + def set_performance_mode(self, mode): + """设置性能模式""" + if mode in self.performance_modes: + self.set('performance', 'mode', value=mode) + + # 更新当前模式索引 + modes = self.get('performance', 'modes') + if modes and mode in modes: + index = modes.index(mode) + self.set('performance', 'current_mode_index', value=index) + + print(f"✓ 性能模式设置为: {self.performance_modes[mode]['name']}") + return True + return False + + def cycle_performance_mode(self): + """循环切换性能模式""" + modes = self.get('performance', 'modes') + if not modes: + modes = ['fast', 'balanced', 'accurate'] + + current_index = self.get('performance', 'current_mode_index') + if current_index is None: + current_index = 0 + + # 计算下一个模式索引 + next_index = (current_index + 1) % len(modes) + next_mode = modes[next_index] + + # 设置新模式 + self.set('performance', 'current_mode_index', value=next_index) + return self.set_performance_mode(next_mode) + + def calibrate_skin_color(self, frame, hand_mask): + """自动校准肤色范围""" + if not self.get('calibration', 'auto_calibrate_skin'): + return + + if len(self.skin_calibration_data) < self.get('calibration', 'skin_calibration_frames'): + # 转换到HSV + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + + # 获取肤色区域的HSV值 + skin_pixels = hsv[hand_mask > 0] + + if len(skin_pixels) > 100: # 确保有足够的像素 + self.skin_calibration_data.append(skin_pixels) + + if len(self.skin_calibration_data) == self.get('calibration', 'skin_calibration_frames'): + # 计算肤色范围 + all_skin_pixels = np.vstack(self.skin_calibration_data) + + h_min, h_max = np.percentile(all_skin_pixels[:, 0], [2, 98]) + s_min, s_max = np.percentile(all_skin_pixels[:, 1], [2, 98]) + v_min, v_max = np.percentile(all_skin_pixels[:, 2], [2, 98]) + + # 更新配置 + self.set('gesture', 'skin_lower_h', value=int(max(0, h_min - 5))) + self.set('gesture', 'skin_upper_h', value=int(min(180, h_max + 5))) + self.set('gesture', 'skin_lower_s', value=int(max(0, s_min - 10))) + self.set('gesture', 'skin_upper_s', value=int(min(255, s_max + 10))) + self.set('gesture', 'skin_lower_v', value=int(max(0, v_min - 10))) + self.set('gesture', 'skin_upper_v', value=int(min(255, v_max + 10))) + + print("✓ 肤色校准完成") + print(f" 肤色范围: H[{self.get('gesture', 'skin_lower_h')}-{self.get('gesture', 'skin_upper_h')}], " + f"S[{self.get('gesture', 'skin_lower_s')}-{self.get('gesture', 'skin_upper_s')}], " + f"V[{self.get('gesture', 'skin_lower_v')}-{self.get('gesture', 'skin_upper_v')}]") + + def calibrate_hand_size(self, hand_area): + """校准手部大小""" + if not self.get('calibration', 'hand_size_calibration') or self.hand_size_calibration_done: + return + + if hand_area > 0: + self.reference_hand_size = hand_area + self.hand_size_calibration_done = True + print(f"✓ 手部大小校准完成: {self.reference_hand_size:.0f} 像素") \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/drone_controller.py b/src/autonomus_drone_hand_gesture_project/drone_controller.py new file mode 100644 index 0000000000..17e8dfea36 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/drone_controller.py @@ -0,0 +1,282 @@ +""" +无人机控制器模块 +负责控制无人机的连接、起飞、降落和移动 +作者: xiaoshiyuan888 +""" + +import time + + +class SimpleDroneController: + """简单的无人机控制器""" + + def __init__(self, airsim_module, speech_manager=None, config=None): + self.airsim = airsim_module + self.client = None + self.connected = False + self.flying = False + self.speech_manager = speech_manager + self.config = config + + # 控制参数 + self.velocity = config.get('drone', 'velocity') + self.duration = config.get('drone', 'duration') + self.altitude = config.get('drone', 'altitude') + self.control_interval = config.get('drone', 'control_interval') + + # 控制状态 + self.last_control_time = 0 + self.last_gesture = None + + # 上次语音提示状态 + self.last_connection_announced = False + self.last_takeoff_announced = False + self.last_land_announced = False + + print("✓ 简单的无人机控制器已初始化") + + def connect(self): + """连接AirSim无人机""" + if self.connected: + return True + + # 语音提示:正在连接 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('connecting') + + if self.airsim is None: + print("⚠ AirSim不可用,使用模拟模式") + + # 语音提示:模拟模式 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('simulation_mode') + + self.connected = True + return True + + print("连接AirSim...") + + try: + self.client = self.airsim.MultirotorClient() + self.client.confirmConnection() + print("✅ 已连接AirSim!") + + # 语音提示:连接成功 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('connected') + + self.client.enableApiControl(True) + print("✅ API控制已启用") + + self.client.armDisarm(True) + print("✅ 无人机已武装") + + self.connected = True + return True + + except Exception as e: + print(f"❌ 连接失败: {e}") + + # 语音提示:连接失败 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('connection_failed') + + print("\n使用模拟模式继续? (y/n)") + choice = input().strip().lower() + if choice == 'y': + self.connected = True + print("✅ 使用模拟模式") + + # 语音提示:模拟模式 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('simulation_mode') + + return True + + return False + + def takeoff(self): + """起飞""" + if not self.connected: + return False + + # 语音提示:正在起飞 + if (self.speech_manager and + self.speech_manager.enabled and + not self.last_takeoff_announced): + self.speech_manager.speak('taking_off') + self.last_takeoff_announced = True + self.last_land_announced = False + + try: + if self.airsim is None or self.client is None: + print("✅ 模拟起飞") + self.flying = True + + # 语音提示:起飞成功 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('takeoff_success') + + return True + + print("起飞中...") + self.client.takeoffAsync().join() + time.sleep(1) + + # 上升到指定高度 + self.client.moveToZAsync(self.altitude, 3).join() + + self.flying = True + print("✅ 无人机成功起飞") + + # 语音提示:起飞成功 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('takeoff_success') + + return True + except Exception as e: + print(f"❌ 起飞失败: {e}") + + # 语音提示:起飞失败 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('takeoff_failed') + + return False + + def land(self): + """降落""" + if not self.connected: + return False + + # 语音提示:正在降落 + if (self.speech_manager and + self.speech_manager.enabled and + not self.last_land_announced): + self.speech_manager.speak('landing') + self.last_land_announced = True + self.last_takeoff_announced = False + + try: + if self.airsim is None or self.client is None: + print("✅ 模拟降落") + self.flying = False + + # 语音提示:降落成功 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('land_success') + + return True + + print("降落中...") + self.client.landAsync().join() + self.flying = False + print("✅ 无人机已降落") + + # 语音提示:降落成功 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('land_success') + + return True + except Exception as e: + print(f"降落失败: {e}") + return False + + def move_by_gesture(self, gesture, confidence): + """根据手势移动""" + if not self.connected or not self.flying: + return False + + # 检查控制间隔 + current_time = time.time() + if current_time - self.last_control_time < self.control_interval: + return False + + # 检查置信度阈值 + min_confidence = self.config.get('gesture', 'min_confidence') + if confidence < min_confidence: + # 低置信度语音提示 + if (self.speech_manager and + self.speech_manager.enabled and + confidence < min_confidence * 0.8): + self.speech_manager.speak('gesture_low_confidence') + return False + + try: + if self.airsim is None or self.client is None: + print(f"模拟移动: {gesture}") + self.last_control_time = current_time + self.last_gesture = gesture + return True + + success = False + + if gesture == "Up": + self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) + success = True + elif gesture == "Down": + self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) + success = True + elif gesture == "Left": + self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) + success = True + elif gesture == "Right": + self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) + success = True + elif gesture == "Forward": + self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) + success = True + elif gesture == "Stop": + self.client.hoverAsync() + success = True + # 悬停语音提示 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('hovering') + elif gesture == "Hover": + self.client.hoverAsync() + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('hovering') + + if success: + self.last_control_time = current_time + self.last_gesture = gesture + + return success + except Exception as e: + print(f"控制命令失败: {e}") + return False + + def emergency_stop(self): + """紧急停止""" + if self.connected: + try: + if self.flying and self.client is not None: + print("紧急降落...") + + # 语音提示:紧急停止 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('emergency_stop') + + self.land() + if self.client is not None: + self.client.armDisarm(False) + self.client.enableApiControl(False) + print("✅ 紧急停止完成") + except: + pass + + self.connected = False + self.flying = False \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index 3cdf48a5cb..fbb45391da 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -18,11 +18,11 @@ }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 176, + "skin_upper_h": 170, "skin_lower_s": 0, - "skin_upper_s": 116, + "skin_upper_s": 70, "skin_lower_v": 0, - "skin_upper_v": 92, + "skin_upper_v": 75, "min_hand_area": 3000, "max_hand_area": 30000, "hand_ratio_threshold": 1.5, diff --git a/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py b/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py new file mode 100644 index 0000000000..ff37f66b83 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py @@ -0,0 +1,952 @@ +""" +手势识别器模块 +负责识别和分析手势 +作者: xiaoshiyuan888 +""" + +import time +import math +import numpy as np +import cv2 +from collections import deque, Counter + + +class EnhancedGestureRecognizer: + """增强的手势识别器 - 支持性能模式选择""" + + def __init__(self, speech_manager=None, config=None): + self.speech_manager = speech_manager + self.config = config + + # 加载性能模式配置 + self.performance_mode = config.get_current_performance_mode() + self.mode_config = config.get_performance_mode_config(self.performance_mode) + + # 根据性能模式初始化参数 + self.history_size = self.mode_config['history_size'] + self.detection_interval = self.mode_config['detection_interval'] + self.smooth_frames = self.mode_config['smooth_frames'] + self.min_confidence = self.mode_config['min_confidence'] + self.resize_factor = self.mode_config['resize_factor'] + + # 增强的手势历史和平滑 + self.gesture_history = deque(maxlen=self.history_size) + self.confidence_history = deque(maxlen=self.history_size) + self.position_history = deque(maxlen=self.history_size) + self.current_gesture = "Waiting" + self.current_confidence = 0.0 + + # 手势状态追踪 + self.gesture_state = "none" + self.gesture_stability_counter = 0 + self.last_stable_gesture = "Waiting" + self.gesture_active_frames = 0 + self.last_gesture_change_time = 0 + + # 记录上次播报的手势 + self.last_announced_gesture = None + self.last_announced_time = 0 + self.last_hand_status_time = 0 + self.gesture_announce_interval = 2.0 + + # 手部跟踪和状态 + self.last_hand_position = None + self.hand_tracking = False + self.track_window = None + self.hand_states = deque(maxlen=15) + self.hand_detected_frames = 0 + self.hand_lost_frames = 0 + + # 性能统计 + self.process_times = deque(maxlen=30) + self.frame_counter = 0 + self.last_performance_report = 0 + + # 手势颜色映射 + self.gesture_colors = { + "Stop": (0, 0, 255), + "Forward": (0, 255, 0), + "Up": (255, 255, 0), + "Down": (255, 0, 255), + "Left": (255, 165, 0), + "Right": (0, 165, 255), + "Waiting": (200, 200, 200), + "Error": (255, 0, 0), + "Hover": (255, 255, 255) + } + + # 手势到语音的映射 + self.gesture_speech_map = { + "Stop": "gesture_stop", + "Forward": "gesture_forward", + "Up": "gesture_up", + "Down": "gesture_down", + "Left": "gesture_left", + "Right": "gesture_right", + "Waiting": "gesture_waiting", + "Error": "gesture_error", + "Hover": "gesture_hover", + } + + # 手势状态颜色 + self.state_colors = { + "none": (100, 100, 100), + "starting": (255, 165, 0), + "active": (0, 255, 0), + "ending": (255, 0, 0), + } + + # 根据性能模式初始化背景减除器 + self.bg_subtractor = None + if self.mode_config['background_subtraction_enabled']: + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( + history=100, varThreshold=25, detectShadows=True + ) + + # 形态学操作核 + self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) + + # 性能监控 + self.avg_process_time = 0 + self.frame_rate = 0 + self.last_fps_check = time.time() + + # 存储手部数据用于轨迹记录 + self.last_hand_data = None + + # 性能模式信息 + self.performance_mode_color = self.mode_config['color'] + self.performance_mode_name = self.mode_config['name'] + + print(f"✓ 增强的手势识别器已初始化 - 性能模式: {self.performance_mode_name}") + + def set_performance_mode(self, mode): + """设置性能模式""" + self.performance_mode = mode + self.mode_config = self.config.get_performance_mode_config(mode) + + # 更新参数 + self.history_size = self.mode_config['history_size'] + self.detection_interval = self.mode_config['detection_interval'] + self.smooth_frames = self.mode_config['smooth_frames'] + self.min_confidence = self.mode_config['min_confidence'] + self.resize_factor = self.mode_config['resize_factor'] + + # 更新背景减除器 + if self.mode_config['background_subtraction_enabled'] and self.bg_subtractor is None: + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( + history=100, varThreshold=25, detectShadows=True + ) + elif not self.mode_config['background_subtraction_enabled']: + self.bg_subtractor = None + + # 更新队列大小 + self.gesture_history = deque(maxlen=self.history_size) + self.confidence_history = deque(maxlen=self.history_size) + self.position_history = deque(maxlen=self.history_size) + + # 更新显示信息 + self.performance_mode_color = self.mode_config['color'] + self.performance_mode_name = self.mode_config['name'] + + print(f"✓ 切换到性能模式: {self.performance_mode_name}") + + def get_skin_mask(self, frame): + """获取肤色掩码""" + h_low = self.config.get('gesture', 'skin_lower_h') + h_high = self.config.get('gesture', 'skin_upper_h') + s_low = self.config.get('gesture', 'skin_lower_s') + s_high = self.config.get('gesture', 'skin_upper_s') + v_low = self.config.get('gesture', 'skin_lower_v') + v_high = self.config.get('gesture', 'skin_upper_v') + + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + lower_skin = np.array([h_low, s_low, v_low], dtype=np.uint8) + upper_skin = np.array([h_high, s_high, v_high], dtype=np.uint8) + skin_mask = cv2.inRange(hsv, lower_skin, upper_skin) + + return skin_mask, hsv + + def enhance_skin_detection(self, frame, skin_mask): + """增强肤色检测""" + if not self.mode_config['skin_detection_enabled']: + return skin_mask + + if self.bg_subtractor is not None: + fg_mask = self.bg_subtractor.apply(frame) + combined_mask = cv2.bitwise_and(skin_mask, fg_mask) + else: + combined_mask = skin_mask + + # 根据性能模式决定形态学操作次数 + if self.performance_mode == 'accurate': + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) + combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) + elif self.performance_mode == 'balanced': + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=1) + combined_mask = cv2.GaussianBlur(combined_mask, (3, 3), 0) + + return combined_mask + + def preprocess_frame(self, frame): + """预处理帧""" + if self.resize_factor != 1.0: + new_width = int(frame.shape[1] * self.resize_factor) + new_height = int(frame.shape[0] * self.resize_factor) + resized_frame = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_AREA) + return resized_frame + return frame + + def find_best_hand_contour(self, mask, frame): + """找到最佳的手部轮廓""" + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + if not contours: + return None, 0.0 + + contours = sorted(contours, key=cv2.contourArea, reverse=True) + best_contour = None + best_score = 0.0 + min_area = self.config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = self.config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) + + # 根据性能模式调整检查的轮廓数量 + max_contours = 3 if self.performance_mode != 'fast' else 1 + + for contour in contours[:max_contours]: + area = cv2.contourArea(contour) + + if area < min_area or area > max_area: + continue + + score = self.rate_contour(contour, frame.shape) + + if score > best_score: + best_score = score + best_contour = contour + + return best_contour, best_score + + def rate_contour(self, contour, frame_shape): + """评估轮廓作为手部的可能性""" + score = 0.0 + area = cv2.contourArea(contour) + min_area = self.config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = self.config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) + + if min_area < area < max_area: + area_ratio = min(area / max_area, 1.0) + score += area_ratio * 0.3 + + perimeter = cv2.arcLength(contour, True) + if perimeter > 100: + score += 0.2 + + if area > 0: + compactness = perimeter ** 2 / area + if 12 < compactness < 25: + compactness_score = 1.0 - abs(compactness - 18) / 6 + score += compactness_score * 0.3 + + x, y, w, h = cv2.boundingRect(contour) + if h > 0: + aspect_ratio = w / h + if 0.4 < aspect_ratio < 2.5: + aspect_score = 1.0 - abs(aspect_ratio - 1.0) / 1.5 + score += aspect_score * 0.2 + + return score + + def analyze_hand_features(self, contour, frame_shape): + """分析手部特征""" + if contour is None: + return None, 0.0 + + area = cv2.contourArea(contour) + M = cv2.moments(contour) + if M["m00"] == 0: + return None, 0.0 + + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + x, y, w, h = cv2.boundingRect(contour) + bbox_area = w * h + palm_center = (cx, cy) + palm_radius = int(w * self.config.get('gesture', 'palm_circle_radius_ratio')) + + # 根据性能模式调整轮廓简化程度 + epsilon = self.mode_config['contour_simplify_epsilon'] * cv2.arcLength(contour, True) + fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius, epsilon) + + direction = self.calculate_hand_direction(contour, cx, cy) + h_img, w_img = frame_shape[:2] + norm_x = cx / w_img + norm_y = cy / h_img + confidence = self.calculate_confidence(area, len(fingers), len(contour), bbox_area) + + result = { + 'contour': contour, + 'center': (cx, cy), + 'bbox': (x, y, x + w, y + h), + 'fingers': fingers, + 'fingertips': fingertips, + 'defects': defects, + 'palm_center': palm_center, + 'palm_radius': palm_radius, + 'direction': direction, + 'area': area, + 'position': (norm_x, norm_y), + 'bbox_size': (w, h), + 'confidence': confidence + } + + return result, confidence + + def analyze_fingers(self, contour, palm_center, palm_radius, epsilon): + """分析手指""" + approx = cv2.approxPolyDP(contour, epsilon, True) + hull = cv2.convexHull(approx, returnPoints=False) + + if hull is None or len(hull) < 3: + return [], [], [] + + defects = cv2.convexityDefects(approx, hull) + fingers = [] + fingertips = [] + defect_points = [] + + if defects is not None: + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(approx[s][0]) + end = tuple(approx[e][0]) + far = tuple(approx[f][0]) + + start_dist = np.linalg.norm(np.array(start) - np.array(palm_center)) + end_dist = np.linalg.norm(np.array(end) - np.array(palm_center)) + far_dist = np.linalg.norm(np.array(far) - np.array(palm_center)) + + if (start_dist > palm_radius * 1.2 and + end_dist > palm_radius * 1.2 and + d > self.config.get('gesture', 'defect_distance_threshold') * 256): + + a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + + if b * c != 0: + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + angle_degrees = math.degrees(angle) + + if angle_degrees < 90: + finger = { + 'start': start, + 'end': end, + 'far': far, + 'depth': d, + 'angle': angle_degrees + } + fingers.append(finger) + + for point in [start, end]: + if point not in fingertips: + point_dist = np.linalg.norm(np.array(point) - np.array(palm_center)) + if point_dist > palm_radius * 1.5: + fingertips.append(point) + + defect_points.append((start, end, far, d)) + + if len(fingertips) == 0: + hull_points = cv2.convexHull(approx, returnPoints=True) + if len(hull_points) > 0: + hull_points = hull_points.reshape(-1, 2) + + for point in hull_points: + point_tuple = tuple(point) + point_dist = np.linalg.norm(point - np.array(palm_center)) + if point_dist > palm_radius * 1.5 and point_tuple not in fingertips: + fingertips.append(point_tuple) + + fingertips = fingertips[:5] + + return fingers, fingertips, defect_points + + def calculate_hand_direction(self, contour, cx, cy): + """计算手部方向""" + if len(contour) < 5: + return 0.0 + + points = contour.reshape(-1, 2).astype(np.float32) + mean = np.empty((0)) + mean, eigenvectors, eigenvalues = cv2.PCACompute2(points, mean) + direction = math.degrees(math.atan2(eigenvectors[0, 1], eigenvectors[0, 0])) + + return direction + + def calculate_confidence(self, area, finger_count, contour_length, bbox_area): + """计算手势置信度""" + confidence = 0.5 + min_area = self.config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = self.config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) + + if min_area < area < max_area: + area_norm = (area - min_area) / (max_area - min_area) + confidence += area_norm * 0.2 + + if 0 <= finger_count <= 5: + confidence += 0.2 + + if contour_length > 200: + confidence += 0.1 + + if bbox_area > 0: + fill_ratio = area / bbox_area + if 0.2 < fill_ratio < 0.8: + fill_score = 1.0 - abs(fill_ratio - 0.5) / 0.3 + confidence += fill_score * 0.1 + + return min(confidence, 1.0) + + def recognize_gesture_improved(self, hand_data): + """改进的手势识别逻辑""" + if hand_data is None: + return "Waiting", 0.3 + + finger_count = len(hand_data.get('fingers', [])) + fingertips = hand_data.get('fingertips', []) + norm_x, norm_y = hand_data['position'] + direction = hand_data.get('direction', 0.0) + confidence = hand_data['confidence'] + w, h = hand_data['bbox_size'] + aspect_ratio = w / h if h > 0 else 1.0 + + # 根据手指数量分类 + if finger_count == 0: + if len(fingertips) == 0: + return "Stop", confidence * 0.9 + else: + return "Stop", confidence * 0.7 + + elif finger_count == 1: + if len(fingertips) >= 1: + cx, cy = hand_data['center'] + fingertip = fingertips[0] + dx = fingertip[0] - cx + dy = fingertip[1] - cy + + if abs(dx) > abs(dy): + if dx > 0: + return "Right", confidence * 0.8 + else: + return "Left", confidence * 0.8 + else: + if dy < 0: + return "Up", confidence * 0.8 + else: + return "Forward", confidence * 0.8 + return "Forward", confidence * 0.7 + + elif finger_count == 2: + return "Forward", confidence * 0.7 + + elif finger_count == 3: + if -45 <= direction <= 45: + if direction > 0: + return "Right", confidence * 0.7 + else: + return "Left", confidence * 0.7 + else: + if direction > 0: + return "Down", confidence * 0.7 + else: + return "Up", confidence * 0.7 + + elif finger_count >= 4: + if norm_x < 0.4: + return "Left", confidence * 0.8 + elif norm_x > 0.6: + return "Right", confidence * 0.8 + elif norm_y < 0.4: + return "Up", confidence * 0.8 + elif norm_y > 0.6: + return "Down", confidence * 0.8 + else: + if -45 <= direction <= 45: + return "Forward", confidence * 0.7 + else: + return "Stop", confidence * 0.7 + + return "Waiting", confidence * 0.5 + + def smooth_gesture_enhanced(self, new_gesture, new_confidence, hand_data): + """增强的手势平滑处理""" + current_time = time.time() + + # 检查手势冷却时间 + if current_time - self.last_gesture_change_time < self.config.get('gesture', 'gesture_cooldown'): + return self.current_gesture, self.current_confidence + + # 添加到历史 + self.gesture_history.append(new_gesture) + self.confidence_history.append(new_confidence) + + if hand_data is not None: + self.position_history.append(hand_data['position']) + + # 计算手势稳定性 + if len(self.gesture_history) >= 3: + recent_gestures = list(self.gesture_history)[-3:] + gesture_counter = Counter(recent_gestures) + most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] + + # 计算位置稳定性 + position_stability = 1.0 + if len(self.position_history) >= 2 and hand_data is not None: + current_pos = hand_data['position'] + prev_pos = self.position_history[-2] if len(self.position_history) >= 2 else current_pos + position_diff = math.sqrt((current_pos[0] - prev_pos[0]) ** 2 + (current_pos[1] - prev_pos[1]) ** 2) + position_stability = max(0, 1.0 - position_diff * 5.0) + + # 增强的稳定性检查 + stability_threshold = self.mode_config['gesture_stability_threshold'] + transition_threshold = self.config.get('gesture', 'transition_threshold') + position_weight = self.config.get('gesture', 'position_stability_weight') + + # 计算综合稳定性得分 + gesture_stability = most_common_count / 3.0 + overall_stability = gesture_stability * (1.0 - position_weight) + position_stability * position_weight + + # 手势状态转换逻辑 + if overall_stability >= transition_threshold: + if most_common_gesture != self.last_stable_gesture: + self.gesture_stability_counter += 1 + else: + self.gesture_stability_counter = max(0, self.gesture_stability_counter - 1) + + if self.gesture_stability_counter >= stability_threshold: + self.current_gesture = most_common_gesture + self.current_confidence = np.mean(list(self.confidence_history)[-3:]) + self.last_stable_gesture = most_common_gesture + self.gesture_stability_counter = 0 + self.last_gesture_change_time = current_time + else: + self.gesture_stability_counter = max(0, self.gesture_stability_counter - 2) + + return self.current_gesture, self.current_confidence + + def update_gesture_state(self, hand_data, gesture, confidence): + """更新手势状态""" + current_time = time.time() + + if hand_data is None: + # 手部丢失 + self.hand_lost_frames += 1 + self.hand_detected_frames = max(0, self.hand_detected_frames - 1) + + if self.hand_lost_frames > 10 and self.gesture_state != "none": + self.gesture_state = "none" + if (self.speech_manager and + self.speech_manager.enabled and + current_time - self.last_hand_status_time > 3.0): + self.speech_manager.speak('hand_lost', immediate=True) + self.last_hand_status_time = current_time + return + + # 手部检测到 + self.hand_detected_frames += 1 + self.hand_lost_frames = 0 + + # 检查手部大小和位置 + hand_area = hand_data['area'] + min_area = self.config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = self.config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) + + # 提供手部位置反馈 + if (self.speech_manager and + self.speech_manager.enabled and + current_time - self.last_hand_status_time > 5.0): + + if hand_area < min_area * 0.8: + self.speech_manager.speak('move_closer', immediate=True) + self.last_hand_status_time = current_time + elif hand_area > max_area * 1.2: + self.speech_manager.speak('move_away', immediate=True) + self.last_hand_status_time = current_time + elif self.hand_detected_frames == 5: + self.speech_manager.speak('hand_detected', immediate=True) + self.last_hand_status_time = current_time + + # 手势状态转换 + if self.gesture_state == "none" and gesture != "Waiting" and confidence > 0.6: + self.gesture_state = "starting" + self.gesture_active_frames = 0 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_start', immediate=True) + + elif self.gesture_state == "starting": + self.gesture_active_frames += 1 + if self.gesture_active_frames >= self.config.get('speech', 'gesture_start_threshold'): + self.gesture_state = "active" + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_stable', immediate=True) + + elif self.gesture_state == "active": + if gesture == "Waiting" or confidence < 0.5: + self.gesture_active_frames = max(0, self.gesture_active_frames - 2) + if self.gesture_active_frames <= 0: + self.gesture_state = "ending" + else: + self.gesture_active_frames = min(20, self.gesture_active_frames + 1) + + elif self.gesture_state == "ending": + self.gesture_active_frames -= 1 + if self.gesture_active_frames <= 0: + self.gesture_state = "none" + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_end', immediate=True) + + def visualize_detection(self, frame, hand_data, gesture, confidence): + """可视化检测结果""" + if hand_data is None: + return frame + + show_contours = self.config.get('display', 'show_contours') + show_bbox = self.config.get('display', 'show_bbox') + show_fingertips = self.config.get('display', 'show_fingertips') + show_palm_center = self.config.get('display', 'show_palm_center') + show_hand_direction = self.config.get('display', 'show_hand_direction') + show_debug_info = self.config.get('display', 'show_debug_info') + show_gesture_history = self.config.get('display', 'show_gesture_history') + show_stability_indicator = self.config.get('display', 'show_stability_indicator') + + # 绘制轮廓 + if show_contours and 'contour' in hand_data: + cv2.drawContours(frame, [hand_data['contour']], -1, (0, 255, 0), 2) + + # 绘制边界框 + if show_bbox and 'bbox' in hand_data: + x1, y1, x2, y2 = hand_data['bbox'] + color = self.gesture_colors.get(gesture, (255, 255, 255)) + + # 根据手势状态调整边界框颜色 + state_color = self.state_colors.get(self.gesture_state, color) + if self.gesture_state != "none": + color = state_color + + cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) + + # 显示手势标签 + label = f"{gesture}" + if self.config.get('display', 'show_confidence'): + label += f" ({confidence:.0%})" + + # 显示手势状态 + if self.gesture_state != "none": + state_text = {"starting": "开始", "active": "活跃", "ending": "结束"} + label += f" [{state_text.get(self.gesture_state, '')}]" + + # 计算文本大小 + (text_width, text_height), baseline = cv2.getTextSize( + label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 + ) + + # 绘制文本背景 + cv2.rectangle(frame, + (x1, y1 - text_height - 10), + (x1 + text_width, y1), + color, -1) + + # 绘制文本 + cv2.putText(frame, label, (x1, y1 - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) + + # 绘制手掌中心 + if show_palm_center and 'palm_center' in hand_data: + cx, cy = hand_data['palm_center'] + palm_radius = hand_data.get('palm_radius', 20) + cv2.circle(frame, (cx, cy), 5, (0, 0, 255), -1) + cv2.circle(frame, (cx, cy), palm_radius, (0, 0, 255), 1) + cv2.putText(frame, "Palm", (cx + 10, cy), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) + + # 绘制指尖 + if show_fingertips and 'fingertips' in hand_data: + for i, point in enumerate(hand_data['fingertips']): + cv2.circle(frame, point, 4, (255, 0, 0), -1) + cv2.putText(frame, f"F{i + 1}", (point[0] + 5, point[1]), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1) + + # 绘制手部方向 + if show_hand_direction and 'direction' in hand_data and 'center' in hand_data: + cx, cy = hand_data['center'] + direction = hand_data['direction'] + length = 50 + + dx = length * math.cos(math.radians(direction)) + dy = length * math.sin(math.radians(direction)) + + end_point = (int(cx + dx), int(cy + dy)) + cv2.arrowedLine(frame, (cx, cy), end_point, (255, 255, 0), 2) + + angle_text = f"Dir: {direction:.0f}°" + cv2.putText(frame, angle_text, (cx, cy - 20), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 1) + + # 绘制稳定性指示器 + if show_stability_indicator: + h, w = frame.shape[:2] + indicator_x = w - 100 + indicator_y = 30 + + # 绘制稳定性背景 + cv2.rectangle(frame, (indicator_x, indicator_y), + (indicator_x + 80, indicator_y + 15), (50, 50, 50), -1) + + # 计算稳定性指示条长度 + stability_level = min(1.0, + self.gesture_stability_counter / self.mode_config['gesture_stability_threshold']) + bar_length = int(70 * stability_level) + + # 根据稳定性级别选择颜色 + if stability_level > 0.7: + bar_color = (0, 255, 0) + elif stability_level > 0.4: + bar_color = (255, 165, 0) + else: + bar_color = (255, 0, 0) + + # 绘制稳定性指示条 + cv2.rectangle(frame, (indicator_x + 5, indicator_y + 5), + (indicator_x + 5 + bar_length, indicator_y + 10), bar_color, -1) + + # 绘制稳定性文本 + cv2.putText(frame, "稳定度", (indicator_x, indicator_y - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 绘制手势历史 + if show_gesture_history and len(self.gesture_history) > 0: + h, w = frame.shape[:2] + history_y = h - 50 + + # 绘制历史背景 + cv2.rectangle(frame, (10, history_y - 20), (200, history_y + 10), (0, 0, 0), -1) + cv2.putText(frame, "手势历史:", (15, history_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 显示最近几个手势 + recent_gestures = list(self.gesture_history)[-5:] if len(self.gesture_history) >= 5 else list( + self.gesture_history) + for i, gest in enumerate(recent_gestures): + color = self.gesture_colors.get(gest, (255, 255, 255)) + cv2.putText(frame, gest[0], (85 + i * 20, history_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + + # 显示调试信息 + if show_debug_info: + finger_count = len(hand_data.get('fingers', [])) + finger_text = f"Fingers: {finger_count}" + cv2.putText(frame, finger_text, (10, frame.shape[0] - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + pos_text = f"Pos: ({hand_data['position'][0]:.2f}, {hand_data['position'][1]:.2f})" + cv2.putText(frame, pos_text, (10, frame.shape[0] - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + state_text = f"State: {self.gesture_state}" + cv2.putText(frame, state_text, (150, frame.shape[0] - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + stability_text = f"Stability: {self.gesture_stability_counter}" + cv2.putText(frame, stability_text, (150, frame.shape[0] - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + return frame + + def recognize(self, frame): + """识别手势""" + start_time = time.time() + + try: + # 预处理帧 + processed_frame = cv2.flip(frame, 1) + + # 根据性能模式调整图像大小 + original_frame = processed_frame.copy() + if self.resize_factor != 1.0: + processed_frame = self.preprocess_frame(processed_frame) + + # 每隔几帧检测一次以提高性能 + if self.frame_counter % self.detection_interval != 0: + self.frame_counter += 1 + return self.current_gesture, self.current_confidence, original_frame + + # 获取肤色掩码 + skin_mask, hsv = self.get_skin_mask(processed_frame) + + # 增强肤色检测 + enhanced_mask = self.enhance_skin_detection(processed_frame, skin_mask) + + # 找到最佳的手部轮廓 + hand_contour, contour_score = self.find_best_hand_contour(enhanced_mask, processed_frame) + + # 分析手部特征 + hand_data, confidence = self.analyze_hand_features(hand_contour, processed_frame.shape) + + # 保存手部数据用于轨迹记录 + self.last_hand_data = hand_data + + # 识别手势 + if hand_data is not None: + # 校准肤色 + self.config.calibrate_skin_color(processed_frame, enhanced_mask) + + # 校准手部大小 + self.config.calibrate_hand_size(hand_data['area']) + + # 识别手势 + gesture, raw_confidence = self.recognize_gesture_improved(hand_data) + confidence = max(confidence, raw_confidence) + + # 更新手势状态 + self.update_gesture_state(hand_data, gesture, confidence) + + # 增强的手势平滑 + final_gesture, final_confidence = self.smooth_gesture_enhanced(gesture, confidence, hand_data) + else: + gesture, confidence = "Waiting", 0.3 + self.update_gesture_state(None, gesture, confidence) + final_gesture, final_confidence = gesture, confidence + + # 手势语音播报 + if (self.speech_manager and + self.speech_manager.enabled): + + current_time = time.time() + + # 根据置信度提供反馈 + if confidence >= 0.8 and current_time - self.last_hand_status_time > 5.0: + self.speech_manager.speak('gesture_good_confidence', immediate=True) + self.last_hand_status_time = current_time + elif confidence < 0.5 and current_time - self.last_hand_status_time > 5.0: + self.speech_manager.speak('gesture_low_confidence', immediate=True) + self.last_hand_status_time = current_time + + # 手势语音播报 + if (final_gesture != "Waiting" and + final_gesture != "Error" and + final_gesture != "摄像头错误" and + final_confidence >= self.min_confidence and + current_time - self.last_announced_time > self.gesture_announce_interval): + + if final_gesture in self.gesture_speech_map: + speech_key = self.gesture_speech_map[final_gesture] + self.speech_manager.speak(speech_key) + else: + self.speech_manager.speak_direct(f"手势{final_gesture}") + + self.last_announced_gesture = final_gesture + self.last_announced_time = current_time + + # 性能报告 + current_time = time.time() + if current_time - self.last_performance_report > 30.0: + if (self.speech_manager and + self.speech_manager.enabled): + + if self.avg_process_time < 20: + self.speech_manager.speak('performance_good', immediate=True) + elif self.avg_process_time > 50: + self.speech_manager.speak('performance_warning', immediate=True) + + self.last_performance_report = current_time + + # 可视化结果 + if hand_data is not None: + # 需要将坐标转换回原始图像大小 + if self.resize_factor != 1.0: + scale_factor = 1.0 / self.resize_factor + if 'center' in hand_data: + hand_data['center'] = (int(hand_data['center'][0] * scale_factor), + int(hand_data['center'][1] * scale_factor)) + if 'bbox' in hand_data: + x1, y1, x2, y2 = hand_data['bbox'] + hand_data['bbox'] = (int(x1 * scale_factor), int(y1 * scale_factor), + int(x2 * scale_factor), int(y2 * scale_factor)) + if 'fingertips' in hand_data: + hand_data['fingertips'] = [(int(x * scale_factor), int(y * scale_factor)) + for (x, y) in hand_data['fingertips']] + if 'palm_center' in hand_data: + hand_data['palm_center'] = (int(hand_data['palm_center'][0] * scale_factor), + int(hand_data['palm_center'][1] * scale_factor)) + + original_frame = self.visualize_detection( + original_frame, hand_data, final_gesture, final_confidence + ) + else: + original_frame = self.visualize_detection( + original_frame, None, final_gesture, final_confidence + ) + + # 更新计数器 + self.frame_counter += 1 + + # 计算处理时间 + process_time = (time.time() - start_time) * 1000 + self.process_times.append(process_time) + + # 更新平均处理时间 + if len(self.process_times) > 0: + self.avg_process_time = np.mean(list(self.process_times)) + + # 更新帧率 + current_time = time.time() + if current_time - self.last_fps_check >= 1.0: + self.frame_rate = self.frame_counter + self.frame_counter = 0 + self.last_fps_check = current_time + + return final_gesture, final_confidence, original_frame + + except Exception as e: + print(f"⚠ 手势识别错误: {e}") + return "Error", 0.0, frame + + def get_performance_stats(self): + """获取性能统计""" + if len(self.process_times) == 0: + return 0.0, self.frame_rate + + return np.mean(list(self.process_times)), self.frame_rate + + def get_performance_mode_info(self): + """获取性能模式信息""" + return { + 'name': self.performance_mode_name, + 'mode': self.performance_mode, + 'color': self.performance_mode_color, + 'detection_interval': self.detection_interval, + 'resize_factor': self.resize_factor, + 'smooth_frames': self.smooth_frames, + 'min_confidence': self.min_confidence + } + + def set_simulated_gesture(self, gesture): + """设置模拟的手势""" + self.current_gesture = gesture + self.current_confidence = 0.9 + + # 模拟手势也触发语音提示 + if (self.speech_manager and + self.speech_manager.enabled): + + if gesture in self.gesture_speech_map: + self.speech_manager.speak(self.gesture_speech_map[gesture]) + else: + self.speech_manager.speak_direct(f"手势{gesture}") + + self.last_announced_gesture = gesture + self.last_announced_time = time.time() \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 7e4ae44fbf..0a63edfb6c 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 详细性能统计优化版 -新增完整的性能监控、统计和报告功能 +手势控制无人机系统 - 主入口模块 +协调所有子模块,实现完整的手势控制无人机系统 作者: xiaoshiyuan888 """ @@ -9,67 +9,59 @@ import os import time import traceback -import json -import math -import threading -import tempfile -import pickle -from datetime import datetime -from PIL import Image, ImageDraw, ImageFont import cv2 import numpy as np -from collections import deque, Counter -import psutil # 新增:用于系统资源监控 -import csv # 新增:用于性能日志导出 -print("=" * 60) -print("Gesture Controlled Drone - Performance Statistics Enhanced") -print("详细性能统计优化版!") -print("=" * 60) - -# ========== 修复导入路径 ========== +# 添加当前目录到路径 current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) +# 导入自定义模块 +from config_manager import ConfigManager +from performance_analyzer import PerformanceAnalyzer +from speech_manager import EnhancedSpeechFeedbackManager +from gesture_recognizer import EnhancedGestureRecognizer +from drone_controller import SimpleDroneController +from ui_renderer import ChineseUIRenderer +from trajectory_recorder import GestureTrajectoryRecorder -# ========== 核心模块导入 ========== -def safe_import(): - """安全导入所有模块""" - modules_status = {} - try: - from PIL import Image, ImageDraw, ImageFont - modules_status['PIL'] = True - print("[PIL] ✓ 图像处理库就绪") - except Exception as e: - modules_status['PIL'] = False - print(f"[PIL] ✗ 导入失败: {e}") - return None, modules_status +# 安全导入所需库 +def safe_import_libs(): + """安全导入所有需要的库""" + libs = {} + status = {} try: import cv2 import numpy as np - modules_status['OpenCV'] = True + libs['cv2'] = cv2 + libs['np'] = np + status['OpenCV'] = True print("[OpenCV] ✓ 计算机视觉库就绪") except Exception as e: - modules_status['OpenCV'] = False + status['OpenCV'] = False print(f"[OpenCV] ✗ 导入失败: {e}") - return None, modules_status + return None, status try: - import psutil - modules_status['psutil'] = True - print("[psutil] ✓ 系统资源监控库就绪") + from PIL import Image, ImageDraw, ImageFont + libs['PIL'] = {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont} + status['PIL'] = True + print("[PIL] ✓ 图像处理库就绪") except Exception as e: - modules_status['psutil'] = False - print(f"[psutil] ✗ 导入失败: {e}") - # 继续运行,psutil不是必需的 + status['PIL'] = False + print(f"[PIL] ✗ 导入失败: {e}") + return None, status + # 尝试导入AirSim airsim_module = None try: - airsim_module = __import__('airsim') - modules_status['AirSim'] = True - print(f"[AirSim] ✓ 成功导入") + import airsim + airsim_module = airsim + libs['airsim'] = airsim_module + status['AirSim'] = True + print("[AirSim] ✓ 成功导入") except ImportError: print("\n" + "!" * 60) print("⚠ AirSim库未找到!") @@ -86,3202 +78,56 @@ def safe_import(): if choice != 'y': sys.exit(1) - # 尝试导入语音合成库 + # 尝试导入语音库 speech_module = None try: - # 尝试导入pyttsx3(离线TTS) import pyttsx3 speech_module = pyttsx3 - modules_status['Speech'] = True + status['Speech'] = True print("[Speech] ✓ pyttsx3语音库就绪 (离线)") except ImportError: print("\n" + "!" * 60) print("⚠ pyttsx3语音库未找到!") print("!" * 60) - print("安装语音库 (使用清华大学镜像源):") - print("1. 安装离线语音库: pip install pyttsx3 -i https://pypi.tuna.tsinghua.edu.cn/simple") - print("2. 或者安装在线语音库: pip install gtts pygame -i https://pypi.tuna.tsinghua.edu.cn/simple") + print("安装语音库:") + print("1. 安装离线语音库: pip install pyttsx3") + print("2. 或者安装在线语音库: pip install gtts pygame") print("!" * 60) - # 尝试其他语音库 try: - # 尝试使用gTTS(需要网络) from gtts import gTTS speech_module = {'gTTS': gTTS, 'type': 'gtts'} - modules_status['Speech'] = True + status['Speech'] = True print("[Speech] ✓ gTTS语音库就绪 (需要网络连接)") - # 尝试导入音频播放库 try: import pygame pygame.mixer.init() speech_module['pygame'] = pygame print("[Speech] ✓ pygame音频播放库就绪") except ImportError: - # 尝试其他播放方式 - try: - import pydub - from pydub import AudioSegment - from pydub.playback import play - speech_module['pydub'] = pydub - speech_module['AudioSegment'] = AudioSegment - speech_module['play'] = play - print("[Speech] ✓ pydub音频播放库就绪") - except ImportError: - # 最后尝试使用系统命令 - if os.name == 'nt': # Windows - speech_module['play_method'] = 'windows' - print("[Speech] ✓ 使用Windows系统命令播放音频") - elif os.name == 'posix': # Linux/Mac - speech_module['play_method'] = 'posix' - print("[Speech] ✓ 使用系统命令播放音频") - else: - print("[Speech] ✗ 所有音频播放库导入失败,语音功能将不可用") - speech_module = None - modules_status['Speech'] = False - + speech_module['play_method'] = 'system' except ImportError: print("[Speech] ✗ 所有语音库导入失败,语音功能将不可用") speech_module = None - modules_status['Speech'] = False - - return { - 'cv2': cv2, - 'np': np, - 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, - 'airsim': airsim_module, - 'speech': speech_module, - 'psutil': psutil if 'psutil' in modules_status and modules_status['psutil'] else None - }, modules_status - - -# 执行导入 -libs, status = safe_import() -if not status.get('OpenCV', False) or not status.get('PIL', False): - print("\n❌ 核心库缺失,无法启动。") - input("按回车键退出...") - sys.exit(1) - -print("-" * 60) -print("✅ 环境检查通过,正在初始化...") -print("-" * 60) - -# 解包库 -cv2, np = libs['cv2'], libs['np'] -Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] - - -# ========== 性能分析器类 ========== -class PerformanceAnalyzer: - """性能分析器 - 监控和报告系统性能""" - - def __init__(self, speech_manager=None): - self.speech_manager = speech_manager - self.start_time = time.time() - self.session_start_time = time.time() - - # 帧率统计 - self.frame_times = deque(maxlen=300) # 保留最近300帧的时间戳 - self.frame_count = 0 - self.fps_history = deque(maxlen=100) # FPS历史记录 - - # 手势识别性能 - self.gesture_recognition_times = deque(maxlen=100) - self.avg_recognition_time = 0 - self.max_recognition_time = 0 - - # 系统资源监控 - self.cpu_usage_history = deque(maxlen=100) - self.memory_usage_history = deque(maxlen=100) - - # 性能事件记录 - self.performance_events = [] - self.performance_snapshots = [] - - # 手势统计 - self.gesture_counts = {} - self.gesture_confidence_sum = {} - self.gesture_confidence_count = {} - - # 错误统计 - self.error_count = 0 - self.warning_count = 0 - - # 无人机控制统计 - self.drone_commands = 0 - self.successful_commands = 0 - self.failed_commands = 0 - - # 轨迹记录统计 - self.recording_sessions = 0 - self.total_trajectory_points = 0 - - # 性能日志 - self.performance_log = [] - self.log_file = os.path.join(current_dir, 'performance_log.csv') - - # 性能阈值 - self.performance_thresholds = { - 'fps_warning': 15, - 'fps_critical': 5, - 'cpu_warning': 80, - 'cpu_critical': 90, - 'memory_warning': 85, - 'memory_critical': 95, - 'recognition_warning': 50, # ms - 'recognition_critical': 100 # ms - } - - # 性能状态 - self.performance_status = "良好" - self.last_performance_report = 0 - self.auto_report_interval = 60 # 自动报告间隔(秒) - - print("✓ 性能分析器已初始化") - - def update_frame(self): - """更新帧统计""" - current_time = time.time() - self.frame_times.append(current_time) - self.frame_count += 1 - - # 计算当前FPS - if len(self.frame_times) > 1: - time_span = self.frame_times[-1] - self.frame_times[0] - if time_span > 0: - current_fps = (len(self.frame_times) - 1) / time_span - self.fps_history.append(current_fps) - - def update_gesture_recognition_time(self, recognition_time_ms): - """更新手势识别时间""" - self.gesture_recognition_times.append(recognition_time_ms) - - # 更新平均识别时间 - if len(self.gesture_recognition_times) > 0: - self.avg_recognition_time = np.mean(list(self.gesture_recognition_times)) - self.max_recognition_time = max(self.max_recognition_time, recognition_time_ms) - - def update_system_resources(self): - """更新系统资源使用情况""" - try: - if libs['psutil']: - cpu_percent = libs['psutil'].cpu_percent(interval=0.1) - memory_percent = libs['psutil'].virtual_memory().percent - - self.cpu_usage_history.append(cpu_percent) - self.memory_usage_history.append(memory_percent) - - # 检查性能问题 - self.check_performance_issues(cpu_percent, memory_percent) - except: - pass # psutil不可用时跳过 - - def check_performance_issues(self, cpu_percent, memory_percent): - """检查性能问题""" - issues = [] - - # 检查FPS - if len(self.fps_history) > 0: - avg_fps = np.mean(list(self.fps_history[-10:])) if len(self.fps_history) >= 10 else self.fps_history[-1] - - if avg_fps < self.performance_thresholds['fps_critical']: - issues.append(("严重", f"帧率过低: {avg_fps:.1f} FPS")) - self.performance_status = "严重" - elif avg_fps < self.performance_thresholds['fps_warning']: - issues.append(("警告", f"帧率较低: {avg_fps:.1f} FPS")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 检查CPU使用率 - if cpu_percent > self.performance_thresholds['cpu_critical']: - issues.append(("严重", f"CPU使用率过高: {cpu_percent:.1f}%")) - self.performance_status = "严重" - elif cpu_percent > self.performance_thresholds['cpu_warning']: - issues.append(("警告", f"CPU使用率较高: {cpu_percent:.1f}%")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 检查内存使用率 - if memory_percent > self.performance_thresholds['memory_critical']: - issues.append(("严重", f"内存使用率过高: {memory_percent:.1f}%")) - self.performance_status = "严重" - elif memory_percent > self.performance_thresholds['memory_warning']: - issues.append(("警告", f"内存使用率较高: {memory_percent:.1f}%")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 检查手势识别时间 - if self.avg_recognition_time > self.performance_thresholds['recognition_critical']: - issues.append(("严重", f"手势识别时间过长: {self.avg_recognition_time:.1f}ms")) - self.performance_status = "严重" - elif self.avg_recognition_time > self.performance_thresholds['recognition_warning']: - issues.append(("警告", f"手势识别时间较长: {self.avg_recognition_time:.1f}ms")) - if self.performance_status == "良好": - self.performance_status = "警告" - - # 记录性能事件 - if issues: - for level, message in issues: - self.add_performance_event(level, message) - - # 语音提示(仅在状态变化时) - if (self.speech_manager and - self.speech_manager.enabled and - level == "严重"): - current_time = time.time() - if current_time - self.last_performance_report > 10: # 避免频繁提示 - self.speech_manager.speak_direct(f"性能{level}: {message}") - self.last_performance_report = current_time - - def add_performance_event(self, level, message): - """添加性能事件""" - event = { - 'timestamp': time.time(), - 'level': level, - 'message': message, - 'session_time': time.time() - self.session_start_time - } - self.performance_events.append(event) - - # 记录到日志 - self.log_performance_event(event) - - if level == "警告": - self.warning_count += 1 - elif level == "严重": - self.error_count += 1 - - def log_performance_event(self, event): - """记录性能事件到日志""" - log_entry = { - 'timestamp': datetime.fromtimestamp(event['timestamp']).strftime('%Y-%m-%d %H:%M:%S'), - 'session_time': f"{event['session_time']:.1f}", - 'level': event['level'], - 'message': event['message'] - } - self.performance_log.append(log_entry) - - def record_gesture(self, gesture, confidence): - """记录手势统计""" - if gesture not in self.gesture_counts: - self.gesture_counts[gesture] = 0 - self.gesture_confidence_sum[gesture] = 0 - self.gesture_confidence_count[gesture] = 0 - - self.gesture_counts[gesture] += 1 - self.gesture_confidence_sum[gesture] += confidence - self.gesture_confidence_count[gesture] += 1 - - def record_drone_command(self, success=True): - """记录无人机命令""" - self.drone_commands += 1 - if success: - self.successful_commands += 1 - else: - self.failed_commands += 1 - - def record_recording_session(self, points_count=0): - """记录录制会话""" - self.recording_sessions += 1 - self.total_trajectory_points += points_count - - def take_snapshot(self, label=""): - """拍摄性能快照""" - snapshot = { - 'timestamp': time.time(), - 'label': label, - 'fps': self.get_current_fps(), - 'avg_fps': self.get_average_fps(), - 'avg_recognition_time': self.avg_recognition_time, - 'max_recognition_time': self.max_recognition_time, - 'cpu_usage': self.get_current_cpu_usage(), - 'memory_usage': self.get_current_memory_usage(), - 'gesture_counts': dict(self.gesture_counts), - 'performance_status': self.performance_status, - 'frame_count': self.frame_count, - 'session_duration': time.time() - self.session_start_time - } - self.performance_snapshots.append(snapshot) - - print(f"📸 性能快照已保存: {label}") - return snapshot - - def get_current_fps(self): - """获取当前FPS""" - if len(self.fps_history) > 0: - return self.fps_history[-1] - return 0 - - def get_average_fps(self): - """获取平均FPS""" - if len(self.fps_history) > 0: - return np.mean(list(self.fps_history)) - return 0 - - def get_current_cpu_usage(self): - """获取当前CPU使用率""" - if len(self.cpu_usage_history) > 0: - return self.cpu_usage_history[-1] - return 0 - - def get_current_memory_usage(self): - """获取当前内存使用率""" - if len(self.memory_usage_history) > 0: - return self.memory_usage_history[-1] - return 0 - - def generate_report(self, detailed=True): - """生成性能报告""" - report_time = time.time() - session_duration = report_time - self.session_start_time - - # 基础报告 - report = { - '生成时间': datetime.fromtimestamp(report_time).strftime('%Y-%m-%d %H:%M:%S'), - '会话时长': f"{session_duration:.1f}秒", - '总帧数': self.frame_count, - '平均FPS': f"{self.get_average_fps():.1f}", - '当前FPS': f"{self.get_current_fps():.1f}", - '平均手势识别时间': f"{self.avg_recognition_time:.1f}ms", - '最大手势识别时间': f"{self.max_recognition_time:.1f}ms", - '当前CPU使用率': f"{self.get_current_cpu_usage():.1f}%", - '当前内存使用率': f"{self.get_current_memory_usage():.1f}%", - '性能状态': self.performance_status, - '警告数量': self.warning_count, - '错误数量': self.error_count, - '无人机命令': { - '总数': self.drone_commands, - '成功': self.successful_commands, - '失败': self.failed_commands, - '成功率': f"{(self.successful_commands / self.drone_commands * 100 if self.drone_commands > 0 else 0):.1f}%" - }, - '录制统计': { - '会话数': self.recording_sessions, - '总轨迹点数': self.total_trajectory_points - } - } - - # 详细报告 - if detailed: - # 手势统计 - gesture_stats = {} - for gesture in self.gesture_counts: - count = self.gesture_counts[gesture] - if gesture in self.gesture_confidence_count and self.gesture_confidence_count[gesture] > 0: - avg_confidence = self.gesture_confidence_sum[gesture] / self.gesture_confidence_count[gesture] - else: - avg_confidence = 0 - - gesture_stats[gesture] = { - '次数': count, - '占比': f"{(count / self.frame_count * 100 if self.frame_count > 0 else 0):.1f}%", - '平均置信度': f"{avg_confidence:.1%}" - } - - report['手势统计'] = gesture_stats - - # 性能事件 - if self.performance_events: - recent_events = list(self.performance_events)[-10:] # 最近10个事件 - report['最近性能事件'] = [ - { - '时间': datetime.fromtimestamp(e['timestamp']).strftime('%H:%M:%S'), - '级别': e['level'], - '消息': e['message'] - } - for e in recent_events - ] - - # 性能快照 - if self.performance_snapshots: - report['性能快照数'] = len(self.performance_snapshots) - - # 系统建议 - suggestions = self.generate_suggestions() - if suggestions: - report['优化建议'] = suggestions - - return report - - def generate_suggestions(self): - """生成优化建议""" - suggestions = [] - - # 检查FPS - avg_fps = self.get_average_fps() - if avg_fps < self.performance_thresholds['fps_warning']: - suggestions.append(f"帧率较低({avg_fps:.1f}FPS),建议切换到'最快'性能模式") - - # 检查CPU - cpu_usage = self.get_current_cpu_usage() - if cpu_usage > self.performance_thresholds['cpu_warning']: - suggestions.append(f"CPU使用率较高({cpu_usage:.1f}%),请关闭其他占用CPU的程序") - - # 检查内存 - memory_usage = self.get_current_memory_usage() - if memory_usage > self.performance_thresholds['memory_warning']: - suggestions.append(f"内存使用率较高({memory_usage:.1f}%),请关闭不必要的程序") - - # 检查识别时间 - if self.avg_recognition_time > self.performance_thresholds['recognition_warning']: - suggestions.append(f"手势识别时间较长({self.avg_recognition_time:.1f}ms),建议调整摄像头位置或光线") - - return suggestions - - def print_report(self, detailed=True): - """打印性能报告""" - report = self.generate_report(detailed) - - print("\n" + "=" * 80) - print("📊 性能分析报告") - print("=" * 80) - - # 基础信息 - print(f"生成时间: {report['生成时间']}") - print(f"会话时长: {report['会话时长']}") - print(f"总帧数: {report['总帧数']}") - print(f"平均FPS: {report['平均FPS']}") - print(f"当前FPS: {report['当前FPS']}") - print(f"平均手势识别时间: {report['平均手势识别时间']}") - print(f"最大手势识别时间: {report['最大手势识别时间']}") - print(f"当前CPU使用率: {report['当前CPU使用率']}") - print(f"当前内存使用率: {report['当前内存使用率']}") - print(f"性能状态: {report['性能状态']}") - - # 无人机命令统计 - cmd_stats = report['无人机命令'] - print(f"\n无人机命令统计:") - print(f" 总数: {cmd_stats['总数']}") - print(f" 成功: {cmd_stats['成功']}") - print(f" 失败: {cmd_stats['失败']}") - print(f" 成功率: {cmd_stats['成功率']}") - - # 录制统计 - rec_stats = report['录制统计'] - print(f"\n录制统计:") - print(f" 会话数: {rec_stats['会话数']}") - print(f" 总轨迹点数: {rec_stats['总轨迹点数']}") - - # 详细报告 - if detailed and '手势统计' in report: - print(f"\n手势统计:") - for gesture, stats in report['手势统计'].items(): - print(f" {gesture}: {stats['次数']}次 ({stats['占比']}), 平均置信度: {stats['平均置信度']}") - - # 性能事件 - if detailed and '最近性能事件' in report and report['最近性能事件']: - print(f"\n最近性能事件:") - for event in report['最近性能事件']: - print(f" [{event['时间']}] {event['级别']}: {event['消息']}") - - # 优化建议 - if detailed and '优化建议' in report and report['优化建议']: - print(f"\n优化建议:") - for i, suggestion in enumerate(report['优化建议'], 1): - print(f" {i}. {suggestion}") - - print("=" * 80) - - # 语音播报摘要 - if self.speech_manager and self.speech_manager.enabled: - summary = (f"性能报告: 平均帧率{report['平均FPS']},识别时间{report['平均手势识别时间']}," - f"性能状态{report['性能状态']},无人机命令成功率{cmd_stats['成功率']}") - self.speech_manager.speak_direct(summary) - - def export_log(self, filename=None): - """导出性能日志""" - if filename is None: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - filename = os.path.join(current_dir, f'performance_log_{timestamp}.csv') - - try: - with open(filename, 'w', newline='', encoding='utf-8') as f: - if self.performance_log: - fieldnames = self.performance_log[0].keys() - writer = csv.DictWriter(f, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(self.performance_log) - - print(f"📤 性能日志已导出到: {filename}") - return True - except Exception as e: - print(f"❌ 导出性能日志失败: {e}") - return False - - def auto_report(self): - """自动性能报告(定期执行)""" - current_time = time.time() - if current_time - self.last_performance_report > self.auto_report_interval: - # 生成简要报告 - report = self.generate_report(detailed=False) - - # 检查是否需要报告 - if (self.performance_status == "严重" or - self.warning_count > 5 or - self.error_count > 0): - - print(f"⚠ 自动性能检查: {report['性能状态']}, FPS: {report['当前FPS']}, " - f"CPU: {report['当前CPU使用率']}, 内存: {report['当前内存使用率']}") - - # 语音提示 - if (self.speech_manager and - self.speech_manager.enabled and - self.performance_status == "严重"): - self.speech_manager.speak_direct(f"系统性能{self.performance_status},建议检查") - - self.last_performance_report = current_time - - def reset_session(self): - """重置会话统计""" - self.session_start_time = time.time() - self.performance_events = [] - self.performance_snapshots = [] - self.gesture_counts = {} - self.gesture_confidence_sum = {} - self.gesture_confidence_count = {} - self.error_count = 0 - self.warning_count = 0 - self.drone_commands = 0 - self.successful_commands = 0 - self.failed_commands = 0 - self.performance_status = "良好" - - print("✓ 性能统计会话已重置") - - def get_stats_summary(self): - """获取统计摘要""" - return { - 'fps': self.get_current_fps(), - 'avg_fps': self.get_average_fps(), - 'recognition_time': self.avg_recognition_time, - 'cpu_usage': self.get_current_cpu_usage(), - 'memory_usage': self.get_current_memory_usage(), - 'performance_status': self.performance_status, - 'gesture_count': sum(self.gesture_counts.values()), - 'unique_gestures': len(self.gesture_counts) - } - - -# ========== 增强语音反馈管理器 ========== -class EnhancedSpeechFeedbackManager: - """增强的语音反馈管理器""" - - def __init__(self, speech_lib): - self.speech_lib = speech_lib - self.enabled = True - self.volume = 1.0 - self.rate = 150 - self.voice_id = None - self.last_speech_time = {} - self.min_interval = 1.5 # 缩短相同语音的最小间隔(秒) - - # 语音队列,避免语音重叠 - self.speech_queue = [] - self.is_speaking = False - self.queue_thread = None - - # 音频播放方法 - self.audio_method = None - - # 新增:手势状态追踪 - self.last_gesture_state = "none" # 记录上次手势状态 - self.gesture_active_time = 0 # 手势持续活跃时间 - - # 增强的语音消息映射 - self.messages = { - # 连接相关 - 'connecting': "正在连接无人机,请稍候", - 'connected': "无人机连接成功", - 'connection_failed': "无人机连接失败,进入模拟模式", - - # 飞行相关 - 'taking_off': "无人机正在起飞", - 'takeoff_success': "起飞成功", - 'takeoff_failed': "起飞失败", - 'landing': "无人机正在降落", - 'land_success': "降落成功", - 'emergency_stop': "紧急停止,无人机已降落", - 'hovering': "无人机悬停中", - - # 手势相关 - 增强 - 'gesture_detected': "手势识别就绪,请开始手势", - 'gesture_start': "开始识别手势", - 'gesture_end': "手势识别结束", - 'gesture_stop': "停止", - 'gesture_up': "向上", - 'gesture_down': "向下", - 'gesture_left': "向左", - 'gesture_right': "向右", - 'gesture_forward': "向前", - 'gesture_backward': "向后", - 'gesture_waiting': "等待手势", - 'gesture_error': "手势识别错误", - 'gesture_stable': "手势稳定", - 'gesture_change': "手势变化", - 'gesture_low_confidence': "手势识别置信度低", - 'gesture_good_confidence': "手势识别置信度高", - 'gesture_hover': "悬停", - - # 系统相关 - 'program_start': "手势控制无人机系统已启动", - 'program_exit': "程序退出,感谢使用", - 'camera_error': "摄像头错误,请检查连接", - 'camera_ready': "摄像头就绪", - 'system_ready': "系统准备就绪", - - # 模式相关 - 'simulation_mode': "进入模拟模式", - 'debug_mode_on': "调试模式已开启", - 'debug_mode_off': "调试模式已关闭", - 'display_mode_changed': "显示模式已切换", - 'help_toggled': "帮助信息已切换", - 'performance_mode_fast': "切换到最快性能模式", - 'performance_mode_balanced': "切换到平衡性能模式", - 'performance_mode_accurate': "切换到最准确性能模式", - - # 新增:性能相关 - 'performance_good': "系统运行流畅", - 'performance_warning': "系统性能警告", - 'performance_critical': "系统性能严重警告", - 'performance_report': "性能报告生成完成", - 'performance_snapshot': "性能快照已保存", - 'performance_log_exported': "性能日志已导出", - - # 新增:手势指导 - 'move_closer': "请将手靠近摄像头", - 'move_away': "请将手移远一些", - 'good_position': "手部位置良好", - 'hand_detected': "手部已检测到", - 'hand_lost': "手部丢失,请重新放置", - - # 新增:录制相关 - 'recording_start': "开始录制手势轨迹", - 'recording_stop': "停止录制", - 'recording_saved': "轨迹已保存", - 'recording_loaded': "轨迹已加载", - 'recording_playback_start': "开始回放手势轨迹", - 'recording_playback_stop': "回放结束", - 'recording_cleared': "轨迹已清除", - 'recording_paused': "回放已暂停", - 'recording_resumed': "回放继续", - 'recording_not_found': "未找到轨迹数据", - 'recording_frame_count': "轨迹帧数", - } - - # 初始化语音引擎 - self.init_speech_engine() - - def init_speech_engine(self): - """初始化语音引擎""" - if self.speech_lib is None: - print("⚠ 语音库未找到,语音功能禁用") - self.enabled = False - return - - try: - if hasattr(self.speech_lib, 'init'): # pyttsx3 - self.engine = self.speech_lib.init() - self.audio_method = 'pyttsx3' - - # 设置语音参数 - voices = self.engine.getProperty('voices') - - # 尝试寻找中文语音 - for voice in voices: - # 检查语音名称是否包含中文相关标识 - if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower() or 'zh_CN' in voice.name.lower(): - self.engine.setProperty('voice', voice.id) - self.voice_id = voice.id - print(f"[Speech] 使用中文语音: {voice.name}") - break - - # 如果没找到中文语音,使用第一个可用语音 - if self.voice_id is None and len(voices) > 0: - self.engine.setProperty('voice', voices[0].id) - print(f"[Speech] 使用默认语音: {voices[0].name}") - - # 设置语速和音量 - self.engine.setProperty('rate', self.rate) - self.engine.setProperty('volume', self.volume) - - print("✅ 语音引擎初始化成功 (pyttsx3)") - - elif isinstance(self.speech_lib, dict) and self.speech_lib.get('type') == 'gtts': - print("✅ 语音引擎初始化成功 (gTTS,需要网络连接)") - self.audio_method = 'gtts' - - # 确定播放方法 - if 'pygame' in self.speech_lib: - self.audio_method = 'gtts_pygame' - print("✅ 使用pygame播放音频") - elif 'pydub' in self.speech_lib: - self.audio_method = 'gtts_pydub' - print("✅ 使用pydub播放音频") - elif 'play_method' in self.speech_lib: - self.audio_method = f"gtts_{self.speech_lib['play_method']}" - print(f"✅ 使用系统命令播放音频") - else: - self.audio_method = 'gtts_system' - print("✅ 使用默认系统播放器") - - else: - print("⚠ 未知语音库类型,语音功能可能不正常") - self.enabled = False - - except Exception as e: - print(f"⚠ 语音引擎初始化失败: {e}") - self.enabled = False - - def play_audio_file(self, audio_file): - """播放音频文件(根据可用库选择方法)""" - try: - if self.audio_method == 'gtts_pygame' and 'pygame' in self.speech_lib: - pygame = self.speech_lib['pygame'] - pygame.mixer.music.load(audio_file) - pygame.mixer.music.play() - - # 等待播放完成 - while pygame.mixer.music.get_busy(): - pygame.time.Clock().tick(10) - - elif self.audio_method == 'gtts_pydub' and 'pydub' in self.speech_lib: - AudioSegment = self.speech_lib['AudioSegment'] - play = self.speech_lib['play'] - - audio = AudioSegment.from_mp3(audio_file) - play(audio) - - elif self.audio_method == 'gtts_windows': - # Windows系统命令 - os.startfile(audio_file) - # 等待播放完成(简单等待) - time.sleep(1.5) - - elif self.audio_method == 'gtts_posix': - # Linux/Mac系统命令 - import subprocess - if sys.platform == 'darwin': # macOS - subprocess.call(['afplay', audio_file]) - else: # Linux - subprocess.call(['xdg-open', audio_file]) - - else: - # 通用方法:使用系统默认播放器 - import subprocess - if sys.platform == 'win32': - os.startfile(audio_file) - elif sys.platform == 'darwin': - subprocess.call(['open', audio_file]) - else: - subprocess.call(['xdg-open', audio_file]) - - return True - - except Exception as e: - print(f"⚠ 音频播放失败: {e}") - return False - - def speak(self, message_key, force=False, immediate=False): - """播放语音""" - if not self.enabled: - return - - # 检查是否在最小间隔内 - current_time = time.time() - if not force and message_key in self.last_speech_time: - if current_time - self.last_speech_time[message_key] < self.min_interval: - return - - # 获取消息文本 - if message_key in self.messages: - text = self.messages[message_key] - else: - text = message_key # 直接使用传入的文本 - - # 如果是立即播放,直接在新线程中播放 - if immediate: - self.speak_direct(text) - else: - # 添加到语音队列 - self.speech_queue.append(text) - - # 如果没有在播放,启动播放线程 - if not self.is_speaking and self.queue_thread is None: - self.queue_thread = threading.Thread(target=self._process_speech_queue) - self.queue_thread.daemon = True - self.queue_thread.start() - - # 更新时间戳 - self.last_speech_time[message_key] = current_time - - def _process_speech_queue(self): - """处理语音队列""" - while self.speech_queue and self.enabled: - self.is_speaking = True - - text = self.speech_queue.pop(0) - - try: - if self.audio_method == 'pyttsx3': - # 清除之前的语音 - self.engine.stop() - # 播报新语音 - self.engine.say(text) - self.engine.runAndWait() - - elif self.audio_method.startswith('gtts'): - # gTTS需要网络连接 - tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') - - # 保存临时文件 - with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: - temp_file = f.name - tts.save(temp_file) - - # 播放音频 - self.play_audio_file(temp_file) - - # 删除临时文件 - try: - os.unlink(temp_file) - except: - pass # 忽略删除错误 - - except Exception as e: - print(f"⚠ 语音播放失败: {e}") - - time.sleep(0.05) # 减少等待时间,避免卡顿 - - self.is_speaking = False - self.queue_thread = None - - def speak_direct(self, text): - """直接播放文本(不通过消息映射)""" - if not self.enabled: - return - - # 在新线程中播放 - thread = threading.Thread(target=self._speak_thread, args=(text,)) - thread.daemon = True - thread.start() - - def _speak_thread(self, text): - """语音播放线程""" - try: - if self.audio_method == 'pyttsx3': - self.engine.say(text) - self.engine.runAndWait() - - elif self.audio_method.startswith('gtts'): - tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') - - with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: - temp_file = f.name - tts.save(temp_file) - - self.play_audio_file(temp_file) - - try: - os.unlink(temp_file) - except: - pass - - except Exception as e: - print(f"⚠ 直接语音播放失败: {e}") - - def stop(self): - """停止所有语音""" - if hasattr(self, 'engine'): - self.engine.stop() - - self.speech_queue.clear() - self.is_speaking = False - - def set_enabled(self, enabled): - """启用/禁用语音""" - self.enabled = enabled - if not enabled: - self.stop() - - def toggle_enabled(self): - """切换语音启用状态""" - self.enabled = not self.enabled - status = "启用" if self.enabled else "禁用" - self.speak_direct(f"语音反馈已{status}") - return self.enabled - - def get_status(self): - """获取语音状态""" - return { - 'enabled': self.enabled, - 'engine': 'pyttsx3' if self.audio_method == 'pyttsx3' else - 'gTTS' if self.audio_method.startswith('gtts') else - 'None', - 'queue_size': len(self.speech_queue), - 'is_speaking': self.is_speaking, - 'audio_method': self.audio_method - } - - -# ========== 手势轨迹记录器 ========== -class GestureTrajectoryRecorder: - """手势轨迹记录器 - 记录、保存、加载和回放手势轨迹""" - - def __init__(self, speech_manager=None): - self.speech_manager = speech_manager - self.trajectory_data = [] - self.is_recording = False - self.is_playing = False - self.playback_index = 0 - self.playback_paused = False - self.max_trajectory_points = 1000 # 最大轨迹点数 - self.recording_start_time = 0 - self.last_save_time = 0 - self.save_interval = 5 # 自动保存间隔(秒) - - # 轨迹文件路径 - self.trajectory_dir = os.path.join(current_dir, 'trajectories') - if not os.path.exists(self.trajectory_dir): - os.makedirs(self.trajectory_dir) - - # 默认轨迹文件名 - self.default_filename = os.path.join(self.trajectory_dir, - f'trajectory_{datetime.now().strftime("%Y%m%d_%H%M%S")}.pkl') - - # 轨迹可视化设置 - self.trajectory_colors = [ - (255, 0, 0), # 红色 - 起点 - (0, 255, 0), # 绿色 - 正常点 - (0, 0, 255), # 蓝色 - 终点 - (255, 255, 0), # 青色 - 特殊点 - (255, 0, 255) # 紫色 - 特殊点 - ] - - self.show_trajectory = True - self.trajectory_thickness = 2 - self.trajectory_max_length = 100 # 显示的最大轨迹长度 - - print("✓ 手势轨迹记录器已初始化") - - def start_recording(self): - """开始录制手势轨迹""" - if self.is_recording: - return False - - self.trajectory_data = [] - self.is_recording = True - self.recording_start_time = time.time() - self.last_save_time = time.time() - - print("🎬 开始录制手势轨迹") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_start', immediate=True) - - return True - - def stop_recording(self): - """停止录制手势轨迹""" - if not self.is_recording: - return False - - self.is_recording = False - recording_duration = time.time() - self.recording_start_time - - print(f"⏹️ 停止录制手势轨迹") - print(f" 录制时长: {recording_duration:.1f}秒") - print(f" 轨迹点数: {len(self.trajectory_data)}") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_stop', immediate=True) - if len(self.trajectory_data) > 0: - self.speech_manager.speak_direct(f"录制了{len(self.trajectory_data)}个轨迹点") - - return True - - def add_trajectory_point(self, hand_data, gesture, confidence, frame_shape): - """添加轨迹点""" - if not self.is_recording or len(self.trajectory_data) >= self.max_trajectory_points: - return False - - if hand_data is None: - return False - - # 创建轨迹点数据 - trajectory_point = { - 'timestamp': time.time(), - 'hand_position': hand_data['position'] if 'position' in hand_data else (0.5, 0.5), - 'hand_center': hand_data['center'] if 'center' in hand_data else (0, 0), - 'gesture': gesture, - 'confidence': confidence, - 'fingertips': hand_data.get('fingertips', []), - 'frame_shape': frame_shape - } - - self.trajectory_data.append(trajectory_point) - - # 自动保存检查 - current_time = time.time() - if current_time - self.last_save_time >= self.save_interval and len(self.trajectory_data) > 10: - self.auto_save() - self.last_save_time = current_time - - return True - - def auto_save(self): - """自动保存轨迹(临时文件)""" - if len(self.trajectory_data) == 0: - return - - temp_file = os.path.join(self.trajectory_dir, 'trajectory_temp.pkl') - try: - with open(temp_file, 'wb') as f: - pickle.dump(self.trajectory_data, f) - print(f"💾 自动保存轨迹到临时文件 ({len(self.trajectory_data)}个点)") - except Exception as e: - print(f"⚠ 自动保存轨迹失败: {e}") - - def save_trajectory(self, filename=None): - """保存轨迹到文件""" - if len(self.trajectory_data) == 0: - print("⚠ 没有轨迹数据可保存") - return False - - if filename is None: - filename = self.default_filename - - try: - with open(filename, 'wb') as f: - pickle.dump(self.trajectory_data, f) - - print(f"💾 轨迹已保存到: {filename}") - print(f" 轨迹点数: {len(self.trajectory_data)}") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_saved', immediate=True) - self.speech_manager.speak_direct(f"保存了{len(self.trajectory_data)}个轨迹点") - - return True - except Exception as e: - print(f"❌ 保存轨迹失败: {e}") - return False - - def load_trajectory(self, filename): - """从文件加载轨迹""" - try: - if not os.path.exists(filename): - print(f"❌ 轨迹文件不存在: {filename}") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_not_found', immediate=True) - - return False - - with open(filename, 'rb') as f: - self.trajectory_data = pickle.load(f) - - print(f"📂 轨迹已从文件加载: {filename}") - print(f" 轨迹点数: {len(self.trajectory_data)}") + status['Speech'] = False - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_loaded', immediate=True) - self.speech_manager.speak_direct(f"加载了{len(self.trajectory_data)}个轨迹点") - - return True - except Exception as e: - print(f"❌ 加载轨迹失败: {e}") - return False - - def start_playback(self): - """开始回放轨迹""" - if len(self.trajectory_data) == 0: - print("⚠ 没有轨迹数据可回放") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_not_found', immediate=True) - - return False - - self.is_playing = True - self.playback_index = 0 - self.playback_paused = False - - print(f"▶️ 开始回放手势轨迹") - print(f" 总帧数: {len(self.trajectory_data)}") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_playback_start', immediate=True) - - return True - - def stop_playback(self): - """停止回放轨迹""" - if not self.is_playing: - return False - - self.is_playing = False - self.playback_paused = False - - print("⏹️ 停止回放手势轨迹") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_playback_stop', immediate=True) - - return True - - def pause_playback(self): - """暂停/继续回放""" - self.playback_paused = not self.playback_paused - - status = "暂停" if self.playback_paused else "继续" - print(f"⏸️ 回放已{status}") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - if self.playback_paused: - self.speech_manager.speak('recording_paused', immediate=True) - else: - self.speech_manager.speak('recording_resumed', immediate=True) - - return self.playback_paused - - def get_next_playback_point(self): - """获取下一个回放点""" - if not self.is_playing or self.playback_paused or len(self.trajectory_data) == 0: - return None - - if self.playback_index >= len(self.trajectory_data): - self.stop_playback() - return None - - point = self.trajectory_data[self.playback_index] - self.playback_index += 1 - - # 检查是否到达末尾 - if self.playback_index >= len(self.trajectory_data): - self.stop_playback() - - return point - - def clear_trajectory(self): - """清除轨迹数据""" - self.trajectory_data = [] - self.is_recording = False - self.is_playing = False - self.playback_index = 0 - - print("🗑️ 轨迹数据已清除") - - # 语音提示 - if self.speech_manager and self.speech_manager.enabled: - self.speech_manager.speak('recording_cleared', immediate=True) - - return True - - def draw_trajectory(self, frame): - """在帧上绘制轨迹""" - if not self.show_trajectory or len(self.trajectory_data) == 0: - return frame - - h, w = frame.shape[:2] - - # 限制显示的轨迹点数 - display_points = min(len(self.trajectory_data), self.trajectory_max_length) - start_idx = max(0, len(self.trajectory_data) - display_points) - - # 绘制轨迹线 - for i in range(start_idx, len(self.trajectory_data) - 1): - point1 = self.trajectory_data[i] - point2 = self.trajectory_data[i + 1] - - # 获取手部中心位置(转换为图像坐标) - if 'hand_center' in point1 and 'hand_center' in point2: - x1, y1 = point1['hand_center'] - x2, y2 = point2['hand_center'] - - # 确保坐标在图像范围内 - x1 = max(0, min(w - 1, x1)) - y1 = max(0, min(h - 1, y1)) - x2 = max(0, min(w - 1, x2)) - y2 = max(0, min(h - 1, y2)) - - # 根据索引计算颜色(渐变色) - color_idx = int((i - start_idx) / display_points * (len(self.trajectory_colors) - 1)) - color = self.trajectory_colors[color_idx] - - # 绘制线条 - cv2.line(frame, (int(x1), int(y1)), (int(x2), int(y2)), - color, self.trajectory_thickness) - - # 绘制当前点(如果正在录制或回放) - if self.is_recording or self.is_playing: - current_idx = len(self.trajectory_data) - 1 if self.is_recording else self.playback_index - 1 - if 0 <= current_idx < len(self.trajectory_data): - point = self.trajectory_data[current_idx] - if 'hand_center' in point: - x, y = point['hand_center'] - x = max(0, min(w - 1, x)) - y = max(0, min(h - 1, y)) - - # 绘制当前点 - cv2.circle(frame, (int(x), int(y)), 8, (0, 255, 255), -1) - cv2.circle(frame, (int(x), int(y)), 8, (0, 0, 0), 2) - - return frame - - def get_status(self): - """获取录制状态""" - return { - 'is_recording': self.is_recording, - 'is_playing': self.is_playing, - 'playback_paused': self.playback_paused, - 'trajectory_points': len(self.trajectory_data), - 'playback_index': self.playback_index, - 'playback_total': len(self.trajectory_data), - 'recording_duration': time.time() - self.recording_start_time if self.is_recording else 0 - } - - def list_saved_trajectories(self): - """列出保存的轨迹文件""" - try: - files = [f for f in os.listdir(self.trajectory_dir) if f.endswith('.pkl')] - return sorted(files, reverse=True) - except: - return [] - - -# ========== 配置管理器 ========== -class ConfigManager: - """配置管理器""" - - def __init__(self): - self.config_file = os.path.join(current_dir, 'gesture_config.json') - - # 性能模式配置 - self.performance_modes = { - 'fast': { - 'name': '最快', - 'description': '性能优先,降低识别精度换取更高帧率', - 'detection_interval': 2, # 每2帧检测一次 - 'smooth_frames': 3, # 平滑帧数 - 'min_confidence': 0.5, # 最小置信度阈值 - 'resize_factor': 0.5, # 图像缩放因子 - 'skin_detection_enabled': True, - 'background_subtraction_enabled': False, - 'contour_simplify_epsilon': 0.03, - 'history_size': 10, - 'gesture_stability_threshold': 3, - 'color': (0, 255, 0), # 绿色 - }, - 'balanced': { - 'name': '平衡', - 'description': '平衡性能与精度,适用于大多数场景', - 'detection_interval': 1, # 每帧检测 - 'smooth_frames': 5, # 平滑帧数 - 'min_confidence': 0.6, # 最小置信度阈值 - 'resize_factor': 0.75, # 图像缩放因子 - 'skin_detection_enabled': True, - 'background_subtraction_enabled': True, - 'contour_simplify_epsilon': 0.02, - 'history_size': 15, - 'gesture_stability_threshold': 5, - 'color': (255, 165, 0), # 橙色 - }, - 'accurate': { - 'name': '最准', - 'description': '精度优先,提供最准确的手势识别', - 'detection_interval': 1, # 每帧检测 - 'smooth_frames': 7, # 平滑帧数 - 'min_confidence': 0.7, # 最小置信度阈值 - 'resize_factor': 1.0, # 不缩放 - 'skin_detection_enabled': True, - 'background_subtraction_enabled': True, - 'contour_simplify_epsilon': 0.01, - 'history_size': 20, - 'gesture_stability_threshold': 7, - 'color': (255, 0, 0), # 红色 - } - } - - self.default_config = { - 'camera': { - 'index': 0, - 'width': 640, - 'height': 480, - 'fps': 30 - }, - 'gesture': { - 'skin_lower_h': 0, - 'skin_upper_h': 25, - 'skin_lower_s': 30, - 'skin_upper_s': 255, - 'skin_lower_v': 60, - 'skin_upper_v': 255, - 'min_hand_area': 2000, - 'max_hand_area': 30000, - 'hand_ratio_threshold': 1.5, - 'defect_distance_threshold': 20, - 'palm_circle_radius_ratio': 0.3, - 'transition_threshold': 0.3, - 'position_stability_weight': 0.4, - 'gesture_cooldown': 0.5, - }, - 'drone': { - 'velocity': 2.5, - 'duration': 0.3, - 'altitude': -10.0, - 'control_interval': 0.3 - }, - 'display': { - 'show_fps': True, - 'show_confidence': True, - 'show_help': True, - 'show_contours': True, - 'show_bbox': True, - 'show_fingertips': True, - 'show_palm_center': True, - 'show_hand_direction': True, - 'show_debug_info': False, - 'show_speech_status': True, - 'show_gesture_history': True, - 'show_stability_indicator': True, - 'show_trajectory': True, - 'show_recording_status': True, - 'show_performance_mode': True, # 新增:显示性能模式 - 'show_performance_stats': True, # 新增:显示性能统计 - 'show_system_resources': True, # 新增:显示系统资源 - }, - 'performance': { - 'target_fps': 30, - 'resize_factor': 1.0, - 'enable_multiprocessing': False, - 'mode': 'balanced', # 新增:性能模式选择 - 'current_mode_index': 1, # 当前模式索引 - 'modes': ['fast', 'balanced', 'accurate'], # 可用模式列表 - 'auto_report_interval': 60, # 自动报告间隔 - 'enable_performance_monitor': True, # 启用性能监控 - }, - 'calibration': { - 'auto_calibrate_skin': True, - 'skin_calibration_frames': 30, - 'hand_size_calibration': True - }, - 'speech': { - 'enabled': True, - 'volume': 1.0, - 'rate': 150, - 'announce_gestures': True, - 'announce_connections': True, - 'announce_flight_events': True, - 'announce_gesture_changes': True, - 'announce_hand_status': True, - 'announce_performance': True, - 'announce_recording_events': True, - 'announce_performance_mode': True, # 新增:播报性能模式切换 - 'announce_performance_events': True, # 新增:播报性能事件 - 'min_gesture_confidence': 0.7, - 'gesture_start_threshold': 3, - 'gesture_end_threshold': 10, - }, - 'recording': { - 'auto_save_interval': 5, - 'max_trajectory_points': 1000, - 'show_trajectory': True, - 'trajectory_thickness': 2, - 'trajectory_max_length': 100, - 'default_save_dir': 'trajectories', - } - } - self.config = self.load_config() - self.skin_calibration_data = [] - self.hand_size_calibration_done = False - self.reference_hand_size = 0 - - def load_config(self): - """加载配置""" - if os.path.exists(self.config_file): - try: - with open(self.config_file, 'r', encoding='utf-8') as f: - loaded_config = json.load(f) - config = self.default_config.copy() - self._merge_config(config, loaded_config) - print("✓ 从文件加载配置") - return config - except Exception as e: - print(f"⚠ 加载配置失败: {e}, 使用默认配置") - return self.default_config.copy() - else: - print("✓ 使用默认配置") - return self.default_config.copy() - - def _merge_config(self, base, update): - """递归合并配置""" - for key, value in update.items(): - if key in base and isinstance(base[key], dict) and isinstance(value, dict): - self._merge_config(base[key], value) - else: - base[key] = value - - def save_config(self): - """保存配置""" - try: - with open(self.config_file, 'w', encoding='utf-8') as f: - json.dump(self.config, f, indent=2, ensure_ascii=False) - print("✓ 配置已保存") - except Exception as e: - print(f"⚠ 保存配置失败: {e}") - - def get(self, *keys): - """获取配置值""" - value = self.config - for key in keys: - if isinstance(value, dict) and key in value: - value = value[key] - else: - return None - return value - - def set(self, *keys, value): - """设置配置值""" - if len(keys) == 0: - return - - config = self.config - for key in keys[:-1]: - if key not in config: - config[key] = {} - config = config[key] - - config[keys[-1]] = value - self.save_config() - - def get_performance_mode_config(self, mode=None): - """获取性能模式配置""" - if mode is None: - mode = self.get('performance', 'mode') - - if mode in self.performance_modes: - return self.performance_modes[mode] - else: - return self.performance_modes['balanced'] - - def get_current_performance_mode(self): - """获取当前性能模式""" - mode = self.get('performance', 'mode') - if mode in self.performance_modes: - return mode - return 'balanced' - - def set_performance_mode(self, mode): - """设置性能模式""" - if mode in self.performance_modes: - self.set('performance', 'mode', value=mode) - - # 更新当前模式索引 - modes = self.get('performance', 'modes') - if modes and mode in modes: - index = modes.index(mode) - self.set('performance', 'current_mode_index', value=index) - - print(f"✓ 性能模式设置为: {self.performance_modes[mode]['name']}") - return True - return False - - def cycle_performance_mode(self): - """循环切换性能模式""" - modes = self.get('performance', 'modes') - if not modes: - modes = ['fast', 'balanced', 'accurate'] - - current_index = self.get('performance', 'current_mode_index') - if current_index is None: - current_index = 0 - - # 计算下一个模式索引 - next_index = (current_index + 1) % len(modes) - next_mode = modes[next_index] - - # 设置新模式 - self.set('performance', 'current_mode_index', value=next_index) - return self.set_performance_mode(next_mode) - - def calibrate_skin_color(self, frame, hand_mask): - """自动校准肤色范围""" - if not self.get('calibration', 'auto_calibrate_skin'): - return - - if len(self.skin_calibration_data) < self.get('calibration', 'skin_calibration_frames'): - # 转换到HSV - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - - # 获取肤色区域的HSV值 - skin_pixels = hsv[hand_mask > 0] - - if len(skin_pixels) > 100: # 确保有足够的像素 - self.skin_calibration_data.append(skin_pixels) - - if len(self.skin_calibration_data) == self.get('calibration', 'skin_calibration_frames'): - # 计算肤色范围 - all_skin_pixels = np.vstack(self.skin_calibration_data) - - h_min, h_max = np.percentile(all_skin_pixels[:, 0], [2, 98]) - s_min, s_max = np.percentile(all_skin_pixels[:, 1], [2, 98]) - v_min, v_max = np.percentile(all_skin_pixels[:, 2], [2, 98]) - - # 更新配置 - self.set('gesture', 'skin_lower_h', value=int(max(0, h_min - 5))) - self.set('gesture', 'skin_upper_h', value=int(min(180, h_max + 5))) - self.set('gesture', 'skin_lower_s', value=int(max(0, s_min - 10))) - self.set('gesture', 'skin_upper_s', value=int(min(255, s_max + 10))) - self.set('gesture', 'skin_lower_v', value=int(max(0, v_min - 10))) - self.set('gesture', 'skin_upper_v', value=int(min(255, v_max + 10))) - - print("✓ 肤色校准完成") - print(f" 肤色范围: H[{self.get('gesture', 'skin_lower_h')}-{self.get('gesture', 'skin_upper_h')}], " - f"S[{self.get('gesture', 'skin_lower_s')}-{self.get('gesture', 'skin_upper_s')}], " - f"V[{self.get('gesture', 'skin_lower_v')}-{self.get('gesture', 'skin_upper_v')}]") - - def calibrate_hand_size(self, hand_area): - """校准手部大小""" - if not self.get('calibration', 'hand_size_calibration') or self.hand_size_calibration_done: - return - - if hand_area > 0: - self.reference_hand_size = hand_area - self.hand_size_calibration_done = True - print(f"✓ 手部大小校准完成: {self.reference_hand_size:.0f} 像素") - - -config = ConfigManager() - - -# ========== 改进的手势识别器(支持性能模式) ========== -class EnhancedGestureRecognizer: - """增强的手势识别器 - 支持性能模式选择""" - - def __init__(self, speech_manager=None): - self.speech_manager = speech_manager - - # 加载性能模式配置 - self.performance_mode = config.get_current_performance_mode() - self.mode_config = config.get_performance_mode_config(self.performance_mode) - - # 根据性能模式初始化参数 - self.history_size = self.mode_config['history_size'] - self.detection_interval = self.mode_config['detection_interval'] - self.smooth_frames = self.mode_config['smooth_frames'] - self.min_confidence = self.mode_config['min_confidence'] - self.resize_factor = self.mode_config['resize_factor'] - - # 增强的手势历史和平滑 - self.gesture_history = deque(maxlen=self.history_size) - self.confidence_history = deque(maxlen=self.history_size) - self.position_history = deque(maxlen=self.history_size) - self.current_gesture = "Waiting" - self.current_confidence = 0.0 - - # 新增:手势状态追踪 - self.gesture_state = "none" - self.gesture_stability_counter = 0 - self.last_stable_gesture = "Waiting" - self.gesture_active_frames = 0 - self.last_gesture_change_time = 0 - - # 记录上次播报的手势 - self.last_announced_gesture = None - self.last_announced_time = 0 - self.last_hand_status_time = 0 - self.gesture_announce_interval = 2.0 - - # 手部跟踪和状态 - self.last_hand_position = None - self.hand_tracking = False - self.track_window = None - self.hand_states = deque(maxlen=15) - self.hand_detected_frames = 0 - self.hand_lost_frames = 0 - - # 性能统计 - self.process_times = deque(maxlen=30) - self.frame_counter = 0 - self.last_performance_report = 0 - - # 手势颜色映射 - self.gesture_colors = { - "Stop": (0, 0, 255), - "Forward": (0, 255, 0), - "Up": (255, 255, 0), - "Down": (255, 0, 255), - "Left": (255, 165, 0), - "Right": (0, 165, 255), - "Waiting": (200, 200, 200), - "Error": (255, 0, 0), - "Hover": (255, 255, 255) - } - - # 手势到语音的映射 - self.gesture_speech_map = { - "Stop": "gesture_stop", - "Forward": "gesture_forward", - "Up": "gesture_up", - "Down": "gesture_down", - "Left": "gesture_left", - "Right": "gesture_right", - "Waiting": "gesture_waiting", - "Error": "gesture_error", - "Hover": "gesture_hover", - } - - # 手势状态颜色 - self.state_colors = { - "none": (100, 100, 100), - "starting": (255, 165, 0), - "active": (0, 255, 0), - "ending": (255, 0, 0), - } - - # 根据性能模式初始化背景减除器 - self.bg_subtractor = None - if self.mode_config['background_subtraction_enabled']: - self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( - history=100, varThreshold=25, detectShadows=True - ) - - # 形态学操作核 - self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - - # 性能监控 - self.avg_process_time = 0 - self.frame_rate = 0 - self.last_fps_check = time.time() - - # 存储手部数据用于轨迹记录 - self.last_hand_data = None - - # 性能模式信息 - self.performance_mode_color = self.mode_config['color'] - self.performance_mode_name = self.mode_config['name'] - - print(f"✓ 增强的手势识别器已初始化 - 性能模式: {self.performance_mode_name}") - - def set_performance_mode(self, mode): - """设置性能模式""" - self.performance_mode = mode - self.mode_config = config.get_performance_mode_config(mode) - - # 更新参数 - self.history_size = self.mode_config['history_size'] - self.detection_interval = self.mode_config['detection_interval'] - self.smooth_frames = self.mode_config['smooth_frames'] - self.min_confidence = self.mode_config['min_confidence'] - self.resize_factor = self.mode_config['resize_factor'] - - # 更新背景减除器 - if self.mode_config['background_subtraction_enabled'] and self.bg_subtractor is None: - self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( - history=100, varThreshold=25, detectShadows=True - ) - elif not self.mode_config['background_subtraction_enabled']: - self.bg_subtractor = None - - # 更新队列大小 - self.gesture_history = deque(maxlen=self.history_size) - self.confidence_history = deque(maxlen=self.history_size) - self.position_history = deque(maxlen=self.history_size) - - # 更新显示信息 - self.performance_mode_color = self.mode_config['color'] - self.performance_mode_name = self.mode_config['name'] - - print(f"✓ 切换到性能模式: {self.performance_mode_name}") - - def get_skin_mask(self, frame): - """获取肤色掩码""" - h_low = config.get('gesture', 'skin_lower_h') - h_high = config.get('gesture', 'skin_upper_h') - s_low = config.get('gesture', 'skin_lower_s') - s_high = config.get('gesture', 'skin_upper_s') - v_low = config.get('gesture', 'skin_lower_v') - v_high = config.get('gesture', 'skin_upper_v') - - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - lower_skin = np.array([h_low, s_low, v_low], dtype=np.uint8) - upper_skin = np.array([h_high, s_high, v_high], dtype=np.uint8) - skin_mask = cv2.inRange(hsv, lower_skin, upper_skin) - - return skin_mask, hsv - - def enhance_skin_detection(self, frame, skin_mask): - """增强肤色检测""" - if not self.mode_config['skin_detection_enabled']: - return skin_mask - - if self.bg_subtractor is not None: - fg_mask = self.bg_subtractor.apply(frame) - combined_mask = cv2.bitwise_and(skin_mask, fg_mask) - else: - combined_mask = skin_mask - - # 根据性能模式决定形态学操作次数 - if self.performance_mode == 'accurate': - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) - elif self.performance_mode == 'balanced': - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=1) - combined_mask = cv2.GaussianBlur(combined_mask, (3, 3), 0) - # fast模式不进行额外的形态学操作 - - return combined_mask - - def preprocess_frame(self, frame): - """预处理帧(根据性能模式调整大小)""" - if self.resize_factor != 1.0: - new_width = int(frame.shape[1] * self.resize_factor) - new_height = int(frame.shape[0] * self.resize_factor) - resized_frame = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_AREA) - return resized_frame - return frame - - def find_best_hand_contour(self, mask, frame): - """找到最佳的手部轮廓""" - contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - - if not contours: - return None, 0.0 - - contours = sorted(contours, key=cv2.contourArea, reverse=True) - best_contour = None - best_score = 0.0 - min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) - max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) - - # 根据性能模式调整检查的轮廓数量 - max_contours = 3 if self.performance_mode != 'fast' else 1 - - for contour in contours[:max_contours]: - area = cv2.contourArea(contour) - - if area < min_area or area > max_area: - continue - - score = self.rate_contour(contour, frame.shape) - - if score > best_score: - best_score = score - best_contour = contour - - return best_contour, best_score - - def rate_contour(self, contour, frame_shape): - """评估轮廓作为手部的可能性""" - score = 0.0 - area = cv2.contourArea(contour) - min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) - max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) - - if min_area < area < max_area: - area_ratio = min(area / max_area, 1.0) - score += area_ratio * 0.3 - - perimeter = cv2.arcLength(contour, True) - if perimeter > 100: - score += 0.2 - - if area > 0: - compactness = perimeter ** 2 / area - if 12 < compactness < 25: - compactness_score = 1.0 - abs(compactness - 18) / 6 - score += compactness_score * 0.3 - - x, y, w, h = cv2.boundingRect(contour) - if h > 0: - aspect_ratio = w / h - if 0.4 < aspect_ratio < 2.5: - aspect_score = 1.0 - abs(aspect_ratio - 1.0) / 1.5 - score += aspect_score * 0.2 - - return score - - def analyze_hand_features(self, contour, frame_shape): - """分析手部特征""" - if contour is None: - return None, 0.0 - - area = cv2.contourArea(contour) - M = cv2.moments(contour) - if M["m00"] == 0: - return None, 0.0 - - cx = int(M["m10"] / M["m00"]) - cy = int(M["m01"] / M["m00"]) - x, y, w, h = cv2.boundingRect(contour) - bbox_area = w * h - palm_center = (cx, cy) - palm_radius = int(w * config.get('gesture', 'palm_circle_radius_ratio')) - - # 根据性能模式调整轮廓简化程度 - epsilon = self.mode_config['contour_simplify_epsilon'] * cv2.arcLength(contour, True) - fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius, epsilon) - - direction = self.calculate_hand_direction(contour, cx, cy) - h_img, w_img = frame_shape[:2] - norm_x = cx / w_img - norm_y = cy / h_img - confidence = self.calculate_confidence(area, len(fingers), len(contour), bbox_area) - - result = { - 'contour': contour, - 'center': (cx, cy), - 'bbox': (x, y, x + w, y + h), - 'fingers': fingers, - 'fingertips': fingertips, - 'defects': defects, - 'palm_center': palm_center, - 'palm_radius': palm_radius, - 'direction': direction, - 'area': area, - 'position': (norm_x, norm_y), - 'bbox_size': (w, h), - 'confidence': confidence - } - - return result, confidence - - def analyze_fingers(self, contour, palm_center, palm_radius, epsilon): - """分析手指""" - approx = cv2.approxPolyDP(contour, epsilon, True) - hull = cv2.convexHull(approx, returnPoints=False) - - if hull is None or len(hull) < 3: - return [], [], [] - - defects = cv2.convexityDefects(approx, hull) - fingers = [] - fingertips = [] - defect_points = [] - - if defects is not None: - for i in range(defects.shape[0]): - s, e, f, d = defects[i, 0] - start = tuple(approx[s][0]) - end = tuple(approx[e][0]) - far = tuple(approx[f][0]) - - start_dist = np.linalg.norm(np.array(start) - np.array(palm_center)) - end_dist = np.linalg.norm(np.array(end) - np.array(palm_center)) - far_dist = np.linalg.norm(np.array(far) - np.array(palm_center)) - - if (start_dist > palm_radius * 1.2 and - end_dist > palm_radius * 1.2 and - d > config.get('gesture', 'defect_distance_threshold') * 256): - - a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) - b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) - c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) - - if b * c != 0: - angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) - angle_degrees = math.degrees(angle) - - if angle_degrees < 90: - finger = { - 'start': start, - 'end': end, - 'far': far, - 'depth': d, - 'angle': angle_degrees - } - fingers.append(finger) - - for point in [start, end]: - if point not in fingertips: - point_dist = np.linalg.norm(np.array(point) - np.array(palm_center)) - if point_dist > palm_radius * 1.5: - fingertips.append(point) - - defect_points.append((start, end, far, d)) - - if len(fingertips) == 0: - hull_points = cv2.convexHull(approx, returnPoints=True) - if len(hull_points) > 0: - hull_points = hull_points.reshape(-1, 2) - - for point in hull_points: - point_tuple = tuple(point) - point_dist = np.linalg.norm(point - np.array(palm_center)) - if point_dist > palm_radius * 1.5 and point_tuple not in fingertips: - fingertips.append(point_tuple) - - fingertips = fingertips[:5] - - return fingers, fingertips, defect_points - - def calculate_hand_direction(self, contour, cx, cy): - """计算手部方向""" - if len(contour) < 5: - return 0.0 - - points = contour.reshape(-1, 2).astype(np.float32) - mean = np.empty((0)) - mean, eigenvectors, eigenvalues = cv2.PCACompute2(points, mean) - direction = math.degrees(math.atan2(eigenvectors[0, 1], eigenvectors[0, 0])) - - return direction - - def calculate_confidence(self, area, finger_count, contour_length, bbox_area): - """计算手势置信度""" - confidence = 0.5 - min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) - max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) - - if min_area < area < max_area: - area_norm = (area - min_area) / (max_area - min_area) - confidence += area_norm * 0.2 - - if 0 <= finger_count <= 5: - confidence += 0.2 - - if contour_length > 200: - confidence += 0.1 - - if bbox_area > 0: - fill_ratio = area / bbox_area - if 0.2 < fill_ratio < 0.8: - fill_score = 1.0 - abs(fill_ratio - 0.5) / 0.3 - confidence += fill_score * 0.1 - - return min(confidence, 1.0) - - def recognize_gesture_improved(self, hand_data): - """改进的手势识别逻辑""" - if hand_data is None: - return "Waiting", 0.3 - - finger_count = len(hand_data.get('fingers', [])) - fingertips = hand_data.get('fingertips', []) - norm_x, norm_y = hand_data['position'] - direction = hand_data.get('direction', 0.0) - confidence = hand_data['confidence'] - w, h = hand_data['bbox_size'] - aspect_ratio = w / h if h > 0 else 1.0 - - # 根据手指数量分类 - if finger_count == 0: - if len(fingertips) == 0: - return "Stop", confidence * 0.9 - else: - return "Stop", confidence * 0.7 - - elif finger_count == 1: - if len(fingertips) >= 1: - cx, cy = hand_data['center'] - fingertip = fingertips[0] - dx = fingertip[0] - cx - dy = fingertip[1] - cy - - if abs(dx) > abs(dy): - if dx > 0: - return "Right", confidence * 0.8 - else: - return "Left", confidence * 0.8 - else: - if dy < 0: - return "Up", confidence * 0.8 - else: - return "Forward", confidence * 0.8 - return "Forward", confidence * 0.7 - - elif finger_count == 2: - return "Forward", confidence * 0.7 - - elif finger_count == 3: - if -45 <= direction <= 45: - if direction > 0: - return "Right", confidence * 0.7 - else: - return "Left", confidence * 0.7 - else: - if direction > 0: - return "Down", confidence * 0.7 - else: - return "Up", confidence * 0.7 - - elif finger_count >= 4: - if norm_x < 0.4: - return "Left", confidence * 0.8 - elif norm_x > 0.6: - return "Right", confidence * 0.8 - elif norm_y < 0.4: - return "Up", confidence * 0.8 - elif norm_y > 0.6: - return "Down", confidence * 0.8 - else: - if -45 <= direction <= 45: - return "Forward", confidence * 0.7 - else: - return "Stop", confidence * 0.7 - - return "Waiting", confidence * 0.5 - - def smooth_gesture_enhanced(self, new_gesture, new_confidence, hand_data): - """增强的手势平滑处理""" - current_time = time.time() - - # 检查手势冷却时间 - if current_time - self.last_gesture_change_time < config.get('gesture', 'gesture_cooldown'): - return self.current_gesture, self.current_confidence - - # 添加到历史 - self.gesture_history.append(new_gesture) - self.confidence_history.append(new_confidence) - - if hand_data is not None: - self.position_history.append(hand_data['position']) - - # 计算手势稳定性 - if len(self.gesture_history) >= 3: - # 检查最近N个手势是否一致 - recent_gestures = list(self.gesture_history)[-3:] - gesture_counter = Counter(recent_gestures) - most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] - - # 计算位置稳定性 - position_stability = 1.0 - if len(self.position_history) >= 2 and hand_data is not None: - current_pos = hand_data['position'] - prev_pos = self.position_history[-2] if len(self.position_history) >= 2 else current_pos - position_diff = math.sqrt((current_pos[0] - prev_pos[0]) ** 2 + (current_pos[1] - prev_pos[1]) ** 2) - position_stability = max(0, 1.0 - position_diff * 5.0) - - # 增强的稳定性检查 - stability_threshold = self.mode_config['gesture_stability_threshold'] - transition_threshold = config.get('gesture', 'transition_threshold') - position_weight = config.get('gesture', 'position_stability_weight') - - # 计算综合稳定性得分 - gesture_stability = most_common_count / 3.0 - overall_stability = gesture_stability * (1.0 - position_weight) + position_stability * position_weight - - # 手势状态转换逻辑 - if overall_stability >= transition_threshold: - if most_common_gesture != self.last_stable_gesture: - self.gesture_stability_counter += 1 - else: - self.gesture_stability_counter = max(0, self.gesture_stability_counter - 1) - - if self.gesture_stability_counter >= stability_threshold: - # 手势稳定,更新当前手势 - self.current_gesture = most_common_gesture - self.current_confidence = np.mean(list(self.confidence_history)[-3:]) - self.last_stable_gesture = most_common_gesture - self.gesture_stability_counter = 0 - self.last_gesture_change_time = current_time - else: - # 手势不稳定,重置计数器 - self.gesture_stability_counter = max(0, self.gesture_stability_counter - 2) - - return self.current_gesture, self.current_confidence - - def update_gesture_state(self, hand_data, gesture, confidence): - """更新手势状态""" - current_time = time.time() - - if hand_data is None: - # 手部丢失 - self.hand_lost_frames += 1 - self.hand_detected_frames = max(0, self.hand_detected_frames - 1) - - if self.hand_lost_frames > 10 and self.gesture_state != "none": - self.gesture_state = "none" - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_hand_status') and - current_time - self.last_hand_status_time > 3.0): - self.speech_manager.speak('hand_lost', immediate=True) - self.last_hand_status_time = current_time - return - - # 手部检测到 - self.hand_detected_frames += 1 - self.hand_lost_frames = 0 - - # 检查手部大小和位置 - hand_area = hand_data['area'] - min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) - max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) - - # 提供手部位置反馈 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_hand_status') and - current_time - self.last_hand_status_time > 5.0): - - if hand_area < min_area * 0.8: - self.speech_manager.speak('move_closer', immediate=True) - self.last_hand_status_time = current_time - elif hand_area > max_area * 1.2: - self.speech_manager.speak('move_away', immediate=True) - self.last_hand_status_time = current_time - elif self.hand_detected_frames == 5: # 首次稳定检测 - self.speech_manager.speak('hand_detected', immediate=True) - self.last_hand_status_time = current_time - - # 手势状态转换 - if self.gesture_state == "none" and gesture != "Waiting" and confidence > 0.6: - # 手势开始 - self.gesture_state = "starting" - self.gesture_active_frames = 0 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gesture_changes')): - self.speech_manager.speak('gesture_start', immediate=True) - - elif self.gesture_state == "starting": - self.gesture_active_frames += 1 - if self.gesture_active_frames >= config.get('speech', 'gesture_start_threshold'): - self.gesture_state = "active" - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gesture_changes')): - self.speech_manager.speak('gesture_stable', immediate=True) - - elif self.gesture_state == "active": - if gesture == "Waiting" or confidence < 0.5: - self.gesture_active_frames = max(0, self.gesture_active_frames - 2) - if self.gesture_active_frames <= 0: - self.gesture_state = "ending" - else: - self.gesture_active_frames = min(20, self.gesture_active_frames + 1) - - elif self.gesture_state == "ending": - self.gesture_active_frames -= 1 - if self.gesture_active_frames <= 0: - self.gesture_state = "none" - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gesture_changes')): - self.speech_manager.speak('gesture_end', immediate=True) - - def visualize_detection(self, frame, hand_data, gesture, confidence): - """可视化检测结果""" - if hand_data is None: - return frame - - show_contours = config.get('display', 'show_contours') - show_bbox = config.get('display', 'show_bbox') - show_fingertips = config.get('display', 'show_fingertips') - show_palm_center = config.get('display', 'show_palm_center') - show_hand_direction = config.get('display', 'show_hand_direction') - show_debug_info = config.get('display', 'show_debug_info') - show_gesture_history = config.get('display', 'show_gesture_history') - show_stability_indicator = config.get('display', 'show_stability_indicator') - - # 绘制轮廓 - if show_contours and 'contour' in hand_data: - cv2.drawContours(frame, [hand_data['contour']], -1, (0, 255, 0), 2) - - # 绘制边界框 - if show_bbox and 'bbox' in hand_data: - x1, y1, x2, y2 = hand_data['bbox'] - color = self.gesture_colors.get(gesture, (255, 255, 255)) - - # 根据手势状态调整边界框颜色 - state_color = self.state_colors.get(self.gesture_state, color) - if self.gesture_state != "none": - color = state_color - - cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) - - # 显示手势标签 - label = f"{gesture}" - if config.get('display', 'show_confidence'): - label += f" ({confidence:.0%})" - - # 显示手势状态 - if self.gesture_state != "none": - state_text = {"starting": "开始", "active": "活跃", "ending": "结束"} - label += f" [{state_text.get(self.gesture_state, '')}]" - - # 计算文本大小 - (text_width, text_height), baseline = cv2.getTextSize( - label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 - ) - - # 绘制文本背景 - cv2.rectangle(frame, - (x1, y1 - text_height - 10), - (x1 + text_width, y1), - color, -1) - - # 绘制文本 - cv2.putText(frame, label, (x1, y1 - 5), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) - - # 绘制手掌中心 - if show_palm_center and 'palm_center' in hand_data: - cx, cy = hand_data['palm_center'] - palm_radius = hand_data.get('palm_radius', 20) - cv2.circle(frame, (cx, cy), 5, (0, 0, 255), -1) - cv2.circle(frame, (cx, cy), palm_radius, (0, 0, 255), 1) - cv2.putText(frame, "Palm", (cx + 10, cy), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) - - # 绘制指尖 - if show_fingertips and 'fingertips' in hand_data: - for i, point in enumerate(hand_data['fingertips']): - cv2.circle(frame, point, 4, (255, 0, 0), -1) - cv2.putText(frame, f"F{i + 1}", (point[0] + 5, point[1]), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1) - - # 绘制手部方向 - if show_hand_direction and 'direction' in hand_data and 'center' in hand_data: - cx, cy = hand_data['center'] - direction = hand_data['direction'] - length = 50 - - dx = length * math.cos(math.radians(direction)) - dy = length * math.sin(math.radians(direction)) - - end_point = (int(cx + dx), int(cy + dy)) - cv2.arrowedLine(frame, (cx, cy), end_point, (255, 255, 0), 2) - - angle_text = f"Dir: {direction:.0f}°" - cv2.putText(frame, angle_text, (cx, cy - 20), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 1) - - # 绘制稳定性指示器 - if show_stability_indicator: - h, w = frame.shape[:2] - indicator_x = w - 100 - indicator_y = 30 - - # 绘制稳定性背景 - cv2.rectangle(frame, (indicator_x, indicator_y), - (indicator_x + 80, indicator_y + 15), (50, 50, 50), -1) - - # 计算稳定性指示条长度 - stability_level = min(1.0, - self.gesture_stability_counter / self.mode_config['gesture_stability_threshold']) - bar_length = int(70 * stability_level) - - # 根据稳定性级别选择颜色 - if stability_level > 0.7: - bar_color = (0, 255, 0) # 绿色 - elif stability_level > 0.4: - bar_color = (255, 165, 0) # 橙色 - else: - bar_color = (255, 0, 0) # 红色 - - # 绘制稳定性指示条 - cv2.rectangle(frame, (indicator_x + 5, indicator_y + 5), - (indicator_x + 5 + bar_length, indicator_y + 10), bar_color, -1) - - # 绘制稳定性文本 - cv2.putText(frame, "稳定度", (indicator_x, indicator_y - 5), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - # 绘制手势历史(如果启用) - if show_gesture_history and len(self.gesture_history) > 0: - h, w = frame.shape[:2] - history_y = h - 50 - - # 绘制历史背景 - cv2.rectangle(frame, (10, history_y - 20), (200, history_y + 10), (0, 0, 0), -1) - cv2.putText(frame, "手势历史:", (15, history_y), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - # 显示最近几个手势 - recent_gestures = list(self.gesture_history)[-5:] if len(self.gesture_history) >= 5 else list( - self.gesture_history) - for i, gest in enumerate(recent_gestures): - color = self.gesture_colors.get(gest, (255, 255, 255)) - cv2.putText(frame, gest[0], (85 + i * 20, history_y), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) - - # 显示调试信息 - if show_debug_info: - # 显示手指数量 - finger_count = len(hand_data.get('fingers', [])) - finger_text = f"Fingers: {finger_count}" - cv2.putText(frame, finger_text, (10, frame.shape[0] - 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - # 显示手部位置 - pos_text = f"Pos: ({hand_data['position'][0]:.2f}, {hand_data['position'][1]:.2f})" - cv2.putText(frame, pos_text, (10, frame.shape[0] - 10), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - # 显示手势状态 - state_text = f"State: {self.gesture_state}" - cv2.putText(frame, state_text, (150, frame.shape[0] - 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - # 显示稳定性计数器 - stability_text = f"Stability: {self.gesture_stability_counter}" - cv2.putText(frame, stability_text, (150, frame.shape[0] - 10), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) - - return frame - - def recognize(self, frame): - """识别手势""" - start_time = time.time() - - try: - # 预处理帧 - processed_frame = cv2.flip(frame, 1) - - # 根据性能模式调整图像大小 - original_frame = processed_frame.copy() - if self.resize_factor != 1.0: - processed_frame = self.preprocess_frame(processed_frame) - - # 每隔几帧检测一次以提高性能 - if self.frame_counter % self.detection_interval != 0: - self.frame_counter += 1 - return self.current_gesture, self.current_confidence, original_frame - - # 获取肤色掩码 - skin_mask, hsv = self.get_skin_mask(processed_frame) - - # 增强肤色检测 - enhanced_mask = self.enhance_skin_detection(processed_frame, skin_mask) - - # 找到最佳的手部轮廓 - hand_contour, contour_score = self.find_best_hand_contour(enhanced_mask, processed_frame) - - # 分析手部特征 - hand_data, confidence = self.analyze_hand_features(hand_contour, processed_frame.shape) - - # 保存手部数据用于轨迹记录 - self.last_hand_data = hand_data - - # 识别手势 - if hand_data is not None: - # 校准肤色(如果需要) - config.calibrate_skin_color(processed_frame, enhanced_mask) - - # 校准手部大小(如果需要) - config.calibrate_hand_size(hand_data['area']) - - # 识别手势 - gesture, raw_confidence = self.recognize_gesture_improved(hand_data) - confidence = max(confidence, raw_confidence) - - # 更新手势状态 - self.update_gesture_state(hand_data, gesture, confidence) - - # 增强的手势平滑 - final_gesture, final_confidence = self.smooth_gesture_enhanced(gesture, confidence, hand_data) - else: - gesture, confidence = "Waiting", 0.3 - self.update_gesture_state(None, gesture, confidence) - final_gesture, final_confidence = gesture, confidence - - # 手势语音播报 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gestures')): - - current_time = time.time() - - # 根据置信度提供反馈 - if confidence >= 0.8 and current_time - self.last_hand_status_time > 5.0: - self.speech_manager.speak('gesture_good_confidence', immediate=True) - self.last_hand_status_time = current_time - elif confidence < 0.5 and current_time - self.last_hand_status_time > 5.0: - self.speech_manager.speak('gesture_low_confidence', immediate=True) - self.last_hand_status_time = current_time - - # 手势语音播报 - if (final_gesture != "Waiting" and - final_gesture != "Error" and - final_gesture != "摄像头错误" and - final_confidence >= self.min_confidence and - current_time - self.last_announced_time > self.gesture_announce_interval): - - if final_gesture in self.gesture_speech_map: - speech_key = self.gesture_speech_map[final_gesture] - self.speech_manager.speak(speech_key) - else: - self.speech_manager.speak_direct(f"手势{final_gesture}") - - self.last_announced_gesture = final_gesture - self.last_announced_time = current_time - - # 性能报告 - current_time = time.time() - if current_time - self.last_performance_report > 30.0: - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_performance')): - - if self.avg_process_time < 20: - self.speech_manager.speak('performance_good', immediate=True) - elif self.avg_process_time > 50: - self.speech_manager.speak('performance_warning', immediate=True) - - self.last_performance_report = current_time - - # 可视化结果(在原图上) - if hand_data is not None: - # 需要将坐标转换回原始图像大小 - if self.resize_factor != 1.0: - # 缩放坐标 - scale_factor = 1.0 / self.resize_factor - if 'center' in hand_data: - hand_data['center'] = (int(hand_data['center'][0] * scale_factor), - int(hand_data['center'][1] * scale_factor)) - if 'bbox' in hand_data: - x1, y1, x2, y2 = hand_data['bbox'] - hand_data['bbox'] = (int(x1 * scale_factor), int(y1 * scale_factor), - int(x2 * scale_factor), int(y2 * scale_factor)) - if 'fingertips' in hand_data: - hand_data['fingertips'] = [(int(x * scale_factor), int(y * scale_factor)) - for (x, y) in hand_data['fingertips']] - if 'palm_center' in hand_data: - hand_data['palm_center'] = (int(hand_data['palm_center'][0] * scale_factor), - int(hand_data['palm_center'][1] * scale_factor)) - - original_frame = self.visualize_detection( - original_frame, hand_data, final_gesture, final_confidence - ) - else: - original_frame = self.visualize_detection( - original_frame, None, final_gesture, final_confidence - ) - - # 更新计数器 - self.frame_counter += 1 - - # 计算处理时间 - process_time = (time.time() - start_time) * 1000 - self.process_times.append(process_time) - - # 更新平均处理时间 - if len(self.process_times) > 0: - self.avg_process_time = np.mean(list(self.process_times)) - - # 更新帧率 - current_time = time.time() - if current_time - self.last_fps_check >= 1.0: - self.frame_rate = self.frame_counter - self.frame_counter = 0 - self.last_fps_check = current_time - - return final_gesture, final_confidence, original_frame - - except Exception as e: - print(f"⚠ 手势识别错误: {e}") - return "Error", 0.0, frame - - def get_performance_stats(self): - """获取性能统计""" - if len(self.process_times) == 0: - return 0.0, self.frame_rate - - return np.mean(list(self.process_times)), self.frame_rate - - def get_performance_mode_info(self): - """获取性能模式信息""" - return { - 'name': self.performance_mode_name, - 'mode': self.performance_mode, - 'color': self.performance_mode_color, - 'detection_interval': self.detection_interval, - 'resize_factor': self.resize_factor, - 'smooth_frames': self.smooth_frames, - 'min_confidence': self.min_confidence - } - - def set_simulated_gesture(self, gesture): - """设置模拟的手势""" - self.current_gesture = gesture - self.current_confidence = 0.9 - - # 模拟手势也触发语音提示 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gestures')): - - if gesture in self.gesture_speech_map: - self.speech_manager.speak(self.gesture_speech_map[gesture]) - else: - self.speech_manager.speak_direct(f"手势{gesture}") - - self.last_announced_gesture = gesture - self.last_announced_time = time.time() - - -# ========== 简单的无人机控制器 ========== -class SimpleDroneController: - """简单的无人机控制器""" - - def __init__(self, airsim_module, speech_manager=None): - self.airsim = airsim_module - self.client = None - self.connected = False - self.flying = False - self.speech_manager = speech_manager - - # 控制参数 - self.velocity = config.get('drone', 'velocity') - self.duration = config.get('drone', 'duration') - self.altitude = config.get('drone', 'altitude') - self.control_interval = config.get('drone', 'control_interval') - - # 控制状态 - self.last_control_time = 0 - self.last_gesture = None - - # 上次语音提示状态 - self.last_connection_announced = False - self.last_takeoff_announced = False - self.last_land_announced = False - - print("✓ 简单的无人机控制器已初始化") - - def connect(self): - """连接AirSim无人机""" - if self.connected: - return True - - # 语音提示:正在连接 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_connections')): - self.speech_manager.speak('connecting') - - if self.airsim is None: - print("⚠ AirSim不可用,使用模拟模式") - - # 语音提示:模拟模式 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_connections')): - self.speech_manager.speak('simulation_mode') - - self.connected = True - return True - - print("连接AirSim...") - - try: - self.client = self.airsim.MultirotorClient() - self.client.confirmConnection() - print("✅ 已连接AirSim!") - - # 语音提示:连接成功 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_connections')): - self.speech_manager.speak('connected') - - self.client.enableApiControl(True) - print("✅ API控制已启用") - - self.client.armDisarm(True) - print("✅ 无人机已武装") - - self.connected = True - return True - - except Exception as e: - print(f"❌ 连接失败: {e}") - - # 语音提示:连接失败 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_connections')): - self.speech_manager.speak('connection_failed') - - print("\n使用模拟模式继续? (y/n)") - choice = input().strip().lower() - if choice == 'y': - self.connected = True - print("✅ 使用模拟模式") - - # 语音提示:模拟模式 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_connections')): - self.speech_manager.speak('simulation_mode') - - return True - - return False - - def takeoff(self): - """起飞""" - if not self.connected: - return False - - # 语音提示:正在起飞 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events') and - not self.last_takeoff_announced): - self.speech_manager.speak('taking_off') - self.last_takeoff_announced = True - self.last_land_announced = False - - try: - if self.airsim is None or self.client is None: - print("✅ 模拟起飞") - self.flying = True - - # 语音提示:起飞成功 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('takeoff_success') - - return True - - print("起飞中...") - self.client.takeoffAsync().join() - time.sleep(1) - - # 上升到指定高度 - self.client.moveToZAsync(self.altitude, 3).join() - - self.flying = True - print("✅ 无人机成功起飞") - - # 语音提示:起飞成功 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('takeoff_success') - - return True - except Exception as e: - print(f"❌ 起飞失败: {e}") - - # 语音提示:起飞失败 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('takeoff_failed') - - return False - - def land(self): - """降落""" - if not self.connected: - return False - - # 语音提示:正在降落 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events') and - not self.last_land_announced): - self.speech_manager.speak('landing') - self.last_land_announced = True - self.last_takeoff_announced = False - - try: - if self.airsim is None or self.client is None: - print("✅ 模拟降落") - self.flying = False - - # 语音提示:降落成功 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('land_success') - - return True - - print("降落中...") - self.client.landAsync().join() - self.flying = False - print("✅ 无人机已降落") - - # 语音提示:降落成功 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('land_success') - - return True - except Exception as e: - print(f"降落失败: {e}") - return False - - def move_by_gesture(self, gesture, confidence): - """根据手势移动""" - if not self.connected or not self.flying: - return False - - # 检查控制间隔 - current_time = time.time() - if current_time - self.last_control_time < self.control_interval: - return False - - # 检查置信度阈值 - min_confidence = config.get('gesture', 'min_confidence') - if confidence < min_confidence: - # 低置信度语音提示 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_gestures') and - confidence < min_confidence * 0.8): - self.speech_manager.speak('gesture_low_confidence') - return False - - try: - if self.airsim is None or self.client is None: - print(f"模拟移动: {gesture}") - self.last_control_time = current_time - self.last_gesture = gesture - return True - - success = False - - if gesture == "Up": - self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) - success = True - elif gesture == "Down": - self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) - success = True - elif gesture == "Left": - self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) - success = True - elif gesture == "Right": - self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) - success = True - elif gesture == "Forward": - self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) - success = True - elif gesture == "Stop": - self.client.hoverAsync() - success = True - # 悬停语音提示 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('hovering') - elif gesture == "Hover": - self.client.hoverAsync() - success = True - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('hovering') - - if success: - self.last_control_time = current_time - self.last_gesture = gesture - - return success - except Exception as e: - print(f"控制命令失败: {e}") - return False - - def emergency_stop(self): - """紧急停止""" - if self.connected: - try: - if self.flying and self.client is not None: - print("紧急降落...") - - # 语音提示:紧急停止 - if (self.speech_manager and - config.get('speech', 'enabled') and - config.get('speech', 'announce_flight_events')): - self.speech_manager.speak('emergency_stop') - - self.land() - if self.client is not None: - self.client.armDisarm(False) - self.client.enableApiControl(False) - print("✅ 紧急停止完成") - except: - pass - - self.connected = False - self.flying = False - - -# ========== 中文UI渲染器(增强版) ========== -class ChineseUIRenderer: - """中文UI渲染器""" - - def __init__(self, speech_manager=None): - self.fonts = {} - self.speech_manager = speech_manager - self.load_fonts() - - # 颜色定义 - self.colors = { - 'title': (0, 255, 255), - 'connected': (0, 255, 0), - 'disconnected': (0, 0, 255), - 'flying': (0, 255, 0), - 'landed': (255, 165, 0), - 'warning': (0, 165, 255), - 'info': (255, 255, 255), - 'help': (255, 200, 100), - 'speech_enabled': (0, 255, 0), - 'speech_disabled': (255, 0, 0), - 'performance_good': (0, 255, 0), - 'performance_warning': (255, 165, 0), - 'performance_bad': (255, 0, 0), - 'recording': (255, 50, 50), - 'playback': (50, 50, 255), - 'paused': (255, 255, 0), - 'performance_fast': (0, 255, 0), # 绿色 - 'performance_balanced': (255, 165, 0), # 橙色 - 'performance_accurate': (255, 0, 0), # 红色 - 'cpu_good': (0, 255, 0), # CPU使用率颜色 - 'cpu_warning': (255, 165, 0), - 'cpu_critical': (255, 0, 0), - 'memory_good': (0, 255, 0), # 内存使用率颜色 - 'memory_warning': (255, 165, 0), - 'memory_critical': (255, 0, 0), - } - - print("✓ 中文UI渲染器已初始化") - - def load_fonts(self): - """加载字体""" - font_paths = [ - 'simhei.ttf', - 'C:/Windows/Fonts/simhei.ttf', - 'C:/Windows/Fonts/msyh.ttc', - '/System/Library/Fonts/PingFang.ttc', - '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' - ] - - for path in font_paths: - try: - self.fonts[14] = ImageFont.truetype(path, 14) - self.fonts[16] = ImageFont.truetype(path, 16) - self.fonts[18] = ImageFont.truetype(path, 18) - self.fonts[20] = ImageFont.truetype(path, 20) - self.fonts[24] = ImageFont.truetype(path, 24) - print(f"✓ 字体已加载: {path}") - return - except: - continue - - print("⚠ 未找到字体,使用默认") - - def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): - """在图像上绘制文本""" - try: - img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - pil_img = Image.fromarray(img_rgb) - draw = ImageDraw.Draw(pil_img) - - font = self.fonts.get(size, self.fonts.get(16)) - - # 绘制阴影 - shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 1, pos[1] + 1) - draw.text(shadow_pos, text, font=font, fill=shadow_color) - - # 绘制文字 - rgb_color = color[::-1] # BGR to RGB - draw.text(pos, text, font=font, fill=rgb_color) - - return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) - except: - # 备用方案:使用OpenCV绘制英文 - cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, - size / 25, color, 1) - return frame - - def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, - trajectory_recorder=None, gesture_recognizer=None, performance_analyzer=None): - """绘制状态栏(增强版)""" - h, w = frame.shape[:2] - - # 绘制半透明背景 - overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (w, 130), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) - - # 标题 - title = "手势控制无人机系统 - 性能统计版" - frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) - - # 连接状态 - status_color = self.colors['connected'] if drone_controller.connected else self.colors['disconnected'] - status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" - frame = self.draw_text(frame, status_text, (10, 40), size=16, color=status_color) - - # 飞行状态 - flight_color = self.colors['flying'] if drone_controller.flying else self.colors['landed'] - flight_text = f"飞行状态: {'飞行中' if drone_controller.flying else '已降落'}" - frame = self.draw_text(frame, flight_text, (10, 65), size=16, color=flight_color) - - # 手势信息 - if confidence > 0.7: - gesture_color = (0, 255, 0) - elif confidence > 0.5: - gesture_color = (255, 165, 0) - else: - gesture_color = (200, 200, 200) - - gesture_text = f"当前手势: {gesture}" - if config.get('display', 'show_confidence'): - gesture_text += f" ({confidence:.0%})" - - frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) - - # 性能模式显示 - if gesture_recognizer and config.get('display', 'show_performance_mode'): - mode_info = gesture_recognizer.get_performance_mode_info() - - # 根据模式选择颜色 - if mode_info['mode'] == 'fast': - mode_color = self.colors['performance_fast'] - elif mode_info['mode'] == 'balanced': - mode_color = self.colors['performance_balanced'] - else: # accurate - mode_color = self.colors['performance_accurate'] - - mode_text = f"性能模式: {mode_info['name']}" - frame = self.draw_text(frame, mode_text, (w // 2, 65), size=14, color=mode_color) - - # 录制/回放状态 - elif trajectory_recorder and config.get('display', 'show_recording_status'): - recorder_status = trajectory_recorder.get_status() - - if recorder_status['is_recording']: - status_color = self.colors['recording'] - status_text = f"录制中: {recorder_status['trajectory_points']}点" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) - elif recorder_status['is_playing']: - if recorder_status['playback_paused']: - status_color = self.colors['paused'] - status_text = f"回放暂停: {recorder_status['playback_index']}/{recorder_status['playback_total']}" - else: - status_color = self.colors['playback'] - status_text = f"回放中: {recorder_status['playback_index']}/{recorder_status['playback_total']}" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) - else: - status_text = f"轨迹点: {recorder_status['trajectory_points']}" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=12, color=self.colors['info']) - - # 性能统计显示 - if performance_analyzer and config.get('display', 'show_performance_stats'): - perf_stats = performance_analyzer.get_stats_summary() - - # 性能状态颜色 - if perf_stats['performance_status'] == "良好": - perf_color = self.colors['performance_good'] - elif perf_stats['performance_status'] == "警告": - perf_color = self.colors['performance_warning'] - else: - perf_color = self.colors['performance_bad'] - - perf_text = f"性能: {perf_stats['performance_status']} ({perf_stats['fps']:.0f}FPS)" - frame = self.draw_text(frame, perf_text, (w // 2, 90), size=12, color=perf_color) - - # 系统资源显示 - if performance_analyzer and config.get('display', 'show_system_resources'): - cpu_usage = performance_analyzer.get_current_cpu_usage() - memory_usage = performance_analyzer.get_current_memory_usage() - - # CPU颜色 - if cpu_usage < 70: - cpu_color = self.colors['cpu_good'] - elif cpu_usage < 85: - cpu_color = self.colors['cpu_warning'] - else: - cpu_color = self.colors['cpu_critical'] - - # 内存颜色 - if memory_usage < 70: - mem_color = self.colors['memory_good'] - elif memory_usage < 85: - mem_color = self.colors['memory_warning'] - else: - mem_color = self.colors['memory_critical'] - - # 绘制系统资源信息 - sys_text = f"CPU:{cpu_usage:.0f}% 内存:{memory_usage:.0f}%" - frame = self.draw_text(frame, sys_text, (w // 2, 110), size=11, color=self.colors['info']) - - # 语音状态 - if config.get('display', 'show_speech_status') and self.speech_manager: - speech_status = self.speech_manager.get_status() - speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] - speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" - frame = self.draw_text(frame, speech_text, (w - 200, 90), size=12, color=speech_color) - - # 性能信息 - if config.get('display', 'show_fps'): - perf_text = f"帧率: {fps:.1f}" - if process_time > 0: - perf_text += f" | 延迟: {process_time:.1f}ms" - - # 根据处理时间选择颜色 - if process_time < 20: - perf_color = self.colors['performance_good'] - elif process_time < 50: - perf_color = self.colors['performance_warning'] - else: - perf_color = self.colors['performance_bad'] - else: - perf_color = self.colors['info'] - - frame = self.draw_text(frame, perf_text, (w - 200, 65), size=12, color=perf_color) - - # 控制提示 - control_text = "提示: 确保手部完全进入画面,保持稳定手势" - frame = self.draw_text(frame, control_text, (10, 90), size=12, color=self.colors['info']) - - # 性能快捷键提示 - if performance_analyzer: - perf_tip = "P:报告 L:快照 K:导出" - frame = self.draw_text(frame, perf_tip, (10, 110), size=11, color=self.colors['help']) - - return frame - - def draw_help_bar(self, frame): - """绘制帮助栏""" - if not config.get('display', 'show_help'): - return frame - - h, w = frame.shape[:2] - - # 绘制底部帮助栏 - cv2.rectangle(frame, (0, h - 120), (w, h), (0, 0, 0), -1) - - # 帮助文本(增强版,包含性能统计快捷键) - help_lines = [ - "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", - "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音 P:性能报告 L:性能快照 K:导出日志", - "O:切换性能模式 P:详细报告 R:重置统计", - "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" - ] - - for i, line in enumerate(help_lines): - y_pos = h - 105 + i * 20 - frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) - - return frame - - def draw_warning(self, frame, message): - """绘制警告信息""" - h, w = frame.shape[:2] - - # 在顶部绘制警告 - warning_bg = np.zeros((40, w, 3), dtype=np.uint8) - warning_bg[:, :] = (0, 69, 255) - - frame[130:170, 0:w] = cv2.addWeighted( - frame[130:170, 0:w], 0.3, - warning_bg, 0.7, 0 - ) - - # 绘制警告文本 - frame = self.draw_text(frame, message, (10, 145), - size=16, color=self.colors['warning']) - - return frame - - def draw_performance_chart(self, frame, performance_analyzer): - """绘制性能图表""" - if not performance_analyzer: - return frame - - h, w = frame.shape[:2] - - # 绘制图表区域背景 - chart_x = w - 250 - chart_y = 130 - chart_width = 240 - chart_height = 150 - - cv2.rectangle(frame, (chart_x, chart_y), - (chart_x + chart_width, chart_y + chart_height), - (30, 30, 30), -1) - cv2.rectangle(frame, (chart_x, chart_y), - (chart_x + chart_width, chart_y + chart_height), - (100, 100, 100), 1) - - # 绘制标题 - frame = self.draw_text(frame, "性能监控", (chart_x + 10, chart_y + 20), - size=14, color=self.colors['title']) - - # 绘制FPS图表 - if len(performance_analyzer.fps_history) > 1: - fps_values = list(performance_analyzer.fps_history)[-20:] # 最近20个值 - - # 计算最大值和最小值 - if fps_values: - max_fps = max(fps_values) - min_fps = min(fps_values) - if max_fps == min_fps: - max_fps = min_fps + 1 - - # 绘制FPS曲线 - points = [] - for i, fps in enumerate(fps_values): - x = chart_x + 10 + i * (chart_width - 20) / len(fps_values) - y = chart_y + chart_height - 30 - (fps - min_fps) * (chart_height - 50) / (max_fps - min_fps) - points.append((int(x), int(y))) - - # 连接点形成曲线 - for i in range(len(points) - 1): - cv2.line(frame, points[i], points[i + 1], (0, 255, 0), 2) - - # 显示当前FPS值 - current_fps = fps_values[-1] if fps_values else 0 - fps_color = self.colors['performance_good'] if current_fps > 20 else \ - self.colors['performance_warning'] if current_fps > 10 else \ - self.colors['performance_bad'] - - frame = self.draw_text(frame, f"FPS: {current_fps:.1f}", - (chart_x + 120, chart_y + 40), - size=12, color=fps_color) - - return frame - - -# ========== 主程序 ========== -def main(): - """主函数""" - # 初始化语音管理器 - print("初始化语音反馈系统...") - speech_manager = EnhancedSpeechFeedbackManager(libs['speech']) - - # 程序启动语音提示 - if speech_manager.enabled: - speech_manager.speak('program_start', force=True, immediate=True) - speech_manager.speak('system_ready', immediate=True) - - # 初始化组件 - print("初始化组件...") + # 尝试导入psutil + try: + import psutil + libs['psutil'] = psutil + status['psutil'] = True + print("[psutil] ✓ 系统资源监控库就绪") + except ImportError: + print("[psutil] ⚠ 未找到,性能监控功能受限") + libs['psutil'] = None - gesture_recognizer = EnhancedGestureRecognizer(speech_manager) - drone_controller = SimpleDroneController(libs['airsim'], speech_manager) - ui_renderer = ChineseUIRenderer(speech_manager) + libs['speech'] = speech_module + return libs, status - # 初始化性能分析器 - print("初始化性能分析器...") - performance_analyzer = PerformanceAnalyzer(speech_manager) - # 手势轨迹记录器 - print("初始化手势轨迹记录器...") - trajectory_recorder = GestureTrajectoryRecorder(speech_manager) - - # 初始化摄像头 +def init_camera(config): + """初始化摄像头""" cap = None try: cap = cv2.VideoCapture(0) @@ -3290,7 +136,6 @@ def main(): cap.set(cv2.CAP_PROP_FRAME_HEIGHT, config.get('camera', 'height')) cap.set(cv2.CAP_PROP_FPS, config.get('camera', 'fps')) - # 获取实际参数 actual_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) actual_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) actual_fps = int(cap.get(cv2.CAP_PROP_FPS)) @@ -3298,28 +143,18 @@ def main(): print(f"✓ 摄像头已初始化") print(f" 分辨率: {actual_width}x{actual_height}") print(f" 帧率: {actual_fps}") - - # 摄像头就绪语音提示 - if speech_manager.enabled: - speech_manager.speak('camera_ready', immediate=True) else: print("❌ 摄像头不可用,使用模拟模式") - - # 摄像头错误语音提示 - if speech_manager.enabled: - speech_manager.speak('camera_error', immediate=True) - cap = None except Exception as e: print(f"⚠ 摄像头初始化失败: {e}") + cap = None - # 摄像头错误语音提示 - if speech_manager.enabled: - speech_manager.speak('camera_error', immediate=True) + return cap - cap = None - # 显示欢迎信息 +def print_welcome_message(cap, speech_manager, libs): + """打印欢迎信息""" print("\n" + "=" * 80) print("手势控制无人机系统 - 详细性能统计版") print("=" * 80) @@ -3327,13 +162,14 @@ def main(): print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") - print(f" 性能模式: {gesture_recognizer.performance_mode_name}") - print(f" 性能监控: {'已启用' if config.get('performance', 'enable_performance_monitor') else '已禁用'}") + print(f" 性能监控: 已启用") print(f" 轨迹记录: 支持录制/回放功能") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 80) - # 显示操作说明 + +def print_instructions(): + """打印操作说明""" print("\n操作说明:") print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") @@ -3363,6 +199,64 @@ def main(): print("程序启动成功!") print("-" * 80) + +def main(): + """主函数""" + print("=" * 60) + print("手势控制无人机系统 - 详细性能统计优化版") + print("详细性能统计优化版!") + print("=" * 60) + + # 导入库 + libs, status = safe_import_libs() + if not status.get('OpenCV', False) or not status.get('PIL', False): + print("\n❌ 核心库缺失,无法启动。") + input("按回车键退出...") + sys.exit(1) + + print("-" * 60) + print("✅ 环境检查通过,正在初始化...") + print("-" * 60) + + # 初始化配置管理器 + config = ConfigManager() + + # 初始化语音管理器 + print("初始化语音反馈系统...") + speech_manager = EnhancedSpeechFeedbackManager(libs['speech'], config) + + # 程序启动语音提示 + if speech_manager.enabled: + speech_manager.speak('program_start', force=True, immediate=True) + speech_manager.speak('system_ready', immediate=True) + + # 初始化其他组件 + print("初始化组件...") + gesture_recognizer = EnhancedGestureRecognizer(speech_manager, config) + drone_controller = SimpleDroneController(libs['airsim'], speech_manager, config) + ui_renderer = ChineseUIRenderer(speech_manager, config) + + # 初始化性能分析器 + print("初始化性能分析器...") + performance_analyzer = PerformanceAnalyzer(speech_manager, libs.get('psutil'), config) + + # 手势轨迹记录器 + print("初始化手势轨迹记录器...") + trajectory_recorder = GestureTrajectoryRecorder(speech_manager, config) + + # 初始化摄像头 + cap = init_camera(config) + if cap and speech_manager.enabled: + speech_manager.speak('camera_ready', immediate=True) + elif not cap and speech_manager.enabled: + speech_manager.speak('camera_error', immediate=True) + + # 显示欢迎信息 + print_welcome_message(cap, speech_manager, libs) + + # 显示操作说明 + print_instructions() + # 键盘手势映射 key_to_gesture = { ord('w'): "Up", ord('W'): "Up", @@ -3480,7 +374,7 @@ def main(): warning_msg = f"⚠ 性能状态: {performance_analyzer.performance_status}" frame = ui_renderer.draw_warning(frame, warning_msg) - # 显示图像(窗口标题用英文) + # 显示图像 cv2.imshow('Gesture Controlled Drone - Performance Statistics', frame) # ========== 键盘控制 ========== @@ -3515,7 +409,7 @@ def main(): elif key == ord('r') or key == ord('R'): # 重置手势识别 print("重置手势识别...") - gesture_recognizer = EnhancedGestureRecognizer(speech_manager) + gesture_recognizer = EnhancedGestureRecognizer(speech_manager, config) print("✓ 手势识别已重置") # 语音提示 @@ -3576,7 +470,7 @@ def main(): elif key == ord('l') or key == ord('L'): # 性能快照 - snapshot_label = f"快照_{datetime.now().strftime('%H:%M:%S')}" + snapshot_label = f"快照_{time.strftime('%H:%M:%S')}" snapshot = performance_analyzer.take_snapshot(snapshot_label) print(f"📸 性能快照已保存: {snapshot_label}") @@ -3617,7 +511,7 @@ def main(): print(f"✓ 已切换到性能模式: {gesture_recognizer.performance_mode_name}") - # 新增:重置性能统计 + # 重置性能统计 elif key == ord('R') and chr(key).isupper(): # 大写R print("重置性能统计...") performance_analyzer.reset_session() @@ -3627,7 +521,7 @@ def main(): if speech_manager.enabled: speech_manager.speak_direct("性能统计已重置") - # 新增:轨迹记录控制 + # 轨迹记录控制 elif key == ord('1'): # 开始录制轨迹 if trajectory_recorder.start_recording(): @@ -3755,6 +649,5 @@ def main(): input("按回车键退出...") -# ========== 程序入口 ========== if __name__ == "__main__": main() \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/performance_analyzer.py b/src/autonomus_drone_hand_gesture_project/performance_analyzer.py new file mode 100644 index 0000000000..6bf7ca3d08 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/performance_analyzer.py @@ -0,0 +1,506 @@ +""" +性能分析器模块 +负责监控和报告系统性能 +作者: xiaoshiyuan888 +""" + +import time +import os +import csv +import numpy as np +from datetime import datetime +from collections import deque, Counter + + +class PerformanceAnalyzer: + """性能分析器 - 监控和报告系统性能""" + + def __init__(self, speech_manager=None, psutil_lib=None, config=None): + self.speech_manager = speech_manager + self.psutil_lib = psutil_lib + self.config = config + self.start_time = time.time() + self.session_start_time = time.time() + + # 帧率统计 + self.frame_times = deque(maxlen=300) + self.frame_count = 0 + self.fps_history = deque(maxlen=100) + + # 手势识别性能 + self.gesture_recognition_times = deque(maxlen=100) + self.avg_recognition_time = 0 + self.max_recognition_time = 0 + + # 系统资源监控 + self.cpu_usage_history = deque(maxlen=100) + self.memory_usage_history = deque(maxlen=100) + + # 性能事件记录 + self.performance_events = [] + self.performance_snapshots = [] + + # 手势统计 + self.gesture_counts = {} + self.gesture_confidence_sum = {} + self.gesture_confidence_count = {} + + # 错误统计 + self.error_count = 0 + self.warning_count = 0 + + # 无人机控制统计 + self.drone_commands = 0 + self.successful_commands = 0 + self.failed_commands = 0 + + # 轨迹记录统计 + self.recording_sessions = 0 + self.total_trajectory_points = 0 + + # 性能日志 + self.performance_log = [] + self.log_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'performance_log.csv') + + # 性能阈值 + self.performance_thresholds = { + 'fps_warning': 15, + 'fps_critical': 5, + 'cpu_warning': 80, + 'cpu_critical': 90, + 'memory_warning': 85, + 'memory_critical': 95, + 'recognition_warning': 50, + 'recognition_critical': 100 + } + + # 性能状态 + self.performance_status = "良好" + self.last_performance_report = 0 + self.auto_report_interval = 60 + + print("✓ 性能分析器已初始化") + + def update_frame(self): + """更新帧统计""" + current_time = time.time() + self.frame_times.append(current_time) + self.frame_count += 1 + + # 计算当前FPS + if len(self.frame_times) > 1: + time_span = self.frame_times[-1] - self.frame_times[0] + if time_span > 0: + current_fps = (len(self.frame_times) - 1) / time_span + self.fps_history.append(current_fps) + + def update_gesture_recognition_time(self, recognition_time_ms): + """更新手势识别时间""" + self.gesture_recognition_times.append(recognition_time_ms) + + # 更新平均识别时间 + if len(self.gesture_recognition_times) > 0: + self.avg_recognition_time = np.mean(list(self.gesture_recognition_times)) + self.max_recognition_time = max(self.max_recognition_time, recognition_time_ms) + + def update_system_resources(self): + """更新系统资源使用情况""" + try: + if self.psutil_lib: + cpu_percent = self.psutil_lib.cpu_percent(interval=0.1) + memory_percent = self.psutil_lib.virtual_memory().percent + + self.cpu_usage_history.append(cpu_percent) + self.memory_usage_history.append(memory_percent) + + # 检查性能问题 + self.check_performance_issues(cpu_percent, memory_percent) + except: + pass + + def check_performance_issues(self, cpu_percent, memory_percent): + """检查性能问题""" + issues = [] + + # 检查FPS + if len(self.fps_history) > 0: + avg_fps = np.mean(list(self.fps_history[-10:])) if len(self.fps_history) >= 10 else self.fps_history[-1] + + if avg_fps < self.performance_thresholds['fps_critical']: + issues.append(("严重", f"帧率过低: {avg_fps:.1f} FPS")) + self.performance_status = "严重" + elif avg_fps < self.performance_thresholds['fps_warning']: + issues.append(("警告", f"帧率较低: {avg_fps:.1f} FPS")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查CPU使用率 + if cpu_percent > self.performance_thresholds['cpu_critical']: + issues.append(("严重", f"CPU使用率过高: {cpu_percent:.1f}%")) + self.performance_status = "严重" + elif cpu_percent > self.performance_thresholds['cpu_warning']: + issues.append(("警告", f"CPU使用率较高: {cpu_percent:.1f}%")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查内存使用率 + if memory_percent > self.performance_thresholds['memory_critical']: + issues.append(("严重", f"内存使用率过高: {memory_percent:.1f}%")) + self.performance_status = "严重" + elif memory_percent > self.performance_thresholds['memory_warning']: + issues.append(("警告", f"内存使用率较高: {memory_percent:.1f}%")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 检查手势识别时间 + if self.avg_recognition_time > self.performance_thresholds['recognition_critical']: + issues.append(("严重", f"手势识别时间过长: {self.avg_recognition_time:.1f}ms")) + self.performance_status = "严重" + elif self.avg_recognition_time > self.performance_thresholds['recognition_warning']: + issues.append(("警告", f"手势识别时间较长: {self.avg_recognition_time:.1f}ms")) + if self.performance_status == "良好": + self.performance_status = "警告" + + # 记录性能事件 + if issues: + for level, message in issues: + self.add_performance_event(level, message) + + # 语音提示(仅在状态变化时) + if (self.speech_manager and + self.speech_manager.enabled and + level == "严重"): + current_time = time.time() + if current_time - self.last_performance_report > 10: + self.speech_manager.speak_direct(f"性能{level}: {message}") + self.last_performance_report = current_time + + def add_performance_event(self, level, message): + """添加性能事件""" + event = { + 'timestamp': time.time(), + 'level': level, + 'message': message, + 'session_time': time.time() - self.session_start_time + } + self.performance_events.append(event) + + # 记录到日志 + self.log_performance_event(event) + + if level == "警告": + self.warning_count += 1 + elif level == "严重": + self.error_count += 1 + + def log_performance_event(self, event): + """记录性能事件到日志""" + log_entry = { + 'timestamp': datetime.fromtimestamp(event['timestamp']).strftime('%Y-%m-%d %H:%M:%S'), + 'session_time': f"{event['session_time']:.1f}", + 'level': event['level'], + 'message': event['message'] + } + self.performance_log.append(log_entry) + + def record_gesture(self, gesture, confidence): + """记录手势统计""" + if gesture not in self.gesture_counts: + self.gesture_counts[gesture] = 0 + self.gesture_confidence_sum[gesture] = 0 + self.gesture_confidence_count[gesture] = 0 + + self.gesture_counts[gesture] += 1 + self.gesture_confidence_sum[gesture] += confidence + self.gesture_confidence_count[gesture] += 1 + + def record_drone_command(self, success=True): + """记录无人机命令""" + self.drone_commands += 1 + if success: + self.successful_commands += 1 + else: + self.failed_commands += 1 + + def record_recording_session(self, points_count=0): + """记录录制会话""" + self.recording_sessions += 1 + self.total_trajectory_points += points_count + + def take_snapshot(self, label=""): + """拍摄性能快照""" + snapshot = { + 'timestamp': time.time(), + 'label': label, + 'fps': self.get_current_fps(), + 'avg_fps': self.get_average_fps(), + 'avg_recognition_time': self.avg_recognition_time, + 'max_recognition_time': self.max_recognition_time, + 'cpu_usage': self.get_current_cpu_usage(), + 'memory_usage': self.get_current_memory_usage(), + 'gesture_counts': dict(self.gesture_counts), + 'performance_status': self.performance_status, + 'frame_count': self.frame_count, + 'session_duration': time.time() - self.session_start_time + } + self.performance_snapshots.append(snapshot) + + print(f"📸 性能快照已保存: {label}") + return snapshot + + def get_current_fps(self): + """获取当前FPS""" + if len(self.fps_history) > 0: + return self.fps_history[-1] + return 0 + + def get_average_fps(self): + """获取平均FPS""" + if len(self.fps_history) > 0: + return np.mean(list(self.fps_history)) + return 0 + + def get_current_cpu_usage(self): + """获取当前CPU使用率""" + if len(self.cpu_usage_history) > 0: + return self.cpu_usage_history[-1] + return 0 + + def get_current_memory_usage(self): + """获取当前内存使用率""" + if len(self.memory_usage_history) > 0: + return self.memory_usage_history[-1] + return 0 + + def generate_report(self, detailed=True): + """生成性能报告""" + report_time = time.time() + session_duration = report_time - self.session_start_time + + # 基础报告 + report = { + '生成时间': datetime.fromtimestamp(report_time).strftime('%Y-%m-%d %H:%M:%S'), + '会话时长': f"{session_duration:.1f}秒", + '总帧数': self.frame_count, + '平均FPS': f"{self.get_average_fps():.1f}", + '当前FPS': f"{self.get_current_fps():.1f}", + '平均手势识别时间': f"{self.avg_recognition_time:.1f}ms", + '最大手势识别时间': f"{self.max_recognition_time:.1f}ms", + '当前CPU使用率': f"{self.get_current_cpu_usage():.1f}%", + '当前内存使用率': f"{self.get_current_memory_usage():.1f}%", + '性能状态': self.performance_status, + '警告数量': self.warning_count, + '错误数量': self.error_count, + '无人机命令': { + '总数': self.drone_commands, + '成功': self.successful_commands, + '失败': self.failed_commands, + '成功率': f"{(self.successful_commands / self.drone_commands * 100 if self.drone_commands > 0 else 0):.1f}%" + }, + '录制统计': { + '会话数': self.recording_sessions, + '总轨迹点数': self.total_trajectory_points + } + } + + # 详细报告 + if detailed: + # 手势统计 + gesture_stats = {} + for gesture in self.gesture_counts: + count = self.gesture_counts[gesture] + if gesture in self.gesture_confidence_count and self.gesture_confidence_count[gesture] > 0: + avg_confidence = self.gesture_confidence_sum[gesture] / self.gesture_confidence_count[gesture] + else: + avg_confidence = 0 + + gesture_stats[gesture] = { + '次数': count, + '占比': f"{(count / self.frame_count * 100 if self.frame_count > 0 else 0):.1f}%", + '平均置信度': f"{avg_confidence:.1%}" + } + + report['手势统计'] = gesture_stats + + # 性能事件 + if self.performance_events: + recent_events = list(self.performance_events)[-10:] + report['最近性能事件'] = [ + { + '时间': datetime.fromtimestamp(e['timestamp']).strftime('%H:%M:%S'), + '级别': e['level'], + '消息': e['message'] + } + for e in recent_events + ] + + # 性能快照 + if self.performance_snapshots: + report['性能快照数'] = len(self.performance_snapshots) + + # 系统建议 + suggestions = self.generate_suggestions() + if suggestions: + report['优化建议'] = suggestions + + return report + + def generate_suggestions(self): + """生成优化建议""" + suggestions = [] + + # 检查FPS + avg_fps = self.get_average_fps() + if avg_fps < self.performance_thresholds['fps_warning']: + suggestions.append(f"帧率较低({avg_fps:.1f}FPS),建议切换到'最快'性能模式") + + # 检查CPU + cpu_usage = self.get_current_cpu_usage() + if cpu_usage > self.performance_thresholds['cpu_warning']: + suggestions.append(f"CPU使用率较高({cpu_usage:.1f}%),请关闭其他占用CPU的程序") + + # 检查内存 + memory_usage = self.get_current_memory_usage() + if memory_usage > self.performance_thresholds['memory_warning']: + suggestions.append(f"内存使用率较高({memory_usage:.1f}%),请关闭不必要的程序") + + # 检查识别时间 + if self.avg_recognition_time > self.performance_thresholds['recognition_warning']: + suggestions.append(f"手势识别时间较长({self.avg_recognition_time:.1f}ms),建议调整摄像头位置或光线") + + return suggestions + + def print_report(self, detailed=True): + """打印性能报告""" + report = self.generate_report(detailed) + + print("\n" + "=" * 80) + print("📊 性能分析报告") + print("=" * 80) + + # 基础信息 + print(f"生成时间: {report['生成时间']}") + print(f"会话时长: {report['会话时长']}") + print(f"总帧数: {report['总帧数']}") + print(f"平均FPS: {report['平均FPS']}") + print(f"当前FPS: {report['当前FPS']}") + print(f"平均手势识别时间: {report['平均手势识别时间']}") + print(f"最大手势识别时间: {report['最大手势识别时间']}") + print(f"当前CPU使用率: {report['当前CPU使用率']}") + print(f"当前内存使用率: {report['当前内存使用率']}") + print(f"性能状态: {report['性能状态']}") + + # 无人机命令统计 + cmd_stats = report['无人机命令'] + print(f"\n无人机命令统计:") + print(f" 总数: {cmd_stats['总数']}") + print(f" 成功: {cmd_stats['成功']}") + print(f" 失败: {cmd_stats['失败']}") + print(f" 成功率: {cmd_stats['成功率']}") + + # 录制统计 + rec_stats = report['录制统计'] + print(f"\n录制统计:") + print(f" 会话数: {rec_stats['会话数']}") + print(f" 总轨迹点数: {rec_stats['总轨迹点数']}") + + # 详细报告 + if detailed and '手势统计' in report: + print(f"\n手势统计:") + for gesture, stats in report['手势统计'].items(): + print(f" {gesture}: {stats['次数']}次 ({stats['占比']}), 平均置信度: {stats['平均置信度']}") + + # 性能事件 + if detailed and '最近性能事件' in report and report['最近性能事件']: + print(f"\n最近性能事件:") + for event in report['最近性能事件']: + print(f" [{event['时间']}] {event['级别']}: {event['消息']}") + + # 优化建议 + if detailed and '优化建议' in report and report['优化建议']: + print(f"\n优化建议:") + for i, suggestion in enumerate(report['优化建议'], 1): + print(f" {i}. {suggestion}") + + print("=" * 80) + + # 语音播报摘要 + if self.speech_manager and self.speech_manager.enabled: + summary = (f"性能报告: 平均帧率{report['平均FPS']},识别时间{report['平均手势识别时间']}," + f"性能状态{report['性能状态']},无人机命令成功率{cmd_stats['成功率']}") + self.speech_manager.speak_direct(summary) + + def export_log(self, filename=None): + """导出性能日志""" + import os + current_dir = os.path.dirname(os.path.abspath(__file__)) + if filename is None: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = os.path.join(current_dir, f'performance_log_{timestamp}.csv') + + try: + with open(filename, 'w', newline='', encoding='utf-8') as f: + if self.performance_log: + fieldnames = self.performance_log[0].keys() + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(self.performance_log) + + print(f"📤 性能日志已导出到: {filename}") + return True + except Exception as e: + print(f"❌ 导出性能日志失败: {e}") + return False + + def auto_report(self): + """自动性能报告(定期执行)""" + current_time = time.time() + if current_time - self.last_performance_report > self.auto_report_interval: + # 生成简要报告 + report = self.generate_report(detailed=False) + + # 检查是否需要报告 + if (self.performance_status == "严重" or + self.warning_count > 5 or + self.error_count > 0): + + print(f"⚠ 自动性能检查: {report['性能状态']}, FPS: {report['当前FPS']}, " + f"CPU: {report['当前CPU使用率']}, 内存: {report['当前内存使用率']}") + + # 语音提示 + if (self.speech_manager and + self.speech_manager.enabled and + self.performance_status == "严重"): + self.speech_manager.speak_direct(f"系统性能{self.performance_status},建议检查") + + self.last_performance_report = current_time + + def reset_session(self): + """重置会话统计""" + self.session_start_time = time.time() + self.performance_events = [] + self.performance_snapshots = [] + self.gesture_counts = {} + self.gesture_confidence_sum = {} + self.gesture_confidence_count = {} + self.error_count = 0 + self.warning_count = 0 + self.drone_commands = 0 + self.successful_commands = 0 + self.failed_commands = 0 + self.performance_status = "良好" + + print("✓ 性能统计会话已重置") + + def get_stats_summary(self): + """获取统计摘要""" + return { + 'fps': self.get_current_fps(), + 'avg_fps': self.get_average_fps(), + 'recognition_time': self.avg_recognition_time, + 'cpu_usage': self.get_current_cpu_usage(), + 'memory_usage': self.get_current_memory_usage(), + 'performance_status': self.performance_status, + 'gesture_count': sum(self.gesture_counts.values()), + 'unique_gestures': len(self.gesture_counts) + } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/speech_manager.py b/src/autonomus_drone_hand_gesture_project/speech_manager.py new file mode 100644 index 0000000000..f2f54a7309 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/speech_manager.py @@ -0,0 +1,361 @@ +""" +语音反馈管理器模块 +负责语音提示和音频播放 +作者: xiaoshiyuan888 +""" + +import threading +import time +import os +import sys +import tempfile + + +class EnhancedSpeechFeedbackManager: + """增强的语音反馈管理器""" + + def __init__(self, speech_lib, config=None): + self.speech_lib = speech_lib + self.config = config + self.enabled = True + self.volume = 1.0 + self.rate = 150 + self.voice_id = None + self.last_speech_time = {} + self.min_interval = 1.5 + + # 语音队列 + self.speech_queue = [] + self.is_speaking = False + self.queue_thread = None + + # 音频播放方法 + self.audio_method = None + + # 手势状态追踪 + self.last_gesture_state = "none" + self.gesture_active_time = 0 + + # 语音消息映射 + self.messages = { + # 连接相关 + 'connecting': "正在连接无人机,请稍候", + 'connected': "无人机连接成功", + 'connection_failed': "无人机连接失败,进入模拟模式", + + # 飞行相关 + 'taking_off': "无人机正在起飞", + 'takeoff_success': "起飞成功", + 'takeoff_failed': "起飞失败", + 'landing': "无人机正在降落", + 'land_success': "降落成功", + 'emergency_stop': "紧急停止,无人机已降落", + 'hovering': "无人机悬停中", + + # 手势相关 + 'gesture_detected': "手势识别就绪,请开始手势", + 'gesture_start': "开始识别手势", + 'gesture_end': "手势识别结束", + 'gesture_stop': "停止", + 'gesture_up': "向上", + 'gesture_down': "向下", + 'gesture_left': "向左", + 'gesture_right': "向右", + 'gesture_forward': "向前", + 'gesture_backward': "向后", + 'gesture_waiting': "等待手势", + 'gesture_error': "手势识别错误", + 'gesture_stable': "手势稳定", + 'gesture_change': "手势变化", + 'gesture_low_confidence': "手势识别置信度低", + 'gesture_good_confidence': "手势识别置信度高", + 'gesture_hover': "悬停", + + # 系统相关 + 'program_start': "手势控制无人机系统已启动", + 'program_exit': "程序退出,感谢使用", + 'camera_error': "摄像头错误,请检查连接", + 'camera_ready': "摄像头就绪", + 'system_ready': "系统准备就绪", + + # 模式相关 + 'simulation_mode': "进入模拟模式", + 'debug_mode_on': "调试模式已开启", + 'debug_mode_off': "调试模式已关闭", + 'display_mode_changed': "显示模式已切换", + 'help_toggled': "帮助信息已切换", + 'performance_mode_fast': "切换到最快性能模式", + 'performance_mode_balanced': "切换到平衡性能模式", + 'performance_mode_accurate': "切换到最准确性能模式", + + # 性能相关 + 'performance_good': "系统运行流畅", + 'performance_warning': "系统性能警告", + 'performance_critical': "系统性能严重警告", + 'performance_report': "性能报告生成完成", + 'performance_snapshot': "性能快照已保存", + 'performance_log_exported': "性能日志已导出", + + # 手势指导 + 'move_closer': "请将手靠近摄像头", + 'move_away': "请将手移远一些", + 'good_position': "手部位置良好", + 'hand_detected': "手部已检测到", + 'hand_lost': "手部丢失,请重新放置", + + # 录制相关 + 'recording_start': "开始录制手势轨迹", + 'recording_stop': "停止录制", + 'recording_saved': "轨迹已保存", + 'recording_loaded': "轨迹已加载", + 'recording_playback_start': "开始回放手势轨迹", + 'recording_playback_stop': "回放结束", + 'recording_cleared': "轨迹已清除", + 'recording_paused': "回放已暂停", + 'recording_resumed': "回放继续", + 'recording_not_found': "未找到轨迹数据", + 'recording_frame_count': "轨迹帧数", + } + + # 初始化语音引擎 + self.init_speech_engine() + + def init_speech_engine(self): + """初始化语音引擎""" + if self.speech_lib is None: + print("⚠ 语音库未找到,语音功能禁用") + self.enabled = False + return + + try: + if hasattr(self.speech_lib, 'init'): # pyttsx3 + self.engine = self.speech_lib.init() + self.audio_method = 'pyttsx3' + + # 设置语音参数 + voices = self.engine.getProperty('voices') + + # 尝试寻找中文语音 + for voice in voices: + if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower() or 'zh_CN' in voice.name.lower(): + self.engine.setProperty('voice', voice.id) + self.voice_id = voice.id + print(f"[Speech] 使用中文语音: {voice.name}") + break + + # 如果没找到中文语音,使用第一个可用语音 + if self.voice_id is None and len(voices) > 0: + self.engine.setProperty('voice', voices[0].id) + print(f"[Speech] 使用默认语音: {voices[0].name}") + + # 设置语速和音量 + self.engine.setProperty('rate', self.rate) + self.engine.setProperty('volume', self.volume) + + print("✅ 语音引擎初始化成功 (pyttsx3)") + + elif isinstance(self.speech_lib, dict) and self.speech_lib.get('type') == 'gtts': + print("✅ 语音引擎初始化成功 (gTTS,需要网络连接)") + self.audio_method = 'gtts' + + # 确定播放方法 + if 'pygame' in self.speech_lib: + self.audio_method = 'gtts_pygame' + print("✅ 使用pygame播放音频") + elif 'pydub' in self.speech_lib: + self.audio_method = 'gtts_pydub' + print("✅ 使用pydub播放音频") + elif 'play_method' in self.speech_lib: + self.audio_method = f"gtts_{self.speech_lib['play_method']}" + print(f"✅ 使用系统命令播放音频") + else: + self.audio_method = 'gtts_system' + print("✅ 使用默认系统播放器") + + else: + print("⚠ 未知语音库类型,语音功能可能不正常") + self.enabled = False + + except Exception as e: + print(f"⚠ 语音引擎初始化失败: {e}") + self.enabled = False + + def play_audio_file(self, audio_file): + """播放音频文件""" + try: + if self.audio_method == 'gtts_pygame' and 'pygame' in self.speech_lib: + pygame = self.speech_lib['pygame'] + pygame.mixer.music.load(audio_file) + pygame.mixer.music.play() + + # 等待播放完成 + while pygame.mixer.music.get_busy(): + pygame.time.Clock().tick(10) + + elif self.audio_method == 'gtts_pydub' and 'pydub' in self.speech_lib: + AudioSegment = self.speech_lib['AudioSegment'] + play = self.speech_lib['play'] + + audio = AudioSegment.from_mp3(audio_file) + play(audio) + + elif self.audio_method == 'gtts_windows': + # Windows系统命令 + os.startfile(audio_file) + time.sleep(1.5) + + elif self.audio_method == 'gtts_posix': + # Linux/Mac系统命令 + import subprocess + if sys.platform == 'darwin': + subprocess.call(['afplay', audio_file]) + else: + subprocess.call(['xdg-open', audio_file]) + + else: + # 通用方法 + import subprocess + if sys.platform == 'win32': + os.startfile(audio_file) + elif sys.platform == 'darwin': + subprocess.call(['open', audio_file]) + else: + subprocess.call(['xdg-open', audio_file]) + + return True + + except Exception as e: + print(f"⚠ 音频播放失败: {e}") + return False + + def speak(self, message_key, force=False, immediate=False): + """播放语音""" + if not self.enabled: + return + + # 检查是否在最小间隔内 + current_time = time.time() + if not force and message_key in self.last_speech_time: + if current_time - self.last_speech_time[message_key] < self.min_interval: + return + + # 获取消息文本 + if message_key in self.messages: + text = self.messages[message_key] + else: + text = message_key + + # 立即播放或加入队列 + if immediate: + self.speak_direct(text) + else: + self.speech_queue.append(text) + + if not self.is_speaking and self.queue_thread is None: + self.queue_thread = threading.Thread(target=self._process_speech_queue) + self.queue_thread.daemon = True + self.queue_thread.start() + + # 更新时间戳 + self.last_speech_time[message_key] = current_time + + def _process_speech_queue(self): + """处理语音队列""" + while self.speech_queue and self.enabled: + self.is_speaking = True + + text = self.speech_queue.pop(0) + + try: + if self.audio_method == 'pyttsx3': + self.engine.stop() + self.engine.say(text) + self.engine.runAndWait() + + elif self.audio_method.startswith('gtts'): + tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') + + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: + temp_file = f.name + tts.save(temp_file) + + self.play_audio_file(temp_file) + + try: + os.unlink(temp_file) + except: + pass + + except Exception as e: + print(f"⚠ 语音播放失败: {e}") + + time.sleep(0.05) + + self.is_speaking = False + self.queue_thread = None + + def speak_direct(self, text): + """直接播放文本""" + if not self.enabled: + return + + thread = threading.Thread(target=self._speak_thread, args=(text,)) + thread.daemon = True + thread.start() + + def _speak_thread(self, text): + """语音播放线程""" + try: + if self.audio_method == 'pyttsx3': + self.engine.say(text) + self.engine.runAndWait() + + elif self.audio_method.startswith('gtts'): + tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') + + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: + temp_file = f.name + tts.save(temp_file) + + self.play_audio_file(temp_file) + + try: + os.unlink(temp_file) + except: + pass + + except Exception as e: + print(f"⚠ 直接语音播放失败: {e}") + + def stop(self): + """停止所有语音""" + if hasattr(self, 'engine'): + self.engine.stop() + + self.speech_queue.clear() + self.is_speaking = False + + def set_enabled(self, enabled): + """启用/禁用语音""" + self.enabled = enabled + if not enabled: + self.stop() + + def toggle_enabled(self): + """切换语音启用状态""" + self.enabled = not self.enabled + status = "启用" if self.enabled else "禁用" + self.speak_direct(f"语音反馈已{status}") + return self.enabled + + def get_status(self): + """获取语音状态""" + return { + 'enabled': self.enabled, + 'engine': 'pyttsx3' if self.audio_method == 'pyttsx3' else + 'gTTS' if self.audio_method.startswith('gtts') else + 'None', + 'queue_size': len(self.speech_queue), + 'is_speaking': self.is_speaking, + 'audio_method': self.audio_method + } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/trajectory_recorder.py b/src/autonomus_drone_hand_gesture_project/trajectory_recorder.py new file mode 100644 index 0000000000..ffeabf09d5 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/trajectory_recorder.py @@ -0,0 +1,347 @@ +""" +手势轨迹记录器模块 +负责录制、保存、加载和回放手势轨迹 +作者: xiaoshiyuan888 +""" + +import os +import time +import pickle +import cv2 +from datetime import datetime + + +class GestureTrajectoryRecorder: + """手势轨迹记录器 - 记录、保存、加载和回放手势轨迹""" + + def __init__(self, speech_manager=None, config=None): + self.speech_manager = speech_manager + self.config = config + self.trajectory_data = [] + self.is_recording = False + self.is_playing = False + self.playback_index = 0 + self.playback_paused = False + self.max_trajectory_points = 1000 + self.recording_start_time = 0 + self.last_save_time = 0 + self.save_interval = 5 + + # 轨迹文件路径 + current_dir = os.path.dirname(os.path.abspath(__file__)) + self.trajectory_dir = os.path.join(current_dir, 'trajectories') + if not os.path.exists(self.trajectory_dir): + os.makedirs(self.trajectory_dir) + + # 默认轨迹文件名 + self.default_filename = os.path.join(self.trajectory_dir, + f'trajectory_{datetime.now().strftime("%Y%m%d_%H%M%S")}.pkl') + + # 轨迹可视化设置 + self.trajectory_colors = [ + (255, 0, 0), # 红色 - 起点 + (0, 255, 0), # 绿色 - 正常点 + (0, 0, 255), # 蓝色 - 终点 + (255, 255, 0), # 青色 - 特殊点 + (255, 0, 255) # 紫色 - 特殊点 + ] + + self.show_trajectory = True + self.trajectory_thickness = 2 + self.trajectory_max_length = 100 + + print("✓ 手势轨迹记录器已初始化") + + def start_recording(self): + """开始录制手势轨迹""" + if self.is_recording: + return False + + self.trajectory_data = [] + self.is_recording = True + self.recording_start_time = time.time() + self.last_save_time = time.time() + + print("🎬 开始录制手势轨迹") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_start', immediate=True) + + return True + + def stop_recording(self): + """停止录制手势轨迹""" + if not self.is_recording: + return False + + self.is_recording = False + recording_duration = time.time() - self.recording_start_time + + print(f"⏹️ 停止录制手势轨迹") + print(f" 录制时长: {recording_duration:.1f}秒") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_stop', immediate=True) + if len(self.trajectory_data) > 0: + self.speech_manager.speak_direct(f"录制了{len(self.trajectory_data)}个轨迹点") + + return True + + def add_trajectory_point(self, hand_data, gesture, confidence, frame_shape): + """添加轨迹点""" + if not self.is_recording or len(self.trajectory_data) >= self.max_trajectory_points: + return False + + if hand_data is None: + return False + + # 创建轨迹点数据 + trajectory_point = { + 'timestamp': time.time(), + 'hand_position': hand_data['position'] if 'position' in hand_data else (0.5, 0.5), + 'hand_center': hand_data['center'] if 'center' in hand_data else (0, 0), + 'gesture': gesture, + 'confidence': confidence, + 'fingertips': hand_data.get('fingertips', []), + 'frame_shape': frame_shape + } + + self.trajectory_data.append(trajectory_point) + + # 自动保存检查 + current_time = time.time() + if current_time - self.last_save_time >= self.save_interval and len(self.trajectory_data) > 10: + self.auto_save() + self.last_save_time = current_time + + return True + + def auto_save(self): + """自动保存轨迹(临时文件)""" + if len(self.trajectory_data) == 0: + return + + temp_file = os.path.join(self.trajectory_dir, 'trajectory_temp.pkl') + try: + with open(temp_file, 'wb') as f: + pickle.dump(self.trajectory_data, f) + print(f"💾 自动保存轨迹到临时文件 ({len(self.trajectory_data)}个点)") + except Exception as e: + print(f"⚠ 自动保存轨迹失败: {e}") + + def save_trajectory(self, filename=None): + """保存轨迹到文件""" + if len(self.trajectory_data) == 0: + print("⚠ 没有轨迹数据可保存") + return False + + if filename is None: + filename = self.default_filename + + try: + with open(filename, 'wb') as f: + pickle.dump(self.trajectory_data, f) + + print(f"💾 轨迹已保存到: {filename}") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_saved', immediate=True) + self.speech_manager.speak_direct(f"保存了{len(self.trajectory_data)}个轨迹点") + + return True + except Exception as e: + print(f"❌ 保存轨迹失败: {e}") + return False + + def load_trajectory(self, filename): + """从文件加载轨迹""" + try: + if not os.path.exists(filename): + print(f"❌ 轨迹文件不存在: {filename}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_not_found', immediate=True) + + return False + + with open(filename, 'rb') as f: + self.trajectory_data = pickle.load(f) + + print(f"📂 轨迹已从文件加载: {filename}") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_loaded', immediate=True) + self.speech_manager.speak_direct(f"加载了{len(self.trajectory_data)}个轨迹点") + + return True + except Exception as e: + print(f"❌ 加载轨迹失败: {e}") + return False + + def start_playback(self): + """开始回放轨迹""" + if len(self.trajectory_data) == 0: + print("⚠ 没有轨迹数据可回放") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_not_found', immediate=True) + + return False + + self.is_playing = True + self.playback_index = 0 + self.playback_paused = False + + print(f"▶️ 开始回放手势轨迹") + print(f" 总帧数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_playback_start', immediate=True) + + return True + + def stop_playback(self): + """停止回放轨迹""" + if not self.is_playing: + return False + + self.is_playing = False + self.playback_paused = False + + print("⏹️ 停止回放手势轨迹") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_playback_stop', immediate=True) + + return True + + def pause_playback(self): + """暂停/继续回放""" + self.playback_paused = not self.playback_paused + + status = "暂停" if self.playback_paused else "继续" + print(f"⏸️ 回放已{status}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + if self.playback_paused: + self.speech_manager.speak('recording_paused', immediate=True) + else: + self.speech_manager.speak('recording_resumed', immediate=True) + + return self.playback_paused + + def get_next_playback_point(self): + """获取下一个回放点""" + if not self.is_playing or self.playback_paused or len(self.trajectory_data) == 0: + return None + + if self.playback_index >= len(self.trajectory_data): + self.stop_playback() + return None + + point = self.trajectory_data[self.playback_index] + self.playback_index += 1 + + # 检查是否到达末尾 + if self.playback_index >= len(self.trajectory_data): + self.stop_playback() + + return point + + def clear_trajectory(self): + """清除轨迹数据""" + self.trajectory_data = [] + self.is_recording = False + self.is_playing = False + self.playback_index = 0 + + print("🗑️ 轨迹数据已清除") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_cleared', immediate=True) + + return True + + def draw_trajectory(self, frame): + """在帧上绘制轨迹""" + if not self.show_trajectory or len(self.trajectory_data) == 0: + return frame + + h, w = frame.shape[:2] + + # 限制显示的轨迹点数 + display_points = min(len(self.trajectory_data), self.trajectory_max_length) + start_idx = max(0, len(self.trajectory_data) - display_points) + + # 绘制轨迹线 + for i in range(start_idx, len(self.trajectory_data) - 1): + point1 = self.trajectory_data[i] + point2 = self.trajectory_data[i + 1] + + # 获取手部中心位置 + if 'hand_center' in point1 and 'hand_center' in point2: + x1, y1 = point1['hand_center'] + x2, y2 = point2['hand_center'] + + # 确保坐标在图像范围内 + x1 = max(0, min(w - 1, x1)) + y1 = max(0, min(h - 1, y1)) + x2 = max(0, min(w - 1, x2)) + y2 = max(0, min(h - 1, y2)) + + # 根据索引计算颜色 + color_idx = int((i - start_idx) / display_points * (len(self.trajectory_colors) - 1)) + color = self.trajectory_colors[color_idx] + + # 绘制线条 + cv2.line(frame, (int(x1), int(y1)), (int(x2), int(y2)), + color, self.trajectory_thickness) + + # 绘制当前点 + if self.is_recording or self.is_playing: + current_idx = len(self.trajectory_data) - 1 if self.is_recording else self.playback_index - 1 + if 0 <= current_idx < len(self.trajectory_data): + point = self.trajectory_data[current_idx] + if 'hand_center' in point: + x, y = point['hand_center'] + x = max(0, min(w - 1, x)) + y = max(0, min(h - 1, y)) + + # 绘制当前点 + cv2.circle(frame, (int(x), int(y)), 8, (0, 255, 255), -1) + cv2.circle(frame, (int(x), int(y)), 8, (0, 0, 0), 2) + + return frame + + def get_status(self): + """获取录制状态""" + return { + 'is_recording': self.is_recording, + 'is_playing': self.is_playing, + 'playback_paused': self.playback_paused, + 'trajectory_points': len(self.trajectory_data), + 'playback_index': self.playback_index, + 'playback_total': len(self.trajectory_data), + 'recording_duration': time.time() - self.recording_start_time if self.is_recording else 0 + } + + def list_saved_trajectories(self): + """列出保存的轨迹文件""" + try: + files = [f for f in os.listdir(self.trajectory_dir) if f.endswith('.pkl')] + return sorted(files, reverse=True) + except: + return [] \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/ui_renderer.py b/src/autonomus_drone_hand_gesture_project/ui_renderer.py new file mode 100644 index 0000000000..bd2e396fc4 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/ui_renderer.py @@ -0,0 +1,350 @@ +""" +用户界面渲染器模块 +负责绘制中文界面和可视化元素 +作者: xiaoshiyuan888 +""" + +import cv2 +import numpy as np +from PIL import Image, ImageDraw, ImageFont + + +class ChineseUIRenderer: + """中文UI渲染器""" + + def __init__(self, speech_manager=None, config=None): + self.fonts = {} + self.speech_manager = speech_manager + self.config = config + self.load_fonts() + + # 颜色定义 + self.colors = { + 'title': (0, 255, 255), + 'connected': (0, 255, 0), + 'disconnected': (0, 0, 255), + 'flying': (0, 255, 0), + 'landed': (255, 165, 0), + 'warning': (0, 165, 255), + 'info': (255, 255, 255), + 'help': (255, 200, 100), + 'speech_enabled': (0, 255, 0), + 'speech_disabled': (255, 0, 0), + 'performance_good': (0, 255, 0), + 'performance_warning': (255, 165, 0), + 'performance_bad': (255, 0, 0), + 'recording': (255, 50, 50), + 'playback': (50, 50, 255), + 'paused': (255, 255, 0), + 'performance_fast': (0, 255, 0), + 'performance_balanced': (255, 165, 0), + 'performance_accurate': (255, 0, 0), + 'cpu_good': (0, 255, 0), + 'cpu_warning': (255, 165, 0), + 'cpu_critical': (255, 0, 0), + 'memory_good': (0, 255, 0), + 'memory_warning': (255, 165, 0), + 'memory_critical': (255, 0, 0), + } + + print("✓ 中文UI渲染器已初始化") + + def load_fonts(self): + """加载字体""" + font_paths = [ + 'simhei.ttf', + 'C:/Windows/Fonts/simhei.ttf', + 'C:/Windows/Fonts/msyh.ttc', + '/System/Library/Fonts/PingFang.ttc', + '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' + ] + + for path in font_paths: + try: + self.fonts[14] = ImageFont.truetype(path, 14) + self.fonts[16] = ImageFont.truetype(path, 16) + self.fonts[18] = ImageFont.truetype(path, 18) + self.fonts[20] = ImageFont.truetype(path, 20) + self.fonts[24] = ImageFont.truetype(path, 24) + print(f"✓ 字体已加载: {path}") + return + except: + continue + + print("⚠ 未找到字体,使用默认") + + def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): + """在图像上绘制文本""" + try: + img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + pil_img = Image.fromarray(img_rgb) + draw = ImageDraw.Draw(pil_img) + + font = self.fonts.get(size, self.fonts.get(16)) + + # 绘制阴影 + shadow_color = (0, 0, 0) + shadow_pos = (pos[0] + 1, pos[1] + 1) + draw.text(shadow_pos, text, font=font, fill=shadow_color) + + # 绘制文字 + rgb_color = color[::-1] # BGR to RGB + draw.text(pos, text, font=font, fill=rgb_color) + + return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) + except: + # 备用方案 + cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, + size / 25, color, 1) + return frame + + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, + trajectory_recorder=None, gesture_recognizer=None, performance_analyzer=None): + """绘制状态栏(增强版)""" + h, w = frame.shape[:2] + + # 绘制半透明背景 + overlay = frame.copy() + cv2.rectangle(overlay, (0, 0), (w, 130), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + + # 标题 + title = "手势控制无人机系统 - 性能统计版" + frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) + + # 连接状态 + status_color = self.colors['connected'] if drone_controller.connected else self.colors['disconnected'] + status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" + frame = self.draw_text(frame, status_text, (10, 40), size=16, color=status_color) + + # 飞行状态 + flight_color = self.colors['flying'] if drone_controller.flying else self.colors['landed'] + flight_text = f"飞行状态: {'飞行中' if drone_controller.flying else '已降落'}" + frame = self.draw_text(frame, flight_text, (10, 65), size=16, color=flight_color) + + # 手势信息 + if confidence > 0.7: + gesture_color = (0, 255, 0) + elif confidence > 0.5: + gesture_color = (255, 165, 0) + else: + gesture_color = (200, 200, 200) + + gesture_text = f"当前手势: {gesture}" + if self.config.get('display', 'show_confidence'): + gesture_text += f" ({confidence:.0%})" + + frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) + + # 性能模式显示 + if gesture_recognizer and self.config.get('display', 'show_performance_mode'): + mode_info = gesture_recognizer.get_performance_mode_info() + + # 根据模式选择颜色 + if mode_info['mode'] == 'fast': + mode_color = self.colors['performance_fast'] + elif mode_info['mode'] == 'balanced': + mode_color = self.colors['performance_balanced'] + else: # accurate + mode_color = self.colors['performance_accurate'] + + mode_text = f"性能模式: {mode_info['name']}" + frame = self.draw_text(frame, mode_text, (w // 2, 65), size=14, color=mode_color) + + # 录制/回放状态 + elif trajectory_recorder and self.config.get('display', 'show_recording_status'): + recorder_status = trajectory_recorder.get_status() + + if recorder_status['is_recording']: + status_color = self.colors['recording'] + status_text = f"录制中: {recorder_status['trajectory_points']}点" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) + elif recorder_status['is_playing']: + if recorder_status['playback_paused']: + status_color = self.colors['paused'] + status_text = f"回放暂停: {recorder_status['playback_index']}/{recorder_status['playback_total']}" + else: + status_color = self.colors['playback'] + status_text = f"回放中: {recorder_status['playback_index']}/{recorder_status['playback_total']}" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) + else: + status_text = f"轨迹点: {recorder_status['trajectory_points']}" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=12, color=self.colors['info']) + + # 性能统计显示 + if performance_analyzer and self.config.get('display', 'show_performance_stats'): + perf_stats = performance_analyzer.get_stats_summary() + + # 性能状态颜色 + if perf_stats['performance_status'] == "良好": + perf_color = self.colors['performance_good'] + elif perf_stats['performance_status'] == "警告": + perf_color = self.colors['performance_warning'] + else: + perf_color = self.colors['performance_bad'] + + perf_text = f"性能: {perf_stats['performance_status']} ({perf_stats['fps']:.0f}FPS)" + frame = self.draw_text(frame, perf_text, (w // 2, 90), size=12, color=perf_color) + + # 系统资源显示 + if performance_analyzer and self.config.get('display', 'show_system_resources'): + cpu_usage = performance_analyzer.get_current_cpu_usage() + memory_usage = performance_analyzer.get_current_memory_usage() + + # CPU颜色 + if cpu_usage < 70: + cpu_color = self.colors['cpu_good'] + elif cpu_usage < 85: + cpu_color = self.colors['cpu_warning'] + else: + cpu_color = self.colors['cpu_critical'] + + # 内存颜色 + if memory_usage < 70: + mem_color = self.colors['memory_good'] + elif memory_usage < 85: + mem_color = self.colors['memory_warning'] + else: + mem_color = self.colors['memory_critical'] + + # 绘制系统资源信息 + sys_text = f"CPU:{cpu_usage:.0f}% 内存:{memory_usage:.0f}%" + frame = self.draw_text(frame, sys_text, (w // 2, 110), size=11, color=self.colors['info']) + + # 语音状态 + if self.config.get('display', 'show_speech_status') and self.speech_manager: + speech_status = self.speech_manager.get_status() + speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] + speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" + frame = self.draw_text(frame, speech_text, (w - 200, 90), size=12, color=speech_color) + + # 性能信息 + if self.config.get('display', 'show_fps'): + perf_text = f"帧率: {fps:.1f}" + if process_time > 0: + perf_text += f" | 延迟: {process_time:.1f}ms" + + # 根据处理时间选择颜色 + if process_time < 20: + perf_color = self.colors['performance_good'] + elif process_time < 50: + perf_color = self.colors['performance_warning'] + else: + perf_color = self.colors['performance_bad'] + else: + perf_color = self.colors['info'] + + frame = self.draw_text(frame, perf_text, (w - 200, 65), size=12, color=perf_color) + + # 控制提示 + control_text = "提示: 确保手部完全进入画面,保持稳定手势" + frame = self.draw_text(frame, control_text, (10, 90), size=12, color=self.colors['info']) + + # 性能快捷键提示 + if performance_analyzer: + perf_tip = "P:报告 L:快照 K:导出" + frame = self.draw_text(frame, perf_tip, (10, 110), size=11, color=self.colors['help']) + + return frame + + def draw_help_bar(self, frame): + """绘制帮助栏""" + if not self.config.get('display', 'show_help'): + return frame + + h, w = frame.shape[:2] + + # 绘制底部帮助栏 + cv2.rectangle(frame, (0, h - 120), (w, h), (0, 0, 0), -1) + + # 帮助文本 + help_lines = [ + "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", + "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", + "V:切换语音反馈 M:测试语音 P:性能报告 L:性能快照 K:导出日志", + "O:切换性能模式 P:详细报告 R:重置统计", + "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" + ] + + for i, line in enumerate(help_lines): + y_pos = h - 105 + i * 20 + frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) + + return frame + + def draw_warning(self, frame, message): + """绘制警告信息""" + h, w = frame.shape[:2] + + # 在顶部绘制警告 + warning_bg = np.zeros((40, w, 3), dtype=np.uint8) + warning_bg[:, :] = (0, 69, 255) + + frame[130:170, 0:w] = cv2.addWeighted( + frame[130:170, 0:w], 0.3, + warning_bg, 0.7, 0 + ) + + # 绘制警告文本 + frame = self.draw_text(frame, message, (10, 145), + size=16, color=self.colors['warning']) + + return frame + + def draw_performance_chart(self, frame, performance_analyzer): + """绘制性能图表""" + if not performance_analyzer: + return frame + + h, w = frame.shape[:2] + + # 绘制图表区域背景 + chart_x = w - 250 + chart_y = 130 + chart_width = 240 + chart_height = 150 + + cv2.rectangle(frame, (chart_x, chart_y), + (chart_x + chart_width, chart_y + chart_height), + (30, 30, 30), -1) + cv2.rectangle(frame, (chart_x, chart_y), + (chart_x + chart_width, chart_y + chart_height), + (100, 100, 100), 1) + + # 绘制标题 + frame = self.draw_text(frame, "性能监控", (chart_x + 10, chart_y + 20), + size=14, color=self.colors['title']) + + # 绘制FPS图表 + if len(performance_analyzer.fps_history) > 1: + fps_values = list(performance_analyzer.fps_history)[-20:] # 最近20个值 + + # 计算最大值和最小值 + if fps_values: + max_fps = max(fps_values) + min_fps = min(fps_values) + if max_fps == min_fps: + max_fps = min_fps + 1 + + # 绘制FPS曲线 + points = [] + for i, fps in enumerate(fps_values): + x = chart_x + 10 + i * (chart_width - 20) / len(fps_values) + y = chart_y + chart_height - 30 - (fps - min_fps) * (chart_height - 50) / (max_fps - min_fps) + points.append((int(x), int(y))) + + # 连接点形成曲线 + for i in range(len(points) - 1): + cv2.line(frame, points[i], points[i + 1], (0, 255, 0), 2) + + # 显示当前FPS值 + current_fps = fps_values[-1] if fps_values else 0 + fps_color = self.colors['performance_good'] if current_fps > 20 else \ + self.colors['performance_warning'] if current_fps > 10 else \ + self.colors['performance_bad'] + + frame = self.draw_text(frame, f"FPS: {current_fps:.1f}", + (chart_x + 120, chart_y + 40), + size=12, color=fps_color) + + return frame \ No newline at end of file From fb9744bab5ea4cee6005d9376b3ea6fff45006cd Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sun, 21 Dec 2025 20:47:37 +0800 Subject: [PATCH 16/18] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=88=91=E7=9A=84READM?= =?UTF-8?q?E=E6=96=87=E4=BB=B6=E5=92=8C=E4=BE=9D=E8=B5=96=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../README.md | 84 +++++++++++++------ .../requirements.txt | 16 ++-- 2 files changed, 69 insertions(+), 31 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/README.md b/src/autonomus_drone_hand_gesture_project/README.md index 0b5c35f915..45f02648d4 100644 --- a/src/autonomus_drone_hand_gesture_project/README.md +++ b/src/autonomus_drone_hand_gesture_project/README.md @@ -1,40 +1,74 @@ # 手势控制AirSim无人机系统 -[项目简介] -基于计算机视觉和语音反馈技术的手势控制无人机系统。使用纯OpenCV实现手部检测和手势识别,结合语音反馈系统,提供直观的无人机控制体验。系统支持AirSim仿真环境和真实无人机控制,具备手势平滑处理、状态追踪和多模式显示功能。 +## 项目简介 -[核心功能] +基于计算机视觉和语音反馈技术的手势控制无人机系统。使用纯OpenCV实现手部检测和手势识别,结合语音反馈系统,提供直观的无人机控制体验。系统支持AirSim仿真环境和真实无人机控制,具备手势平滑处理、状态追踪、性能监控和轨迹记录等完整功能。 -1. 手势识别与控制 - - 纯OpenCV实现:不依赖深度学习框架,使用肤色检测、轮廓分析和凸缺陷检测识别手势 - - 多种手势支持:支持停止、前进、后退、上升、下降、左转、右转等多种控制手势 - - 实时手势追踪:跟踪手部位置和状态,提供连续控制体验 +## 核心功能 -2. 智能语音反馈系统 - - 多引擎支持:兼容pyttsx3(离线)和gTTS(在线)语音引擎 - - 丰富语音提示:提供连接状态、飞行状态、手势识别、系统状态等多种语音反馈 - - 智能语音队列:避免语音重叠,支持优先级播放和即时播放 +### 1. 增强手势识别与控制 +- **纯OpenCV实现**:不依赖深度学习框架,使用肤色检测、轮廓分析和凸缺陷检测识别手势 +- **多种手势支持**:支持停止、前进、上升、下降、左转、右转、悬停等多种控制手势 +- **智能手势平滑**:使用手势稳定性检测和位置平滑算法,减少误判和抖动 +- **实时状态追踪**:跟踪手势生命周期(开始/活跃/结束),提供精准控制反馈 -3. 增强手势平滑处理 - - 手势状态追踪:实现手势状态机(none/starting/active/ending),跟踪手势生命周期 - - 稳定性检测:使用手势稳定性计数器和位置稳定性权重,减少误判 - - 冷却时间机制:防止手势切换过于频繁,提高控制精度 +### 2. 智能语音反馈系统 +- **多引擎支持**:兼容pyttsx3(离线)和gTTS(在线)语音引擎 +- **丰富语音提示**:提供连接状态、飞行状态、手势识别、系统状态等多种语音反馈 +- **智能语音队列**:避免语音重叠,支持优先级播放和即时播放 +- **中文语音优化**:自动检测并优先使用中文语音引擎 -4. 多平台兼容性 - - 跨平台音频播放:支持pygame、pydub和系统默认播放器 - - 自动库检测:自动检测可用库并选择最优方案 - - 详细错误处理:完善的错误处理和降级方案 +### 3. 详细性能监控与分析 +- **实时性能监控**:监控帧率、CPU使用率、内存使用率、手势识别时间 +- **智能性能报告**:生成详细性能报告,包括手势统计、无人机命令成功率等 +- **性能阈值警告**:自动检测性能问题并提供优化建议 +- **日志记录导出**:支持导出CSV格式性能日志,便于分析和优化 + +### 4. 多模式性能优化 +- **三种性能模式**: + - **最快模式**:性能优先,降低识别精度换取更高帧率 + - **平衡模式**:平衡性能与精度,适用于大多数场景 + - **最准模式**:精度优先,提供最准确的手势识别 +- **动态参数调整**:根据不同模式自动调整检测间隔、图像缩放等参数 + +### 5. 手势轨迹记录与回放 +- **轨迹录制**:实时记录手势轨迹,支持保存到文件 +- **轨迹回放**:加载并回放录制的手势轨迹 +- **轨迹可视化**:在视频画面上实时显示手势轨迹 +- **轨迹管理**:支持开始/停止录制、保存/加载轨迹、暂停/继续回放 + +### 6. 高级用户界面 +- **中文界面**:完整的中文用户界面,包括状态栏、帮助栏和警告信息 +- **多显示模式**:支持普通、详细和最小化三种显示模式 +- **性能图表**:实时显示FPS性能图表 +- **调试信息**:可选的详细调试信息显示 + +### 7. AirSim无人机控制 +- **模拟与真实控制**:支持AirSim模拟器和真实无人机控制 +- **安全控制逻辑**:包含手势冷却时间、控制间隔等安全机制 +- **紧急停止**:一键紧急停止和降落功能 +- **连接管理**:自动检测AirSim连接状态,支持模拟模式 + +## 环境依赖 -[环境依赖] -在运行代码之前,请确保已安装以下Python库: ```bash -# 基础依赖 -pip install opencv-python pillow numpy +# 基础库 +pip install opencv-python>=4.5.0 +pip install numpy>=1.19.0 +pip install Pillow>=8.0.0 + +# 可选但推荐 +pip install psutil>=5.8.0 # 系统资源监控 # AirSim无人机控制(可选) pip install airsim +pip install msgpack-rpc-python>=0.4.0 # AirSim依赖 # 语音反馈系统(选择安装) -pip install pyttsx3 # 离线语音(推荐) -pip install gtts pygame # 在线语音(需要网络) \ No newline at end of file +pip install pyttsx3>=2.90 # 离线语音(推荐) +pip install gtts>=2.2.0 +pip install pygame>=2.0.0 # 在线语音(需要网络) + +# 完整安装(推荐) +pip install -r requirements.txt \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/requirements.txt b/src/autonomus_drone_hand_gesture_project/requirements.txt index 2d313aa3a7..1921bea688 100644 --- a/src/autonomus_drone_hand_gesture_project/requirements.txt +++ b/src/autonomus_drone_hand_gesture_project/requirements.txt @@ -1,6 +1,10 @@ -mediapipe==0.10.7 -opencv-python==4.8.1.78 -numpy==1.24.4 -tensorflow==2.14.0 -pandas==2.1.4 -pillow==10.1.0 \ No newline at end of file +opencv-python>=4.5.0 +numpy>=1.19.0 +Pillow>=8.0.0 +psutil>=5.8.0 +pyttsx3>=2.90 +gtts>=2.2.0 +pygame>=2.0.0 +pydub>=0.25.1 +airsim>=1.7.0 +msgpack-rpc-python>=0.4.0 \ No newline at end of file From 4e9e03b429917b53a14a587ae34b6722477c799b Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sun, 21 Dec 2025 23:50:18 +0800 Subject: [PATCH 17/18] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=AB=98=E7=BA=A7?= =?UTF-8?q?=E6=89=8B=E5=8A=BF=E8=AF=86=E5=88=AB=E5=8A=9F=E8=83=BD=EF=BC=8C?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=97=8B=E8=BD=AC=E3=80=81=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E3=80=81=E6=8B=8D=E7=85=A7=E7=AD=89=E5=A4=8D=E6=9D=82=E6=89=8B?= =?UTF-8?q?=E5=8A=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../advanced_gesture_analyzer.py | 317 ++++++++++++++++++ .../config_manager.py | 1 + .../drone_controller.py | 50 +++ .../gesture_config.json | 19 +- .../gesture_recognizer.py | 16 +- .../main.py | 44 ++- .../performance_analyzer.py | 1 - .../speech_manager.py | 7 + .../ui_renderer.py | 7 + 9 files changed, 444 insertions(+), 18 deletions(-) create mode 100644 src/autonomus_drone_hand_gesture_project/advanced_gesture_analyzer.py diff --git a/src/autonomus_drone_hand_gesture_project/advanced_gesture_analyzer.py b/src/autonomus_drone_hand_gesture_project/advanced_gesture_analyzer.py new file mode 100644 index 0000000000..bf90090708 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/advanced_gesture_analyzer.py @@ -0,0 +1,317 @@ +""" +高级手势分析器模块 +负责分析复杂手势和提供额外的手势识别功能 +作者: xiaoshiyuan888 +""" + +import cv2 +import numpy as np +import time +import math +from collections import deque + + +class AdvancedGestureAnalyzer: + """高级手势分析器 - 识别复杂手势""" + + def __init__(self, speech_manager=None, config=None): + self.speech_manager = speech_manager + self.config = config + + # 手势轨迹追踪 + self.trajectory_history = deque(maxlen=30) + self.fingertip_history = deque(maxlen=20) + + # 手势状态 + self.rotation_direction = None # 旋转方向: 'cw' 顺时针, 'ccw' 逆时针 + self.grab_state = False # 抓取状态 + self.photo_state = False # 拍照状态 + + # 颜色定义 + self.colors = { + 'rotation_cw': (0, 255, 255), # 黄色 - 顺时针旋转 + 'rotation_ccw': (255, 255, 0), # 青色 - 逆时针旋转 + 'grab': (255, 0, 255), # 紫色 - 抓取 + 'release': (0, 255, 0), # 绿色 - 释放 + 'photo': (255, 165, 0), # 橙色 - 拍照 + 'auto_flight': (128, 0, 128), # 紫色 - 自动飞行 + 'return_home': (0, 128, 128), # 深青色 - 返航 + } + + # 语音消息映射 + self.messages = { + 'gesture_rotate_cw': "顺时针旋转", + 'gesture_rotate_ccw': "逆时针旋转", + 'gesture_grab': "抓取", + 'gesture_release': "释放", + 'gesture_photo': "拍照", + 'gesture_auto_flight': "自动飞行模式", + 'gesture_return_home': "返航", + } + + # 初始化语音消息 + if self.speech_manager: + self.speech_manager.messages.update(self.messages) + + print("✓ 高级手势分析器已初始化") + + def analyze(self, frame, hand_data): + """分析手势数据,识别复杂手势""" + if hand_data is None: + return None + + result = {} + + # 分析旋转手势 + rotation_result = self.detect_rotation(hand_data) + if rotation_result: + result.update(rotation_result) + + # 分析抓取手势 + grab_result = self.detect_grab(hand_data) + if grab_result: + result.update(grab_result) + + # 分析拍照手势 + photo_result = self.detect_photo_gesture(hand_data) + if photo_result: + result.update(photo_result) + + # 分析复杂手势 + complex_result = self.detect_complex_gestures(hand_data) + if complex_result: + result.update(complex_result) + + return result if result else None + + def detect_rotation(self, hand_data): + """检测旋转手势""" + if 'fingertips' not in hand_data or len(hand_data['fingertips']) < 1: + return None + + # 获取食指指尖 + index_fingertip = hand_data['fingertips'][0] if len(hand_data['fingertips']) > 0 else None + + if index_fingertip is None: + return None + + # 添加指尖到历史 + self.fingertip_history.append(index_fingertip) + + if len(self.fingertip_history) < 15: + return None + + # 分析指尖运动轨迹 + trajectory = list(self.fingertip_history) + + # 计算轨迹的中心点 + points = np.array(trajectory) + center = np.mean(points, axis=0) + + # 计算每个点相对于中心的角度 + angles = [] + for point in trajectory: + dx = point[0] - center[0] + dy = point[1] - center[1] + angle = math.atan2(dy, dx) + angles.append(angle) + + # 分析角度变化趋势 + if len(angles) >= 10: + # 计算角度变化量 + angle_changes = [] + for i in range(1, len(angles)): + change = angles[i] - angles[i-1] + # 处理角度跨越2π边界的情况 + if change > math.pi: + change -= 2 * math.pi + elif change < -math.pi: + change += 2 * math.pi + angle_changes.append(change) + + avg_change = np.mean(angle_changes) + + # 判断旋转方向 + if abs(avg_change) > 0.05: # 阈值 + if avg_change > 0: + self.rotation_direction = 'ccw' + return { + 'gesture': 'RotateCCW', + 'confidence': min(0.9, abs(avg_change) * 5), + 'direction': 'ccw', + 'angle_change': avg_change + } + else: + self.rotation_direction = 'cw' + return { + 'gesture': 'RotateCW', + 'confidence': min(0.9, abs(avg_change) * 5), + 'direction': 'cw', + 'angle_change': avg_change + } + + return None + + def detect_grab(self, hand_data): + """检测抓取手势(握拳)""" + if 'contour' not in hand_data or 'bbox' not in hand_data: + return None + + contour = hand_data['contour'] + bbox = hand_data['bbox'] + + # 计算轮廓面积和边界框面积 + contour_area = cv2.contourArea(contour) + x1, y1, x2, y2 = bbox + bbox_area = (x2 - x1) * (y2 - y1) + + if bbox_area == 0: + return None + + # 计算面积比(握拳时轮廓更接近边界框) + area_ratio = contour_area / bbox_area + + # 获取手指数量 + finger_count = len(hand_data.get('fingers', [])) + + # 握拳的特征:面积比高,手指数量少 + if area_ratio > 0.7 and finger_count <= 1: + # 检查是否之前不是抓取状态 + if not self.grab_state: + self.grab_state = True + return { + 'gesture': 'Grab', + 'confidence': min(0.95, area_ratio * 1.2), + 'state': 'grab' + } + else: + if self.grab_state: + self.grab_state = False + return { + 'gesture': 'Release', + 'confidence': 0.8, + 'state': 'release' + } + + return None + + def detect_photo_gesture(self, hand_data): + """检测拍照手势(OK手势)""" + if 'fingers' not in hand_data or 'fingertips' not in hand_data: + return None + + fingers = hand_data['fingers'] + fingertips = hand_data['fingertips'] + + # OK手势的特征:拇指和食指形成圆圈,其他手指伸直 + if len(fingers) == 3 and len(fingertips) >= 2: + # 检查拇指和食指的距离 + if len(fingertips) >= 2: + thumb_tip = fingertips[0] + index_tip = fingertips[1] + + distance = math.sqrt((thumb_tip[0] - index_tip[0])**2 + (thumb_tip[1] - index_tip[1])**2) + + # 如果拇指和食指距离较近,可能是OK手势 + if distance < 50: # 阈值 + if not self.photo_state: + self.photo_state = True + return { + 'gesture': 'TakePhoto', + 'confidence': max(0.7, 1.0 - distance/100), + 'distance': distance + } + + self.photo_state = False + return None + + def detect_complex_gestures(self, hand_data): + """检测复杂手势(返航、自动飞行等)""" + if 'fingers' not in hand_data or 'fingertips' not in hand_data: + return None + + finger_count = len(hand_data.get('fingers', [])) + fingertips = hand_data.get('fingertips', []) + + # 返航手势:五指张开然后缓慢握拳 + if finger_count == 5 and len(fingertips) == 5: + # 检查是否保持这个姿势一段时间 + if hasattr(self, 'five_finger_start_time'): + elapsed = time.time() - self.five_finger_start_time + if elapsed > 2.0: # 保持2秒 + delattr(self, 'five_finger_start_time') + return { + 'gesture': 'ReturnHome', + 'confidence': 0.85, + 'hold_time': elapsed + } + else: + self.five_finger_start_time = time.time() + elif hasattr(self, 'five_finger_start_time'): + delattr(self, 'five_finger_start_time') + + # 自动飞行手势:特定手指组合 + if finger_count == 4 and len(fingertips) == 4: + # 检查是否为特定的手指组合(例如,缺少小指) + return { + 'gesture': 'AutoFlight', + 'confidence': 0.75, + 'finger_pattern': 'four_fingers' + } + + return None + + def draw_gesture_info(self, frame): + """在帧上绘制手势信息""" + h, w = frame.shape[:2] + + # 绘制旋转指示器 + if self.rotation_direction: + center_x, center_y = w - 80, 100 + radius = 30 + + if self.rotation_direction == 'cw': + color = self.colors['rotation_cw'] + # 绘制顺时针箭头 + cv2.circle(frame, (center_x, center_y), radius, color, 2) + cv2.putText(frame, "CW", (center_x - 15, center_y + 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + else: + color = self.colors['rotation_ccw'] + # 绘制逆时针箭头 + cv2.circle(frame, (center_x, center_y), radius, color, 2) + cv2.putText(frame, "CCW", (center_x - 20, center_y + 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + + # 绘制抓取状态 + if self.grab_state: + grab_x, grab_y = w - 80, 150 + color = self.colors['grab'] + cv2.rectangle(frame, (grab_x - 20, grab_y - 20), + (grab_x + 20, grab_y + 20), color, 2) + cv2.putText(frame, "抓取", (grab_x - 20, grab_y - 25), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + + # 绘制指尖轨迹 + if len(self.fingertip_history) > 1: + trajectory_color = (0, 255, 255) # 黄色 + for i in range(1, len(self.fingertip_history)): + pt1 = self.fingertip_history[i-1] + pt2 = self.fingertip_history[i] + cv2.line(frame, pt1, pt2, trajectory_color, 2) + + # 绘制当前指尖 + if self.fingertip_history: + last_point = self.fingertip_history[-1] + cv2.circle(frame, last_point, 5, (0, 0, 255), -1) + + return frame + + def get_gesture_info(self): + """获取手势信息""" + return { + 'rotation_direction': self.rotation_direction, + 'grab_state': self.grab_state, + 'photo_state': self.photo_state, + 'trajectory_length': len(self.fingertip_history) + } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/config_manager.py b/src/autonomus_drone_hand_gesture_project/config_manager.py index bfd2c46f73..7c6baff3bb 100644 --- a/src/autonomus_drone_hand_gesture_project/config_manager.py +++ b/src/autonomus_drone_hand_gesture_project/config_manager.py @@ -111,6 +111,7 @@ def __init__(self): 'show_performance_mode': True, 'show_performance_stats': True, 'show_system_resources': True, + 'show_advanced_gestures': True, # 新增:显示高级手势信息 }, 'performance': { 'target_fps': 30, diff --git a/src/autonomus_drone_hand_gesture_project/drone_controller.py b/src/autonomus_drone_hand_gesture_project/drone_controller.py index 17e8dfea36..2f1edeadb0 100644 --- a/src/autonomus_drone_hand_gesture_project/drone_controller.py +++ b/src/autonomus_drone_hand_gesture_project/drone_controller.py @@ -248,6 +248,56 @@ def move_by_gesture(self, gesture, confidence): if (self.speech_manager and self.speech_manager.enabled): self.speech_manager.speak('hovering') + elif gesture == "Grab": + # 抓取动作(模拟) + print("执行抓取动作") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_grab') + elif gesture == "Release": + # 释放动作(模拟) + print("执行释放动作") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_release') + elif gesture == "RotateCW": + # 顺时针旋转 + print("顺时针旋转") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_rotate_cw') + elif gesture == "RotateCCW": + # 逆时针旋转 + print("逆时针旋转") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_rotate_ccw') + elif gesture == "TakePhoto": + # 拍照/截图 + print("执行拍照") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_photo') + elif gesture == "ReturnHome": + # 返航 + print("执行返航") + self.client.moveToPositionAsync(0, 0, self.altitude, 5) + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_return_home') + elif gesture == "AutoFlight": + # 自动飞行模式 + print("启动自动飞行模式") + success = True + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('gesture_auto_flight') if success: self.last_control_time = current_time diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index fbb45391da..0a14cccb19 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -18,11 +18,11 @@ }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 170, + "skin_upper_h": 180, "skin_lower_s": 0, - "skin_upper_s": 70, - "skin_lower_v": 0, - "skin_upper_v": 75, + "skin_upper_s": 127, + "skin_lower_v": 22, + "skin_upper_v": 178, "min_hand_area": 3000, "max_hand_area": 30000, "hand_ratio_threshold": 1.5, @@ -79,7 +79,7 @@ "show_fingertips": true, "show_palm_center": true, "show_hand_direction": true, - "show_debug_info": false, + "show_debug_info": true, "show_speech_status": true, "show_gesture_history": true, "show_stability_indicator": true, @@ -88,6 +88,9 @@ "show_performance_mode": true, "show_performance_stats": true, "show_system_resources": true, + "show_advanced_gestures": true, + "show_hotkey_hints": true, + "show_operation_hints": true, "font_size_large": 20, "font_size_medium": 16, "font_size_small": 14, @@ -168,6 +171,12 @@ "loop_playback": false, "playback_speed": 1.0 }, + "hotkeys": { + "quick_save_enabled": true, + "quick_load_enabled": true, + "hint_duration": 2.0, + "hint_fade_speed": 0.05 + }, "drone_control": { "altitude": -10.0, "base_speed": 2.0, diff --git a/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py b/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py index ff37f66b83..7f7c93840c 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py +++ b/src/autonomus_drone_hand_gesture_project/gesture_recognizer.py @@ -72,7 +72,14 @@ def __init__(self, speech_manager=None, config=None): "Right": (0, 165, 255), "Waiting": (200, 200, 200), "Error": (255, 0, 0), - "Hover": (255, 255, 255) + "Hover": (255, 255, 255), + "Grab": (255, 0, 255), # 新增:紫色 + "Release": (0, 255, 0), # 新增:绿色 + "RotateCW": (0, 255, 255), # 新增:黄色 + "RotateCCW": (255, 255, 0), # 新增:青色 + "TakePhoto": (255, 165, 0), # 新增:橙色 + "ReturnHome": (0, 128, 128), # 新增:深青色 + "AutoFlight": (128, 0, 128), # 新增:紫色 } # 手势到语音的映射 @@ -86,6 +93,13 @@ def __init__(self, speech_manager=None, config=None): "Waiting": "gesture_waiting", "Error": "gesture_error", "Hover": "gesture_hover", + "Grab": "gesture_grab", # 新增 + "Release": "gesture_release", # 新增 + "RotateCW": "gesture_rotate_cw", # 新增 + "RotateCCW": "gesture_rotate_ccw", # 新增 + "TakePhoto": "gesture_photo", # 新增 + "ReturnHome": "gesture_return_home", # 新增 + "AutoFlight": "gesture_auto_flight", # 新增 } # 手势状态颜色 diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 0a63edfb6c..a04228c60f 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -24,6 +24,7 @@ from drone_controller import SimpleDroneController from ui_renderer import ChineseUIRenderer from trajectory_recorder import GestureTrajectoryRecorder +from advanced_gesture_analyzer import AdvancedGestureAnalyzer # 安全导入所需库 @@ -156,11 +157,11 @@ def init_camera(config): def print_welcome_message(cap, speech_manager, libs): """打印欢迎信息""" print("\n" + "=" * 80) - print("手势控制无人机系统 - 详细性能统计版") + print("手势控制无人机系统 - 增强手势识别版") print("=" * 80) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") - print(f" 手势识别: 增强的平滑处理算法") + print(f" 手势识别: 增强的平滑处理算法 + 新手势支持") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") print(f" 性能监控: 已启用") print(f" 轨迹记录: 支持录制/回放功能") @@ -181,15 +182,15 @@ def print_instructions(): print(" - 系统自动监控: FPS, CPU, 内存, 识别时间等") print("4. 性能模式选择:") print(" - 按 [O] 键循环切换性能模式: 最快(fast) → 平衡(balanced) → 最准(accurate)") - print("5. 手势控制:") - print(" - 手势识别后会有语音提示: 向上、向下、向左、向右、向前、停止") - print(" - 手势稳定性越高,识别越准确") - print(" - 手部距离摄像头适中时效果最佳") - print(" * 手势识别置信度 > 60% 时才会执行") + print("5. 新手势控制:") + print(" - 握拳手势: 抓取/释放物体 (模拟)") + print(" - 旋转手势: 顺时针旋转 (模拟)") + print(" - OK手势: 拍照/截图") + print(" - 复杂手势: 返航、自动飞行等") print("6. 轨迹记录功能:") print(" [1]开始录制 [2]停止录制 [3]保存轨迹 [4]回放轨迹 [5]清除轨迹 [6]暂停/继续") print("7. 键盘控制:") - print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止 [H]悬停 [G]返航") print("8. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") print("9. 语音控制:") @@ -203,8 +204,7 @@ def print_instructions(): def main(): """主函数""" print("=" * 60) - print("手势控制无人机系统 - 详细性能统计优化版") - print("详细性能统计优化版!") + print("手势控制无人机系统 - 增强手势识别版") print("=" * 60) # 导入库 @@ -244,6 +244,10 @@ def main(): print("初始化手势轨迹记录器...") trajectory_recorder = GestureTrajectoryRecorder(speech_manager, config) + # 高级手势分析器 + print("初始化高级手势分析器...") + advanced_gesture_analyzer = AdvancedGestureAnalyzer(speech_manager, config) + # 初始化摄像头 cap = init_camera(config) if cap and speech_manager.enabled: @@ -266,6 +270,11 @@ def main(): ord('f'): "Forward", ord('F'): "Forward", ord('x'): "Stop", ord('X'): "Stop", ord('h'): "Hover", ord('H'): "Hover", + ord('g'): "ReturnHome", ord('G'): "ReturnHome", + ord('b'): "AutoFlight", ord('B'): "AutoFlight", + ord('p'): "TakePhoto", ord('P'): "TakePhoto", + ord('r'): "RotateCW", ord('R'): "RotateCW", + ord('l'): "RotateCCW", ord('L'): "RotateCCW", } # 显示模式 @@ -293,6 +302,12 @@ def main(): # 手势识别 gesture, confidence, frame = gesture_recognizer.recognize(frame) + # 使用高级手势分析器进行补充分析 + advanced_result = advanced_gesture_analyzer.analyze(frame, gesture_recognizer.last_hand_data) + if advanced_result and advanced_result.get('confidence', 0) > confidence: + gesture = advanced_result.get('gesture', gesture) + confidence = advanced_result.get('confidence', confidence) + # 记录手势统计 if gesture and gesture != "Waiting" and gesture != "摄像头错误": performance_analyzer.record_gesture(gesture, confidence) @@ -319,6 +334,7 @@ def main(): config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_performance_stats', value=True) config.set('display', 'show_system_resources', value=True) + config.set('display', 'show_advanced_gestures', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -333,6 +349,7 @@ def main(): config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_performance_stats', value=True) config.set('display', 'show_system_resources', value=True) + config.set('display', 'show_advanced_gestures', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) @@ -345,12 +362,17 @@ def main(): config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_performance_stats', value=True) config.set('display', 'show_system_resources', value=True) + config.set('display', 'show_advanced_gestures', value=False) config.set('display', 'show_debug_info', value=False) # 绘制轨迹(如果启用) if config.get('display', 'show_trajectory'): frame = trajectory_recorder.draw_trajectory(frame) + # 绘制高级手势信息 + if config.get('display', 'show_advanced_gestures'): + frame = advanced_gesture_analyzer.draw_gesture_info(frame) + # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, @@ -375,7 +397,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像 - cv2.imshow('Gesture Controlled Drone - Performance Statistics', frame) + cv2.imshow('Gesture Controlled Drone - Enhanced Gestures', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF diff --git a/src/autonomus_drone_hand_gesture_project/performance_analyzer.py b/src/autonomus_drone_hand_gesture_project/performance_analyzer.py index 6bf7ca3d08..6ae0ab4c1c 100644 --- a/src/autonomus_drone_hand_gesture_project/performance_analyzer.py +++ b/src/autonomus_drone_hand_gesture_project/performance_analyzer.py @@ -11,7 +11,6 @@ from datetime import datetime from collections import deque, Counter - class PerformanceAnalyzer: """性能分析器 - 监控和报告系统性能""" diff --git a/src/autonomus_drone_hand_gesture_project/speech_manager.py b/src/autonomus_drone_hand_gesture_project/speech_manager.py index f2f54a7309..67f5f740c1 100644 --- a/src/autonomus_drone_hand_gesture_project/speech_manager.py +++ b/src/autonomus_drone_hand_gesture_project/speech_manager.py @@ -70,6 +70,13 @@ def __init__(self, speech_lib, config=None): 'gesture_low_confidence': "手势识别置信度低", 'gesture_good_confidence': "手势识别置信度高", 'gesture_hover': "悬停", + 'gesture_grab': "抓取", # 新增 + 'gesture_release': "释放", # 新增 + 'gesture_rotate_cw': "顺时针旋转", # 新增 + 'gesture_rotate_ccw': "逆时针旋转", # 新增 + 'gesture_photo': "拍照", # 新增 + 'gesture_auto_flight': "自动飞行模式", # 新增 + 'gesture_return_home': "返航", # 新增 # 系统相关 'program_start': "手势控制无人机系统已启动", diff --git a/src/autonomus_drone_hand_gesture_project/ui_renderer.py b/src/autonomus_drone_hand_gesture_project/ui_renderer.py index bd2e396fc4..048366bc4c 100644 --- a/src/autonomus_drone_hand_gesture_project/ui_renderer.py +++ b/src/autonomus_drone_hand_gesture_project/ui_renderer.py @@ -45,6 +45,13 @@ def __init__(self, speech_manager=None, config=None): 'memory_good': (0, 255, 0), 'memory_warning': (255, 165, 0), 'memory_critical': (255, 0, 0), + # 新增手势颜色 + 'gesture_grab': (255, 0, 255), + 'gesture_rotate_cw': (0, 255, 255), + 'gesture_rotate_ccw': (255, 255, 0), + 'gesture_photo': (255, 165, 0), + 'gesture_return_home': (0, 128, 128), + 'gesture_auto_flight': (128, 0, 128), } print("✓ 中文UI渲染器已初始化") From 7cb4ecabab5446ed60bee44845bca2b5ed77510e Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Mon, 22 Dec 2025 10:35:09 +0800 Subject: [PATCH 18/18] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=AB=98=E7=BA=A7?= =?UTF-8?q?=E9=A3=9E=E8=A1=8C=E6=8E=A7=E5=88=B6=E5=8A=9F=E8=83=BD=20-=20?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E9=A3=9E=E8=A1=8C=E6=A8=A1=E5=BC=8F=E3=80=81?= =?UTF-8?q?=E5=9C=86=E5=BD=A2=E7=9B=98=E6=97=8B=E3=80=818=E5=AD=97?= =?UTF-8?q?=E5=BD=A2=E9=A3=9E=E8=A1=8C=E3=80=81=E9=AB=98=E5=BA=A6=E6=8E=A7?= =?UTF-8?q?=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../drone_controller.py | 446 +++++++++++++++++- .../gesture_config.json | 13 +- .../main.py | 21 +- .../speech_manager.py | 18 + .../ui_renderer.py | 60 ++- 5 files changed, 528 insertions(+), 30 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/drone_controller.py b/src/autonomus_drone_hand_gesture_project/drone_controller.py index 2f1edeadb0..4e121b9b29 100644 --- a/src/autonomus_drone_hand_gesture_project/drone_controller.py +++ b/src/autonomus_drone_hand_gesture_project/drone_controller.py @@ -5,6 +5,8 @@ """ import time +import threading +import math class SimpleDroneController: @@ -27,13 +29,23 @@ def __init__(self, airsim_module, speech_manager=None, config=None): # 控制状态 self.last_control_time = 0 self.last_gesture = None + self.flight_mode = "manual" # manual, auto, circle, eight, square, return_home + + # 飞行任务状态 + self.auto_flying = False + self.auto_flight_thread = None + self.stop_auto_flight = False + + # 飞行高度控制 + self.target_altitude = self.altitude + self.altitude_change_step = 2.0 # 米 # 上次语音提示状态 self.last_connection_announced = False self.last_takeoff_announced = False self.last_land_announced = False - print("✓ 简单的无人机控制器已初始化") + print("✓ 增强的无人机控制器已初始化") def connect(self): """连接AirSim无人机""" @@ -130,9 +142,11 @@ def takeoff(self): time.sleep(1) # 上升到指定高度 + self.target_altitude = self.altitude self.client.moveToZAsync(self.altitude, 3).join() self.flying = True + self.flight_mode = "manual" print("✅ 无人机成功起飞") # 语音提示:起飞成功 @@ -156,6 +170,9 @@ def land(self): if not self.connected: return False + # 停止自动飞行任务 + self.stop_auto_flight_task() + # 语音提示:正在降落 if (self.speech_manager and self.speech_manager.enabled and @@ -168,6 +185,7 @@ def land(self): if self.airsim is None or self.client is None: print("✅ 模拟降落") self.flying = False + self.flight_mode = "manual" # 语音提示:降落成功 if (self.speech_manager and @@ -179,6 +197,7 @@ def land(self): print("降落中...") self.client.landAsync().join() self.flying = False + self.flight_mode = "manual" print("✅ 无人机已降落") # 语音提示:降落成功 @@ -191,6 +210,385 @@ def land(self): print(f"降落失败: {e}") return False + def hover(self): + """悬停""" + if not self.connected or not self.flying: + return False + + try: + if self.airsim is None or self.client is None: + print("模拟悬停") + self.flight_mode = "manual" + return True + + self.client.hoverAsync().join() + self.flight_mode = "manual" + print("✅ 无人机悬停") + + # 语音提示:悬停 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('hovering') + + return True + except Exception as e: + print(f"悬停失败: {e}") + return False + + def return_home(self): + """返航到起飞点""" + if not self.connected or not self.flying: + return False + + # 停止当前自动飞行任务 + self.stop_auto_flight_task() + + # 语音提示:返航 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('returning_home') + + try: + if self.airsim is None or self.client is None: + print("✅ 模拟返航") + self.flight_mode = "return_home" + return True + + print("返航中...") + self.flight_mode = "return_home" + + # 返回到起点 (0, 0, altitude) + self.client.moveToPositionAsync(0, 0, self.target_altitude, 5).join() + + # 悬停 + self.client.hoverAsync().join() + self.flight_mode = "manual" + print("✅ 返航完成") + + # 语音提示:返航成功 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('return_home_success') + + return True + except Exception as e: + print(f"❌ 返航失败: {e}") + return False + + def start_auto_flight(self): + """启动自动飞行模式(正方形轨迹)""" + if not self.connected or not self.flying: + return False + + # 停止之前的自动飞行任务 + self.stop_auto_flight_task() + + # 语音提示:自动飞行 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('auto_flight_start') + + try: + if self.airsim is None or self.client is None: + print("✅ 模拟自动飞行") + self.flight_mode = "auto" + return True + + # 启动自动飞行线程 + self.stop_auto_flight = False + self.auto_flight_thread = threading.Thread(target=self._square_flight_task) + self.auto_flight_thread.daemon = True + self.auto_flight_thread.start() + + self.flight_mode = "auto" + print("✅ 自动飞行模式启动") + + return True + except Exception as e: + print(f"❌ 启动自动飞行失败: {e}") + return False + + def _square_flight_task(self): + """正方形飞行任务""" + if self.airsim is None or self.client is None: + return + + try: + side_length = 10.0 # 正方形边长(米) + speed = 3.0 # 飞行速度 + + # 四个顶点 + points = [ + (side_length/2, side_length/2, self.target_altitude), + (side_length/2, -side_length/2, self.target_altitude), + (-side_length/2, -side_length/2, self.target_altitude), + (-side_length/2, side_length/2, self.target_altitude), + (0, 0, self.target_altitude) # 返回起点 + ] + + for i, point in enumerate(points): + if self.stop_auto_flight: + break + + print(f"自动飞行: 前往点 {i+1}/{len(points)}: {point}") + self.client.moveToPositionAsync(point[0], point[1], point[2], speed).join() + + # 在每个顶点悬停1秒 + if not self.stop_auto_flight: + time.sleep(1) + + if not self.stop_auto_flight: + # 悬停 + self.client.hoverAsync().join() + self.flight_mode = "manual" + print("✅ 自动飞行完成") + + # 语音提示 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('auto_flight_complete') + except Exception as e: + print(f"自动飞行任务错误: {e}") + + def start_circle_flight(self): + """启动圆形盘旋飞行""" + if not self.connected or not self.flying: + return False + + # 停止之前的自动飞行任务 + self.stop_auto_flight_task() + + # 语音提示:圆形盘旋 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('circle_flight_start') + + try: + if self.airsim is None or self.client is None: + print("✅ 模拟圆形盘旋") + self.flight_mode = "circle" + return True + + # 启动圆形飞行线程 + self.stop_auto_flight = False + self.auto_flight_thread = threading.Thread(target=self._circle_flight_task) + self.auto_flight_thread.daemon = True + self.auto_flight_thread.start() + + self.flight_mode = "circle" + print("✅ 圆形盘旋模式启动") + + return True + except Exception as e: + print(f"❌ 启动圆形盘旋失败: {e}") + return False + + def _circle_flight_task(self): + """圆形飞行任务""" + if self.airsim is None or self.client is None: + return + + try: + radius = 5.0 # 圆半径(米) + duration = 30.0 # 飞行时间(秒) + start_time = time.time() + + while time.time() - start_time < duration and not self.stop_auto_flight: + # 计算圆上的点 + t = time.time() - start_time + angle = t * 2 * math.pi / 5 # 每5秒一圈 + + x = radius * math.cos(angle) + y = radius * math.sin(angle) + + # 飞向圆上的点 + self.client.moveToPositionAsync(x, y, self.target_altitude, 2).join() + + # 短暂延迟 + time.sleep(0.1) + + if not self.stop_auto_flight: + # 返回中心并悬停 + self.client.moveToPositionAsync(0, 0, self.target_altitude, 3).join() + self.client.hoverAsync().join() + self.flight_mode = "manual" + print("✅ 圆形盘旋完成") + + # 语音提示 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('circle_flight_complete') + except Exception as e: + print(f"圆形飞行任务错误: {e}") + + def start_eight_flight(self): + """启动8字形飞行""" + if not self.connected or not self.flying: + return False + + # 停止之前的自动飞行任务 + self.stop_auto_flight_task() + + # 语音提示:8字形飞行 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('eight_flight_start') + + try: + if self.airsim is None or self.client is None: + print("✅ 模拟8字形飞行") + self.flight_mode = "eight" + return True + + # 启动8字形飞行线程 + self.stop_auto_flight = False + self.auto_flight_thread = threading.Thread(target=self._eight_flight_task) + self.auto_flight_thread.daemon = True + self.auto_flight_thread.start() + + self.flight_mode = "eight" + print("✅ 8字形飞行模式启动") + + return True + except Exception as e: + print(f"❌ 启动8字形飞行失败: {e}") + return False + + def _eight_flight_task(self): + """8字形飞行任务""" + if self.airsim is None or self.client is None: + return + + try: + radius = 5.0 # 半径(米) + duration = 40.0 # 飞行时间(秒) + start_time = time.time() + + while time.time() - start_time < duration and not self.stop_auto_flight: + # 计算8字形参数方程 + t = time.time() - start_time + scale = t * 2 * math.pi / 10 # 每10秒一个完整8字 + + # 8字形参数方程 + x = radius * math.sin(scale) + y = radius * math.sin(scale) * math.cos(scale) + + # 飞向8字形上的点 + self.client.moveToPositionAsync(x, y, self.target_altitude, 2).join() + + # 短暂延迟 + time.sleep(0.1) + + if not self.stop_auto_flight: + # 返回中心并悬停 + self.client.moveToPositionAsync(0, 0, self.target_altitude, 3).join() + self.client.hoverAsync().join() + self.flight_mode = "manual" + print("✅ 8字形飞行完成") + + # 语音提示 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('eight_flight_complete') + except Exception as e: + print(f"8字形飞行任务错误: {e}") + + def increase_altitude(self): + """增加飞行高度""" + if not self.connected or not self.flying: + return False + + try: + self.target_altitude -= self.altitude_change_step # AirSim中Z轴向下为负 + + # 语音提示:增加高度 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('altitude_increasing') + + if self.airsim is None or self.client is None: + print(f"✅ 模拟增加高度到: {-self.target_altitude:.1f}米") + return True + + print(f"增加高度到: {-self.target_altitude:.1f}米") + self.client.moveToZAsync(self.target_altitude, 3).join() + print("✅ 高度调整完成") + + return True + except Exception as e: + print(f"❌ 调整高度失败: {e}") + return False + + def decrease_altitude(self): + """降低飞行高度""" + if not self.connected or not self.flying: + return False + + try: + self.target_altitude += self.altitude_change_step # AirSim中Z轴向下为负 + + # 确保高度不低于最小值 + min_altitude = -20.0 + if self.target_altitude > min_altitude: + self.target_altitude = min_altitude + + # 语音提示:降低高度 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak('altitude_decreasing') + + if self.airsim is None or self.client is None: + print(f"✅ 模拟降低高度到: {-self.target_altitude:.1f}米") + return True + + print(f"降低高度到: {-self.target_altitude:.1f}米") + self.client.moveToZAsync(self.target_altitude, 3).join() + print("✅ 高度调整完成") + + return True + except Exception as e: + print(f"❌ 调整高度失败: {e}") + return False + + def set_altitude(self, altitude=None): + """设置特定飞行高度""" + if not self.connected or not self.flying: + return False + + try: + if altitude is None: + # 使用默认高度 + altitude = self.altitude + + self.target_altitude = altitude + + # 语音提示:设置高度 + if (self.speech_manager and + self.speech_manager.enabled): + self.speech_manager.speak_direct(f"设置高度到{-altitude:.1f}米") + + if self.airsim is None or self.client is None: + print(f"✅ 模拟设置高度到: {-altitude:.1f}米") + return True + + print(f"设置高度到: {-altitude:.1f}米") + self.client.moveToZAsync(altitude, 3).join() + print("✅ 高度设置完成") + + return True + except Exception as e: + print(f"❌ 设置高度失败: {e}") + return False + + def stop_auto_flight_task(self): + """停止自动飞行任务""" + if self.auto_flight_thread and self.auto_flight_thread.is_alive(): + self.stop_auto_flight = True + self.auto_flight_thread.join(timeout=2.0) + self.auto_flight_thread = None + self.flight_mode = "manual" + print("✅ 自动飞行任务已停止") + def move_by_gesture(self, gesture, confidence): """根据手势移动""" if not self.connected or not self.flying: @@ -235,9 +633,13 @@ def move_by_gesture(self, gesture, confidence): elif gesture == "Forward": self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) success = True + elif gesture == "Backward": + self.client.moveByVelocityAsync(0, self.velocity, 0, self.duration) + success = True elif gesture == "Stop": self.client.hoverAsync() success = True + self.flight_mode = "manual" # 悬停语音提示 if (self.speech_manager and self.speech_manager.enabled): @@ -245,9 +647,26 @@ def move_by_gesture(self, gesture, confidence): elif gesture == "Hover": self.client.hoverAsync() success = True + self.flight_mode = "manual" if (self.speech_manager and self.speech_manager.enabled): self.speech_manager.speak('hovering') + elif gesture == "ReturnHome": + success = self.return_home() + elif gesture == "AutoFlight": + success = self.start_auto_flight() + elif gesture == "CircleFlight": + success = self.start_circle_flight() + elif gesture == "EightFlight": + success = self.start_eight_flight() + elif gesture == "SquareFlight": + success = self.start_auto_flight() # 复用自动飞行 + elif gesture == "IncreaseAltitude": + success = self.increase_altitude() + elif gesture == "DecreaseAltitude": + success = self.decrease_altitude() + elif gesture == "SetAltitude": + success = self.set_altitude() elif gesture == "Grab": # 抓取动作(模拟) print("执行抓取动作") @@ -283,21 +702,6 @@ def move_by_gesture(self, gesture, confidence): if (self.speech_manager and self.speech_manager.enabled): self.speech_manager.speak('gesture_photo') - elif gesture == "ReturnHome": - # 返航 - print("执行返航") - self.client.moveToPositionAsync(0, 0, self.altitude, 5) - success = True - if (self.speech_manager and - self.speech_manager.enabled): - self.speech_manager.speak('gesture_return_home') - elif gesture == "AutoFlight": - # 自动飞行模式 - print("启动自动飞行模式") - success = True - if (self.speech_manager and - self.speech_manager.enabled): - self.speech_manager.speak('gesture_auto_flight') if success: self.last_control_time = current_time @@ -312,6 +716,9 @@ def emergency_stop(self): """紧急停止""" if self.connected: try: + # 停止自动飞行任务 + self.stop_auto_flight_task() + if self.flying and self.client is not None: print("紧急降落...") @@ -329,4 +736,9 @@ def emergency_stop(self): pass self.connected = False - self.flying = False \ No newline at end of file + self.flying = False + self.flight_mode = "manual" + + def get_flight_mode(self): + """获取当前飞行模式""" + return self.flight_mode \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index 0a14cccb19..7286a11db4 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -20,9 +20,9 @@ "skin_lower_h": 0, "skin_upper_h": 180, "skin_lower_s": 0, - "skin_upper_s": 127, - "skin_lower_v": 22, - "skin_upper_v": 178, + "skin_upper_s": 165, + "skin_lower_v": 16, + "skin_upper_v": 255, "min_hand_area": 3000, "max_hand_area": 30000, "hand_ratio_threshold": 1.5, @@ -104,14 +104,15 @@ "show_trajectory_heatmap": false, "theme": "dark", "trajectory_max_points": 100, - "ui_opacity": 0.7 + "ui_opacity": 0.7, + "show_flight_mode": true }, "performance": { "target_fps": 30, "resize_factor": 1.0, "enable_multiprocessing": false, - "mode": "accurate", - "current_mode_index": 2, + "mode": "balanced", + "current_mode_index": 1, "modes": [ "fast", "balanced", diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index a04228c60f..0c193452d7 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -165,6 +165,7 @@ def print_welcome_message(cap, speech_manager, libs): print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") print(f" 性能监控: 已启用") print(f" 轨迹记录: 支持录制/回放功能") + print(f" 高级飞行模式: 方形轨迹、圆形盘旋、8字形飞行") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 80) @@ -190,7 +191,9 @@ def print_instructions(): print("6. 轨迹记录功能:") print(" [1]开始录制 [2]停止录制 [3]保存轨迹 [4]回放轨迹 [5]清除轨迹 [6]暂停/继续") print("7. 键盘控制:") - print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止 [H]悬停 [G]返航") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [B]向后 [X]停止 [H]悬停") + print(" [G]返航 [Q]自动飞行模式 [E]圆形盘旋 [8]8字形飞行 [9]方形轨迹") + print(" [T]增加高度 [Y]降低高度 [U]设定高度") print("8. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") print("9. 语音控制:") @@ -268,13 +271,20 @@ def main(): ord('a'): "Left", ord('A'): "Left", ord('d'): "Right", ord('D'): "Right", ord('f'): "Forward", ord('F'): "Forward", + ord('b'): "Backward", ord('B'): "Backward", ord('x'): "Stop", ord('X'): "Stop", ord('h'): "Hover", ord('H'): "Hover", ord('g'): "ReturnHome", ord('G'): "ReturnHome", - ord('b'): "AutoFlight", ord('B'): "AutoFlight", + ord('q'): "AutoFlight", ord('Q'): "AutoFlight", ord('p'): "TakePhoto", ord('P'): "TakePhoto", ord('r'): "RotateCW", ord('R'): "RotateCW", ord('l'): "RotateCCW", ord('L'): "RotateCCW", + ord('e'): "CircleFlight", ord('E'): "CircleFlight", + ord('8'): "EightFlight", ord('*'): "EightFlight", + ord('9'): "SquareFlight", ord('('): "SquareFlight", + ord('t'): "IncreaseAltitude", ord('T'): "IncreaseAltitude", + ord('y'): "DecreaseAltitude", ord('Y'): "DecreaseAltitude", + ord('u'): "SetAltitude", ord('U'): "SetAltitude", } # 显示模式 @@ -335,6 +345,7 @@ def main(): config.set('display', 'show_performance_stats', value=True) config.set('display', 'show_system_resources', value=True) config.set('display', 'show_advanced_gestures', value=True) + config.set('display', 'show_flight_mode', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -350,6 +361,7 @@ def main(): config.set('display', 'show_performance_stats', value=True) config.set('display', 'show_system_resources', value=True) config.set('display', 'show_advanced_gestures', value=True) + config.set('display', 'show_flight_mode', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) @@ -363,6 +375,7 @@ def main(): config.set('display', 'show_performance_stats', value=True) config.set('display', 'show_system_resources', value=True) config.set('display', 'show_advanced_gestures', value=False) + config.set('display', 'show_flight_mode', value=True) config.set('display', 'show_debug_info', value=False) # 绘制轨迹(如果启用) @@ -380,6 +393,10 @@ def main(): trajectory_recorder, gesture_recognizer, performance_analyzer ) + # 绘制飞行模式信息 + if config.get('display', 'show_flight_mode'): + frame = ui_renderer.draw_flight_mode(frame, drone_controller) + # 绘制性能图表(如果启用详细模式) if display_modes[current_display_mode] == 'detailed': frame = ui_renderer.draw_performance_chart(frame, performance_analyzer) diff --git a/src/autonomus_drone_hand_gesture_project/speech_manager.py b/src/autonomus_drone_hand_gesture_project/speech_manager.py index 67f5f740c1..c9a585f5c4 100644 --- a/src/autonomus_drone_hand_gesture_project/speech_manager.py +++ b/src/autonomus_drone_hand_gesture_project/speech_manager.py @@ -52,6 +52,18 @@ def __init__(self, speech_lib, config=None): 'emergency_stop': "紧急停止,无人机已降落", 'hovering': "无人机悬停中", + # 高级飞行模式 + 'returning_home': "正在返航", + 'return_home_success': "返航成功", + 'auto_flight_start': "开始自动飞行模式", + 'auto_flight_complete': "自动飞行完成", + 'circle_flight_start': "开始圆形盘旋", + 'circle_flight_complete': "圆形盘旋完成", + 'eight_flight_start': "开始8字形飞行", + 'eight_flight_complete': "8字形飞行完成", + 'altitude_increasing': "正在增加高度", + 'altitude_decreasing': "正在降低高度", + # 手势相关 'gesture_detected': "手势识别就绪,请开始手势", 'gesture_start': "开始识别手势", @@ -77,6 +89,12 @@ def __init__(self, speech_lib, config=None): 'gesture_photo': "拍照", # 新增 'gesture_auto_flight': "自动飞行模式", # 新增 'gesture_return_home': "返航", # 新增 + 'gesture_circle_flight': "圆形盘旋", # 新增 + 'gesture_eight_flight': "8字形飞行", # 新增 + 'gesture_square_flight': "方形轨迹", # 新增 + 'gesture_increase_altitude': "增加高度", # 新增 + 'gesture_decrease_altitude': "降低高度", # 新增 + 'gesture_set_altitude': "设置高度", # 新增 # 系统相关 'program_start': "手势控制无人机系统已启动", diff --git a/src/autonomus_drone_hand_gesture_project/ui_renderer.py b/src/autonomus_drone_hand_gesture_project/ui_renderer.py index 048366bc4c..cb52a20513 100644 --- a/src/autonomus_drone_hand_gesture_project/ui_renderer.py +++ b/src/autonomus_drone_hand_gesture_project/ui_renderer.py @@ -52,6 +52,13 @@ def __init__(self, speech_manager=None, config=None): 'gesture_photo': (255, 165, 0), 'gesture_return_home': (0, 128, 128), 'gesture_auto_flight': (128, 0, 128), + # 飞行模式颜色 + 'flight_mode_manual': (0, 255, 0), + 'flight_mode_auto': (255, 165, 0), + 'flight_mode_circle': (0, 255, 255), + 'flight_mode_eight': (255, 0, 255), + 'flight_mode_square': (255, 255, 0), + 'flight_mode_return_home': (0, 128, 128), } print("✓ 中文UI渲染器已初始化") @@ -116,7 +123,7 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 性能统计版" + title = "手势控制无人机系统 - 高级飞行模式版" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -254,6 +261,48 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro return frame + def draw_flight_mode(self, frame, drone_controller): + """绘制飞行模式信息""" + if not drone_controller.connected: + return frame + + h, w = frame.shape[:2] + + # 飞行模式显示 + flight_mode = drone_controller.get_flight_mode() + + # 根据飞行模式选择颜色和文本 + mode_texts = { + 'manual': ('手动模式', self.colors['flight_mode_manual']), + 'auto': ('自动飞行', self.colors['flight_mode_auto']), + 'circle': ('圆形盘旋', self.colors['flight_mode_circle']), + 'eight': ('8字形飞行', self.colors['flight_mode_eight']), + 'square': ('方形轨迹', self.colors['flight_mode_square']), + 'return_home': ('返航中', self.colors['flight_mode_return_home']) + } + + if flight_mode in mode_texts: + text, color = mode_texts[flight_mode] + mode_text = f"飞行模式: {text}" + + # 绘制背景 + text_size = cv2.getTextSize(mode_text, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2)[0] + bg_x = w - text_size[0] - 20 + bg_y = 150 + bg_w = text_size[0] + 10 + bg_h = text_size[1] + 10 + + # 绘制半透明背景 + overlay = frame.copy() + cv2.rectangle(overlay, (bg_x, bg_y), (bg_x + bg_w, bg_y + bg_h), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + + # 绘制文本 + frame = self.draw_text(frame, mode_text, (bg_x + 5, bg_y + 5), + size=14, color=color) + + return frame + def draw_help_bar(self, frame): """绘制帮助栏""" if not self.config.get('display', 'show_help'): @@ -262,19 +311,20 @@ def draw_help_bar(self, frame): h, w = frame.shape[:2] # 绘制底部帮助栏 - cv2.rectangle(frame, (0, h - 120), (w, h), (0, 0, 0), -1) + cv2.rectangle(frame, (0, h - 140), (w, h), (0, 0, 0), -1) # 帮助文本 help_lines = [ - "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", + "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/B:键盘控制", "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", "V:切换语音反馈 M:测试语音 P:性能报告 L:性能快照 K:导出日志", - "O:切换性能模式 P:详细报告 R:重置统计", + "O:切换性能模式 Q:自动飞行 E:圆形盘旋 8:8字形 9:方形轨迹", + "T/Y/U:高度控制 G:返航 X:停止 H:悬停", "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" ] for i, line in enumerate(help_lines): - y_pos = h - 105 + i * 20 + y_pos = h - 125 + i * 20 frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame