From 18d483948dbb0f21f8bb2973bd18fa60d66d44b5 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Thu, 18 Dec 2025 16:16:41 +0800 Subject: [PATCH 01/11] =?UTF-8?q?new=E5=88=86=E6=94=AF=EF=BC=9A1.=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0.gitignore=E5=BF=BD=E7=95=A5Lib=E7=9B=AE=E5=BD=95=202.?= =?UTF-8?q?=E4=BF=AE=E6=94=B9main.py=E4=BC=98=E5=8C=96=E6=97=A0=E4=BA=BA?= =?UTF-8?q?=E6=9C=BA=E6=8E=A7=E5=88=B6=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 6 + .../main.py | 546 ++++++++++++++++++ 2 files changed, 552 insertions(+) create mode 100644 src/autonomus_drone_hand_gesture_project/.gitignore diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore new file mode 100644 index 0000000000..ad298e3e2d --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -0,0 +1,6 @@ +# 忽略整个Lib目录(包含所有子文件/子目录) +Lib/ +# 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 +__pycache__/ +*.pyc +*.dll.a \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index e69de29bb2..b77d05ba12 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -0,0 +1,546 @@ +# -*- coding: utf-8 -*- +""" +手势控制AirSim无人机 - 修复连接问题版 +确保正确连接AirSim,移除模拟模式 +作者: xiaoshiyuan888 +""" + +import sys +import os +import time +import traceback +import subprocess +from PIL import Image, ImageDraw, ImageFont +import cv2 +import numpy as np + +print("=" * 60) +print("Gesture Controlled Drone - Fixed Connection Version") +print("=" * 60) + +# ========== 修复导入路径 ========== +# 添加当前目录到Python路径 +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + + +# ========== 核心模块导入与异常处理 ========== +def safe_import(): + """安全导入所有模块""" + modules_status = {} + + # 1. 导入PIL + try: + from PIL import Image, ImageDraw, ImageFont + modules_status['PIL'] = True + print("[PIL] ✓ Image processing library ready") + except Exception as e: + modules_status['PIL'] = False + print(f"[PIL] ✗ Import failed: {e}") + return None, modules_status + + # 2. 导入OpenCV + try: + import cv2 + import numpy as np + modules_status['OpenCV'] = True + print("[OpenCV] ✓ Computer vision library ready") + except Exception as e: + modules_status['OpenCV'] = False + print(f"[OpenCV] ✗ Import failed: {e}") + return None, modules_status + + # 3. 导入MediaPipe + mp = None + mp_hands = None + mp_drawing = None + try: + import mediapipe as mp + mp_hands = mp.solutions.hands + mp_drawing = mp.solutions.drawing_utils + modules_status['MediaPipe'] = True + print("[MediaPipe] ✓ Gesture recognition library ready") + except Exception as e: + modules_status['MediaPipe'] = False + print(f"[MediaPipe] ✗ Import failed: {e}") + print(" Please install: pip install mediapipe") + + # 4. 尝试导入AirSim - 关键修复部分 + airsim_module = None + airsim_client = None + + # 尝试多种导入方式 + import_methods = [ + lambda: __import__('airsim'), + lambda: __import__('AirSim'), + ] + + for method in import_methods: + try: + airsim_module = method() + modules_status['AirSim'] = True + print(f"[AirSim] ✓ Successfully imported airsim from {airsim_module.__file__}") + break + except ImportError: + continue + except Exception as e: + print(f"[AirSim] Import error: {e}") + continue + + if not modules_status.get('AirSim', False): + print("\n" + "!" * 60) + print("⚠ AirSim library NOT FOUND!") + print("!" * 60) + print("To install AirSim, run:") + print("1. First install: pip install msgpack-rpc-python") + print("2. Then install: pip install airsim") + print("\nOr from source:") + print(" pip install git+https://github.com/microsoft/AirSim.git") + print("!" * 60) + + # 询问是否自动安装 + print("\nDo you want to try automatic installation? (y/n)") + choice = input().strip().lower() + if choice == 'y': + try: + print("Installing msgpack-rpc-python...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "msgpack-rpc-python"]) + print("Installing AirSim...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "airsim"]) + print("\n✅ Installation complete! Please restart the program.") + except Exception as e: + print(f"Installation failed: {e}") + sys.exit(1) + + return { + 'cv2': cv2, + 'np': np, + 'mp': mp, + 'mp_hands': mp_hands, + 'mp_drawing': mp_drawing, + 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, + 'airsim': airsim_module + }, modules_status + + +# 执行导入 +libs, status = safe_import() +if not status.get('OpenCV', False) or not status.get('PIL', False): + print("\n❌ Core libraries missing, cannot start.") + input("Press Enter to exit...") + sys.exit(1) + +print("-" * 60) +print("✅ Environment check passed, initializing...") +print("-" * 60) + +# 解包库 +cv2, np, mp = libs['cv2'], libs['np'], libs['mp'] +Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] +mp_hands, mp_drawing = libs['mp_hands'], libs['mp_drawing'] + + +# ========== 中文显示模块 ========== +class ChineseTextRenderer: + """中文文本渲染器""" + + def __init__(self): + self.fonts = {} + self.load_fonts() + + def load_fonts(self): + """加载字体""" + font_paths = [ + 'simhei.ttf', + 'C:/Windows/Fonts/simhei.ttf', + 'C:/Windows/Fonts/msyh.ttc' + ] + for path in font_paths: + try: + self.fonts[20] = ImageFont.truetype(path, 20) + self.fonts[24] = ImageFont.truetype(path, 24) + self.fonts[30] = ImageFont.truetype(path, 30) + print(f"✓ Chinese fonts loaded: {path}") + return + except: + continue + print("⚠ No Chinese fonts found, using default") + + def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): + """在图像上绘制中文文本""" + try: + img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + pil_img = Image.fromarray(img_rgb) + draw = ImageDraw.Draw(pil_img) + + font = self.fonts.get(size, self.fonts.get(20)) + + # 绘制阴影 + shadow_color = (0, 0, 0) + shadow_pos = (pos[0] + 1, pos[1] + 1) + draw.text(shadow_pos, text, font=font, fill=shadow_color) + + # 绘制文字 + rgb_color = color[::-1] + draw.text(pos, text, font=font, fill=rgb_color) + + return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) + except: + cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, + size / 30, color, 2) + return frame + + +# 初始化中文渲染器 +chinese_renderer = ChineseTextRenderer() + + +# ========== 手势识别模块 ========== +class GestureRecognizer: + """手势识别器""" + + def __init__(self): + try: + self.hands = mp_hands.Hands( + max_num_hands=1, + min_detection_confidence=0.7, + min_tracking_confidence=0.5 + ) + print("✓ MediaPipe gesture detector initialized") + except Exception as e: + print(f"⚠ MediaPipe init failed: {e}") + self.hands = None + + self.current_gesture = "等待手势" + + def recognize(self, frame): + """识别手势""" + if self.hands is None: + return self.current_gesture + + try: + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + results = self.hands.process(rgb_frame) + + if results.multi_hand_landmarks: + for hand_landmarks in results.multi_hand_landmarks: + # 绘制手部关键点 + mp_drawing.draw_landmarks( + frame, hand_landmarks, mp_hands.HAND_CONNECTIONS, + mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2), + mp_drawing.DrawingSpec(color=(255, 0, 0), thickness=2) + ) + + # 简化的手势判断 + landmarks = hand_landmarks.landmark + if len(landmarks) >= 9: + wrist = landmarks[0] + index_tip = landmarks[8] + + # 手势判断逻辑 + if index_tip.y < wrist.y - 0.2: + self.current_gesture = "向上" + elif index_tip.y > wrist.y + 0.2: + self.current_gesture = "向下" + elif index_tip.x < wrist.x - 0.2: + self.current_gesture = "向左" + elif index_tip.x > wrist.x + 0.2: + self.current_gesture = "向右" + elif abs(index_tip.y - wrist.y) < 0.1: + self.current_gesture = "向前" + else: + self.current_gesture = "停止" + + return self.current_gesture + except Exception as e: + print(f"Gesture recognition error: {e}") + return "识别异常" + + def set_simulated_gesture(self, gesture): + """设置模拟的手势""" + self.current_gesture = gesture + + +# ========== 无人机控制模块 ========== +class DroneController: + """无人机控制器""" + + def __init__(self, airsim_module): + self.airsim = airsim_module + self.client = None + self.connected = False + self.flying = False + self.connection_attempted = False + + def connect(self): + """连接AirSim无人机""" + if self.connection_attempted: + return self.connected + + self.connection_attempted = True + print("Connecting to AirSim...") + + try: + # 创建客户端 + self.client = self.airsim.MultirotorClient() + self.client.confirmConnection() + print("✅ Connected to AirSim!") + + # 启用API控制 + self.client.enableApiControl(True) + print("✅ API control enabled") + + # 解锁无人机 + self.client.armDisarm(True) + print("✅ Drone armed") + + self.connected = True + return True + + except Exception as e: + print(f"❌ Connection failed: {e}") + print("\nPlease ensure:") + print("1. AirSim simulator is running") + print("2. In simulator: Settings → Computer Vision mode is OFF") + print("3. Drone is spawned in the world") + print("4. Press 'R' in simulator to reset drone if needed") + return False + + def takeoff(self): + """起飞""" + if not self.connected: + print("❌ Drone not connected") + return False + + try: + print("Taking off...") + # 起飞到5米高度 + self.client.takeoffAsync().join() + # 悬停1秒稳定 + time.sleep(1) + self.flying = True + print("✅ Drone took off successfully") + return True + except Exception as e: + print(f"❌ Takeoff failed: {e}") + return False + + def land(self): + """降落""" + if not self.connected: + return False + + try: + print("Landing...") + self.client.landAsync().join() + self.flying = False + print("✅ Drone landed") + return True + except Exception as e: + print(f"Landing failed: {e}") + return False + + def move_by_gesture(self, gesture): + """根据手势移动""" + if not self.connected or not self.flying: + return + + try: + # 设置速度参数(单位:米/秒) + velocity = 3 + duration = 0.5 # 持续0.5秒 + + if gesture == "向上": + self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() + elif gesture == "向下": + self.client.moveByVelocityZAsync(0, 0, velocity, duration).join() + elif gesture == "向左": + self.client.moveByVelocityAsync(-velocity, 0, 0, duration).join() + elif gesture == "向右": + self.client.moveByVelocityAsync(velocity, 0, 0, duration).join() + elif gesture == "向前": + self.client.moveByVelocityAsync(0, -velocity, 0, duration).join() + elif gesture == "停止": + self.client.hoverAsync().join() + except Exception as e: + print(f"Control command failed: {e}") + + def emergency_stop(self): + """紧急停止""" + if self.connected: + try: + if self.flying: + print("Emergency landing...") + self.land() + self.client.armDisarm(False) + self.client.enableApiControl(False) + print("✅ Emergency stop complete") + except: + pass + self.connected = False + self.flying = False + + +# ========== 主程序 ========== +def main(): + """主函数""" + # 初始化组件 + gesture_recognizer = GestureRecognizer() + drone_controller = DroneController(libs['airsim']) + + # 初始化摄像头 + cap = None + try: + cap = cv2.VideoCapture(0) + if not cap.isOpened(): + print("❌ Camera not available") + cap = None + else: + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + print("✓ Camera initialized") + except: + print("⚠ Camera init failed") + cap = None + + # 显示说明 + print("\n" + "=" * 60) + print("INSTRUCTIONS") + print("=" * 60) + print("1. Start AirSim simulator first") + print("2. Choose environment (e.g., Landscape Mountains)") + print("3. Press [C] in this window to connect drone") + print("4. Press [SPACE] to takeoff/land") + if cap: + print("5. Gesture control with hand in front of camera") + else: + print("5. Keyboard control:") + print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") + print("6. Press [ESC] to exit safely") + print("=" * 60) + print("Program started successfully!") + print("-" * 60) + + # 键盘手势映射 + key_to_gesture = { + ord('w'): "向上", ord('W'): "向上", + ord('s'): "向下", ord('S'): "向下", + ord('a'): "向左", ord('A'): "向左", + ord('d'): "向右", ord('D'): "向右", + ord('f'): "向前", ord('F'): "向前", + ord('x'): "停止", ord('X'): "停止", + } + + # 主循环 + while True: + frame = None + + # 读取摄像头画面 + if cap: + ret, frame = cap.read() + if ret: + frame = cv2.flip(frame, 1) + current_gesture = gesture_recognizer.recognize(frame) + else: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + current_gesture = "摄像头错误" + else: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + current_gesture = gesture_recognizer.current_gesture + + # ========== 绘制界面 ========== + # 标题栏 + overlay = frame.copy() + cv2.rectangle(overlay, (0, 0), (640, 100), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + + # 主标题 + frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), + size=30, color=(0, 255, 255)) + + # 状态信息 + status_color = (0, 255, 0) if drone_controller.connected else (0, 0, 255) + status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" + frame = chinese_renderer.put_text(frame, status_text, (10, 50), + size=24, color=status_color) + + flight_color = (0, 255, 0) if drone_controller.flying else (255, 255, 0) + flight_status = f"飞行: {'是' if drone_controller.flying else '否'}" + frame = chinese_renderer.put_text(frame, flight_status, (10, 80), + size=24, color=flight_color) + + # 手势信息 + gesture_color = (0, 255, 0) if current_gesture != "等待手势" else (200, 200, 200) + gesture_text = f"手势: {current_gesture}" + frame = chinese_renderer.put_text(frame, gesture_text, (10, 120), + size=24, color=gesture_color) + + # 操作提示 + help_text = "C:连接 空格:起飞/降落 ESC:退出" + if not cap: + help_text += " W/A/S/D/F/X:控制" + + frame = chinese_renderer.put_text(frame, help_text, (10, 450), + size=20, color=(255, 255, 255)) + + # AirSim连接状态 + if not drone_controller.connected: + warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" + frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + size=18, color=(0, 165, 255)) + + # 显示窗口 + cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) + + # ========== 键盘控制 ========== + key = cv2.waitKey(30) & 0xFF + + if key == 27: # ESC键 + print("\nExiting...") + break + + elif key == ord('c') or key == ord('C'): # 连接无人机 + if not drone_controller.connected: + drone_controller.connect() + + elif key == 32: # 空格键 + if drone_controller.connected: + if drone_controller.flying: + drone_controller.land() + else: + drone_controller.takeoff() + time.sleep(0.3) + + elif key in key_to_gesture: # 键盘控制 + gesture = key_to_gesture[key] + gesture_recognizer.set_simulated_gesture(gesture) + if drone_controller.connected and drone_controller.flying: + drone_controller.move_by_gesture(gesture) + + # 真实手势控制 + if (current_gesture and current_gesture != "等待手势" and + current_gesture != "摄像头错误" and + drone_controller.connected and drone_controller.flying): + drone_controller.move_by_gesture(current_gesture) + + # ========== 清理资源 ========== + print("\nCleaning up resources...") + if cap: + cap.release() + cv2.destroyAllWindows() + drone_controller.emergency_stop() + print("Program safely exited") + print("=" * 60) + + +# ========== 程序入口 ========== +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nProgram interrupted by user") + except Exception as e: + print(f"\nProgram error: {e}") + traceback.print_exc() + finally: + input("\nPress Enter to exit...") \ No newline at end of file From dd54e834a48d55daa8c8f6dd313c074efcea1d05 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Thu, 18 Dec 2025 22:05:02 +0800 Subject: [PATCH 02/11] =?UTF-8?q?=E6=94=B9=E8=BF=9B=E6=97=A0=E4=BA=BA?= =?UTF-8?q?=E6=9C=BA=E6=89=8B=E5=8A=BF=E6=8E=A7=E5=88=B6=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=EF=BC=8C=E4=BC=98=E5=8C=96=E6=91=84=E5=83=8F=E5=A4=B4=E5=A4=84?= =?UTF-8?q?=E7=90=86=E5=92=8C=E6=89=8B=E5=8A=BF=E7=A8=B3=E5=AE=9A=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 4 +- .../main.py | 362 ++++++++++++------ 2 files changed, 246 insertions(+), 120 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index ad298e3e2d..05dab76400 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -3,4 +3,6 @@ Lib/ # 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 __pycache__/ *.pyc -*.dll.a \ No newline at end of file +*.dll.a +# 忽略IDEA配置文件 +.idea/ \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index b77d05ba12..f2a7020806 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 修复连接问题版 -确保正确连接AirSim,移除模拟模式 +手势控制AirSim无人机 - 修复手势识别版 +优化了手势识别算法,增加更多手势判断逻辑 作者: xiaoshiyuan888 """ @@ -15,21 +15,19 @@ import numpy as np print("=" * 60) -print("Gesture Controlled Drone - Fixed Connection Version") +print("Gesture Controlled Drone - Enhanced Gesture Recognition") print("=" * 60) # ========== 修复导入路径 ========== -# 添加当前目录到Python路径 current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -# ========== 核心模块导入与异常处理 ========== +# ========== 核心模块导入 ========== def safe_import(): """安全导入所有模块""" modules_status = {} - # 1. 导入PIL try: from PIL import Image, ImageDraw, ImageFont modules_status['PIL'] = True @@ -39,7 +37,6 @@ def safe_import(): print(f"[PIL] ✗ Import failed: {e}") return None, modules_status - # 2. 导入OpenCV try: import cv2 import numpy as np @@ -50,7 +47,6 @@ def safe_import(): print(f"[OpenCV] ✗ Import failed: {e}") return None, modules_status - # 3. 导入MediaPipe mp = None mp_hands = None mp_drawing = None @@ -65,11 +61,7 @@ def safe_import(): print(f"[MediaPipe] ✗ Import failed: {e}") print(" Please install: pip install mediapipe") - # 4. 尝试导入AirSim - 关键修复部分 airsim_module = None - airsim_client = None - - # 尝试多种导入方式 import_methods = [ lambda: __import__('airsim'), lambda: __import__('AirSim'), @@ -79,7 +71,7 @@ def safe_import(): try: airsim_module = method() modules_status['AirSim'] = True - print(f"[AirSim] ✓ Successfully imported airsim from {airsim_module.__file__}") + print(f"[AirSim] ✓ Successfully imported") break except ImportError: continue @@ -98,7 +90,6 @@ def safe_import(): print(" pip install git+https://github.com/microsoft/AirSim.git") print("!" * 60) - # 询问是否自动安装 print("\nDo you want to try automatic installation? (y/n)") choice = input().strip().lower() if choice == 'y': @@ -153,7 +144,9 @@ def load_fonts(self): font_paths = [ 'simhei.ttf', 'C:/Windows/Fonts/simhei.ttf', - 'C:/Windows/Fonts/msyh.ttc' + 'C:/Windows/Fonts/msyh.ttc', + '/System/Library/Fonts/PingFang.ttc', + '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' ] for path in font_paths: try: @@ -177,7 +170,7 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): # 绘制阴影 shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 1, pos[1] + 1) + shadow_pos = (pos[0] + 2, pos[1] + 2) draw.text(shadow_pos, text, font=font, fill=shadow_color) # 绘制文字 @@ -186,70 +179,172 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) except: + # 备用方案:使用OpenCV绘制英文 cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, size / 30, color, 2) return frame -# 初始化中文渲染器 chinese_renderer = ChineseTextRenderer() -# ========== 手势识别模块 ========== -class GestureRecognizer: - """手势识别器""" +# ========== 简化的手势识别模块 ========== +class SimpleGestureRecognizer: + """简化的手势识别器,不依赖MediaPipe""" def __init__(self): - try: - self.hands = mp_hands.Hands( - max_num_hands=1, - min_detection_confidence=0.7, - min_tracking_confidence=0.5 - ) - print("✓ MediaPipe gesture detector initialized") - except Exception as e: - print(f"⚠ MediaPipe init failed: {e}") - self.hands = None - self.current_gesture = "等待手势" + self.last_gesture_time = time.time() + self.gesture_history = [] + self.max_history = 5 + print("✓ Simple gesture recognizer initialized (不使用MediaPipe)") + + # 加载OpenCV的预训练模型用于手部检测 + try: + # 尝试加载Haar级联分类器 + cascade_path = cv2.data.haarcascades + 'haarcascade_frontalface_default.xml' + # 我们使用人脸检测器作为替代,实际上应该用手部检测器 + self.face_cascade = cv2.CascadeClassifier(cascade_path) + print("✓ OpenCV cascade classifier loaded") + except: + print("⚠ OpenCV cascade classifier not available") + self.face_cascade = None + + def detect_hand_contour(self, frame): + """使用轮廓检测手部""" + # 转换为HSV颜色空间 + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + + # 定义肤色范围 + lower_skin = np.array([0, 20, 70], dtype=np.uint8) + upper_skin = np.array([20, 255, 255], dtype=np.uint8) + + # 创建肤色掩码 + mask = cv2.inRange(hsv, lower_skin, upper_skin) + + # 形态学操作 + kernel = np.ones((5, 5), np.uint8) + mask = cv2.dilate(mask, kernel, iterations=2) + mask = cv2.erode(mask, kernel, iterations=1) + mask = cv2.GaussianBlur(mask, (5, 5), 0) + + # 查找轮廓 + contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + if contours: + # 找到最大的轮廓(假设是手) + largest_contour = max(contours, key=cv2.contourArea) + + # 计算轮廓面积 + area = cv2.contourArea(largest_contour) + + if area > 1000: # 最小面积阈值 + # 获取边界矩形 + x, y, w, h = cv2.boundingRect(largest_contour) + + # 计算轮廓的中心点 + M = cv2.moments(largest_contour) + if M["m00"] != 0: + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + + # 计算凸包和凸缺陷 + hull = cv2.convexHull(largest_contour, returnPoints=False) + if len(hull) > 3: + defects = cv2.convexityDefects(largest_contour, hull) + + if defects is not None: + # 计算手指数量 + finger_count = 0 + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(largest_contour[s][0]) + end = tuple(largest_contour[e][0]) + far = tuple(largest_contour[f][0]) + + # 计算角度 + a = np.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = np.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = np.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + + angle = np.arccos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + + if angle <= np.pi / 2: + finger_count += 1 + + finger_count = min(finger_count + 1, 5) + + # 绘制轮廓和中心点 + cv2.drawContours(frame, [largest_contour], -1, (0, 255, 0), 2) + cv2.circle(frame, (cx, cy), 7, (255, 0, 0), -1) + cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) + + # 在边界框上方显示手指数量 + cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + + return cx, cy, w, h, finger_count, frame + + return None, None, None, None, 0, frame + + def recognize_simple_gesture(self, frame): + """简化的手势识别""" + # 检测手部轮廓 + cx, cy, w, h, finger_count, frame = self.detect_hand_contour(frame) + + if cx is None or cy is None: + return "等待手势", frame + + # 获取图像尺寸 + height, width = frame.shape[:2] + + # 定义区域 + center_x = width // 2 + center_y = height // 2 + + # 根据手指数量和位置判断手势 + if finger_count == 0: + return "停止", frame + elif finger_count == 1: + return "向前", frame + elif finger_count >= 4: + # 手掌张开,根据位置判断方向 + if cy < center_y - 50: + return "向上", frame + elif cy > center_y + 50: + return "向下", frame + elif cx < center_x - 50: + return "向左", frame + elif cx > center_x + 50: + return "向右", frame + else: + return "停止", frame + else: + return "等待手势", frame def recognize(self, frame): """识别手势""" - if self.hands is None: - return self.current_gesture - try: - rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - results = self.hands.process(rgb_frame) - - if results.multi_hand_landmarks: - for hand_landmarks in results.multi_hand_landmarks: - # 绘制手部关键点 - mp_drawing.draw_landmarks( - frame, hand_landmarks, mp_hands.HAND_CONNECTIONS, - mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2), - mp_drawing.DrawingSpec(color=(255, 0, 0), thickness=2) - ) - - # 简化的手势判断 - landmarks = hand_landmarks.landmark - if len(landmarks) >= 9: - wrist = landmarks[0] - index_tip = landmarks[8] - - # 手势判断逻辑 - if index_tip.y < wrist.y - 0.2: - self.current_gesture = "向上" - elif index_tip.y > wrist.y + 0.2: - self.current_gesture = "向下" - elif index_tip.x < wrist.x - 0.2: - self.current_gesture = "向左" - elif index_tip.x > wrist.x + 0.2: - self.current_gesture = "向右" - elif abs(index_tip.y - wrist.y) < 0.1: - self.current_gesture = "向前" - else: - self.current_gesture = "停止" + # 使用简化的手势识别 + gesture, processed_frame = self.recognize_simple_gesture(frame) + + # 手势历史记录和平滑处理 + self.gesture_history.append(gesture) + if len(self.gesture_history) > self.max_history: + self.gesture_history.pop(0) + + # 使用多数表决决定最终手势 + if self.gesture_history: + gesture_counts = {} + for g in self.gesture_history: + gesture_counts[g] = gesture_counts.get(g, 0) + 1 + + # 找到出现次数最多的手势 + max_gesture = max(gesture_counts, key=gesture_counts.get) + if gesture_counts[max_gesture] > len(self.gesture_history) // 2: + self.current_gesture = max_gesture + else: + self.current_gesture = gesture return self.current_gesture except Exception as e: @@ -281,16 +376,13 @@ def connect(self): print("Connecting to AirSim...") try: - # 创建客户端 self.client = self.airsim.MultirotorClient() self.client.confirmConnection() print("✅ Connected to AirSim!") - # 启用API控制 self.client.enableApiControl(True) print("✅ API control enabled") - # 解锁无人机 self.client.armDisarm(True) print("✅ Drone armed") @@ -299,11 +391,11 @@ def connect(self): except Exception as e: print(f"❌ Connection failed: {e}") - print("\nPlease ensure:") - print("1. AirSim simulator is running") - print("2. In simulator: Settings → Computer Vision mode is OFF") - print("3. Drone is spawned in the world") - print("4. Press 'R' in simulator to reset drone if needed") + print("\n请确保:") + print("1. AirSim模拟器正在运行") + print("2. 选择环境(如:Landscape Mountains)") + print("3. 无人机已在世界中生成") + print("4. 如果需要,按模拟器中的'R'键重置无人机") return False def takeoff(self): @@ -314,9 +406,7 @@ def takeoff(self): try: print("Taking off...") - # 起飞到5米高度 self.client.takeoffAsync().join() - # 悬停1秒稳定 time.sleep(1) self.flying = True print("✅ Drone took off successfully") @@ -346,9 +436,8 @@ def move_by_gesture(self, gesture): return try: - # 设置速度参数(单位:米/秒) - velocity = 3 - duration = 0.5 # 持续0.5秒 + velocity = 2.0 # 降低速度以获得更平滑的控制 + duration = 0.3 # 减少持续时间 if gesture == "向上": self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() @@ -384,8 +473,8 @@ def emergency_stop(self): # ========== 主程序 ========== def main(): """主函数""" - # 初始化组件 - gesture_recognizer = GestureRecognizer() + # 使用简化的手势识别器 + gesture_recognizer = SimpleGestureRecognizer() drone_controller = DroneController(libs['airsim']) # 初始化摄像头 @@ -393,32 +482,46 @@ def main(): try: cap = cv2.VideoCapture(0) if not cap.isOpened(): - print("❌ Camera not available") - cap = None + # 尝试其他摄像头索引 + for i in range(1, 5): + cap = cv2.VideoCapture(i) + if cap.isOpened(): + print(f"✓ Camera {i} initialized") + break + else: + print("❌ Camera not available, using keyboard control only") + cap = None else: + print("✓ Camera 0 initialized") + + if cap: cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - print("✓ Camera initialized") - except: - print("⚠ Camera init failed") + cap.set(cv2.CAP_PROP_FPS, 30) + except Exception as e: + print(f"⚠ Camera init failed: {e}") cap = None # 显示说明 print("\n" + "=" * 60) - print("INSTRUCTIONS") + print("操作说明") print("=" * 60) - print("1. Start AirSim simulator first") - print("2. Choose environment (e.g., Landscape Mountains)") - print("3. Press [C] in this window to connect drone") - print("4. Press [SPACE] to takeoff/land") + print("1. 首先启动AirSim模拟器") + print("2. 选择环境(例如:Landscape Mountains)") + print("3. 按 [C] 连接无人机") + print("4. 按 [空格键] 起飞/降落") if cap: - print("5. Gesture control with hand in front of camera") + print("5. 手势控制:") + print(" - 握拳(0个手指):停止") + print(" - 伸出1个手指:向前") + print(" - 张开手掌(4-5个手指):根据手的位置控制方向") + print(" * 确保手部在摄像头视野内,光线充足,背景简单") else: - print("5. Keyboard control:") - print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") - print("6. Press [ESC] to exit safely") + print("5. 键盘控制:") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print("6. 按 [ESC] 安全退出") print("=" * 60) - print("Program started successfully!") + print("程序启动成功!") print("-" * 60) # 键盘手势映射 @@ -431,14 +534,17 @@ def main(): ord('x'): "停止", ord('X'): "停止", } + last_control_time = time.time() + control_interval = 0.5 # 控制间隔,避免过于频繁 + # 主循环 while True: frame = None - # 读取摄像头画面 if cap: ret, frame = cap.read() if ret: + # 镜像图像以便更直观的控制 frame = cv2.flip(frame, 1) current_gesture = gesture_recognizer.recognize(frame) else: @@ -449,12 +555,16 @@ def main(): current_gesture = gesture_recognizer.current_gesture # ========== 绘制界面 ========== - # 标题栏 + # 创建半透明覆盖层 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (640, 100), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + cv2.rectangle(overlay, (0, 0), (640, 120), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.6, frame, 0.4, 0) - # 主标题 + # 绘制底部操作提示区域 + cv2.rectangle(frame, (0, 400), (640, 480), (0, 0, 0), -1) + frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) + + # 标题 frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), size=30, color=(0, 255, 255)) @@ -470,9 +580,15 @@ def main(): size=24, color=flight_color) # 手势信息 - gesture_color = (0, 255, 0) if current_gesture != "等待手势" else (200, 200, 200) + if current_gesture != "等待手势" and current_gesture != "摄像头错误": + gesture_color = (0, 255, 0) # 绿色 + # 添加手势识别置信度指示 + cv2.circle(frame, (600, 60), 10, gesture_color, -1) + else: + gesture_color = (200, 200, 200) # 灰色 + gesture_text = f"手势: {current_gesture}" - frame = chinese_renderer.put_text(frame, gesture_text, (10, 120), + frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), size=24, color=gesture_color) # 操作提示 @@ -480,26 +596,30 @@ def main(): if not cap: help_text += " W/A/S/D/F/X:控制" - frame = chinese_renderer.put_text(frame, help_text, (10, 450), + frame = chinese_renderer.put_text(frame, help_text, (10, 420), size=20, color=(255, 255, 255)) - # AirSim连接状态 + # 添加手势提示 + if cap: + gesture_hint = "提示: 握拳停止,食指向前,手掌张开移动" + frame = chinese_renderer.put_text(frame, gesture_hint, (10, 450), + size=18, color=(255, 200, 100)) + if not drone_controller.connected: warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" - frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + frame = chinese_renderer.put_text(frame, warning_text, (10, 360), size=18, color=(0, 165, 255)) - # 显示窗口 cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) # ========== 键盘控制 ========== - key = cv2.waitKey(30) & 0xFF + key = cv2.waitKey(1) & 0xFF if key == 27: # ESC键 - print("\nExiting...") + print("\n退出程序...") break - elif key == ord('c') or key == ord('C'): # 连接无人机 + elif key == ord('c') or key == ord('C'): if not drone_controller.connected: drone_controller.connect() @@ -511,25 +631,29 @@ def main(): drone_controller.takeoff() time.sleep(0.3) - elif key in key_to_gesture: # 键盘控制 + elif key in key_to_gesture: gesture = key_to_gesture[key] gesture_recognizer.set_simulated_gesture(gesture) + current_gesture = gesture if drone_controller.connected and drone_controller.flying: drone_controller.move_by_gesture(gesture) - # 真实手势控制 + # 真实手势控制(添加间隔控制避免过于频繁) + current_time = time.time() if (current_gesture and current_gesture != "等待手势" and - current_gesture != "摄像头错误" and - drone_controller.connected and drone_controller.flying): + current_gesture != "摄像头错误" and current_gesture != "识别异常" and + drone_controller.connected and drone_controller.flying and + current_time - last_control_time > control_interval): drone_controller.move_by_gesture(current_gesture) + last_control_time = current_time - # ========== 清理资源 ========== - print("\nCleaning up resources...") + # 清理资源 + print("\n清理资源...") if cap: cap.release() cv2.destroyAllWindows() drone_controller.emergency_stop() - print("Program safely exited") + print("程序安全退出") print("=" * 60) @@ -538,9 +662,9 @@ def main(): try: main() except KeyboardInterrupt: - print("\nProgram interrupted by user") + print("\n程序被用户中断") except Exception as e: - print(f"\nProgram error: {e}") + print(f"\n程序错误: {e}") traceback.print_exc() finally: - input("\nPress Enter to exit...") \ No newline at end of file + input("\n按回车键退出...") \ No newline at end of file From d1726c94f7739505e7cd5e035dd74dc285ac53ba Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 09:57:53 +0800 Subject: [PATCH 03/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E8=AF=86=E5=88=AB=E7=AE=97=E6=B3=95=EF=BC=8C=E6=8F=90=E5=8D=87?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F=E7=A8=B3=E5=AE=9A=E6=80=A7=E5=92=8C=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E4=BD=93=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main.py | 857 ++++++++++++------ 1 file changed, 588 insertions(+), 269 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index f2a7020806..5325f35afa 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 修复手势识别版 -优化了手势识别算法,增加更多手势判断逻辑 +手势控制AirSim无人机 - 深度优化版 +优化了手势识别准确性和系统稳定性 作者: xiaoshiyuan888 """ @@ -10,12 +10,14 @@ import time import traceback import subprocess +import math from PIL import Image, ImageDraw, ImageFont import cv2 import numpy as np +from collections import deque print("=" * 60) -print("Gesture Controlled Drone - Enhanced Gesture Recognition") +print("Gesture Controlled Drone - Advanced Recognition v2.0") print("=" * 60) # ========== 修复导入路径 ========== @@ -47,6 +49,7 @@ def safe_import(): print(f"[OpenCV] ✗ Import failed: {e}") return None, modules_status + # 尝试导入mediapipe,如果失败则使用OpenCV方案 mp = None mp_hands = None mp_drawing = None @@ -58,28 +61,14 @@ def safe_import(): print("[MediaPipe] ✓ Gesture recognition library ready") except Exception as e: modules_status['MediaPipe'] = False - print(f"[MediaPipe] ✗ Import failed: {e}") - print(" Please install: pip install mediapipe") + print(f"[MediaPipe] ✗ Import failed, using OpenCV fallback: {e}") airsim_module = None - import_methods = [ - lambda: __import__('airsim'), - lambda: __import__('AirSim'), - ] - - for method in import_methods: - try: - airsim_module = method() - modules_status['AirSim'] = True - print(f"[AirSim] ✓ Successfully imported") - break - except ImportError: - continue - except Exception as e: - print(f"[AirSim] Import error: {e}") - continue - - if not modules_status.get('AirSim', False): + try: + airsim_module = __import__('airsim') + modules_status['AirSim'] = True + print(f"[AirSim] ✓ Successfully imported") + except ImportError: print("\n" + "!" * 60) print("⚠ AirSim library NOT FOUND!") print("!" * 60) @@ -90,18 +79,10 @@ def safe_import(): print(" pip install git+https://github.com/microsoft/AirSim.git") print("!" * 60) - print("\nDo you want to try automatic installation? (y/n)") + print("\nContinue without AirSim? (y/n)") choice = input().strip().lower() - if choice == 'y': - try: - print("Installing msgpack-rpc-python...") - subprocess.check_call([sys.executable, "-m", "pip", "install", "msgpack-rpc-python"]) - print("Installing AirSim...") - subprocess.check_call([sys.executable, "-m", "pip", "install", "airsim"]) - print("\n✅ Installation complete! Please restart the program.") - except Exception as e: - print(f"Installation failed: {e}") - sys.exit(1) + if choice != 'y': + sys.exit(1) return { 'cv2': cv2, @@ -126,9 +107,8 @@ def safe_import(): print("-" * 60) # 解包库 -cv2, np, mp = libs['cv2'], libs['np'], libs['mp'] +cv2, np = libs['cv2'], libs['np'] Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -mp_hands, mp_drawing = libs['mp_hands'], libs['mp_drawing'] # ========== 中文显示模块 ========== @@ -150,16 +130,18 @@ def load_fonts(self): ] for path in font_paths: try: + self.fonts[16] = ImageFont.truetype(path, 16) self.fonts[20] = ImageFont.truetype(path, 20) self.fonts[24] = ImageFont.truetype(path, 24) - self.fonts[30] = ImageFont.truetype(path, 30) + self.fonts[28] = ImageFont.truetype(path, 28) + self.fonts[32] = ImageFont.truetype(path, 32) print(f"✓ Chinese fonts loaded: {path}") return except: continue print("⚠ No Chinese fonts found, using default") - def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): + def put_text(self, frame, text, pos, size=20, color=(255, 255, 255), bg_color=None): """在图像上绘制中文文本""" try: img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) @@ -168,197 +150,421 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): font = self.fonts.get(size, self.fonts.get(20)) + # 如果有背景色,先绘制背景 + if bg_color: + bbox = draw.textbbox(pos, text, font=font) + padding = 2 + draw.rectangle( + [bbox[0] - padding, bbox[1] - padding, bbox[2] + padding, bbox[3] + padding], + fill=bg_color + ) + # 绘制阴影 shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 2, pos[1] + 2) + shadow_pos = (pos[0] + 1, pos[1] + 1) draw.text(shadow_pos, text, font=font, fill=shadow_color) # 绘制文字 - rgb_color = color[::-1] + rgb_color = color[::-1] # BGR to RGB draw.text(pos, text, font=font, fill=rgb_color) return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) - except: + except Exception as e: # 备用方案:使用OpenCV绘制英文 cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, - size / 30, color, 2) + size / 25, color, 2) return frame chinese_renderer = ChineseTextRenderer() -# ========== 简化的手势识别模块 ========== -class SimpleGestureRecognizer: - """简化的手势识别器,不依赖MediaPipe""" +# ========== 高级手势识别模块 ========== +class AdvancedGestureRecognizer: + """高级手势识别器,结合多种检测方法""" def __init__(self): + self.use_mediapipe = status.get('MediaPipe', False) + + # 初始化MediaPipe(如果可用) + if self.use_mediapipe and libs['mp'] is not None: + try: + self.hands = libs['mp_hands'].Hands( + max_num_hands=2, + min_detection_confidence=0.7, + min_tracking_confidence=0.5, + static_image_mode=False, + model_complexity=1 + ) + print("✓ Using MediaPipe for gesture recognition") + except: + self.use_mediapipe = False + print("⚠ MediaPipe initialization failed, using OpenCV fallback") + else: + self.use_mediapipe = False + + # OpenCV手势识别参数 + self.history_size = 10 + self.gesture_history = deque(maxlen=self.history_size) + self.finger_history = deque(maxlen=self.history_size) self.current_gesture = "等待手势" + self.confidence = 0.0 self.last_gesture_time = time.time() - self.gesture_history = [] - self.max_history = 5 - print("✓ Simple gesture recognizer initialized (不使用MediaPipe)") - # 加载OpenCV的预训练模型用于手部检测 - try: - # 尝试加载Haar级联分类器 - cascade_path = cv2.data.haarcascades + 'haarcascade_frontalface_default.xml' - # 我们使用人脸检测器作为替代,实际上应该用手部检测器 - self.face_cascade = cv2.CascadeClassifier(cascade_path) - print("✓ OpenCV cascade classifier loaded") - except: - print("⚠ OpenCV cascade classifier not available") - self.face_cascade = None - - def detect_hand_contour(self, frame): - """使用轮廓检测手部""" - # 转换为HSV颜色空间 - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + # 肤色检测参数(自适应) + self.skin_lower = np.array([0, 30, 60], dtype=np.uint8) + self.skin_upper = np.array([25, 255, 255], dtype=np.uint8) + + # 背景减除 + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2(history=500, varThreshold=16, detectShadows=True) + + # 手部跟踪 + self.track_window = None + self.track_box = None + self.tracking = False + + print("✓ Advanced gesture recognizer initialized") - # 定义肤色范围 - lower_skin = np.array([0, 20, 70], dtype=np.uint8) - upper_skin = np.array([20, 255, 255], dtype=np.uint8) + def adaptive_skin_detection(self, frame): + """自适应肤色检测""" + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - # 创建肤色掩码 - mask = cv2.inRange(hsv, lower_skin, upper_skin) + # 自适应调整肤色范围 + mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) # 形态学操作 - kernel = np.ones((5, 5), np.uint8) - mask = cv2.dilate(mask, kernel, iterations=2) + kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) mask = cv2.erode(mask, kernel, iterations=1) + mask = cv2.dilate(mask, kernel, iterations=2) mask = cv2.GaussianBlur(mask, (5, 5), 0) + return mask + + def find_hand_contours(self, frame): + """查找手部轮廓""" + # 使用背景减除 + fg_mask = self.bg_subtractor.apply(frame) + + # 肤色检测 + skin_mask = self.adaptive_skin_detection(frame) + + # 结合两种掩码 + combined_mask = cv2.bitwise_and(fg_mask, skin_mask) + + # 形态学操作优化 + kernel = np.ones((7, 7), np.uint8) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, kernel) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, kernel) + # 查找轮廓 - contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours, _ = cv2.findContours(combined_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + if not contours: + return None, combined_mask + + # 找到最大的轮廓(假设是手) + largest_contour = max(contours, key=cv2.contourArea) + + # 过滤掉太小的轮廓 + if cv2.contourArea(largest_contour) < 5000: + return None, combined_mask + + return largest_contour, combined_mask - if contours: - # 找到最大的轮廓(假设是手) - largest_contour = max(contours, key=cv2.contourArea) + def analyze_hand_contour(self, contour): + """分析手部轮廓特征""" + # 计算轮廓特征 + area = cv2.contourArea(contour) + perimeter = cv2.arcLength(contour, True) - # 计算轮廓面积 - area = cv2.contourArea(largest_contour) + # 计算凸包和凸缺陷 + hull = cv2.convexHull(contour, returnPoints=False) + if len(hull) < 3: + return 0, None, None - if area > 1000: # 最小面积阈值 - # 获取边界矩形 - x, y, w, h = cv2.boundingRect(largest_contour) + defects = cv2.convexityDefects(contour, hull) - # 计算轮廓的中心点 - M = cv2.moments(largest_contour) - if M["m00"] != 0: - cx = int(M["m10"] / M["m00"]) - cy = int(M["m01"] / M["m00"]) + if defects is None: + return 0, None, None - # 计算凸包和凸缺陷 - hull = cv2.convexHull(largest_contour, returnPoints=False) - if len(hull) > 3: - defects = cv2.convexityDefects(largest_contour, hull) + # 计算手指数量 + finger_count = 0 + finger_tips = [] - if defects is not None: - # 计算手指数量 - finger_count = 0 - for i in range(defects.shape[0]): - s, e, f, d = defects[i, 0] - start = tuple(largest_contour[s][0]) - end = tuple(largest_contour[e][0]) - far = tuple(largest_contour[f][0]) + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(contour[s][0]) + end = tuple(contour[e][0]) + far = tuple(contour[f][0]) - # 计算角度 - a = np.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) - b = np.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) - c = np.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + # 计算角度 + a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) - angle = np.arccos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) - if angle <= np.pi / 2: - finger_count += 1 + # 如果角度小于90度,可能是指缝 + if angle < math.pi / 2: + finger_count += 1 + finger_tips.append(end) - finger_count = min(finger_count + 1, 5) + # 手指数量加1(大拇指) + finger_count = min(finger_count + 1, 5) - # 绘制轮廓和中心点 - cv2.drawContours(frame, [largest_contour], -1, (0, 255, 0), 2) - cv2.circle(frame, (cx, cy), 7, (255, 0, 0), -1) - cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) + # 计算轮廓中心 + M = cv2.moments(contour) + if M["m00"] != 0: + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + center = (cx, cy) + else: + center = None + + return finger_count, center, finger_tips - # 在边界框上方显示手指数量 - cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + def detect_fingertips_with_contours(self, contour): + """使用轮廓检测指尖""" + # 简化轮廓 + epsilon = 0.02 * cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon, True) - return cx, cy, w, h, finger_count, frame + # 查找凸点(可能是指尖) + hull = cv2.convexHull(approx, returnPoints=True) - return None, None, None, None, 0, frame + fingertips = [] + for point in hull: + fingertips.append(tuple(point[0])) - def recognize_simple_gesture(self, frame): - """简化的手势识别""" - # 检测手部轮廓 - cx, cy, w, h, finger_count, frame = self.detect_hand_contour(frame) + return fingertips[:5] # 最多5个指尖 - if cx is None or cy is None: - return "等待手势", frame + def recognize_gesture_by_fingers(self, finger_count, center, frame_shape): + """根据手指数量识别手势""" + if center is None: + return "等待手势", 0.5 - # 获取图像尺寸 - height, width = frame.shape[:2] + h, w = frame_shape[:2] + cx, cy = center - # 定义区域 - center_x = width // 2 - center_y = height // 2 + # 计算手部位置(归一化) + norm_x = cx / w + norm_y = cy / h - # 根据手指数量和位置判断手势 if finger_count == 0: - return "停止", frame + return "停止", 0.9 elif finger_count == 1: - return "向前", frame + return "向前", 0.8 + elif finger_count == 2: + # V字手势,可能是向前或特殊命令 + return "向前", 0.7 + elif finger_count == 3: + # 三指手势,根据位置判断 + if norm_y < 0.4: + return "向上", 0.7 + elif norm_y > 0.6: + return "向下", 0.7 + else: + return "停止", 0.6 elif finger_count >= 4: # 手掌张开,根据位置判断方向 - if cy < center_y - 50: - return "向上", frame - elif cy > center_y + 50: - return "向下", frame - elif cx < center_x - 50: - return "向左", frame - elif cx > center_x + 50: - return "向右", frame + if norm_x < 0.3: + return "向左", 0.8 + elif norm_x > 0.7: + return "向右", 0.8 + elif norm_y < 0.4: + return "向上", 0.8 + elif norm_y > 0.6: + return "向下", 0.8 else: - return "停止", frame - else: - return "等待手势", frame + return "停止", 0.7 + + return "等待手势", 0.5 + + def recognize_with_mediapipe(self, frame): + """使用MediaPipe进行手势识别""" + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + results = self.hands.process(rgb_frame) + + gesture = "等待手势" + confidence = 0.5 + + if results.multi_hand_landmarks: + for hand_landmarks in results.multi_hand_landmarks: + # 绘制手部关键点 + libs['mp_drawing'].draw_landmarks( + frame, hand_landmarks, libs['mp_hands'].HAND_CONNECTIONS, + libs['mp_drawing'].DrawingSpec(color=(0, 255, 0), thickness=2), + libs['mp_drawing'].DrawingSpec(color=(255, 0, 0), thickness=2) + ) + + # 计算手指状态 + landmarks = hand_landmarks.landmark + + # 简单的手指状态检测 + finger_tips = [4, 8, 12, 16, 20] + finger_pips = [2, 6, 10, 14, 18] + + extended_fingers = 0 + for tip, pip in zip(finger_tips, finger_pips): + if landmarks[tip].y < landmarks[pip].y: + extended_fingers += 1 + + # 根据伸直的手指数量判断手势 + if extended_fingers == 0: + gesture = "停止" + confidence = 0.9 + elif extended_fingers == 1: + gesture = "向前" + confidence = 0.8 + elif extended_fingers >= 4: + # 根据手掌方向判断 + wrist = landmarks[0] + middle_mcp = landmarks[9] + + dx = middle_mcp.x - wrist.x + dy = middle_mcp.y - wrist.y + + if abs(dx) > abs(dy): + if dx > 0: + gesture = "向右" + else: + gesture = "向左" + else: + if dy > 0: + gesture = "向下" + else: + gesture = "向上" + confidence = 0.7 + + return gesture, confidence, frame + + def smooth_gesture(self, gesture, confidence): + """平滑手势输出""" + self.gesture_history.append((gesture, confidence)) + + if len(self.gesture_history) < 3: + return gesture, confidence + + # 统计最近的手势 + gesture_counts = {} + confidence_sum = {} + + for g, c in self.gesture_history: + if g not in gesture_counts: + gesture_counts[g] = 0 + confidence_sum[g] = 0 + gesture_counts[g] += 1 + confidence_sum[g] += c + + # 找到最频繁的手势 + max_count = 0 + best_gesture = gesture + best_confidence = confidence + + for g, count in gesture_counts.items(): + if count > max_count and g != "等待手势": + max_count = count + best_gesture = g + best_confidence = confidence_sum[g] / count + + # 如果手势稳定,更新当前手势 + if max_count >= len(self.gesture_history) // 2: + self.current_gesture = best_gesture + self.confidence = best_confidence + + return self.current_gesture, self.confidence + + def visualize_hand(self, frame, contour, center, finger_count, fingertips): + """可视化手部检测结果""" + if contour is not None: + # 绘制轮廓 + cv2.drawContours(frame, [contour], -1, (0, 255, 0), 2) + + # 绘制边界框 + x, y, w, h = cv2.boundingRect(contour) + cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 0), 2) + + # 绘制中心点 + if center: + cx, cy = center + cv2.circle(frame, (cx, cy), 8, (0, 0, 255), -1) + + # 绘制指尖 + for tip in fingertips[:finger_count]: + cv2.circle(frame, tip, 6, (255, 255, 0), -1) + + # 显示手指数量 + cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + + return frame def recognize(self, frame): """识别手势""" + start_time = time.time() + try: - # 使用简化的手势识别 - gesture, processed_frame = self.recognize_simple_gesture(frame) - - # 手势历史记录和平滑处理 - self.gesture_history.append(gesture) - if len(self.gesture_history) > self.max_history: - self.gesture_history.pop(0) - - # 使用多数表决决定最终手势 - if self.gesture_history: - gesture_counts = {} - for g in self.gesture_history: - gesture_counts[g] = gesture_counts.get(g, 0) + 1 - - # 找到出现次数最多的手势 - max_gesture = max(gesture_counts, key=gesture_counts.get) - if gesture_counts[max_gesture] > len(self.gesture_history) // 2: - self.current_gesture = max_gesture + # 镜像图像以便更直观的控制 + frame = cv2.flip(frame, 1) + + # 方法1:使用MediaPipe(如果可用) + if self.use_mediapipe: + gesture, confidence, processed_frame = self.recognize_with_mediapipe(frame) + frame = processed_frame + else: + # 方法2:使用OpenCV + contour, mask = self.find_hand_contours(frame) + + if contour is not None: + finger_count, center, finger_tips = self.analyze_hand_contour(contour) + + # 如果凸缺陷检测失败,使用轮廓检测指尖 + if finger_tips is None or len(finger_tips) < finger_count: + finger_tips = self.detect_fingertips_with_contours(contour) + + self.finger_history.append(finger_count) + + # 计算平均手指数量(平滑) + if len(self.finger_history) > 0: + avg_fingers = sum(self.finger_history) / len(self.finger_history) + finger_count = int(round(avg_fingers)) + + # 识别手势 + gesture, confidence = self.recognize_gesture_by_fingers( + finger_count, center, frame.shape + ) + + # 可视化结果 + frame = self.visualize_hand(frame, contour, center, finger_count, finger_tips) else: - self.current_gesture = gesture + gesture, confidence = "等待手势", 0.3 + + # 手势平滑处理 + final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) + + # 更新最后检测时间 + self.last_gesture_time = time.time() + + # 显示处理时间 + process_time = (time.time() - start_time) * 1000 + cv2.putText(frame, f'Process: {process_time:.1f}ms', (10, frame.shape[0] - 20), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + return final_gesture, final_confidence, frame - return self.current_gesture except Exception as e: print(f"Gesture recognition error: {e}") - return "识别异常" + return "识别异常", 0.0, frame def set_simulated_gesture(self, gesture): """设置模拟的手势""" self.current_gesture = gesture + self.confidence = 0.9 -# ========== 无人机控制模块 ========== -class DroneController: - """无人机控制器""" +# ========== 增强的无人机控制模块 ========== +class EnhancedDroneController: + """增强的无人机控制器""" def __init__(self, airsim_module): self.airsim = airsim_module @@ -367,12 +573,27 @@ def __init__(self, airsim_module): self.flying = False self.connection_attempted = False + # 控制参数 + self.velocity = 3.0 + self.duration = 0.2 + self.altitude = -10.0 # 初始高度(负值表示向上) + + # 控制历史 + self.control_history = deque(maxlen=10) + self.last_control_time = 0 + def connect(self): """连接AirSim无人机""" if self.connection_attempted: return self.connected self.connection_attempted = True + + if self.airsim is None: + print("⚠ AirSim not available, using simulation mode") + self.connected = True + return True + print("Connecting to AirSim...") try: @@ -396,6 +617,14 @@ def connect(self): print("2. 选择环境(如:Landscape Mountains)") print("3. 无人机已在世界中生成") print("4. 如果需要,按模拟器中的'R'键重置无人机") + + print("\n使用模拟模式继续?(y/n)") + choice = input().strip().lower() + if choice == 'y': + self.connected = True + print("✅ Using simulation mode") + return True + return False def takeoff(self): @@ -405,15 +634,25 @@ def takeoff(self): return False try: + if self.airsim is None or self.client is None: + print("✅ Simulated takeoff") + self.flying = True + return True + print("Taking off...") self.client.takeoffAsync().join() time.sleep(1) + + # 上升到指定高度 + self.client.moveToZAsync(self.altitude, 3).join() + self.flying = True print("✅ Drone took off successfully") return True except Exception as e: print(f"❌ Takeoff failed: {e}") - return False + self.flying = True # 模拟模式 + return True def land(self): """降落""" @@ -421,6 +660,11 @@ def land(self): return False try: + if self.airsim is None or self.client is None: + print("✅ Simulated landing") + self.flying = False + return True + print("Landing...") self.client.landAsync().join() self.flying = False @@ -428,101 +672,155 @@ def land(self): return True except Exception as e: print(f"Landing failed: {e}") + self.flying = False return False - def move_by_gesture(self, gesture): + def move_by_gesture(self, gesture, confidence=0.7): """根据手势移动""" if not self.connected or not self.flying: - return + return False + + # 检查控制间隔 + current_time = time.time() + if current_time - self.last_control_time < 0.1: # 最小控制间隔 + return False + + # 如果置信度太低,不执行动作 + if confidence < 0.5: + return False try: - velocity = 2.0 # 降低速度以获得更平滑的控制 - duration = 0.3 # 减少持续时间 + if self.airsim is None or self.client is None: + print(f"Simulated move: {gesture}") + self.control_history.append((gesture, current_time)) + self.last_control_time = current_time + return True + + success = False if gesture == "向上": - self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() + self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) + success = True elif gesture == "向下": - self.client.moveByVelocityZAsync(0, 0, velocity, duration).join() + self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) + success = True elif gesture == "向左": - self.client.moveByVelocityAsync(-velocity, 0, 0, duration).join() + self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) + success = True elif gesture == "向右": - self.client.moveByVelocityAsync(velocity, 0, 0, duration).join() + self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) + success = True elif gesture == "向前": - self.client.moveByVelocityAsync(0, -velocity, 0, duration).join() + self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) + success = True elif gesture == "停止": - self.client.hoverAsync().join() + self.client.hoverAsync() + success = True + + if success: + self.control_history.append((gesture, current_time)) + self.last_control_time = current_time + + return success except Exception as e: print(f"Control command failed: {e}") + return False def emergency_stop(self): """紧急停止""" if self.connected: try: - if self.flying: + if self.flying and self.client is not None: print("Emergency landing...") self.land() - self.client.armDisarm(False) - self.client.enableApiControl(False) - print("✅ Emergency stop complete") + if self.client is not None: + self.client.armDisarm(False) + self.client.enableApiControl(False) + print("✅ Emergency stop complete") except: pass self.connected = False self.flying = False +# ========== 性能监控模块 ========== +class PerformanceMonitor: + """性能监控器""" + + def __init__(self): + self.frame_times = deque(maxlen=60) + self.fps = 0 + self.last_update = time.time() + + def update(self): + """更新性能数据""" + current_time = time.time() + self.frame_times.append(current_time) + + # 每秒更新一次FPS + if current_time - self.last_update >= 1.0: + if len(self.frame_times) > 1: + self.fps = len(self.frame_times) / (self.frame_times[-1] - self.frame_times[0]) + self.last_update = current_time + + def get_stats(self): + """获取性能统计""" + return { + 'fps': self.fps, + 'frame_count': len(self.frame_times) + } + + # ========== 主程序 ========== def main(): """主函数""" - # 使用简化的手势识别器 - gesture_recognizer = SimpleGestureRecognizer() - drone_controller = DroneController(libs['airsim']) + # 初始化组件 + gesture_recognizer = AdvancedGestureRecognizer() + drone_controller = EnhancedDroneController(libs['airsim']) + performance_monitor = PerformanceMonitor() # 初始化摄像头 cap = None - try: - cap = cv2.VideoCapture(0) - if not cap.isOpened(): - # 尝试其他摄像头索引 - for i in range(1, 5): - cap = cv2.VideoCapture(i) - if cap.isOpened(): - print(f"✓ Camera {i} initialized") - break - else: - print("❌ Camera not available, using keyboard control only") - cap = None - else: - print("✓ Camera 0 initialized") - - if cap: - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - cap.set(cv2.CAP_PROP_FPS, 30) - except Exception as e: - print(f"⚠ Camera init failed: {e}") + available_cameras = [] + + # 检测可用摄像头 + for i in range(0, 4): + temp_cap = cv2.VideoCapture(i) + if temp_cap.isOpened(): + available_cameras.append(i) + temp_cap.release() + + if available_cameras: + print(f"✓ Available cameras: {available_cameras}") + cap = cv2.VideoCapture(available_cameras[0]) + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + cap.set(cv2.CAP_PROP_FPS, 30) + print(f"✓ Using camera {available_cameras[0]}") + else: + print("❌ No cameras available, using keyboard control only") cap = None # 显示说明 print("\n" + "=" * 60) - print("操作说明") + print("手势控制无人机系统 - 高级版") print("=" * 60) - print("1. 首先启动AirSim模拟器") - print("2. 选择环境(例如:Landscape Mountains)") - print("3. 按 [C] 连接无人机") - print("4. 按 [空格键] 起飞/降落") + print("操作说明:") + print("1. 首先启动AirSim模拟器(可选)") + print("2. 按 [C] 连接无人机") + print("3. 按 [空格键] 起飞/降落") if cap: - print("5. 手势控制:") - print(" - 握拳(0个手指):停止") - print(" - 伸出1个手指:向前") - print(" - 张开手掌(4-5个手指):根据手的位置控制方向") - print(" * 确保手部在摄像头视野内,光线充足,背景简单") + print("4. 手势控制:") + print(" - 握拳(0指):停止") + print(" - 食指(1指):向前") + print(" - 手掌张开(4-5指):根据手的位置控制方向") + print(" * 手势识别置信度 > 50% 时才会执行") else: - print("5. 键盘控制:") + print("4. 键盘控制:") print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") - print("6. 按 [ESC] 安全退出") + print("5. 按 [ESC] 安全退出") + print("6. 按 [R] 重置手势识别") print("=" * 60) - print("程序启动成功!") - print("-" * 60) # 键盘手势映射 key_to_gesture = { @@ -534,83 +832,91 @@ def main(): ord('x'): "停止", ord('X'): "停止", } - last_control_time = time.time() - control_interval = 0.5 # 控制间隔,避免过于频繁 - # 主循环 + last_gesture_time = time.time() + gesture_timeout = 2.0 # 手势超时时间 + while True: - frame = None + performance_monitor.update() + # 读取摄像头帧 if cap: ret, frame = cap.read() - if ret: - # 镜像图像以便更直观的控制 - frame = cv2.flip(frame, 1) - current_gesture = gesture_recognizer.recognize(frame) + if not ret: + # 如果读取失败,尝试重新初始化摄像头 + print("⚠ Camera read failed, trying to reinitialize...") + cap.release() + if available_cameras: + cap = cv2.VideoCapture(available_cameras[0]) + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + ret, frame = cap.read() + + if not ret: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + current_gesture = "摄像头错误" + confidence = 0.0 + else: + current_gesture, confidence, frame = gesture_recognizer.recognize(frame) else: - frame = np.zeros((480, 640, 3), dtype=np.uint8) - current_gesture = "摄像头错误" + current_gesture, confidence, frame = gesture_recognizer.recognize(frame) else: frame = np.zeros((480, 640, 3), dtype=np.uint8) current_gesture = gesture_recognizer.current_gesture + confidence = gesture_recognizer.confidence # ========== 绘制界面 ========== - # 创建半透明覆盖层 + # 创建信息覆盖层 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (640, 120), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.6, frame, 0.4, 0) - - # 绘制底部操作提示区域 - cv2.rectangle(frame, (0, 400), (640, 480), (0, 0, 0), -1) - frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) + cv2.rectangle(overlay, (0, 0), (640, 140), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) - # 标题 - frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), - size=30, color=(0, 255, 255)) + # 标题和版本 + title = "手势控制无人机系统 - 高级版" + frame = chinese_renderer.put_text(frame, title, (10, 10), size=28, color=(0, 255, 255)) # 状态信息 status_color = (0, 255, 0) if drone_controller.connected else (0, 0, 255) - status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" - frame = chinese_renderer.put_text(frame, status_text, (10, 50), - size=24, color=status_color) + status_text = f"状态: {'已连接' if drone_controller.connected else '未连接'}" + frame = chinese_renderer.put_text(frame, status_text, (10, 50), size=24, color=status_color) - flight_color = (0, 255, 0) if drone_controller.flying else (255, 255, 0) - flight_status = f"飞行: {'是' if drone_controller.flying else '否'}" - frame = chinese_renderer.put_text(frame, flight_status, (10, 80), - size=24, color=flight_color) + flight_color = (0, 255, 0) if drone_controller.flying else (255, 165, 0) + flight_text = f"飞行: {'是' if drone_controller.flying else '否'}" + frame = chinese_renderer.put_text(frame, flight_text, (10, 80), size=24, color=flight_color) # 手势信息 - if current_gesture != "等待手势" and current_gesture != "摄像头错误": - gesture_color = (0, 255, 0) # 绿色 - # 添加手势识别置信度指示 - cv2.circle(frame, (600, 60), 10, gesture_color, -1) - else: - gesture_color = (200, 200, 200) # 灰色 + gesture_color = (0, 255, 0) if confidence > 0.7 else (255, 165, 0) if confidence > 0.5 else (200, 200, 200) + gesture_text = f"手势: {current_gesture} ({confidence:.1%})" + frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), size=24, color=gesture_color) - gesture_text = f"手势: {current_gesture}" - frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), - size=24, color=gesture_color) + # 性能信息 + stats = performance_monitor.get_stats() + perf_text = f"FPS: {stats['fps']:.1f} | 延迟: {1000 / stats['fps']:.1f}ms" if stats['fps'] > 0 else "FPS: 0" + frame = chinese_renderer.put_text(frame, perf_text, (440, 50), size=20, color=(255, 255, 255)) # 操作提示 - help_text = "C:连接 空格:起飞/降落 ESC:退出" + help_text = "C:连接 空格:起飞/降落 R:重置 ESC:退出" if not cap: help_text += " W/A/S/D/F/X:控制" - frame = chinese_renderer.put_text(frame, help_text, (10, 420), - size=20, color=(255, 255, 255)) + # 绘制底部提示栏 + cv2.rectangle(frame, (0, 440), (640, 480), (0, 0, 0), -1) + frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) + frame = chinese_renderer.put_text(frame, help_text, (10, 450), size=18, color=(255, 255, 255)) - # 添加手势提示 + # 手势提示 if cap: - gesture_hint = "提示: 握拳停止,食指向前,手掌张开移动" - frame = chinese_renderer.put_text(frame, gesture_hint, (10, 450), - size=18, color=(255, 200, 100)) + hint_text = "提示: 保持手势稳定,确保良好光线" + frame = chinese_renderer.put_text(frame, hint_text, (10, 470), size=16, color=(255, 200, 100)) + # 显示连接提示 if not drone_controller.connected: - warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" - frame = chinese_renderer.put_text(frame, warning_text, (10, 360), - size=18, color=(0, 165, 255)) + warning_text = "⚠ 按C键连接无人机,或继续使用模拟模式" + frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + size=18, color=(0, 165, 255), bg_color=(0, 0, 0)) - cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) + # 显示图像 + cv2.imshow('Gesture Controlled Drone - Advanced Version', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -629,23 +935,35 @@ def main(): drone_controller.land() else: drone_controller.takeoff() - time.sleep(0.3) + time.sleep(0.5) + + elif key == ord('r') or key == ord('R'): + print("重置手势识别...") + gesture_recognizer = AdvancedGestureRecognizer() elif key in key_to_gesture: gesture = key_to_gesture[key] gesture_recognizer.set_simulated_gesture(gesture) current_gesture = gesture + confidence = 0.9 if drone_controller.connected and drone_controller.flying: - drone_controller.move_by_gesture(gesture) + drone_controller.move_by_gesture(gesture, confidence) - # 真实手势控制(添加间隔控制避免过于频繁) + # 真实手势控制 current_time = time.time() if (current_gesture and current_gesture != "等待手势" and current_gesture != "摄像头错误" and current_gesture != "识别异常" and drone_controller.connected and drone_controller.flying and - current_time - last_control_time > control_interval): - drone_controller.move_by_gesture(current_gesture) - last_control_time = current_time + confidence > 0.5 and # 只有置信度足够高才执行 + current_time - last_gesture_time > 0.2): # 控制频率限制 + + success = drone_controller.move_by_gesture(current_gesture, confidence) + if success: + last_gesture_time = current_time + + # 检查手势超时 + if current_time - gesture_recognizer.last_gesture_time > gesture_timeout: + gesture_recognizer.current_gesture = "等待手势" # 清理资源 print("\n清理资源...") @@ -667,4 +985,5 @@ def main(): print(f"\n程序错误: {e}") traceback.print_exc() finally: - input("\n按回车键退出...") \ No newline at end of file + print("\n按回车键退出...") + input() \ No newline at end of file From 4b5e52e8e77f1a51b3fadc2b457a45463264baca Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 14:30:43 +0800 Subject: [PATCH 04/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=8B=B1=E6=96=87?= =?UTF-8?q?=E7=95=8C=E9=9D=A2=EF=BC=8C=E6=94=B9=E8=BF=9B=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E8=AF=86=E5=88=AB=E7=AE=97=E6=B3=95=EF=BC=8C=E5=A2=9E=E5=BC=BA?= =?UTF-8?q?=E5=8D=95=E6=8C=87=E8=AF=86=E5=88=AB=E5=87=86=E7=A1=AE=E7=8E=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 9 +- .../config.json | 53 + .../gesture_config.json | 45 + .../main.py | 1409 ++++++++++------- 4 files changed, 906 insertions(+), 610 deletions(-) create mode 100644 src/autonomus_drone_hand_gesture_project/config.json create mode 100644 src/autonomus_drone_hand_gesture_project/gesture_config.json diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index 05dab76400..0ad6fac5ef 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -1,8 +1,13 @@ # 忽略整个Lib目录(包含所有子文件/子目录) Lib/ -# 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 + +# 忽略Python编译文件,避免无关文件干扰 __pycache__/ *.pyc *.dll.a + # 忽略IDEA配置文件 -.idea/ \ No newline at end of file +.idea/ + +# 忽略日志文件夹(临时日志文件,无需提交到仓库) +logs/ \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/config.json b/src/autonomus_drone_hand_gesture_project/config.json new file mode 100644 index 0000000000..1fa5f144eb --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/config.json @@ -0,0 +1,53 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "detection_confidence": 0.7, + "tracking_confidence": 0.5, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.6 + }, + "drone": { + "velocity": 3.0, + "duration": 0.2, + "altitude": -10.0, + "control_interval": 0.2 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_landmarks": true, + "show_help": true + }, + "performance": { + "target_fps": 30, + "resize_factor": 0.5, + "enable_threading": true + }, + "camera_width": 640, + "camera_height": 480, + "camera_fps": 30, + "camera_index": 0, + "mirror_camera": false, + "min_detection_confidence": 0.6, + "min_tracking_confidence": 0.5, + "gesture_smoothness": 5, + "gesture_stability_threshold": 3, + "control_velocity": 3.0, + "control_duration": 0.25, + "control_deadzone": 0.1, + "max_altitude": -20.0, + "min_altitude": -5.0, + "target_fps": 30, + "frame_timeout": 2.0, + "log_level": "INFO", + "ui_theme": "dark", + "show_fps": true, + "show_landmarks": true, + "show_confidence": true +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json new file mode 100644 index 0000000000..c6d931acbe --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -0,0 +1,45 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "skin_lower": [ + 0, + 30, + 60 + ], + "skin_upper": [ + 25, + 255, + 255 + ], + "min_hand_area": 3000, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.6, + "detection_interval": 2, + "single_finger_angle_threshold": 60 + }, + "drone": { + "velocity": 2.5, + "duration": 0.3, + "altitude": -10.0, + "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false + } +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 5325f35afa..4bd8903a38 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,33 +1,34 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 深度优化版 -优化了手势识别准确性和系统稳定性 -作者: xiaoshiyuan888 +Gesture Controlled AirSim Drone - Optimized Version +Pure OpenCV implementation with English interface +Author: xiaoshiyuan888 """ import sys import os import time import traceback -import subprocess +import json import math +from datetime import datetime from PIL import Image, ImageDraw, ImageFont import cv2 import numpy as np -from collections import deque +from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Advanced Recognition v2.0") +print("Gesture Controlled Drone - Optimized Version") print("=" * 60) -# ========== 修复导入路径 ========== +# ========== Fix import path ========== current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -# ========== 核心模块导入 ========== +# ========== Core module imports ========== def safe_import(): - """安全导入所有模块""" + """Safely import all modules""" modules_status = {} try: @@ -49,20 +50,6 @@ def safe_import(): print(f"[OpenCV] ✗ Import failed: {e}") return None, modules_status - # 尝试导入mediapipe,如果失败则使用OpenCV方案 - mp = None - mp_hands = None - mp_drawing = None - try: - import mediapipe as mp - mp_hands = mp.solutions.hands - mp_drawing = mp.solutions.drawing_utils - modules_status['MediaPipe'] = True - print("[MediaPipe] ✓ Gesture recognition library ready") - except Exception as e: - modules_status['MediaPipe'] = False - print(f"[MediaPipe] ✗ Import failed, using OpenCV fallback: {e}") - airsim_module = None try: airsim_module = __import__('airsim') @@ -87,15 +74,12 @@ def safe_import(): return { 'cv2': cv2, 'np': np, - 'mp': mp, - 'mp_hands': mp_hands, - 'mp_drawing': mp_drawing, 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, 'airsim': airsim_module }, modules_status -# 执行导入 +# Execute imports libs, status = safe_import() if not status.get('OpenCV', False) or not status.get('PIL', False): print("\n❌ Core libraries missing, cannot start.") @@ -106,488 +90,608 @@ def safe_import(): print("✅ Environment check passed, initializing...") print("-" * 60) -# 解包库 +# Unpack libraries cv2, np = libs['cv2'], libs['np'] Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -# ========== 中文显示模块 ========== -class ChineseTextRenderer: - """中文文本渲染器""" +# ========== Configuration Manager ========== +class ConfigManager: + """Configuration Manager""" def __init__(self): - self.fonts = {} - self.load_fonts() - - def load_fonts(self): - """加载字体""" - font_paths = [ - 'simhei.ttf', - 'C:/Windows/Fonts/simhei.ttf', - 'C:/Windows/Fonts/msyh.ttc', - '/System/Library/Fonts/PingFang.ttc', - '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' - ] - for path in font_paths: + self.config_file = os.path.join(current_dir, 'gesture_config.json') + self.default_config = { + 'camera': { + 'index': 0, + 'width': 640, + 'height': 480, + 'fps': 30 + }, + 'gesture': { + 'skin_lower': [0, 30, 60], + 'skin_upper': [25, 255, 255], + 'min_hand_area': 3000, + 'history_size': 10, + 'smooth_frames': 5, + 'min_confidence': 0.6, + 'detection_interval': 1, + 'single_finger_angle_threshold': 60 # 新增:单个手指角度阈值 + }, + 'drone': { + 'velocity': 2.5, + 'duration': 0.3, + 'altitude': -10.0, + 'control_interval': 0.3 + }, + 'display': { + 'show_fps': True, + 'show_confidence': True, + 'show_help': True, + 'show_contours': True, + 'show_bbox': True, + 'show_fingertips': True # 新增:显示指尖 + }, + 'performance': { + 'target_fps': 30, + 'resize_factor': 1.0, + 'enable_multiprocessing': False + } + } + self.config = self.load_config() + + def load_config(self): + """Load configuration""" + if os.path.exists(self.config_file): try: - self.fonts[16] = ImageFont.truetype(path, 16) - self.fonts[20] = ImageFont.truetype(path, 20) - self.fonts[24] = ImageFont.truetype(path, 24) - self.fonts[28] = ImageFont.truetype(path, 28) - self.fonts[32] = ImageFont.truetype(path, 32) - print(f"✓ Chinese fonts loaded: {path}") - return - except: - continue - print("⚠ No Chinese fonts found, using default") + with open(self.config_file, 'r', encoding='utf-8') as f: + loaded_config = json.load(f) + config = self.default_config.copy() + self._merge_config(config, loaded_config) + print("✓ Configuration loaded from file") + return config + except Exception as e: + print(f"⚠ Failed to load config: {e}, using defaults") + return self.default_config.copy() + else: + print("✓ Using default configuration") + return self.default_config.copy() + + def _merge_config(self, base, update): + """Recursively merge configurations""" + for key, value in update.items(): + if key in base and isinstance(base[key], dict) and isinstance(value, dict): + self._merge_config(base[key], value) + else: + base[key] = value - def put_text(self, frame, text, pos, size=20, color=(255, 255, 255), bg_color=None): - """在图像上绘制中文文本""" + def save_config(self): + """Save configuration""" try: - img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - pil_img = Image.fromarray(img_rgb) - draw = ImageDraw.Draw(pil_img) - - font = self.fonts.get(size, self.fonts.get(20)) - - # 如果有背景色,先绘制背景 - if bg_color: - bbox = draw.textbbox(pos, text, font=font) - padding = 2 - draw.rectangle( - [bbox[0] - padding, bbox[1] - padding, bbox[2] + padding, bbox[3] + padding], - fill=bg_color - ) + with open(self.config_file, 'w', encoding='utf-8') as f: + json.dump(self.config, f, indent=2, ensure_ascii=False) + print("✓ Configuration saved") + except Exception as e: + print(f"⚠ Failed to save config: {e}") + + def get(self, *keys): + """Get configuration value""" + value = self.config + for key in keys: + if isinstance(value, dict) and key in value: + value = value[key] + else: + return None + return value - # 绘制阴影 - shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 1, pos[1] + 1) - draw.text(shadow_pos, text, font=font, fill=shadow_color) + def set(self, *keys, value): + """Set configuration value""" + if len(keys) == 0: + return - # 绘制文字 - rgb_color = color[::-1] # BGR to RGB - draw.text(pos, text, font=font, fill=rgb_color) + config = self.config + for key in keys[:-1]: + if key not in config: + config[key] = {} + config = config[key] - return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) - except Exception as e: - # 备用方案:使用OpenCV绘制英文 - cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, - size / 25, color, 2) - return frame + config[keys[-1]] = value + self.save_config() -chinese_renderer = ChineseTextRenderer() +config = ConfigManager() -# ========== 高级手势识别模块 ========== -class AdvancedGestureRecognizer: - """高级手势识别器,结合多种检测方法""" +# ========== Robust Gesture Recognizer ========== +class RobustGestureRecognizer: + """Robust Gesture Recognizer - Pure OpenCV Implementation""" def __init__(self): - self.use_mediapipe = status.get('MediaPipe', False) - - # 初始化MediaPipe(如果可用) - if self.use_mediapipe and libs['mp'] is not None: - try: - self.hands = libs['mp_hands'].Hands( - max_num_hands=2, - min_detection_confidence=0.7, - min_tracking_confidence=0.5, - static_image_mode=False, - model_complexity=1 - ) - print("✓ Using MediaPipe for gesture recognition") - except: - self.use_mediapipe = False - print("⚠ MediaPipe initialization failed, using OpenCV fallback") - else: - self.use_mediapipe = False + self.config = config.get('gesture') - # OpenCV手势识别参数 - self.history_size = 10 + # Gesture history and smoothing + self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) - self.finger_history = deque(maxlen=self.history_size) - self.current_gesture = "等待手势" - self.confidence = 0.0 - self.last_gesture_time = time.time() + self.confidence_history = deque(maxlen=self.history_size) + self.current_gesture = "Waiting" + self.current_confidence = 0.0 - # 肤色检测参数(自适应) - self.skin_lower = np.array([0, 30, 60], dtype=np.uint8) - self.skin_upper = np.array([25, 255, 255], dtype=np.uint8) + # Hand tracking + self.last_hand_position = None + self.hand_tracking = False + self.track_window = None - # 背景减除 - self.bg_subtractor = cv2.createBackgroundSubtractorMOG2(history=500, varThreshold=16, detectShadows=True) + # Performance statistics + self.process_times = deque(maxlen=30) + self.frame_counter = 0 + self.detection_interval = self.config['detection_interval'] + + # Gesture color mapping + self.gesture_colors = { + "Stop": (0, 0, 255), # Red + "Forward": (0, 255, 0), # Green + "Up": (255, 255, 0), # Cyan + "Down": (255, 0, 255), # Purple + "Left": (255, 165, 0), # Orange + "Right": (0, 165, 255), # Light Blue + "Waiting": (200, 200, 200), # Gray + "Error": (255, 0, 0) # Blue + } - # 手部跟踪 - self.track_window = None - self.track_box = None - self.tracking = False + # Skin detection parameters + self.skin_lower = np.array(self.config['skin_lower'], dtype=np.uint8) + self.skin_upper = np.array(self.config['skin_upper'], dtype=np.uint8) - print("✓ Advanced gesture recognizer initialized") + # Background subtractor + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( + history=100, varThreshold=25, detectShadows=True + ) - def adaptive_skin_detection(self, frame): - """自适应肤色检测""" - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + # Morphological operation kernels + self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) + + # Fingertip detection + self.prev_fingertips = None + self.fingertip_stability_threshold = 3 - # 自适应调整肤色范围 - mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) + print("✓ Robust gesture recognizer initialized (Pure OpenCV)") + + def preprocess_frame(self, frame): + """Preprocess frame""" + # Flip image horizontally for intuitive control + frame = cv2.flip(frame, 1) + + # Adjust size for performance + resize_factor = config.get('performance', 'resize_factor') + if resize_factor != 1.0: + new_width = int(frame.shape[1] * resize_factor) + new_height = int(frame.shape[0] * resize_factor) + frame = cv2.resize(frame, (new_width, new_height)) + + return frame - # 形态学操作 - kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - mask = cv2.erode(mask, kernel, iterations=1) - mask = cv2.dilate(mask, kernel, iterations=2) - mask = cv2.GaussianBlur(mask, (5, 5), 0) + def detect_skin(self, frame): + """Detect skin regions""" + # Convert to HSV color space + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - return mask + # Skin mask + skin_mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) - def find_hand_contours(self, frame): - """查找手部轮廓""" - # 使用背景减除 + # Apply background subtraction fg_mask = self.bg_subtractor.apply(frame) - # 肤色检测 - skin_mask = self.adaptive_skin_detection(frame) + # Combine skin mask and foreground mask + combined_mask = cv2.bitwise_and(skin_mask, fg_mask) + + # Morphological operations + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - # 结合两种掩码 - combined_mask = cv2.bitwise_and(fg_mask, skin_mask) + # Gaussian blur + combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) - # 形态学操作优化 - kernel = np.ones((7, 7), np.uint8) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, kernel) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, kernel) + return combined_mask - # 查找轮廓 - contours, _ = cv2.findContours(combined_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + def find_hand_contours(self, mask): + """Find hand contours""" + # Find contours + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: - return None, combined_mask + return None - # 找到最大的轮廓(假设是手) - largest_contour = max(contours, key=cv2.contourArea) + # Find the largest contour (assumed to be hand) + contours = sorted(contours, key=cv2.contourArea, reverse=True) - # 过滤掉太小的轮廓 - if cv2.contourArea(largest_contour) < 5000: - return None, combined_mask + for contour in contours: + area = cv2.contourArea(contour) - return largest_contour, combined_mask + # Filter out too small contours + if area < self.config['min_hand_area']: + continue - def analyze_hand_contour(self, contour): - """分析手部轮廓特征""" - # 计算轮廓特征 - area = cv2.contourArea(contour) - perimeter = cv2.arcLength(contour, True) + # Calculate contour perimeter + perimeter = cv2.arcLength(contour, True) + + # Filter out too simple contours + if perimeter < 100: + continue + + return contour + + return None + + def detect_fingertips_advanced(self, contour): + """Advanced fingertip detection""" + # Simplify contour + epsilon = 0.01 * cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon, True) + + # Find convex hull + hull = cv2.convexHull(approx, returnPoints=False) - # 计算凸包和凸缺陷 - hull = cv2.convexHull(contour, returnPoints=False) - if len(hull) < 3: - return 0, None, None + if hull is None or len(hull) < 3: + return [], 0 - defects = cv2.convexityDefects(contour, hull) + # Find convexity defects + defects = cv2.convexityDefects(approx, hull) if defects is None: - return 0, None, None + return [], 0 - # 计算手指数量 - finger_count = 0 - finger_tips = [] + fingertips = [] + defects_points = [] for i in range(defects.shape[0]): s, e, f, d = defects[i, 0] - start = tuple(contour[s][0]) - end = tuple(contour[e][0]) - far = tuple(contour[f][0]) + start = tuple(approx[s][0]) + end = tuple(approx[e][0]) + far = tuple(approx[f][0]) - # 计算角度 + defects_points.append((start, end, far)) + + # Calculate triangle sides a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) - angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + # Calculate angle + if b * c != 0: + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + angle_degrees = math.degrees(angle) + + # If angle is less than 90 degrees, it might be a finger + if angle_degrees < 90: + # Check if this is a deep defect (far from hull) + if d > 1000: # Adjust this threshold based on your needs + # Both start and end points could be fingertips + for point in [start, end]: + if point not in fingertips: + fingertips.append(point) + + # If we found few fingertips, try alternative method + if len(fingertips) <= 2: + # Use convex hull points as fingertips + hull_points = cv2.convexHull(approx, returnPoints=True) + + # Find extreme points in hull + if len(hull_points) > 0: + hull_points = hull_points.reshape(-1, 2) + + # Sort by y-coordinate (top to bottom) + hull_points = hull_points[hull_points[:, 1].argsort()] + + # Take top points as fingertips + num_fingertips = min(5, len(hull_points)) + for i in range(num_fingertips): + point = tuple(hull_points[i]) + if point not in fingertips: + fingertips.append(point) + + return fingertips, len(fingertips) + + def analyze_hand_contour(self, contour, frame_shape): + """Analyze hand contour""" + if contour is None: + return None, 0.0 + + # Calculate contour area + area = cv2.contourArea(contour) - # 如果角度小于90度,可能是指缝 - if angle < math.pi / 2: - finger_count += 1 - finger_tips.append(end) + # Calculate contour center + M = cv2.moments(contour) + if M["m00"] == 0: + return None, 0.0 + + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + + # Calculate bounding box + x, y, w, h = cv2.boundingRect(contour) + + # Advanced fingertip detection + fingertips, finger_count = self.detect_fingertips_advanced(contour) + + # Calculate hand position (normalized) + h_img, w_img = frame_shape[:2] + norm_x = cx / w_img + norm_y = cy / h_img + + # Calculate gesture confidence + confidence = self.calculate_confidence(area, finger_count, len(contour)) + + # Return result + result = { + 'contour': contour, + 'center': (cx, cy), + 'bbox': (x, y, x + w, y + h), + 'finger_count': finger_count, + 'fingertips': fingertips, + 'area': area, + 'position': (norm_x, norm_y), + 'confidence': confidence + } - # 手指数量加1(大拇指) - finger_count = min(finger_count + 1, 5) + return result, confidence - # 计算轮廓中心 - M = cv2.moments(contour) - if M["m00"] != 0: - cx = int(M["m10"] / M["m00"]) - cy = int(M["m01"] / M["m00"]) - center = (cx, cy) - else: - center = None + def calculate_confidence(self, area, finger_count, contour_length): + """Calculate gesture confidence""" + confidence = 0.5 # Base confidence - return finger_count, center, finger_tips + # Confidence based on area + if area > 5000: + confidence += 0.2 + elif area > 3000: + confidence += 0.1 - def detect_fingertips_with_contours(self, contour): - """使用轮廓检测指尖""" - # 简化轮廓 - epsilon = 0.02 * cv2.arcLength(contour, True) - approx = cv2.approxPolyDP(contour, epsilon, True) + # Confidence based on finger count + if 0 <= finger_count <= 5: + confidence += 0.2 - # 查找凸点(可能是指尖) - hull = cv2.convexHull(approx, returnPoints=True) + # Confidence based on contour complexity + if contour_length > 200: + confidence += 0.1 - fingertips = [] - for point in hull: - fingertips.append(tuple(point[0])) + return min(confidence, 1.0) + + def recognize_gesture(self, hand_data): + """Recognize gesture based on hand data""" + if hand_data is None: + return "Waiting", 0.3 - return fingertips[:5] # 最多5个指尖 + finger_count = hand_data['finger_count'] + norm_x, norm_y = hand_data['position'] + confidence = hand_data['confidence'] + fingertips = hand_data.get('fingertips', []) - def recognize_gesture_by_fingers(self, finger_count, center, frame_shape): - """根据手指数量识别手势""" - if center is None: - return "等待手势", 0.5 + # Special case: single finger detection + if finger_count == 1: + # Additional check for single finger gesture + if len(fingertips) == 1: + # Check if the fingertip is above the hand center (typical for pointing) + cx, cy = hand_data['center'] + fingertip = fingertips[0] - h, w = frame_shape[:2] - cx, cy = center + if fingertip[1] < cy - 20: # Fingertip is above hand center + return "Forward", confidence * 0.9 + else: + return "Stop", confidence * 0.7 + return "Forward", confidence * 0.8 - # 计算手部位置(归一化) - norm_x = cx / w - norm_y = cy / h + # No fingers (fist) + elif finger_count == 0: + return "Stop", confidence * 0.9 - if finger_count == 0: - return "停止", 0.9 - elif finger_count == 1: - return "向前", 0.8 + # Two fingers (V sign) elif finger_count == 2: - # V字手势,可能是向前或特殊命令 - return "向前", 0.7 + return "Forward", confidence * 0.7 + + # Three fingers elif finger_count == 3: - # 三指手势,根据位置判断 + # Judge direction based on position if norm_y < 0.4: - return "向上", 0.7 + return "Up", confidence * 0.7 elif norm_y > 0.6: - return "向下", 0.7 + return "Down", confidence * 0.7 else: - return "停止", 0.6 + return "Stop", confidence * 0.6 + + # Four or five fingers (open hand) elif finger_count >= 4: - # 手掌张开,根据位置判断方向 + # Judge direction based on hand position if norm_x < 0.3: - return "向左", 0.8 + return "Left", confidence * 0.8 elif norm_x > 0.7: - return "向右", 0.8 + return "Right", confidence * 0.8 elif norm_y < 0.4: - return "向上", 0.8 + return "Up", confidence * 0.8 elif norm_y > 0.6: - return "向下", 0.8 + return "Down", confidence * 0.8 else: - return "停止", 0.7 + return "Stop", confidence * 0.7 - return "等待手势", 0.5 + return "Waiting", confidence * 0.5 - def recognize_with_mediapipe(self, frame): - """使用MediaPipe进行手势识别""" - rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - results = self.hands.process(rgb_frame) + def smooth_gesture(self, new_gesture, new_confidence): + """Smooth gesture output""" + # Add to history + self.gesture_history.append(new_gesture) + self.confidence_history.append(new_confidence) - gesture = "等待手势" - confidence = 0.5 + # If not enough history, return directly + if len(self.gesture_history) < 3: + self.current_gesture = new_gesture + self.current_confidence = new_confidence + return new_gesture, new_confidence - if results.multi_hand_landmarks: - for hand_landmarks in results.multi_hand_landmarks: - # 绘制手部关键点 - libs['mp_drawing'].draw_landmarks( - frame, hand_landmarks, libs['mp_hands'].HAND_CONNECTIONS, - libs['mp_drawing'].DrawingSpec(color=(0, 255, 0), thickness=2), - libs['mp_drawing'].DrawingSpec(color=(255, 0, 0), thickness=2) - ) + # Count most common gesture + gesture_counter = Counter(self.gesture_history) + most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] - # 计算手指状态 - landmarks = hand_landmarks.landmark - - # 简单的手指状态检测 - finger_tips = [4, 8, 12, 16, 20] - finger_pips = [2, 6, 10, 14, 18] - - extended_fingers = 0 - for tip, pip in zip(finger_tips, finger_pips): - if landmarks[tip].y < landmarks[pip].y: - extended_fingers += 1 - - # 根据伸直的手指数量判断手势 - if extended_fingers == 0: - gesture = "停止" - confidence = 0.9 - elif extended_fingers == 1: - gesture = "向前" - confidence = 0.8 - elif extended_fingers >= 4: - # 根据手掌方向判断 - wrist = landmarks[0] - middle_mcp = landmarks[9] - - dx = middle_mcp.x - wrist.x - dy = middle_mcp.y - wrist.y - - if abs(dx) > abs(dy): - if dx > 0: - gesture = "向右" - else: - gesture = "向左" - else: - if dy > 0: - gesture = "向下" - else: - gesture = "向上" - confidence = 0.7 + # Calculate average confidence + avg_confidence = np.mean(list(self.confidence_history)) - return gesture, confidence, frame + # If the most common gesture appears enough times, use it + smooth_frames = self.config['smooth_frames'] + if most_common_count >= smooth_frames: + self.current_gesture = most_common_gesture + self.current_confidence = avg_confidence - def smooth_gesture(self, gesture, confidence): - """平滑手势输出""" - self.gesture_history.append((gesture, confidence)) + return self.current_gesture, self.current_confidence - if len(self.gesture_history) < 3: - return gesture, confidence - - # 统计最近的手势 - gesture_counts = {} - confidence_sum = {} - - for g, c in self.gesture_history: - if g not in gesture_counts: - gesture_counts[g] = 0 - confidence_sum[g] = 0 - gesture_counts[g] += 1 - confidence_sum[g] += c - - # 找到最频繁的手势 - max_count = 0 - best_gesture = gesture - best_confidence = confidence - - for g, count in gesture_counts.items(): - if count > max_count and g != "等待手势": - max_count = count - best_gesture = g - best_confidence = confidence_sum[g] / count - - # 如果手势稳定,更新当前手势 - if max_count >= len(self.gesture_history) // 2: - self.current_gesture = best_gesture - self.confidence = best_confidence - - return self.current_gesture, self.confidence - - def visualize_hand(self, frame, contour, center, finger_count, fingertips): - """可视化手部检测结果""" - if contour is not None: - # 绘制轮廓 - cv2.drawContours(frame, [contour], -1, (0, 255, 0), 2) - - # 绘制边界框 - x, y, w, h = cv2.boundingRect(contour) - cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 0), 2) - - # 绘制中心点 - if center: - cx, cy = center - cv2.circle(frame, (cx, cy), 8, (0, 0, 255), -1) - - # 绘制指尖 - for tip in fingertips[:finger_count]: - cv2.circle(frame, tip, 6, (255, 255, 0), -1) - - # 显示手指数量 - cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + def visualize_detection(self, frame, hand_data, gesture, confidence): + """Visualize detection results""" + if hand_data is None: + return frame + + # Get display configuration + show_contours = config.get('display', 'show_contours') + show_bbox = config.get('display', 'show_bbox') + show_fingertips = config.get('display', 'show_fingertips') + + # Draw contour + if show_contours and 'contour' in hand_data: + cv2.drawContours(frame, [hand_data['contour']], -1, (0, 255, 0), 2) + + # Draw bounding box + if show_bbox and 'bbox' in hand_data: + x1, y1, x2, y2 = hand_data['bbox'] + color = self.gesture_colors.get(gesture, (255, 255, 255)) + cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) + + # Show gesture label + label = f"{gesture}" + if config.get('display', 'show_confidence'): + label += f" ({confidence:.0%})" + + # Calculate text size + (text_width, text_height), baseline = cv2.getTextSize( + label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 + ) + + # Draw text background + cv2.rectangle(frame, + (x1, y1 - text_height - 10), + (x1 + text_width, y1), + color, -1) + + # Draw text + cv2.putText(frame, label, (x1, y1 - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) + + # Draw center point + if 'center' in hand_data: + cx, cy = hand_data['center'] + cv2.circle(frame, (cx, cy), 5, (0, 0, 255), -1) + cv2.putText(frame, "Center", (cx + 10, cy), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) + + # Draw fingertips + if show_fingertips and 'fingertips' in hand_data: + for i, point in enumerate(hand_data['fingertips']): + cv2.circle(frame, point, 4, (255, 0, 0), -1) + cv2.putText(frame, f"F{i + 1}", (point[0] + 5, point[1]), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1) + + # Show finger count + if 'finger_count' in hand_data: + finger_text = f"Fingers: {hand_data['finger_count']}" + cv2.putText(frame, finger_text, (10, frame.shape[0] - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) return frame def recognize(self, frame): - """识别手势""" + """Recognize gesture""" start_time = time.time() try: - # 镜像图像以便更直观的控制 - frame = cv2.flip(frame, 1) - - # 方法1:使用MediaPipe(如果可用) - if self.use_mediapipe: - gesture, confidence, processed_frame = self.recognize_with_mediapipe(frame) - frame = processed_frame - else: - # 方法2:使用OpenCV - contour, mask = self.find_hand_contours(frame) + # Preprocess frame + processed_frame = self.preprocess_frame(frame) - if contour is not None: - finger_count, center, finger_tips = self.analyze_hand_contour(contour) + # Detect every few frames to improve performance + if self.frame_counter % self.detection_interval != 0: + self.frame_counter += 1 + return self.current_gesture, self.current_confidence, processed_frame - # 如果凸缺陷检测失败,使用轮廓检测指尖 - if finger_tips is None or len(finger_tips) < finger_count: - finger_tips = self.detect_fingertips_with_contours(contour) + # Detect skin + skin_mask = self.detect_skin(processed_frame) - self.finger_history.append(finger_count) + # Find hand contour + hand_contour = self.find_hand_contours(skin_mask) - # 计算平均手指数量(平滑) - if len(self.finger_history) > 0: - avg_fingers = sum(self.finger_history) / len(self.finger_history) - finger_count = int(round(avg_fingers)) + # Analyze hand contour + hand_data, confidence = self.analyze_hand_contour(hand_contour, processed_frame.shape) - # 识别手势 - gesture, confidence = self.recognize_gesture_by_fingers( - finger_count, center, frame.shape - ) - - # 可视化结果 - frame = self.visualize_hand(frame, contour, center, finger_count, finger_tips) - else: - gesture, confidence = "等待手势", 0.3 + # Recognize gesture + if hand_data is not None: + gesture, raw_confidence = self.recognize_gesture(hand_data) + confidence = max(confidence, raw_confidence) + else: + gesture, confidence = "Waiting", 0.3 - # 手势平滑处理 + # Smooth gesture final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) - # 更新最后检测时间 - self.last_gesture_time = time.time() + # Visualize results + if hand_data is not None: + processed_frame = self.visualize_detection( + processed_frame, hand_data, final_gesture, final_confidence + ) + + # Update counter + self.frame_counter += 1 - # 显示处理时间 + # Calculate processing time process_time = (time.time() - start_time) * 1000 - cv2.putText(frame, f'Process: {process_time:.1f}ms', (10, frame.shape[0] - 20), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + self.process_times.append(process_time) - return final_gesture, final_confidence, frame + return final_gesture, final_confidence, processed_frame except Exception as e: - print(f"Gesture recognition error: {e}") - return "识别异常", 0.0, frame + print(f"⚠ Gesture recognition error: {e}") + return "Error", 0.0, frame + + def get_performance_stats(self): + """Get performance statistics""" + if len(self.process_times) == 0: + return 0.0 + + return np.mean(list(self.process_times)) def set_simulated_gesture(self, gesture): - """设置模拟的手势""" + """Set simulated gesture""" self.current_gesture = gesture - self.confidence = 0.9 + self.current_confidence = 0.9 -# ========== 增强的无人机控制模块 ========== -class EnhancedDroneController: - """增强的无人机控制器""" +# ========== Simple Drone Controller ========== +class SimpleDroneController: + """Simple Drone Controller""" def __init__(self, airsim_module): self.airsim = airsim_module self.client = None self.connected = False self.flying = False - self.connection_attempted = False - # 控制参数 - self.velocity = 3.0 - self.duration = 0.2 - self.altitude = -10.0 # 初始高度(负值表示向上) + # Control parameters + self.velocity = config.get('drone', 'velocity') + self.duration = config.get('drone', 'duration') + self.altitude = config.get('drone', 'altitude') + self.control_interval = config.get('drone', 'control_interval') - # 控制历史 - self.control_history = deque(maxlen=10) + # Control state self.last_control_time = 0 + self.last_gesture = None - def connect(self): - """连接AirSim无人机""" - if self.connection_attempted: - return self.connected + print("✓ Simple drone controller initialized") - self.connection_attempted = True + def connect(self): + """Connect to AirSim drone""" + if self.connected: + return True if self.airsim is None: print("⚠ AirSim not available, using simulation mode") @@ -612,13 +716,7 @@ def connect(self): except Exception as e: print(f"❌ Connection failed: {e}") - print("\n请确保:") - print("1. AirSim模拟器正在运行") - print("2. 选择环境(如:Landscape Mountains)") - print("3. 无人机已在世界中生成") - print("4. 如果需要,按模拟器中的'R'键重置无人机") - - print("\n使用模拟模式继续?(y/n)") + print("\nContinue with simulation mode? (y/n)") choice = input().strip().lower() if choice == 'y': self.connected = True @@ -628,9 +726,8 @@ def connect(self): return False def takeoff(self): - """起飞""" + """Take off""" if not self.connected: - print("❌ Drone not connected") return False try: @@ -643,7 +740,7 @@ def takeoff(self): self.client.takeoffAsync().join() time.sleep(1) - # 上升到指定高度 + # Ascend to specified altitude self.client.moveToZAsync(self.altitude, 3).join() self.flying = True @@ -651,11 +748,10 @@ def takeoff(self): return True except Exception as e: print(f"❌ Takeoff failed: {e}") - self.flying = True # 模拟模式 - return True + return False def land(self): - """降落""" + """Land""" if not self.connected: return False @@ -672,54 +768,54 @@ def land(self): return True except Exception as e: print(f"Landing failed: {e}") - self.flying = False return False - def move_by_gesture(self, gesture, confidence=0.7): - """根据手势移动""" + def move_by_gesture(self, gesture, confidence): + """Move based on gesture""" if not self.connected or not self.flying: return False - # 检查控制间隔 + # Check control interval current_time = time.time() - if current_time - self.last_control_time < 0.1: # 最小控制间隔 + if current_time - self.last_control_time < self.control_interval: return False - # 如果置信度太低,不执行动作 - if confidence < 0.5: + # Check confidence threshold + min_confidence = config.get('gesture', 'min_confidence') + if confidence < min_confidence: return False try: if self.airsim is None or self.client is None: print(f"Simulated move: {gesture}") - self.control_history.append((gesture, current_time)) self.last_control_time = current_time + self.last_gesture = gesture return True success = False - if gesture == "向上": + if gesture == "Up": self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) success = True - elif gesture == "向下": + elif gesture == "Down": self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) success = True - elif gesture == "向左": + elif gesture == "Left": self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) success = True - elif gesture == "向右": + elif gesture == "Right": self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) success = True - elif gesture == "向前": + elif gesture == "Forward": self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) success = True - elif gesture == "停止": + elif gesture == "Stop": self.client.hoverAsync() success = True if success: - self.control_history.append((gesture, current_time)) self.last_control_time = current_time + self.last_gesture = gesture return success except Exception as e: @@ -727,7 +823,7 @@ def move_by_gesture(self, gesture, confidence=0.7): return False def emergency_stop(self): - """紧急停止""" + """Emergency stop""" if self.connected: try: if self.flying and self.client is not None: @@ -739,251 +835,348 @@ def emergency_stop(self): print("✅ Emergency stop complete") except: pass + self.connected = False self.flying = False -# ========== 性能监控模块 ========== +# ========== Simple UI Renderer ========== +class SimpleUIRenderer: + """Simple UI Renderer""" + + def __init__(self): + # Color definitions + self.colors = { + 'title': (0, 255, 255), # Cyan + 'connected': (0, 255, 0), # Green + 'disconnected': (0, 0, 255), # Red + 'flying': (0, 255, 0), # Green + 'landed': (255, 165, 0), # Orange + 'warning': (0, 165, 255), # Light Blue + 'info': (255, 255, 255), # White + 'help': (255, 200, 100) # Light Orange + } + + print("✓ Simple UI renderer initialized") + + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time): + """Draw status bar""" + h, w = frame.shape[:2] + + # Draw semi-transparent background + overlay = frame.copy() + cv2.rectangle(overlay, (0, 0), (w, 100), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + + # Title + title = "Gesture Controlled Drone" + cv2.putText(frame, title, (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, self.colors['title'], 2) + + # Connection status + status_color = self.colors['connected'] if drone_controller.connected else self.colors['disconnected'] + status_text = f"Drone: {'Connected' if drone_controller.connected else 'Disconnected'}" + cv2.putText(frame, status_text, (10, 60), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, status_color, 1) + + # Flight status + flight_color = self.colors['flying'] if drone_controller.flying else self.colors['landed'] + flight_text = f"Flight: {'Flying' if drone_controller.flying else 'Landed'}" + cv2.putText(frame, flight_text, (10, 85), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, flight_color, 1) + + # Gesture information + if confidence > 0.7: + gesture_color = (0, 255, 0) # Green + elif confidence > 0.5: + gesture_color = (255, 165, 0) # Orange + else: + gesture_color = (200, 200, 200) # Gray + + gesture_text = f"Gesture: {gesture}" + if config.get('display', 'show_confidence'): + gesture_text += f" ({confidence:.0%})" + + cv2.putText(frame, gesture_text, (w // 2, 60), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, gesture_color, 1) + + # Performance information + if config.get('display', 'show_fps'): + perf_text = f"FPS: {fps:.1f}" + if process_time > 0: + perf_text += f" | Delay: {process_time:.1f}ms" + + cv2.putText(frame, perf_text, (w - 200, 85), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['info'], 1) + + return frame + + def draw_help_bar(self, frame): + """Draw help bar""" + if not config.get('display', 'show_help'): + return frame + + h, w = frame.shape[:2] + + # Draw bottom help bar + cv2.rectangle(frame, (0, h - 50), (w, h), (0, 0, 0), -1) + + # Help text + help_text = "C:Connect Space:Takeoff/Land ESC:Exit W/A/S/D/F/X:Keyboard" + cv2.putText(frame, help_text, (10, h - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['help'], 1) + + return frame + + def draw_warning(self, frame, message): + """Draw warning message""" + h, w = frame.shape[:2] + + # Draw warning at top + warning_bg = np.zeros((40, w, 3), dtype=np.uint8) + warning_bg[:, :] = (0, 69, 255) # Orange + + frame[100:140, 0:w] = cv2.addWeighted( + frame[100:140, 0:w], 0.3, + warning_bg, 0.7, 0 + ) + + # Draw warning text + cv2.putText(frame, message, (10, 125), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, self.colors['warning'], 1) + + return frame + + +# ========== Performance Monitor ========== class PerformanceMonitor: - """性能监控器""" + """Performance Monitor""" def __init__(self): self.frame_times = deque(maxlen=60) - self.fps = 0 self.last_update = time.time() + self.fps = 0 + self.frame_count = 0 def update(self): - """更新性能数据""" + """Update performance data""" current_time = time.time() self.frame_times.append(current_time) + self.frame_count += 1 - # 每秒更新一次FPS + # Calculate FPS once per second if current_time - self.last_update >= 1.0: if len(self.frame_times) > 1: - self.fps = len(self.frame_times) / (self.frame_times[-1] - self.frame_times[0]) + time_diff = self.frame_times[-1] - self.frame_times[0] + if time_diff > 0: + self.fps = len(self.frame_times) / time_diff + else: + self.fps = 0 self.last_update = current_time def get_stats(self): - """获取性能统计""" + """Get performance statistics""" return { 'fps': self.fps, - 'frame_count': len(self.frame_times) + 'frame_count': self.frame_count } -# ========== 主程序 ========== +# ========== Main Program ========== def main(): - """主函数""" - # 初始化组件 - gesture_recognizer = AdvancedGestureRecognizer() - drone_controller = EnhancedDroneController(libs['airsim']) + """Main function""" + # Initialize components + print("Initializing components...") + + gesture_recognizer = RobustGestureRecognizer() + drone_controller = SimpleDroneController(libs['airsim']) + ui_renderer = SimpleUIRenderer() performance_monitor = PerformanceMonitor() - # 初始化摄像头 + # Initialize camera cap = None - available_cameras = [] - - # 检测可用摄像头 - for i in range(0, 4): - temp_cap = cv2.VideoCapture(i) - if temp_cap.isOpened(): - available_cameras.append(i) - temp_cap.release() - - if available_cameras: - print(f"✓ Available cameras: {available_cameras}") - cap = cv2.VideoCapture(available_cameras[0]) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - cap.set(cv2.CAP_PROP_FPS, 30) - print(f"✓ Using camera {available_cameras[0]}") - else: - print("❌ No cameras available, using keyboard control only") + try: + cap = cv2.VideoCapture(0) + if cap.isOpened(): + cap.set(cv2.CAP_PROP_FRAME_WIDTH, config.get('camera', 'width')) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, config.get('camera', 'height')) + cap.set(cv2.CAP_PROP_FPS, config.get('camera', 'fps')) + + # Get actual parameters + actual_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + actual_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + actual_fps = int(cap.get(cv2.CAP_PROP_FPS)) + + print(f"✓ Camera initialized") + print(f" Resolution: {actual_width}x{actual_height}") + print(f" FPS: {actual_fps}") + else: + print("❌ Camera not available, using simulation mode") + cap = None + except Exception as e: + print(f"⚠ Camera init failed: {e}") cap = None - # 显示说明 + # Display welcome message print("\n" + "=" * 60) - print("手势控制无人机系统 - 高级版") + print("Gesture Controlled Drone - Optimized Version") print("=" * 60) - print("操作说明:") - print("1. 首先启动AirSim模拟器(可选)") - print("2. 按 [C] 连接无人机") - print("3. 按 [空格键] 起飞/降落") - if cap: - print("4. 手势控制:") - print(" - 握拳(0指):停止") - print(" - 食指(1指):向前") - print(" - 手掌张开(4-5指):根据手的位置控制方向") - print(" * 手势识别置信度 > 50% 时才会执行") - else: - print("4. 键盘控制:") - print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") - print("5. 按 [ESC] 安全退出") - print("6. 按 [R] 重置手势识别") + print("System Status:") + print(f" Camera: {'Connected' if cap else 'Simulation Mode'}") + print(f" Gesture Recognition: Pure OpenCV") + print(f" AirSim: {'Available' if libs['airsim'] else 'Simulation Mode'}") print("=" * 60) - # 键盘手势映射 + # Display operation instructions + print("\nOperation Instructions:") + print("1. Press [C] to connect drone (AirSim simulator)") + print("2. Press [SPACE] to takeoff/land") + print("3. Gesture Control:") + print(" - Fist (0 fingers): Stop") + print(" - Index finger (1 finger): Forward") + print(" - Open hand (4-5 fingers): Control direction by hand position") + print(" * Gesture confidence > 60% to execute") + print("4. Keyboard Control:") + print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") + print("5. Press [H] to toggle help display") + print("6. Press [R] to reset gesture recognition") + print("7. Press [ESC] to exit safely") + print("=" * 60) + print("Program started successfully!") + print("-" * 60) + + # Keyboard gesture mapping key_to_gesture = { - ord('w'): "向上", ord('W'): "向上", - ord('s'): "向下", ord('S'): "向下", - ord('a'): "向左", ord('A'): "向左", - ord('d'): "向右", ord('D'): "向右", - ord('f'): "向前", ord('F'): "向前", - ord('x'): "停止", ord('X'): "停止", + ord('w'): "Up", ord('W'): "Up", + ord('s'): "Down", ord('S'): "Down", + ord('a'): "Left", ord('A'): "Left", + ord('d'): "Right", ord('D'): "Right", + ord('f'): "Forward", ord('F'): "Forward", + ord('x'): "Stop", ord('X'): "Stop", } - # 主循环 - last_gesture_time = time.time() - gesture_timeout = 2.0 # 手势超时时间 - - while True: - performance_monitor.update() + # Main loop + print("\nEntering main loop, press ESC to exit...") - # 读取摄像头帧 - if cap: - ret, frame = cap.read() - if not ret: - # 如果读取失败,尝试重新初始化摄像头 - print("⚠ Camera read failed, trying to reinitialize...") - cap.release() - if available_cameras: - cap = cv2.VideoCapture(available_cameras[0]) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - ret, frame = cap.read() + try: + while True: + # Update performance monitor + performance_monitor.update() + # Read camera frame + if cap: + ret, frame = cap.read() if not ret: + # Create blank frame frame = np.zeros((480, 640, 3), dtype=np.uint8) - current_gesture = "摄像头错误" - confidence = 0.0 + gesture, confidence = "Camera Error", 0.0 else: - current_gesture, confidence, frame = gesture_recognizer.recognize(frame) + # Gesture recognition + gesture, confidence, frame = gesture_recognizer.recognize(frame) else: - current_gesture, confidence, frame = gesture_recognizer.recognize(frame) - else: - frame = np.zeros((480, 640, 3), dtype=np.uint8) - current_gesture = gesture_recognizer.current_gesture - confidence = gesture_recognizer.confidence - - # ========== 绘制界面 ========== - # 创建信息覆盖层 - overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (640, 140), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) - - # 标题和版本 - title = "手势控制无人机系统 - 高级版" - frame = chinese_renderer.put_text(frame, title, (10, 10), size=28, color=(0, 255, 255)) + # Simulation mode + frame = np.zeros((480, 640, 3), dtype=np.uint8) + gesture, confidence = gesture_recognizer.current_gesture, gesture_recognizer.current_confidence - # 状态信息 - status_color = (0, 255, 0) if drone_controller.connected else (0, 0, 255) - status_text = f"状态: {'已连接' if drone_controller.connected else '未连接'}" - frame = chinese_renderer.put_text(frame, status_text, (10, 50), size=24, color=status_color) + # Get performance statistics + perf_stats = performance_monitor.get_stats() + process_time = gesture_recognizer.get_performance_stats() - flight_color = (0, 255, 0) if drone_controller.flying else (255, 165, 0) - flight_text = f"飞行: {'是' if drone_controller.flying else '否'}" - frame = chinese_renderer.put_text(frame, flight_text, (10, 80), size=24, color=flight_color) + # Draw UI + frame = ui_renderer.draw_status_bar( + frame, drone_controller, gesture, confidence, + perf_stats['fps'], process_time + ) - # 手势信息 - gesture_color = (0, 255, 0) if confidence > 0.7 else (255, 165, 0) if confidence > 0.5 else (200, 200, 200) - gesture_text = f"手势: {current_gesture} ({confidence:.1%})" - frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), size=24, color=gesture_color) + frame = ui_renderer.draw_help_bar(frame) - # 性能信息 - stats = performance_monitor.get_stats() - perf_text = f"FPS: {stats['fps']:.1f} | 延迟: {1000 / stats['fps']:.1f}ms" if stats['fps'] > 0 else "FPS: 0" - frame = chinese_renderer.put_text(frame, perf_text, (440, 50), size=20, color=(255, 255, 255)) - - # 操作提示 - help_text = "C:连接 空格:起飞/降落 R:重置 ESC:退出" - if not cap: - help_text += " W/A/S/D/F/X:控制" - - # 绘制底部提示栏 - cv2.rectangle(frame, (0, 440), (640, 480), (0, 0, 0), -1) - frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) - frame = chinese_renderer.put_text(frame, help_text, (10, 450), size=18, color=(255, 255, 255)) - - # 手势提示 - if cap: - hint_text = "提示: 保持手势稳定,确保良好光线" - frame = chinese_renderer.put_text(frame, hint_text, (10, 470), size=16, color=(255, 200, 100)) - - # 显示连接提示 - if not drone_controller.connected: - warning_text = "⚠ 按C键连接无人机,或继续使用模拟模式" - frame = chinese_renderer.put_text(frame, warning_text, (10, 400), - size=18, color=(0, 165, 255), bg_color=(0, 0, 0)) + # Show connection warning + if not drone_controller.connected: + warning_msg = "⚠ Press C to connect drone, or use simulation mode" + frame = ui_renderer.draw_warning(frame, warning_msg) - # 显示图像 - cv2.imshow('Gesture Controlled Drone - Advanced Version', frame) + # Show image + window_title = "Gesture Controlled Drone - Press ESC to exit" + cv2.imshow(window_title, frame) - # ========== 键盘控制 ========== - key = cv2.waitKey(1) & 0xFF + # ========== Keyboard Control ========== + key = cv2.waitKey(1) & 0xFF - if key == 27: # ESC键 - print("\n退出程序...") - break + if key == 27: # ESC key + print("\nExiting program...") + break - elif key == ord('c') or key == ord('C'): - if not drone_controller.connected: - drone_controller.connect() + elif key == ord('c') or key == ord('C'): + if not drone_controller.connected: + drone_controller.connect() - elif key == 32: # 空格键 - if drone_controller.connected: - if drone_controller.flying: - drone_controller.land() - else: - drone_controller.takeoff() - time.sleep(0.5) - - elif key == ord('r') or key == ord('R'): - print("重置手势识别...") - gesture_recognizer = AdvancedGestureRecognizer() - - elif key in key_to_gesture: - gesture = key_to_gesture[key] - gesture_recognizer.set_simulated_gesture(gesture) - current_gesture = gesture - confidence = 0.9 - if drone_controller.connected and drone_controller.flying: + elif key == 32: # Space key + if drone_controller.connected: + if drone_controller.flying: + drone_controller.land() + else: + drone_controller.takeoff() + time.sleep(0.5) + + elif key == ord('h') or key == ord('H'): + # Toggle help display + current = config.get('display', 'show_help') + config.set('display', 'show_help', value=not current) + print(f"Help display: {'ON' if not current else 'OFF'}") + + elif key == ord('f') or key == ord('F'): + # Toggle FPS display + current = config.get('display', 'show_fps') + config.set('display', 'show_fps', value=not current) + print(f"FPS display: {'ON' if not current else 'OFF'}") + + elif key == ord('r') or key == ord('R'): + # Reset gesture recognition + print("Resetting gesture recognition...") + gesture_recognizer = RobustGestureRecognizer() + + elif key == ord('t') or key == ord('T'): + # Toggle fingertip display + current = config.get('display', 'show_fingertips') + config.set('display', 'show_fingertips', value=not current) + print(f"Fingertip display: {'ON' if not current else 'OFF'}") + + elif key in key_to_gesture: + # Keyboard control + simulated_gesture = key_to_gesture[key] + gesture_recognizer.set_simulated_gesture(simulated_gesture) + gesture = simulated_gesture + confidence = 0.9 + if drone_controller.connected and drone_controller.flying: + drone_controller.move_by_gesture(gesture, confidence) + + # Real gesture control + current_time = time.time() + if (gesture and gesture != "Waiting" and + gesture != "Camera Error" and gesture != "Error" and + drone_controller.connected and drone_controller.flying): drone_controller.move_by_gesture(gesture, confidence) - # 真实手势控制 - current_time = time.time() - if (current_gesture and current_gesture != "等待手势" and - current_gesture != "摄像头错误" and current_gesture != "识别异常" and - drone_controller.connected and drone_controller.flying and - confidence > 0.5 and # 只有置信度足够高才执行 - current_time - last_gesture_time > 0.2): # 控制频率限制 - - success = drone_controller.move_by_gesture(current_gesture, confidence) - if success: - last_gesture_time = current_time - - # 检查手势超时 - if current_time - gesture_recognizer.last_gesture_time > gesture_timeout: - gesture_recognizer.current_gesture = "等待手势" - - # 清理资源 - print("\n清理资源...") - if cap: - cap.release() - cv2.destroyAllWindows() - drone_controller.emergency_stop() - print("程序安全退出") - print("=" * 60) - - -# ========== 程序入口 ========== -if __name__ == "__main__": - try: - main() except KeyboardInterrupt: - print("\n程序被用户中断") + print("\nProgram interrupted by user") except Exception as e: - print(f"\n程序错误: {e}") + print(f"\nProgram error: {e}") traceback.print_exc() finally: - print("\n按回车键退出...") - input() \ No newline at end of file + # Cleanup resources + print("\nCleaning up resources...") + if cap: + cap.release() + cv2.destroyAllWindows() + drone_controller.emergency_stop() + config.save_config() + + print("Program exited safely") + print("=" * 60) + print("\nThank you for using Gesture Controlled Drone System!") + input("Press Enter to exit...") + + +# ========== Program Entry Point ========== +if __name__ == "__main__": + main() \ No newline at end of file From f1897a2c8b60eee2305bb24fc22521495d7de8ab Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 15:46:37 +0800 Subject: [PATCH 05/11] =?UTF-8?q?=E6=8C=81=E7=BB=AD=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E6=89=8B=E5=8A=BF=E8=AF=86=E5=88=AB=E7=AE=97=E6=B3=95=EF=BC=8C?= =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=BD=8D=E7=BD=AE=E6=95=8F=E6=84=9F=E9=97=AE?= =?UTF-8?q?=E9=A2=98=EF=BC=8C=E5=A2=9E=E5=BC=BA=E5=85=A8=E5=B1=8F=E6=89=8B?= =?UTF-8?q?=E5=8A=BF=E6=A3=80=E6=B5=8B=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config.json | 7 +- .../gesture_config_v2.json | 53 + .../gesture_config_v3.json | 53 + .../main.py | 1241 ++++++++++------- 4 files changed, 887 insertions(+), 467 deletions(-) create mode 100644 src/autonomus_drone_hand_gesture_project/gesture_config_v2.json create mode 100644 src/autonomus_drone_hand_gesture_project/gesture_config_v3.json diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index c6d931acbe..d41f020e20 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -21,6 +21,9 @@ "smooth_frames": 5, "min_confidence": 0.6, "detection_interval": 2, + "hand_region_threshold": 0.2, + "face_exclusion_area": 8000, + "palm_aspect_ratio": 1.3, "single_finger_angle_threshold": 60 }, "drone": { @@ -35,7 +38,9 @@ "show_help": true, "show_contours": true, "show_bbox": true, - "show_fingertips": true + "show_fingertips": true, + "show_direction": true, + "show_region_grid": true }, "performance": { "target_fps": 30, diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json new file mode 100644 index 0000000000..b5479855de --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json @@ -0,0 +1,53 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "skin_lower_h": 0, + "skin_upper_h": 178, + "skin_lower_s": 0, + "skin_upper_s": 67, + "skin_lower_v": 36, + "skin_upper_v": 98, + "min_hand_area": 2000, + "max_hand_area": 30000, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.5, + "detection_interval": 1, + "hand_ratio_threshold": 1.5, + "contour_simplify_epsilon": 0.02, + "defect_distance_threshold": 20, + "palm_circle_radius_ratio": 0.3 + }, + "drone": { + "velocity": 2.5, + "duration": 0.3, + "altitude": -10.0, + "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": true, + "show_hand_direction": true, + "show_debug_info": false + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false + }, + "calibration": { + "auto_calibrate_skin": true, + "skin_calibration_frames": 30, + "hand_size_calibration": true + } +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json new file mode 100644 index 0000000000..7491f9cd30 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json @@ -0,0 +1,53 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "skin_lower_h": 0, + "skin_upper_h": 25, + "skin_lower_s": 30, + "skin_upper_s": 255, + "skin_lower_v": 60, + "skin_upper_v": 255, + "min_hand_area": 2000, + "max_hand_area": 30000, + "history_size": 15, + "smooth_frames": 10, + "min_confidence": 0.5, + "detection_interval": 1, + "gesture_lock_frames": 8, + "contour_simplify_epsilon": 0.02, + "defect_distance_threshold": 20, + "palm_circle_radius_ratio": 0.3 + }, + "drone": { + "velocity": 2.5, + "duration": 0.3, + "altitude": -10.0, + "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": false, + "show_hand_direction": false, + "show_debug_info": false + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false + }, + "calibration": { + "auto_calibrate_skin": true, + "skin_calibration_frames": 30, + "hand_size_calibration": true + } +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 4bd8903a38..f6f31be3fe 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- """ -Gesture Controlled AirSim Drone - Optimized Version -Pure OpenCV implementation with English interface -Author: xiaoshiyuan888 +手势控制AirSim无人机 - 手势识别优化版 +优化手势识别算法,解决位置敏感问题 +作者: xiaoshiyuan888 """ import sys @@ -18,55 +18,55 @@ from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Optimized Version") +print("Gesture Controlled Drone - Gesture Recognition Optimized") print("=" * 60) -# ========== Fix import path ========== +# ========== 修复导入路径 ========== current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -# ========== Core module imports ========== +# ========== 核心模块导入 ========== def safe_import(): - """Safely import all modules""" + """安全导入所有模块""" modules_status = {} try: from PIL import Image, ImageDraw, ImageFont modules_status['PIL'] = True - print("[PIL] ✓ Image processing library ready") + print("[PIL] ✓ 图像处理库就绪") except Exception as e: modules_status['PIL'] = False - print(f"[PIL] ✗ Import failed: {e}") + print(f"[PIL] ✗ 导入失败: {e}") return None, modules_status try: import cv2 import numpy as np modules_status['OpenCV'] = True - print("[OpenCV] ✓ Computer vision library ready") + print("[OpenCV] ✓ 计算机视觉库就绪") except Exception as e: modules_status['OpenCV'] = False - print(f"[OpenCV] ✗ Import failed: {e}") + print(f"[OpenCV] ✗ 导入失败: {e}") return None, modules_status airsim_module = None try: airsim_module = __import__('airsim') modules_status['AirSim'] = True - print(f"[AirSim] ✓ Successfully imported") + print(f"[AirSim] ✓ 成功导入") except ImportError: print("\n" + "!" * 60) - print("⚠ AirSim library NOT FOUND!") + print("⚠ AirSim库未找到!") print("!" * 60) - print("To install AirSim, run:") - print("1. First install: pip install msgpack-rpc-python") - print("2. Then install: pip install airsim") - print("\nOr from source:") + print("安装AirSim:") + print("1. 首先安装: pip install msgpack-rpc-python") + print("2. 然后安装: pip install airsim") + print("\n或从源码安装:") print(" pip install git+https://github.com/microsoft/AirSim.git") print("!" * 60) - print("\nContinue without AirSim? (y/n)") + print("\n无AirSim继续运行? (y/n)") choice = input().strip().lower() if choice != 'y': sys.exit(1) @@ -79,28 +79,28 @@ def safe_import(): }, modules_status -# Execute imports +# 执行导入 libs, status = safe_import() if not status.get('OpenCV', False) or not status.get('PIL', False): - print("\n❌ Core libraries missing, cannot start.") - input("Press Enter to exit...") + print("\n❌ 核心库缺失,无法启动。") + input("按回车键退出...") sys.exit(1) print("-" * 60) -print("✅ Environment check passed, initializing...") +print("✅ 环境检查通过,正在初始化...") print("-" * 60) -# Unpack libraries +# 解包库 cv2, np = libs['cv2'], libs['np'] Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -# ========== Configuration Manager ========== +# ========== 配置管理器 ========== class ConfigManager: - """Configuration Manager""" + """配置管理器""" def __init__(self): - self.config_file = os.path.join(current_dir, 'gesture_config.json') + self.config_file = os.path.join(current_dir, 'gesture_config_v2.json') self.default_config = { 'camera': { 'index': 0, @@ -109,14 +109,22 @@ def __init__(self): 'fps': 30 }, 'gesture': { - 'skin_lower': [0, 30, 60], - 'skin_upper': [25, 255, 255], - 'min_hand_area': 3000, + 'skin_lower_h': 0, + 'skin_upper_h': 25, + 'skin_lower_s': 30, + 'skin_upper_s': 255, + 'skin_lower_v': 60, + 'skin_upper_v': 255, + 'min_hand_area': 2000, + 'max_hand_area': 30000, 'history_size': 10, 'smooth_frames': 5, - 'min_confidence': 0.6, + 'min_confidence': 0.5, 'detection_interval': 1, - 'single_finger_angle_threshold': 60 # 新增:单个手指角度阈值 + 'hand_ratio_threshold': 1.5, # 手部长宽比阈值 + 'contour_simplify_epsilon': 0.02, + 'defect_distance_threshold': 20, # 凸缺陷距离阈值 + 'palm_circle_radius_ratio': 0.3, # 手掌半径与边界框的比例 }, 'drone': { 'velocity': 2.5, @@ -130,35 +138,46 @@ def __init__(self): 'show_help': True, 'show_contours': True, 'show_bbox': True, - 'show_fingertips': True # 新增:显示指尖 + 'show_fingertips': True, + 'show_palm_center': True, + 'show_hand_direction': True, + 'show_debug_info': False }, 'performance': { 'target_fps': 30, 'resize_factor': 1.0, 'enable_multiprocessing': False + }, + 'calibration': { + 'auto_calibrate_skin': True, + 'skin_calibration_frames': 30, + 'hand_size_calibration': True } } self.config = self.load_config() + self.skin_calibration_data = [] + self.hand_size_calibration_done = False + self.reference_hand_size = 0 def load_config(self): - """Load configuration""" + """加载配置""" if os.path.exists(self.config_file): try: with open(self.config_file, 'r', encoding='utf-8') as f: loaded_config = json.load(f) config = self.default_config.copy() self._merge_config(config, loaded_config) - print("✓ Configuration loaded from file") + print("✓ 从文件加载配置") return config except Exception as e: - print(f"⚠ Failed to load config: {e}, using defaults") + print(f"⚠ 加载配置失败: {e}, 使用默认配置") return self.default_config.copy() else: - print("✓ Using default configuration") + print("✓ 使用默认配置") return self.default_config.copy() def _merge_config(self, base, update): - """Recursively merge configurations""" + """递归合并配置""" for key, value in update.items(): if key in base and isinstance(base[key], dict) and isinstance(value, dict): self._merge_config(base[key], value) @@ -166,16 +185,16 @@ def _merge_config(self, base, update): base[key] = value def save_config(self): - """Save configuration""" + """保存配置""" try: with open(self.config_file, 'w', encoding='utf-8') as f: json.dump(self.config, f, indent=2, ensure_ascii=False) - print("✓ Configuration saved") + print("✓ 配置已保存") except Exception as e: - print(f"⚠ Failed to save config: {e}") + print(f"⚠ 保存配置失败: {e}") def get(self, *keys): - """Get configuration value""" + """获取配置值""" value = self.config for key in keys: if isinstance(value, dict) and key in value: @@ -185,7 +204,7 @@ def get(self, *keys): return value def set(self, *keys, value): - """Set configuration value""" + """设置配置值""" if len(keys) == 0: return @@ -198,208 +217,223 @@ def set(self, *keys, value): config[keys[-1]] = value self.save_config() + def calibrate_skin_color(self, frame, hand_mask): + """自动校准肤色范围""" + if not self.get('calibration', 'auto_calibrate_skin'): + return + + if len(self.skin_calibration_data) < self.get('calibration', 'skin_calibration_frames'): + # 转换到HSV + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + + # 获取肤色区域的HSV值 + skin_pixels = hsv[hand_mask > 0] + + if len(skin_pixels) > 100: # 确保有足够的像素 + self.skin_calibration_data.append(skin_pixels) + + if len(self.skin_calibration_data) == self.get('calibration', 'skin_calibration_frames'): + # 计算肤色范围 + all_skin_pixels = np.vstack(self.skin_calibration_data) + + h_min, h_max = np.percentile(all_skin_pixels[:, 0], [2, 98]) + s_min, s_max = np.percentile(all_skin_pixels[:, 1], [2, 98]) + v_min, v_max = np.percentile(all_skin_pixels[:, 2], [2, 98]) + + # 更新配置 + self.set('gesture', 'skin_lower_h', value=int(max(0, h_min - 5))) + self.set('gesture', 'skin_upper_h', value=int(min(180, h_max + 5))) + self.set('gesture', 'skin_lower_s', value=int(max(0, s_min - 10))) + self.set('gesture', 'skin_upper_s', value=int(min(255, s_max + 10))) + self.set('gesture', 'skin_lower_v', value=int(max(0, v_min - 10))) + self.set('gesture', 'skin_upper_v', value=int(min(255, v_max + 10))) + + print("✓ 肤色校准完成") + print(f" 肤色范围: H[{self.get('gesture', 'skin_lower_h')}-{self.get('gesture', 'skin_upper_h')}], " + f"S[{self.get('gesture', 'skin_lower_s')}-{self.get('gesture', 'skin_upper_s')}], " + f"V[{self.get('gesture', 'skin_lower_v')}-{self.get('gesture', 'skin_upper_v')}]") + + def calibrate_hand_size(self, hand_area): + """校准手部大小""" + if not self.get('calibration', 'hand_size_calibration') or self.hand_size_calibration_done: + return + + if hand_area > 0: + self.reference_hand_size = hand_area + self.hand_size_calibration_done = True + print(f"✓ 手部大小校准完成: {self.reference_hand_size:.0f} 像素") + config = ConfigManager() -# ========== Robust Gesture Recognizer ========== -class RobustGestureRecognizer: - """Robust Gesture Recognizer - Pure OpenCV Implementation""" +# ========== 改进的手势识别器 ========== +class ImprovedGestureRecognizer: + """改进的手势识别器 - 纯OpenCV实现""" def __init__(self): self.config = config.get('gesture') - # Gesture history and smoothing + # 手势历史和平滑 self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) self.confidence_history = deque(maxlen=self.history_size) - self.current_gesture = "Waiting" + self.current_gesture = "等待手势" self.current_confidence = 0.0 - # Hand tracking + # 手部跟踪和状态 self.last_hand_position = None self.hand_tracking = False self.track_window = None + self.hand_states = deque(maxlen=10) # 手部状态历史 - # Performance statistics + # 性能统计 self.process_times = deque(maxlen=30) self.frame_counter = 0 self.detection_interval = self.config['detection_interval'] - # Gesture color mapping + # 手势颜色映射 self.gesture_colors = { - "Stop": (0, 0, 255), # Red - "Forward": (0, 255, 0), # Green - "Up": (255, 255, 0), # Cyan - "Down": (255, 0, 255), # Purple - "Left": (255, 165, 0), # Orange - "Right": (0, 165, 255), # Light Blue - "Waiting": (200, 200, 200), # Gray - "Error": (255, 0, 0) # Blue + "停止": (0, 0, 255), # 红色 + "向前": (0, 255, 0), # 绿色 + "向上": (255, 255, 0), # 青色 + "向下": (255, 0, 255), # 紫色 + "向左": (255, 165, 0), # 橙色 + "向右": (0, 165, 255), # 浅蓝色 + "等待手势": (200, 200, 200), # 灰色 + "识别异常": (255, 0, 0), # 蓝色 + "悬停": (255, 255, 255) # 白色 } - # Skin detection parameters - self.skin_lower = np.array(self.config['skin_lower'], dtype=np.uint8) - self.skin_upper = np.array(self.config['skin_upper'], dtype=np.uint8) - - # Background subtractor + # 背景减除器 self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( history=100, varThreshold=25, detectShadows=True ) - # Morphological operation kernels + # 形态学操作核 self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - # Fingertip detection - self.prev_fingertips = None - self.fingertip_stability_threshold = 3 + print("✓ 改进的手势识别器已初始化 (纯OpenCV实现)") - print("✓ Robust gesture recognizer initialized (Pure OpenCV)") + def get_skin_mask(self, frame): + """获取肤色掩码""" + # 从配置获取肤色范围 + h_low = config.get('gesture', 'skin_lower_h') + h_high = config.get('gesture', 'skin_upper_h') + s_low = config.get('gesture', 'skin_lower_s') + s_high = config.get('gesture', 'skin_upper_s') + v_low = config.get('gesture', 'skin_lower_v') + v_high = config.get('gesture', 'skin_upper_v') - def preprocess_frame(self, frame): - """Preprocess frame""" - # Flip image horizontally for intuitive control - frame = cv2.flip(frame, 1) + # 转换为HSV颜色空间 + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - # Adjust size for performance - resize_factor = config.get('performance', 'resize_factor') - if resize_factor != 1.0: - new_width = int(frame.shape[1] * resize_factor) - new_height = int(frame.shape[0] * resize_factor) - frame = cv2.resize(frame, (new_width, new_height)) + # 定义肤色范围 + lower_skin = np.array([h_low, s_low, v_low], dtype=np.uint8) + upper_skin = np.array([h_high, s_high, v_high], dtype=np.uint8) - return frame + # 创建肤色掩码 + skin_mask = cv2.inRange(hsv, lower_skin, upper_skin) - def detect_skin(self, frame): - """Detect skin regions""" - # Convert to HSV color space - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + return skin_mask, hsv - # Skin mask - skin_mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) - - # Apply background subtraction + def enhance_skin_detection(self, frame, skin_mask): + """增强肤色检测""" + # 应用背景减除 fg_mask = self.bg_subtractor.apply(frame) - # Combine skin mask and foreground mask + # 结合肤色掩码和前景掩码 combined_mask = cv2.bitwise_and(skin_mask, fg_mask) - # Morphological operations + # 形态学操作 combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - # Gaussian blur + # 高斯模糊 combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) return combined_mask - def find_hand_contours(self, mask): - """Find hand contours""" - # Find contours + def find_best_hand_contour(self, mask, frame): + """找到最佳的手部轮廓""" + # 查找轮廓 contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: - return None + return None, 0.0 - # Find the largest contour (assumed to be hand) + # 按面积排序 contours = sorted(contours, key=cv2.contourArea, reverse=True) - for contour in contours: - area = cv2.contourArea(contour) + best_contour = None + best_score = 0.0 - # Filter out too small contours - if area < self.config['min_hand_area']: - continue + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') - # Calculate contour perimeter - perimeter = cv2.arcLength(contour, True) + for contour in contours[:3]: # 只检查前3个最大轮廓 + area = cv2.contourArea(contour) - # Filter out too simple contours - if perimeter < 100: + # 面积过滤 + if area < min_area or area > max_area: continue - return contour - - return None - - def detect_fingertips_advanced(self, contour): - """Advanced fingertip detection""" - # Simplify contour - epsilon = 0.01 * cv2.arcLength(contour, True) - approx = cv2.approxPolyDP(contour, epsilon, True) - - # Find convex hull - hull = cv2.convexHull(approx, returnPoints=False) + # 计算轮廓评分 + score = self.rate_contour(contour, frame.shape) - if hull is None or len(hull) < 3: - return [], 0 + if score > best_score: + best_score = score + best_contour = contour - # Find convexity defects - defects = cv2.convexityDefects(approx, hull) + return best_contour, best_score - if defects is None: - return [], 0 + def rate_contour(self, contour, frame_shape): + """评估轮廓作为手部的可能性""" + score = 0.0 - fingertips = [] - defects_points = [] - - for i in range(defects.shape[0]): - s, e, f, d = defects[i, 0] - start = tuple(approx[s][0]) - end = tuple(approx[e][0]) - far = tuple(approx[f][0]) - - defects_points.append((start, end, far)) - - # Calculate triangle sides - a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) - b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) - c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) - - # Calculate angle - if b * c != 0: - angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) - angle_degrees = math.degrees(angle) - - # If angle is less than 90 degrees, it might be a finger - if angle_degrees < 90: - # Check if this is a deep defect (far from hull) - if d > 1000: # Adjust this threshold based on your needs - # Both start and end points could be fingertips - for point in [start, end]: - if point not in fingertips: - fingertips.append(point) - - # If we found few fingertips, try alternative method - if len(fingertips) <= 2: - # Use convex hull points as fingertips - hull_points = cv2.convexHull(approx, returnPoints=True) - - # Find extreme points in hull - if len(hull_points) > 0: - hull_points = hull_points.reshape(-1, 2) - - # Sort by y-coordinate (top to bottom) - hull_points = hull_points[hull_points[:, 1].argsort()] - - # Take top points as fingertips - num_fingertips = min(5, len(hull_points)) - for i in range(num_fingertips): - point = tuple(hull_points[i]) - if point not in fingertips: - fingertips.append(point) + # 面积评分 + area = cv2.contourArea(contour) + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') + + if min_area < area < max_area: + # 面积在合理范围内 + area_ratio = min(area / max_area, 1.0) + score += area_ratio * 0.3 + + # 周长评分 + perimeter = cv2.arcLength(contour, True) + if perimeter > 100: # 最小周长 + score += 0.2 + + # 紧凑度评分 (周长^2 / 面积) + if area > 0: + compactness = perimeter ** 2 / area + # 手的紧凑度通常在14-20之间 + if 12 < compactness < 25: + compactness_score = 1.0 - abs(compactness - 18) / 6 + score += compactness_score * 0.3 + + # 长宽比评分 + x, y, w, h = cv2.boundingRect(contour) + if h > 0: + aspect_ratio = w / h + # 手的长宽比通常在0.5-2.0之间 + if 0.4 < aspect_ratio < 2.5: + aspect_score = 1.0 - abs(aspect_ratio - 1.0) / 1.5 + score += aspect_score * 0.2 - return fingertips, len(fingertips) + return score - def analyze_hand_contour(self, contour, frame_shape): - """Analyze hand contour""" + def analyze_hand_features(self, contour, frame_shape): + """分析手部特征""" if contour is None: return None, 0.0 - # Calculate contour area + # 计算轮廓面积 area = cv2.contourArea(contour) - # Calculate contour center + # 计算轮廓中心 M = cv2.moments(contour) if M["m00"] == 0: return None, 0.0 @@ -407,27 +441,41 @@ def analyze_hand_contour(self, contour, frame_shape): cx = int(M["m10"] / M["m00"]) cy = int(M["m01"] / M["m00"]) - # Calculate bounding box + # 计算边界框 x, y, w, h = cv2.boundingRect(contour) + bbox_area = w * h + + # 计算手掌中心(假设是轮廓的中心) + palm_center = (cx, cy) - # Advanced fingertip detection - fingertips, finger_count = self.detect_fingertips_advanced(contour) + # 计算手掌半径(假设为边界框宽度的30%) + palm_radius = int(w * config.get('gesture', 'palm_circle_radius_ratio')) - # Calculate hand position (normalized) + # 分析手指 + fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius) + + # 计算手部方向 + direction = self.calculate_hand_direction(contour, cx, cy) + + # 计算手部位置(归一化) h_img, w_img = frame_shape[:2] norm_x = cx / w_img norm_y = cy / h_img - # Calculate gesture confidence - confidence = self.calculate_confidence(area, finger_count, len(contour)) + # 计算手势置信度 + confidence = self.calculate_confidence(area, len(fingers), len(contour), bbox_area) - # Return result + # 返回结果 result = { 'contour': contour, 'center': (cx, cy), 'bbox': (x, y, x + w, y + h), - 'finger_count': finger_count, - 'fingertips': fingertips, + 'fingers': fingers, # 手指列表 + 'fingertips': fingertips, # 指尖点列表 + 'defects': defects, # 凸缺陷列表 + 'palm_center': palm_center, + 'palm_radius': palm_radius, + 'direction': direction, # 手部方向(角度) 'area': area, 'position': (norm_x, norm_y), 'confidence': confidence @@ -435,104 +483,241 @@ def analyze_hand_contour(self, contour, frame_shape): return result, confidence - def calculate_confidence(self, area, finger_count, contour_length): - """Calculate gesture confidence""" - confidence = 0.5 # Base confidence + def analyze_fingers(self, contour, palm_center, palm_radius): + """分析手指""" + # 简化轮廓 + epsilon = config.get('gesture', 'contour_simplify_epsilon') * cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon, True) - # Confidence based on area - if area > 5000: - confidence += 0.2 - elif area > 3000: - confidence += 0.1 + # 计算凸包 + hull = cv2.convexHull(approx, returnPoints=False) - # Confidence based on finger count + if hull is None or len(hull) < 3: + return [], [], [] + + # 计算凸缺陷 + defects = cv2.convexityDefects(approx, hull) + + fingers = [] + fingertips = [] + defect_points = [] + + if defects is not None: + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(approx[s][0]) + end = tuple(approx[e][0]) + far = tuple(approx[f][0]) + + # 计算距离 + start_dist = np.linalg.norm(np.array(start) - np.array(palm_center)) + end_dist = np.linalg.norm(np.array(end) - np.array(palm_center)) + far_dist = np.linalg.norm(np.array(far) - np.array(palm_center)) + + # 检查是否可能是手指 + if (start_dist > palm_radius * 1.2 and + end_dist > palm_radius * 1.2 and + d > config.get('gesture', 'defect_distance_threshold') * 256): + + # 检查角度 + a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + + if b * c != 0: + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + angle_degrees = math.degrees(angle) + + # 如果角度小于90度,可能是指缝 + if angle_degrees < 90: + # 添加手指 + finger = { + 'start': start, + 'end': end, + 'far': far, + 'depth': d, + 'angle': angle_degrees + } + fingers.append(finger) + + # 添加指尖(开始和结束点都可能是指尖) + for point in [start, end]: + if point not in fingertips: + # 检查点是否在手掌半径之外 + point_dist = np.linalg.norm(np.array(point) - np.array(palm_center)) + if point_dist > palm_radius * 1.5: + fingertips.append(point) + + defect_points.append((start, end, far, d)) + + # 如果没有检测到手指,尝试使用凸包点作为指尖 + if len(fingertips) == 0: + hull_points = cv2.convexHull(approx, returnPoints=True) + if len(hull_points) > 0: + hull_points = hull_points.reshape(-1, 2) + + # 过滤掉离手掌中心太近的点 + for point in hull_points: + point_tuple = tuple(point) + point_dist = np.linalg.norm(point - np.array(palm_center)) + if point_dist > palm_radius * 1.5 and point_tuple not in fingertips: + fingertips.append(point_tuple) + + # 限制指尖数量为5 + fingertips = fingertips[:5] + + return fingers, fingertips, defect_points + + def calculate_hand_direction(self, contour, cx, cy): + """计算手部方向""" + # 使用PCA(主成分分析)计算手部方向 + if len(contour) < 5: + return 0.0 + + # 将轮廓转换为点集 + points = contour.reshape(-1, 2).astype(np.float32) + + # 计算PCA + mean = np.empty((0)) + mean, eigenvectors, eigenvalues = cv2.PCACompute2(points, mean) + + # 计算方向角度(度) + direction = math.degrees(math.atan2(eigenvectors[0, 1], eigenvectors[0, 0])) + + return direction + + def calculate_confidence(self, area, finger_count, contour_length, bbox_area): + """计算手势置信度""" + confidence = 0.5 # 基础置信度 + + # 基于面积的置信度 + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') + + if min_area < area < max_area: + # 面积在合理范围内 + area_norm = (area - min_area) / (max_area - min_area) + confidence += area_norm * 0.2 + + # 基于手指数量的置信度 if 0 <= finger_count <= 5: confidence += 0.2 - # Confidence based on contour complexity + # 基于轮廓复杂度的置信度 if contour_length > 200: confidence += 0.1 + # 基于填充度的置信度(轮廓面积/边界框面积) + if bbox_area > 0: + fill_ratio = area / bbox_area + # 手的填充度通常在0.3-0.7之间 + if 0.2 < fill_ratio < 0.8: + fill_score = 1.0 - abs(fill_ratio - 0.5) / 0.3 + confidence += fill_score * 0.1 + return min(confidence, 1.0) - def recognize_gesture(self, hand_data): - """Recognize gesture based on hand data""" + def recognize_gesture_improved(self, hand_data): + """改进的手势识别逻辑""" if hand_data is None: - return "Waiting", 0.3 + return "等待手势", 0.3 - finger_count = hand_data['finger_count'] + finger_count = len(hand_data.get('fingers', [])) + fingertips = hand_data.get('fingertips', []) norm_x, norm_y = hand_data['position'] + direction = hand_data.get('direction', 0.0) confidence = hand_data['confidence'] - fingertips = hand_data.get('fingertips', []) - # Special case: single finger detection - if finger_count == 1: - # Additional check for single finger gesture - if len(fingertips) == 1: - # Check if the fingertip is above the hand center (typical for pointing) + # 根据手指数量分类 + if finger_count == 0: + # 握拳或没有手指 + if len(fingertips) == 0: + return "停止", confidence * 0.9 + else: + # 有指尖但不是手指,可能是部分握拳 + return "停止", confidence * 0.7 + + elif finger_count == 1: + # 单指手势 + if len(fingertips) >= 1: + # 检查指尖方向 cx, cy = hand_data['center'] fingertip = fingertips[0] - if fingertip[1] < cy - 20: # Fingertip is above hand center - return "Forward", confidence * 0.9 - else: - return "Stop", confidence * 0.7 - return "Forward", confidence * 0.8 + # 计算指尖相对于手中心的方向 + dx = fingertip[0] - cx + dy = fingertip[1] - cy - # No fingers (fist) - elif finger_count == 0: - return "Stop", confidence * 0.9 + # 判断指尖方向 + if abs(dx) > abs(dy): # 主要水平方向 + if dx > 0: + return "向右", confidence * 0.8 + else: + return "向左", confidence * 0.8 + else: # 主要垂直方向 + if dy < 0: # 指尖在手中心上方 + return "向上", confidence * 0.8 + else: + return "向前", confidence * 0.8 + return "向前", confidence * 0.7 - # Two fingers (V sign) elif finger_count == 2: - return "Forward", confidence * 0.7 + # 双指手势 + return "向前", confidence * 0.7 - # Three fingers elif finger_count == 3: - # Judge direction based on position - if norm_y < 0.4: - return "Up", confidence * 0.7 - elif norm_y > 0.6: - return "Down", confidence * 0.7 - else: - return "Stop", confidence * 0.6 + # 三指手势,根据手部方向判断 + if -45 <= direction <= 45: # 大致水平 + if direction > 0: + return "向右", confidence * 0.7 + else: + return "向左", confidence * 0.7 + else: # 大致垂直 + if direction > 0: + return "向下", confidence * 0.7 + else: + return "向上", confidence * 0.7 - # Four or five fingers (open hand) elif finger_count >= 4: - # Judge direction based on hand position - if norm_x < 0.3: - return "Left", confidence * 0.8 - elif norm_x > 0.7: - return "Right", confidence * 0.8 - elif norm_y < 0.4: - return "Up", confidence * 0.8 - elif norm_y > 0.6: - return "Down", confidence * 0.8 - else: - return "Stop", confidence * 0.7 - - return "Waiting", confidence * 0.5 + # 多指手势(手掌张开) + # 根据手部位置判断 + if norm_x < 0.4: # 左侧 + return "向左", confidence * 0.8 + elif norm_x > 0.6: # 右侧 + return "向右", confidence * 0.8 + elif norm_y < 0.4: # 上方 + return "向上", confidence * 0.8 + elif norm_y > 0.6: # 下方 + return "向下", confidence * 0.8 + else: # 中间 + # 根据手部方向判断 + if -45 <= direction <= 45: # 大致水平 + return "向前", confidence * 0.7 + else: # 大致垂直 + return "停止", confidence * 0.7 + + return "等待手势", confidence * 0.5 def smooth_gesture(self, new_gesture, new_confidence): - """Smooth gesture output""" - # Add to history + """平滑手势输出""" + # 添加到历史 self.gesture_history.append(new_gesture) self.confidence_history.append(new_confidence) - # If not enough history, return directly + # 如果历史记录不足,直接返回 if len(self.gesture_history) < 3: self.current_gesture = new_gesture self.current_confidence = new_confidence return new_gesture, new_confidence - # Count most common gesture + # 统计最常见的姿势 gesture_counter = Counter(self.gesture_history) most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] - # Calculate average confidence + # 计算平均置信度 avg_confidence = np.mean(list(self.confidence_history)) - # If the most common gesture appears enough times, use it + # 如果最常见的姿势出现次数足够多,使用它 smooth_frames = self.config['smooth_frames'] if most_common_count >= smooth_frames: self.current_gesture = most_common_gesture @@ -541,134 +726,175 @@ def smooth_gesture(self, new_gesture, new_confidence): return self.current_gesture, self.current_confidence def visualize_detection(self, frame, hand_data, gesture, confidence): - """Visualize detection results""" + """可视化检测结果""" if hand_data is None: return frame - # Get display configuration + # 获取显示配置 show_contours = config.get('display', 'show_contours') show_bbox = config.get('display', 'show_bbox') show_fingertips = config.get('display', 'show_fingertips') + show_palm_center = config.get('display', 'show_palm_center') + show_hand_direction = config.get('display', 'show_hand_direction') + show_debug_info = config.get('display', 'show_debug_info') - # Draw contour + # 绘制轮廓 if show_contours and 'contour' in hand_data: cv2.drawContours(frame, [hand_data['contour']], -1, (0, 255, 0), 2) - # Draw bounding box + # 绘制边界框 if show_bbox and 'bbox' in hand_data: x1, y1, x2, y2 = hand_data['bbox'] color = self.gesture_colors.get(gesture, (255, 255, 255)) cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) - # Show gesture label + # 显示手势标签 label = f"{gesture}" if config.get('display', 'show_confidence'): label += f" ({confidence:.0%})" - # Calculate text size + # 计算文本大小 (text_width, text_height), baseline = cv2.getTextSize( label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 ) - # Draw text background + # 绘制文本背景 cv2.rectangle(frame, (x1, y1 - text_height - 10), (x1 + text_width, y1), color, -1) - # Draw text + # 绘制文本 cv2.putText(frame, label, (x1, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) - # Draw center point - if 'center' in hand_data: - cx, cy = hand_data['center'] + # 绘制手掌中心 + if show_palm_center and 'palm_center' in hand_data: + cx, cy = hand_data['palm_center'] + palm_radius = hand_data.get('palm_radius', 20) cv2.circle(frame, (cx, cy), 5, (0, 0, 255), -1) - cv2.putText(frame, "Center", (cx + 10, cy), + cv2.circle(frame, (cx, cy), palm_radius, (0, 0, 255), 1) + cv2.putText(frame, "Palm", (cx + 10, cy), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) - # Draw fingertips + # 绘制指尖 if show_fingertips and 'fingertips' in hand_data: for i, point in enumerate(hand_data['fingertips']): cv2.circle(frame, point, 4, (255, 0, 0), -1) cv2.putText(frame, f"F{i + 1}", (point[0] + 5, point[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1) - # Show finger count - if 'finger_count' in hand_data: - finger_text = f"Fingers: {hand_data['finger_count']}" - cv2.putText(frame, finger_text, (10, frame.shape[0] - 10), + # 绘制手部方向 + if show_hand_direction and 'direction' in hand_data and 'center' in hand_data: + cx, cy = hand_data['center'] + direction = hand_data['direction'] + length = 50 + + # 计算方向向量 + dx = length * math.cos(math.radians(direction)) + dy = length * math.sin(math.radians(direction)) + + # 绘制方向线 + end_point = (int(cx + dx), int(cy + dy)) + cv2.arrowedLine(frame, (cx, cy), end_point, (255, 255, 0), 2) + + # 显示方向角度 + angle_text = f"Dir: {direction:.0f}°" + cv2.putText(frame, angle_text, (cx, cy - 20), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 1) + + # 显示调试信息 + if show_debug_info: + # 显示手指数量 + finger_count = len(hand_data.get('fingers', [])) + finger_text = f"Fingers: {finger_count}" + cv2.putText(frame, finger_text, (10, frame.shape[0] - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 显示手部位置 + pos_text = f"Pos: ({hand_data['position'][0]:.2f}, {hand_data['position'][1]:.2f})" + cv2.putText(frame, pos_text, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) return frame def recognize(self, frame): - """Recognize gesture""" + """识别手势""" start_time = time.time() try: - # Preprocess frame - processed_frame = self.preprocess_frame(frame) + # 预处理帧 + processed_frame = cv2.flip(frame, 1) - # Detect every few frames to improve performance + # 每隔几帧检测一次以提高性能 if self.frame_counter % self.detection_interval != 0: self.frame_counter += 1 return self.current_gesture, self.current_confidence, processed_frame - # Detect skin - skin_mask = self.detect_skin(processed_frame) + # 获取肤色掩码 + skin_mask, hsv = self.get_skin_mask(processed_frame) + + # 增强肤色检测 + enhanced_mask = self.enhance_skin_detection(processed_frame, skin_mask) - # Find hand contour - hand_contour = self.find_hand_contours(skin_mask) + # 找到最佳的手部轮廓 + hand_contour, contour_score = self.find_best_hand_contour(enhanced_mask, processed_frame) - # Analyze hand contour - hand_data, confidence = self.analyze_hand_contour(hand_contour, processed_frame.shape) + # 分析手部特征 + hand_data, confidence = self.analyze_hand_features(hand_contour, processed_frame.shape) - # Recognize gesture + # 识别手势 if hand_data is not None: - gesture, raw_confidence = self.recognize_gesture(hand_data) + # 校准肤色(如果需要) + config.calibrate_skin_color(processed_frame, enhanced_mask) + + # 校准手部大小(如果需要) + config.calibrate_hand_size(hand_data['area']) + + # 识别手势 + gesture, raw_confidence = self.recognize_gesture_improved(hand_data) confidence = max(confidence, raw_confidence) else: - gesture, confidence = "Waiting", 0.3 + gesture, confidence = "等待手势", 0.3 - # Smooth gesture + # 平滑手势 final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) - # Visualize results + # 可视化结果 if hand_data is not None: processed_frame = self.visualize_detection( processed_frame, hand_data, final_gesture, final_confidence ) - # Update counter + # 更新计数器 self.frame_counter += 1 - # Calculate processing time + # 计算处理时间 process_time = (time.time() - start_time) * 1000 self.process_times.append(process_time) return final_gesture, final_confidence, processed_frame except Exception as e: - print(f"⚠ Gesture recognition error: {e}") - return "Error", 0.0, frame + print(f"⚠ 手势识别错误: {e}") + return "识别异常", 0.0, frame def get_performance_stats(self): - """Get performance statistics""" + """获取性能统计""" if len(self.process_times) == 0: return 0.0 return np.mean(list(self.process_times)) def set_simulated_gesture(self, gesture): - """Set simulated gesture""" + """设置模拟的手势""" self.current_gesture = gesture self.current_confidence = 0.9 -# ========== Simple Drone Controller ========== +# ========== 简单的无人机控制器 ========== class SimpleDroneController: - """Simple Drone Controller""" + """简单的无人机控制器""" def __init__(self, airsim_module): self.airsim = airsim_module @@ -676,140 +902,143 @@ def __init__(self, airsim_module): self.connected = False self.flying = False - # Control parameters + # 控制参数 self.velocity = config.get('drone', 'velocity') self.duration = config.get('drone', 'duration') self.altitude = config.get('drone', 'altitude') self.control_interval = config.get('drone', 'control_interval') - # Control state + # 控制状态 self.last_control_time = 0 self.last_gesture = None - print("✓ Simple drone controller initialized") + print("✓ 简单的无人机控制器已初始化") def connect(self): - """Connect to AirSim drone""" + """连接AirSim无人机""" if self.connected: return True if self.airsim is None: - print("⚠ AirSim not available, using simulation mode") + print("⚠ AirSim不可用,使用模拟模式") self.connected = True return True - print("Connecting to AirSim...") + print("连接AirSim...") try: self.client = self.airsim.MultirotorClient() self.client.confirmConnection() - print("✅ Connected to AirSim!") + print("✅ 已连接AirSim!") self.client.enableApiControl(True) - print("✅ API control enabled") + print("✅ API控制已启用") self.client.armDisarm(True) - print("✅ Drone armed") + print("✅ 无人机已武装") self.connected = True return True except Exception as e: - print(f"❌ Connection failed: {e}") - print("\nContinue with simulation mode? (y/n)") + print(f"❌ 连接失败: {e}") + print("\n使用模拟模式继续? (y/n)") choice = input().strip().lower() if choice == 'y': self.connected = True - print("✅ Using simulation mode") + print("✅ 使用模拟模式") return True return False def takeoff(self): - """Take off""" + """起飞""" if not self.connected: return False try: if self.airsim is None or self.client is None: - print("✅ Simulated takeoff") + print("✅ 模拟起飞") self.flying = True return True - print("Taking off...") + print("起飞中...") self.client.takeoffAsync().join() time.sleep(1) - # Ascend to specified altitude + # 上升到指定高度 self.client.moveToZAsync(self.altitude, 3).join() self.flying = True - print("✅ Drone took off successfully") + print("✅ 无人机成功起飞") return True except Exception as e: - print(f"❌ Takeoff failed: {e}") + print(f"❌ 起飞失败: {e}") return False def land(self): - """Land""" + """降落""" if not self.connected: return False try: if self.airsim is None or self.client is None: - print("✅ Simulated landing") + print("✅ 模拟降落") self.flying = False return True - print("Landing...") + print("降落中...") self.client.landAsync().join() self.flying = False - print("✅ Drone landed") + print("✅ 无人机已降落") return True except Exception as e: - print(f"Landing failed: {e}") + print(f"降落失败: {e}") return False def move_by_gesture(self, gesture, confidence): - """Move based on gesture""" + """根据手势移动""" if not self.connected or not self.flying: return False - # Check control interval + # 检查控制间隔 current_time = time.time() if current_time - self.last_control_time < self.control_interval: return False - # Check confidence threshold + # 检查置信度阈值 min_confidence = config.get('gesture', 'min_confidence') if confidence < min_confidence: return False try: if self.airsim is None or self.client is None: - print(f"Simulated move: {gesture}") + print(f"模拟移动: {gesture}") self.last_control_time = current_time self.last_gesture = gesture return True success = False - if gesture == "Up": + if gesture == "向上": self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) success = True - elif gesture == "Down": + elif gesture == "向下": self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) success = True - elif gesture == "Left": + elif gesture == "向左": self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) success = True - elif gesture == "Right": + elif gesture == "向右": self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) success = True - elif gesture == "Forward": + elif gesture == "向前": self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) success = True - elif gesture == "Stop": + elif gesture == "停止": + self.client.hoverAsync() + success = True + elif gesture == "悬停": self.client.hoverAsync() success = True @@ -819,20 +1048,20 @@ def move_by_gesture(self, gesture, confidence): return success except Exception as e: - print(f"Control command failed: {e}") + print(f"控制命令失败: {e}") return False def emergency_stop(self): - """Emergency stop""" + """紧急停止""" if self.connected: try: if self.flying and self.client is not None: - print("Emergency landing...") + print("紧急降落...") self.land() if self.client is not None: self.client.armDisarm(False) self.client.enableApiControl(False) - print("✅ Emergency stop complete") + print("✅ 紧急停止完成") except: pass @@ -840,117 +1069,173 @@ def emergency_stop(self): self.flying = False -# ========== Simple UI Renderer ========== -class SimpleUIRenderer: - """Simple UI Renderer""" +# ========== 中文UI渲染器 ========== +class ChineseUIRenderer: + """中文UI渲染器""" def __init__(self): - # Color definitions + self.fonts = {} + self.load_fonts() + + # 颜色定义 self.colors = { - 'title': (0, 255, 255), # Cyan - 'connected': (0, 255, 0), # Green - 'disconnected': (0, 0, 255), # Red - 'flying': (0, 255, 0), # Green - 'landed': (255, 165, 0), # Orange - 'warning': (0, 165, 255), # Light Blue - 'info': (255, 255, 255), # White - 'help': (255, 200, 100) # Light Orange + 'title': (0, 255, 255), # 青色 + 'connected': (0, 255, 0), # 绿色 + 'disconnected': (0, 0, 255), # 红色 + 'flying': (0, 255, 0), # 绿色 + 'landed': (255, 165, 0), # 橙色 + 'warning': (0, 165, 255), # 浅蓝色 + 'info': (255, 255, 255), # 白色 + 'help': (255, 200, 100) # 浅橙色 } - print("✓ Simple UI renderer initialized") + print("✓ 中文UI渲染器已初始化") + + def load_fonts(self): + """加载字体""" + font_paths = [ + 'simhei.ttf', + 'C:/Windows/Fonts/simhei.ttf', + 'C:/Windows/Fonts/msyh.ttc', + '/System/Library/Fonts/PingFang.ttc', + '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' + ] + + for path in font_paths: + try: + self.fonts[14] = ImageFont.truetype(path, 14) + self.fonts[16] = ImageFont.truetype(path, 16) + self.fonts[18] = ImageFont.truetype(path, 18) + self.fonts[20] = ImageFont.truetype(path, 20) + self.fonts[24] = ImageFont.truetype(path, 24) + print(f"✓ 字体已加载: {path}") + return + except: + continue + + print("⚠ 未找到字体,使用默认") + + def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): + """在图像上绘制文本""" + try: + img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + pil_img = Image.fromarray(img_rgb) + draw = ImageDraw.Draw(pil_img) + + font = self.fonts.get(size, self.fonts.get(16)) + + # 绘制阴影 + shadow_color = (0, 0, 0) + shadow_pos = (pos[0] + 1, pos[1] + 1) + draw.text(shadow_pos, text, font=font, fill=shadow_color) + + # 绘制文字 + rgb_color = color[::-1] # BGR to RGB + draw.text(pos, text, font=font, fill=rgb_color) + + return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) + except: + # 备用方案:使用OpenCV绘制英文 + cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, + size / 25, color, 1) + return frame def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time): - """Draw status bar""" + """绘制状态栏""" h, w = frame.shape[:2] - # Draw semi-transparent background + # 绘制半透明背景 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (w, 100), (0, 0, 0), -1) + cv2.rectangle(overlay, (0, 0), (w, 120), (0, 0, 0), -1) frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) - # Title - title = "Gesture Controlled Drone" - cv2.putText(frame, title, (10, 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, self.colors['title'], 2) + # 标题 + title = "手势控制无人机系统 - 优化版" + frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) - # Connection status + # 连接状态 status_color = self.colors['connected'] if drone_controller.connected else self.colors['disconnected'] - status_text = f"Drone: {'Connected' if drone_controller.connected else 'Disconnected'}" - cv2.putText(frame, status_text, (10, 60), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, status_color, 1) + status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" + frame = self.draw_text(frame, status_text, (10, 40), size=16, color=status_color) - # Flight status + # 飞行状态 flight_color = self.colors['flying'] if drone_controller.flying else self.colors['landed'] - flight_text = f"Flight: {'Flying' if drone_controller.flying else 'Landed'}" - cv2.putText(frame, flight_text, (10, 85), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, flight_color, 1) + flight_text = f"飞行状态: {'飞行中' if drone_controller.flying else '已降落'}" + frame = self.draw_text(frame, flight_text, (10, 65), size=16, color=flight_color) - # Gesture information + # 手势信息 if confidence > 0.7: - gesture_color = (0, 255, 0) # Green + gesture_color = (0, 255, 0) # 绿色 elif confidence > 0.5: - gesture_color = (255, 165, 0) # Orange + gesture_color = (255, 165, 0) # 橙色 else: - gesture_color = (200, 200, 200) # Gray + gesture_color = (200, 200, 200) # 灰色 - gesture_text = f"Gesture: {gesture}" + gesture_text = f"当前手势: {gesture}" if config.get('display', 'show_confidence'): gesture_text += f" ({confidence:.0%})" - cv2.putText(frame, gesture_text, (w // 2, 60), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, gesture_color, 1) + frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) - # Performance information + # 性能信息 if config.get('display', 'show_fps'): - perf_text = f"FPS: {fps:.1f}" + perf_text = f"帧率: {fps:.1f}" if process_time > 0: - perf_text += f" | Delay: {process_time:.1f}ms" + perf_text += f" | 延迟: {process_time:.1f}ms" + + frame = self.draw_text(frame, perf_text, (w - 200, 65), size=14, color=self.colors['info']) - cv2.putText(frame, perf_text, (w - 200, 85), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['info'], 1) + # 控制提示 + control_text = "提示: 确保手部完全进入画面,光线充足" + frame = self.draw_text(frame, control_text, (10, 90), size=14, color=self.colors['info']) return frame def draw_help_bar(self, frame): - """Draw help bar""" + """绘制帮助栏""" if not config.get('display', 'show_help'): return frame h, w = frame.shape[:2] - # Draw bottom help bar - cv2.rectangle(frame, (0, h - 50), (w, h), (0, 0, 0), -1) + # 绘制底部帮助栏 + cv2.rectangle(frame, (0, h - 60), (w, h), (0, 0, 0), -1) + + # 帮助文本 + help_lines = [ + "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", + "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息" + ] - # Help text - help_text = "C:Connect Space:Takeoff/Land ESC:Exit W/A/S/D/F/X:Keyboard" - cv2.putText(frame, help_text, (10, h - 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['help'], 1) + for i, line in enumerate(help_lines): + y_pos = h - 45 + i * 20 + frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame def draw_warning(self, frame, message): - """Draw warning message""" + """绘制警告信息""" h, w = frame.shape[:2] - # Draw warning at top + # 在顶部绘制警告 warning_bg = np.zeros((40, w, 3), dtype=np.uint8) - warning_bg[:, :] = (0, 69, 255) # Orange + warning_bg[:, :] = (0, 69, 255) # 橙色 - frame[100:140, 0:w] = cv2.addWeighted( - frame[100:140, 0:w], 0.3, + frame[120:160, 0:w] = cv2.addWeighted( + frame[120:160, 0:w], 0.3, warning_bg, 0.7, 0 ) - # Draw warning text - cv2.putText(frame, message, (10, 125), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, self.colors['warning'], 1) + # 绘制警告文本 + frame = self.draw_text(frame, message, (10, 135), + size=16, color=self.colors['warning']) return frame -# ========== Performance Monitor ========== +# ========== 性能监控器 ========== class PerformanceMonitor: - """Performance Monitor""" + """性能监控器""" def __init__(self): self.frame_times = deque(maxlen=60) @@ -959,12 +1244,12 @@ def __init__(self): self.frame_count = 0 def update(self): - """Update performance data""" + """更新性能数据""" current_time = time.time() self.frame_times.append(current_time) self.frame_count += 1 - # Calculate FPS once per second + # 每秒计算一次FPS if current_time - self.last_update >= 1.0: if len(self.frame_times) > 1: time_diff = self.frame_times[-1] - self.frame_times[0] @@ -975,25 +1260,25 @@ def update(self): self.last_update = current_time def get_stats(self): - """Get performance statistics""" + """获取性能统计""" return { 'fps': self.fps, 'frame_count': self.frame_count } -# ========== Main Program ========== +# ========== 主程序 ========== def main(): - """Main function""" - # Initialize components - print("Initializing components...") + """主函数""" + # 初始化组件 + print("初始化组件...") - gesture_recognizer = RobustGestureRecognizer() + gesture_recognizer = ImprovedGestureRecognizer() drone_controller = SimpleDroneController(libs['airsim']) - ui_renderer = SimpleUIRenderer() + ui_renderer = ChineseUIRenderer() performance_monitor = PerformanceMonitor() - # Initialize camera + # 初始化摄像头 cap = None try: cap = cv2.VideoCapture(0) @@ -1002,87 +1287,111 @@ def main(): cap.set(cv2.CAP_PROP_FRAME_HEIGHT, config.get('camera', 'height')) cap.set(cv2.CAP_PROP_FPS, config.get('camera', 'fps')) - # Get actual parameters + # 获取实际参数 actual_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) actual_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) actual_fps = int(cap.get(cv2.CAP_PROP_FPS)) - print(f"✓ Camera initialized") - print(f" Resolution: {actual_width}x{actual_height}") - print(f" FPS: {actual_fps}") + print(f"✓ 摄像头已初始化") + print(f" 分辨率: {actual_width}x{actual_height}") + print(f" 帧率: {actual_fps}") else: - print("❌ Camera not available, using simulation mode") + print("❌ 摄像头不可用,使用模拟模式") cap = None except Exception as e: - print(f"⚠ Camera init failed: {e}") + print(f"⚠ 摄像头初始化失败: {e}") cap = None - # Display welcome message + # 显示欢迎信息 print("\n" + "=" * 60) - print("Gesture Controlled Drone - Optimized Version") + print("手势控制无人机系统 - 手势识别优化版") print("=" * 60) - print("System Status:") - print(f" Camera: {'Connected' if cap else 'Simulation Mode'}") - print(f" Gesture Recognition: Pure OpenCV") - print(f" AirSim: {'Available' if libs['airsim'] else 'Simulation Mode'}") + print("系统状态:") + print(f" 摄像头: {'已连接' if cap else '模拟模式'}") + print(f" 手势识别: 改进的OpenCV算法") + print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) - # Display operation instructions - print("\nOperation Instructions:") - print("1. Press [C] to connect drone (AirSim simulator)") - print("2. Press [SPACE] to takeoff/land") - print("3. Gesture Control:") - print(" - Fist (0 fingers): Stop") - print(" - Index finger (1 finger): Forward") - print(" - Open hand (4-5 fingers): Control direction by hand position") - print(" * Gesture confidence > 60% to execute") - print("4. Keyboard Control:") - print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") - print("5. Press [H] to toggle help display") - print("6. Press [R] to reset gesture recognition") - print("7. Press [ESC] to exit safely") + # 显示操作说明 + print("\n操作说明:") + print("1. 按 [C] 连接无人机 (AirSim模拟器)") + print("2. 按 [空格键] 起飞/降落") + print("3. 手势控制改进:") + print(" - 握拳或握紧: 停止") + print(" - 单指指向: 根据指尖方向判断上下左右") + print(" - 双指: 向前") + print(" - 手掌张开: 根据手的位置判断方向") + print(" * 手势识别置信度 > 50% 时才会执行") + print("4. 键盘控制:") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print("5. 调试功能:") + print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") + print("6. 按 [ESC] 安全退出") print("=" * 60) - print("Program started successfully!") + print("程序启动成功!") print("-" * 60) - # Keyboard gesture mapping + # 键盘手势映射 key_to_gesture = { - ord('w'): "Up", ord('W'): "Up", - ord('s'): "Down", ord('S'): "Down", - ord('a'): "Left", ord('A'): "Left", - ord('d'): "Right", ord('D'): "Right", - ord('f'): "Forward", ord('F'): "Forward", - ord('x'): "Stop", ord('X'): "Stop", + ord('w'): "向上", ord('W'): "向上", + ord('s'): "向下", ord('S'): "向下", + ord('a'): "向左", ord('A'): "向左", + ord('d'): "向右", ord('D'): "向右", + ord('f'): "向前", ord('F'): "向前", + ord('x'): "停止", ord('X'): "停止", } - # Main loop - print("\nEntering main loop, press ESC to exit...") + # 显示模式 + display_modes = ['normal', 'detailed', 'minimal'] + current_display_mode = 0 + + # 主循环 + print("\n进入主循环,按ESC退出...") try: while True: - # Update performance monitor + # 更新性能监控 performance_monitor.update() - # Read camera frame + # 读取摄像头帧 if cap: ret, frame = cap.read() if not ret: - # Create blank frame + # 创建空白帧 frame = np.zeros((480, 640, 3), dtype=np.uint8) - gesture, confidence = "Camera Error", 0.0 + gesture, confidence = "摄像头错误", 0.0 else: - # Gesture recognition + # 手势识别 gesture, confidence, frame = gesture_recognizer.recognize(frame) else: - # Simulation mode + # 模拟模式 frame = np.zeros((480, 640, 3), dtype=np.uint8) gesture, confidence = gesture_recognizer.current_gesture, gesture_recognizer.current_confidence - # Get performance statistics + # 获取性能统计 perf_stats = performance_monitor.get_stats() process_time = gesture_recognizer.get_performance_stats() - # Draw UI + # 根据显示模式调整显示选项 + if display_modes[current_display_mode] == 'normal': + config.set('display', 'show_contours', value=True) + config.set('display', 'show_bbox', value=True) + config.set('display', 'show_fingertips', value=True) + config.set('display', 'show_debug_info', value=False) + elif display_modes[current_display_mode] == 'detailed': + config.set('display', 'show_contours', value=True) + config.set('display', 'show_bbox', value=True) + config.set('display', 'show_fingertips', value=True) + config.set('display', 'show_palm_center', value=True) + config.set('display', 'show_hand_direction', value=True) + config.set('display', 'show_debug_info', value=True) + elif display_modes[current_display_mode] == 'minimal': + config.set('display', 'show_contours', value=False) + config.set('display', 'show_bbox', value=True) + config.set('display', 'show_fingertips', value=False) + config.set('display', 'show_debug_info', value=False) + + # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, perf_stats['fps'], process_time @@ -1090,27 +1399,26 @@ def main(): frame = ui_renderer.draw_help_bar(frame) - # Show connection warning + # 显示连接提示 if not drone_controller.connected: - warning_msg = "⚠ Press C to connect drone, or use simulation mode" + warning_msg = "⚠ 按C键连接无人机,或使用模拟模式" frame = ui_renderer.draw_warning(frame, warning_msg) - # Show image - window_title = "Gesture Controlled Drone - Press ESC to exit" - cv2.imshow(window_title, frame) + # 显示图像(窗口标题用英文) + cv2.imshow('Gesture Controlled Drone - Optimized', frame) - # ========== Keyboard Control ========== + # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF - if key == 27: # ESC key - print("\nExiting program...") + if key == 27: # ESC键 + print("\n退出程序...") break elif key == ord('c') or key == ord('C'): if not drone_controller.connected: drone_controller.connect() - elif key == 32: # Space key + elif key == 32: # 空格键 if drone_controller.connected: if drone_controller.flying: drone_controller.land() @@ -1119,30 +1427,31 @@ def main(): time.sleep(0.5) elif key == ord('h') or key == ord('H'): - # Toggle help display + # 切换帮助显示 current = config.get('display', 'show_help') config.set('display', 'show_help', value=not current) - print(f"Help display: {'ON' if not current else 'OFF'}") - - elif key == ord('f') or key == ord('F'): - # Toggle FPS display - current = config.get('display', 'show_fps') - config.set('display', 'show_fps', value=not current) - print(f"FPS display: {'ON' if not current else 'OFF'}") + print(f"帮助显示: {'开启' if not current else '关闭'}") elif key == ord('r') or key == ord('R'): - # Reset gesture recognition - print("Resetting gesture recognition...") - gesture_recognizer = RobustGestureRecognizer() + # 重置手势识别 + print("重置手势识别...") + gesture_recognizer = ImprovedGestureRecognizer() + print("✓ 手势识别已重置") elif key == ord('t') or key == ord('T'): - # Toggle fingertip display - current = config.get('display', 'show_fingertips') - config.set('display', 'show_fingertips', value=not current) - print(f"Fingertip display: {'ON' if not current else 'OFF'}") + # 切换显示模式 + current_display_mode = (current_display_mode + 1) % len(display_modes) + mode_name = display_modes[current_display_mode] + print(f"显示模式: {mode_name}") + + elif key == ord('d') or key == ord('D'): + # 切换调试信息 + current = config.get('display', 'show_debug_info') + config.set('display', 'show_debug_info', value=not current) + print(f"调试信息: {'开启' if not current else '关闭'}") elif key in key_to_gesture: - # Keyboard control + # 键盘控制 simulated_gesture = key_to_gesture[key] gesture_recognizer.set_simulated_gesture(simulated_gesture) gesture = simulated_gesture @@ -1150,33 +1459,33 @@ def main(): if drone_controller.connected and drone_controller.flying: drone_controller.move_by_gesture(gesture, confidence) - # Real gesture control + # 真实手势控制 current_time = time.time() - if (gesture and gesture != "Waiting" and - gesture != "Camera Error" and gesture != "Error" and + if (gesture and gesture != "等待手势" and + gesture != "摄像头错误" and gesture != "识别异常" and drone_controller.connected and drone_controller.flying): drone_controller.move_by_gesture(gesture, confidence) except KeyboardInterrupt: - print("\nProgram interrupted by user") + print("\n程序被用户中断") except Exception as e: - print(f"\nProgram error: {e}") + print(f"\n程序错误: {e}") traceback.print_exc() finally: - # Cleanup resources - print("\nCleaning up resources...") + # 清理资源 + print("\n清理资源...") if cap: cap.release() cv2.destroyAllWindows() drone_controller.emergency_stop() config.save_config() - print("Program exited safely") + print("程序安全退出") print("=" * 60) - print("\nThank you for using Gesture Controlled Drone System!") - input("Press Enter to exit...") + print("\n感谢使用手势控制无人机系统!") + input("按回车键退出...") -# ========== Program Entry Point ========== +# ========== 程序入口 ========== if __name__ == "__main__": main() \ No newline at end of file From 1c7a07d2a683967d9c6e7563bd8b1eb75a071e31 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 17:15:10 +0800 Subject: [PATCH 06/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E6=A0=87=E7=AD=BE=E6=98=BE=E7=A4=BA=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=B8=AD=E6=96=87=E5=AD=97=E4=BD=93=E7=BC=BA=E5=A4=B1=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=9A=84=E9=97=AE=E5=8F=B7=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config_v2.json | 6 +- .../main.py | 104 +++++++++--------- 2 files changed, 55 insertions(+), 55 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json index b5479855de..57205fa642 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json @@ -9,9 +9,9 @@ "skin_lower_h": 0, "skin_upper_h": 178, "skin_lower_s": 0, - "skin_upper_s": 67, - "skin_lower_v": 36, - "skin_upper_v": 98, + "skin_upper_s": 255, + "skin_lower_v": 0, + "skin_upper_v": 77, "min_hand_area": 2000, "max_hand_area": 30000, "history_size": 10, diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index f6f31be3fe..bfa1b35eef 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -278,7 +278,7 @@ def __init__(self): self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) self.confidence_history = deque(maxlen=self.history_size) - self.current_gesture = "等待手势" + self.current_gesture = "Waiting" self.current_confidence = 0.0 # 手部跟踪和状态 @@ -294,15 +294,15 @@ def __init__(self): # 手势颜色映射 self.gesture_colors = { - "停止": (0, 0, 255), # 红色 - "向前": (0, 255, 0), # 绿色 - "向上": (255, 255, 0), # 青色 - "向下": (255, 0, 255), # 紫色 - "向左": (255, 165, 0), # 橙色 - "向右": (0, 165, 255), # 浅蓝色 - "等待手势": (200, 200, 200), # 灰色 - "识别异常": (255, 0, 0), # 蓝色 - "悬停": (255, 255, 255) # 白色 + "Stop": (0, 0, 255), # 红色 + "Forward": (0, 255, 0), # 绿色 + "Up": (255, 255, 0), # 青色 + "Down": (255, 0, 255), # 紫色 + "Left": (255, 165, 0), # 橙色 + "Right": (0, 165, 255), # 浅蓝色 + "Waiting": (200, 200, 200), # 灰色 + "Error": (255, 0, 0), # 蓝色 + "Hover": (255, 255, 255) # 白色 } # 背景减除器 @@ -620,7 +620,7 @@ def calculate_confidence(self, area, finger_count, contour_length, bbox_area): def recognize_gesture_improved(self, hand_data): """改进的手势识别逻辑""" if hand_data is None: - return "等待手势", 0.3 + return "Waiting", 0.3 finger_count = len(hand_data.get('fingers', [])) fingertips = hand_data.get('fingertips', []) @@ -632,10 +632,10 @@ def recognize_gesture_improved(self, hand_data): if finger_count == 0: # 握拳或没有手指 if len(fingertips) == 0: - return "停止", confidence * 0.9 + return "Stop", confidence * 0.9 else: # 有指尖但不是手指,可能是部分握拳 - return "停止", confidence * 0.7 + return "Stop", confidence * 0.7 elif finger_count == 1: # 单指手势 @@ -651,52 +651,52 @@ def recognize_gesture_improved(self, hand_data): # 判断指尖方向 if abs(dx) > abs(dy): # 主要水平方向 if dx > 0: - return "向右", confidence * 0.8 + return "Right", confidence * 0.8 else: - return "向左", confidence * 0.8 + return "Left", confidence * 0.8 else: # 主要垂直方向 if dy < 0: # 指尖在手中心上方 - return "向上", confidence * 0.8 + return "Up", confidence * 0.8 else: - return "向前", confidence * 0.8 - return "向前", confidence * 0.7 + return "Forward", confidence * 0.8 + return "Forward", confidence * 0.7 elif finger_count == 2: # 双指手势 - return "向前", confidence * 0.7 + return "Forward", confidence * 0.7 elif finger_count == 3: # 三指手势,根据手部方向判断 if -45 <= direction <= 45: # 大致水平 if direction > 0: - return "向右", confidence * 0.7 + return "Right", confidence * 0.7 else: - return "向左", confidence * 0.7 + return "Left", confidence * 0.7 else: # 大致垂直 if direction > 0: - return "向下", confidence * 0.7 + return "Down", confidence * 0.7 else: - return "向上", confidence * 0.7 + return "Up", confidence * 0.7 elif finger_count >= 4: # 多指手势(手掌张开) # 根据手部位置判断 if norm_x < 0.4: # 左侧 - return "向左", confidence * 0.8 + return "Left", confidence * 0.8 elif norm_x > 0.6: # 右侧 - return "向右", confidence * 0.8 + return "Right", confidence * 0.8 elif norm_y < 0.4: # 上方 - return "向上", confidence * 0.8 + return "Up", confidence * 0.8 elif norm_y > 0.6: # 下方 - return "向下", confidence * 0.8 + return "Down", confidence * 0.8 else: # 中间 # 根据手部方向判断 if -45 <= direction <= 45: # 大致水平 - return "向前", confidence * 0.7 + return "Forward", confidence * 0.7 else: # 大致垂直 - return "停止", confidence * 0.7 + return "Stop", confidence * 0.7 - return "等待手势", confidence * 0.5 + return "Waiting", confidence * 0.5 def smooth_gesture(self, new_gesture, new_confidence): """平滑手势输出""" @@ -748,7 +748,7 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): color = self.gesture_colors.get(gesture, (255, 255, 255)) cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) - # 显示手势标签 + # 显示手势标签 - 使用英文避免字体问题 label = f"{gesture}" if config.get('display', 'show_confidence'): label += f" ({confidence:.0%})" @@ -855,7 +855,7 @@ def recognize(self, frame): gesture, raw_confidence = self.recognize_gesture_improved(hand_data) confidence = max(confidence, raw_confidence) else: - gesture, confidence = "等待手势", 0.3 + gesture, confidence = "Waiting", 0.3 # 平滑手势 final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) @@ -877,7 +877,7 @@ def recognize(self, frame): except Exception as e: print(f"⚠ 手势识别错误: {e}") - return "识别异常", 0.0, frame + return "Error", 0.0, frame def get_performance_stats(self): """获取性能统计""" @@ -1020,25 +1020,25 @@ def move_by_gesture(self, gesture, confidence): success = False - if gesture == "向上": + if gesture == "Up": self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) success = True - elif gesture == "向下": + elif gesture == "Down": self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) success = True - elif gesture == "向左": + elif gesture == "Left": self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) success = True - elif gesture == "向右": + elif gesture == "Right": self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) success = True - elif gesture == "向前": + elif gesture == "Forward": self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) success = True - elif gesture == "停止": + elif gesture == "Stop": self.client.hoverAsync() success = True - elif gesture == "悬停": + elif gesture == "Hover": self.client.hoverAsync() success = True @@ -1317,13 +1317,13 @@ def main(): print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") print("3. 手势控制改进:") - print(" - 握拳或握紧: 停止") - print(" - 单指指向: 根据指尖方向判断上下左右") - print(" - 双指: 向前") + print(" - 握拳或握紧: Stop") + print(" - 单指指向: 根据指尖方向判断Up/Down/Left/Right") + print(" - 双指: Forward") print(" - 手掌张开: 根据手的位置判断方向") print(" * 手势识别置信度 > 50% 时才会执行") print("4. 键盘控制:") - print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") print("5. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") print("6. 按 [ESC] 安全退出") @@ -1333,12 +1333,12 @@ def main(): # 键盘手势映射 key_to_gesture = { - ord('w'): "向上", ord('W'): "向上", - ord('s'): "向下", ord('S'): "向下", - ord('a'): "向左", ord('A'): "向左", - ord('d'): "向右", ord('D'): "向右", - ord('f'): "向前", ord('F'): "向前", - ord('x'): "停止", ord('X'): "停止", + ord('w'): "Up", ord('W'): "Up", + ord('s'): "Down", ord('S'): "Down", + ord('a'): "Left", ord('A'): "Left", + ord('d'): "Right", ord('D'): "Right", + ord('f'): "Forward", ord('F'): "Forward", + ord('x'): "Stop", ord('X'): "Stop", } # 显示模式 @@ -1461,8 +1461,8 @@ def main(): # 真实手势控制 current_time = time.time() - if (gesture and gesture != "等待手势" and - gesture != "摄像头错误" and gesture != "识别异常" and + if (gesture and gesture != "Waiting" and + gesture != "摄像头错误" and gesture != "Error" and drone_controller.connected and drone_controller.flying): drone_controller.move_by_gesture(gesture, confidence) From 36f5403394d01f3d3811cc6c2e3e994da5b6e6a7 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 21:57:39 +0800 Subject: [PATCH 07/11] =?UTF-8?q?=E6=96=B0=E5=A2=9E=EF=BC=9A=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E5=8F=8D=E9=A6=88=E7=B3=BB=E7=BB=9F=EF=BC=8C=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=A4=9A=E5=B9=B3=E5=8F=B0=E9=9F=B3=E9=A2=91=E6=92=AD?= =?UTF-8?q?=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config.json | 108 ++- .../gesture_config_v2.json | 20 +- .../gesture_config_v3.json | 83 ++- .../main.py | 650 +++++++++++++++++- 4 files changed, 800 insertions(+), 61 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index d41f020e20..f07c4b5817 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -1,11 +1,95 @@ { + "system": { + "version": "1.0", + "last_updated": "2025-12-19 20:51:44", + "debug_mode": false + }, "camera": { + "device_id": 0, + "resolution": { + "width": 640, + "height": 480 + }, + "fps": 30, + "flip_horizontal": true, + "use_virtual_camera": false, "index": 0, "width": 640, - "height": 480, - "fps": 30 + "height": 480 + }, + "gesture_detection": { + "skin_hsv_ranges": { + "h_low": 0, + "h_high": 25, + "s_low": 30, + "s_high": 255, + "v_low": 60, + "v_high": 255 + }, + "min_hand_area": 2000, + "max_hand_area": 30000, + "confidence_threshold": 0.5, + "smoothing_window": 5, + "detection_interval": 1 + }, + "gesture_mapping": { + "0": "stop", + "1": "forward", + "2": "backward", + "3": "up", + "4": "down", + "5": "left", + "6": "right", + "7": "hover" + }, + "drone_control": { + "base_speed": 2.0, + "altitude": -10.0, + "control_delay": 0.3, + "takeoff_height": 5.0, + "emergency_stop_enabled": true + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_gesture_info": true, + "show_control_help": true, + "show_detection_overlay": true, + "theme": "dark", + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": false, + "show_hand_direction": false, + "show_debug_info": false, + "show_direction": true, + "show_region_grid": true + }, + "performance": { + "target_fps": 30, + "enable_multithreading": false, + "enable_hardware_acceleration": true, + "resize_factor": 1.0, + "enable_multiprocessing": false }, "gesture": { + "skin_lower_h": 0, + "skin_upper_h": 25, + "skin_lower_s": 30, + "skin_upper_s": 255, + "skin_lower_v": 60, + "skin_upper_v": 255, + "min_hand_area": 3000, + "max_hand_area": 30000, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.6, + "detection_interval": 2, + "hand_ratio_threshold": 1.5, + "contour_simplify_epsilon": 0.02, + "defect_distance_threshold": 20, + "palm_circle_radius_ratio": 0.3, "skin_lower": [ 0, 30, @@ -16,11 +100,6 @@ 255, 255 ], - "min_hand_area": 3000, - "history_size": 10, - "smooth_frames": 5, - "min_confidence": 0.6, - "detection_interval": 2, "hand_region_threshold": 0.2, "face_exclusion_area": 8000, "palm_aspect_ratio": 1.3, @@ -31,20 +110,5 @@ "duration": 0.3, "altitude": -10.0, "control_interval": 0.3 - }, - "display": { - "show_fps": true, - "show_confidence": true, - "show_help": true, - "show_contours": true, - "show_bbox": true, - "show_fingertips": true, - "show_direction": true, - "show_region_grid": true - }, - "performance": { - "target_fps": 30, - "resize_factor": 1.0, - "enable_multiprocessing": false } } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json index 57205fa642..fca5800e11 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json @@ -7,11 +7,11 @@ }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 178, + "skin_upper_h": 175, "skin_lower_s": 0, - "skin_upper_s": 255, + "skin_upper_s": 57, "skin_lower_v": 0, - "skin_upper_v": 77, + "skin_upper_v": 53, "min_hand_area": 2000, "max_hand_area": 30000, "history_size": 10, @@ -21,7 +21,15 @@ "hand_ratio_threshold": 1.5, "contour_simplify_epsilon": 0.02, "defect_distance_threshold": 20, - "palm_circle_radius_ratio": 0.3 + "palm_circle_radius_ratio": 0.3, + "validation_buffer_size": 5, + "validation_threshold": 0.8, + "min_hold_frames": 8, + "false_trigger_cooldown": 0.5, + "min_gesture_duration": 0.3, + "stability_threshold": 3, + "position_change_threshold": 0.15, + "area_change_threshold": 0.3 }, "drone": { "velocity": 2.5, @@ -38,7 +46,9 @@ "show_fingertips": true, "show_palm_center": true, "show_hand_direction": true, - "show_debug_info": false + "show_debug_info": false, + "show_validation_info": true, + "show_stability_indicator": true }, "performance": { "target_fps": 30, diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json index 7491f9cd30..0f240d3668 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json @@ -3,31 +3,56 @@ "index": 0, "width": 640, "height": 480, - "fps": 30 + "fps": 30, + "flip_horizontal": true, + "auto_exposure": true }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 25, - "skin_lower_s": 30, - "skin_upper_s": 255, - "skin_lower_v": 60, - "skin_upper_v": 255, + "skin_upper_h": 176, + "skin_lower_s": 0, + "skin_upper_s": 72, + "skin_lower_v": 1, + "skin_upper_v": 96, "min_hand_area": 2000, "max_hand_area": 30000, "history_size": 15, "smooth_frames": 10, "min_confidence": 0.5, "detection_interval": 1, - "gesture_lock_frames": 8, + "hand_ratio_threshold": 1.5, "contour_simplify_epsilon": 0.02, "defect_distance_threshold": 20, - "palm_circle_radius_ratio": 0.3 + "palm_circle_radius_ratio": 0.3, + "validation_buffer_size": 8, + "validation_threshold": 0.75, + "min_hold_frames": 10, + "false_trigger_cooldown": 0.8, + "min_gesture_duration": 0.5, + "stability_threshold": 5, + "position_change_threshold": 0.1, + "area_change_threshold": 0.25, + "use_background_subtraction": true, + "bg_history": 50, + "bg_var_threshold": 16, + "morph_close_iterations": 2, + "morph_open_iterations": 2, + "use_roi": true, + "roi_margin": 50, + "finger_min_length_ratio": 0.15, + "finger_max_length_ratio": 0.8, + "enable_multi_hand": false, + "hand_tracking_smoothness": 0.7, + "gesture_lock_frames": 8 }, "drone": { "velocity": 2.5, "duration": 0.3, "altitude": -10.0, - "control_interval": 0.3 + "control_interval": 0.3, + "angular_velocity": 45.0, + "smooth_movement": true, + "emergency_landing_timeout": 10.0 }, "display": { "show_fps": true, @@ -36,18 +61,48 @@ "show_contours": true, "show_bbox": true, "show_fingertips": true, - "show_palm_center": false, - "show_hand_direction": false, - "show_debug_info": false + "show_palm_center": true, + "show_hand_direction": true, + "show_debug_info": false, + "show_speech_status": true, + "show_validation_info": true, + "show_stability_indicator": true, + "use_english_text": true, + "show_skin_mask": false, + "show_trajectory": false, + "show_gesture_history": false, + "ui_theme": "dark", + "font_size": 16 }, "performance": { "target_fps": 30, "resize_factor": 1.0, - "enable_multiprocessing": false + "enable_multiprocessing": false, + "optimize_for_low_end": false, + "max_frame_skip": 2 }, "calibration": { "auto_calibrate_skin": true, "skin_calibration_frames": 30, - "hand_size_calibration": true + "hand_size_calibration": true, + "calibration_done": true, + "environment_adaptation": true, + "adaptive_threshold": true + }, + "speech": { + "enabled": true, + "volume": 1.0, + "rate": 150, + "announce_gestures": true, + "announce_connections": true, + "announce_flight_events": true, + "min_gesture_confidence": 0.7 + }, + "advanced": { + "use_ml_backup": false, + "enable_gesture_learning": false, + "save_gesture_samples": false, + "log_level": "INFO", + "auto_save_config": true } } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index bfa1b35eef..4e33bea6dd 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 手势识别优化版 -优化手势识别算法,解决位置敏感问题 +手势控制AirSim无人机 - 手势识别优化版(添加语音反馈) +优化手势识别算法,解决位置敏感问题,添加语音反馈功能 作者: xiaoshiyuan888 """ @@ -11,6 +11,8 @@ import traceback import json import math +import threading +import tempfile from datetime import datetime from PIL import Image, ImageDraw, ImageFont import cv2 @@ -19,6 +21,7 @@ print("=" * 60) print("Gesture Controlled Drone - Gesture Recognition Optimized") +print("现在添加了语音反馈功能!") print("=" * 60) # ========== 修复导入路径 ========== @@ -71,11 +74,71 @@ def safe_import(): if choice != 'y': sys.exit(1) + # 尝试导入语音合成库 + speech_module = None + try: + # 尝试导入pyttsx3(离线TTS) + import pyttsx3 + speech_module = pyttsx3 + modules_status['Speech'] = True + print("[Speech] ✓ pyttsx3语音库就绪 (离线)") + except ImportError: + print("\n" + "!" * 60) + print("⚠ pyttsx3语音库未找到!") + print("!" * 60) + print("安装语音库 (使用清华大学镜像源):") + print("1. 安装离线语音库: pip install pyttsx3 -i https://pypi.tuna.tsinghua.edu.cn/simple") + print("2. 或者安装在线语音库: pip install gtts pygame -i https://pypi.tuna.tsinghua.edu.cn/simple") + print("!" * 60) + + # 尝试其他语音库 + try: + # 尝试使用gTTS(需要网络) + from gtts import gTTS + speech_module = {'gTTS': gTTS, 'type': 'gtts'} + modules_status['Speech'] = True + print("[Speech] ✓ gTTS语音库就绪 (需要网络连接)") + + # 尝试导入音频播放库 + try: + import pygame + pygame.mixer.init() + speech_module['pygame'] = pygame + print("[Speech] ✓ pygame音频播放库就绪") + except ImportError: + # 尝试其他播放方式 + try: + import pydub + from pydub import AudioSegment + from pydub.playback import play + speech_module['pydub'] = pydub + speech_module['AudioSegment'] = AudioSegment + speech_module['play'] = play + print("[Speech] ✓ pydub音频播放库就绪") + except ImportError: + # 最后尝试使用系统命令 + if os.name == 'nt': # Windows + speech_module['play_method'] = 'windows' + print("[Speech] ✓ 使用Windows系统命令播放音频") + elif os.name == 'posix': # Linux/Mac + speech_module['play_method'] = 'posix' + print("[Speech] ✓ 使用系统命令播放音频") + else: + print("[Speech] ✗ 所有音频播放库导入失败,语音功能将不可用") + speech_module = None + modules_status['Speech'] = False + + except ImportError: + print("[Speech] ✗ 所有语音库导入失败,语音功能将不可用") + speech_module = None + modules_status['Speech'] = False + return { 'cv2': cv2, 'np': np, 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, - 'airsim': airsim_module + 'airsim': airsim_module, + 'speech': speech_module }, modules_status @@ -95,12 +158,324 @@ def safe_import(): Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] +# ========== 语音反馈管理器 ========== +class SpeechFeedbackManager: + """语音反馈管理器""" + + def __init__(self, speech_lib): + self.speech_lib = speech_lib + self.enabled = True + self.volume = 1.0 + self.rate = 150 + self.voice_id = None + self.last_speech_time = {} + self.min_interval = 2.0 # 相同语音的最小间隔(秒) + + # 语音队列,避免语音重叠 + self.speech_queue = [] + self.is_speaking = False + self.queue_thread = None + + # 音频播放方法 + self.audio_method = None + + # 语音消息映射 + self.messages = { + # 连接相关 + 'connecting': "正在连接无人机,请稍候", + 'connected': "无人机连接成功", + 'connection_failed': "无人机连接失败,进入模拟模式", + + # 飞行相关 + 'taking_off': "无人机正在起飞", + 'takeoff_success': "起飞成功", + 'takeoff_failed': "起飞失败", + 'landing': "无人机正在降落", + 'land_success': "降落成功", + 'emergency_stop': "紧急停止,无人机已降落", + + # 手势相关 + 'gesture_detected': "手势识别就绪", + 'gesture_stop': "停止", + 'gesture_up': "向上", + 'gesture_down': "向下", + 'gesture_left': "向左", + 'gesture_right': "向右", + 'gesture_forward': "向前", + 'gesture_waiting': "等待手势", + 'gesture_error': "手势识别错误", + + # 系统相关 + 'program_start': "手势控制无人机系统已启动", + 'program_exit': "程序退出,感谢使用", + 'camera_error': "摄像头错误,请检查连接", + 'low_confidence': "置信度过低,请调整手势", + + # 模式相关 + 'simulation_mode': "进入模拟模式", + 'debug_mode_on': "调试模式已开启", + 'debug_mode_off': "调试模式已关闭", + 'display_mode_changed': "显示模式已切换", + 'help_toggled': "帮助信息已切换", + } + + # 初始化语音引擎 + self.init_speech_engine() + + def init_speech_engine(self): + """初始化语音引擎""" + if self.speech_lib is None: + print("⚠ 语音库未找到,语音功能禁用") + self.enabled = False + return + + try: + if hasattr(self.speech_lib, 'init'): # pyttsx3 + self.engine = self.speech_lib.init() + self.audio_method = 'pyttsx3' + + # 设置语音参数 + voices = self.engine.getProperty('voices') + + # 尝试寻找中文语音 + for voice in voices: + # 检查语音名称是否包含中文相关标识 + if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower(): + self.engine.setProperty('voice', voice.id) + self.voice_id = voice.id + print(f"[Speech] 使用中文语音: {voice.name}") + break + + # 如果没找到中文语音,使用第一个可用语音 + if self.voice_id is None and len(voices) > 0: + self.engine.setProperty('voice', voices[0].id) + print(f"[Speech] 使用默认语音: {voices[0].name}") + + # 设置语速和音量 + self.engine.setProperty('rate', self.rate) + self.engine.setProperty('volume', self.volume) + + print("✅ 语音引擎初始化成功 (pyttsx3)") + + elif isinstance(self.speech_lib, dict) and self.speech_lib.get('type') == 'gtts': + print("✅ 语音引擎初始化成功 (gTTS,需要网络连接)") + self.audio_method = 'gtts' + + # 确定播放方法 + if 'pygame' in self.speech_lib: + self.audio_method = 'gtts_pygame' + print("✅ 使用pygame播放音频") + elif 'pydub' in self.speech_lib: + self.audio_method = 'gtts_pydub' + print("✅ 使用pydub播放音频") + elif 'play_method' in self.speech_lib: + self.audio_method = f"gtts_{self.speech_lib['play_method']}" + print(f"✅ 使用系统命令播放音频") + else: + self.audio_method = 'gtts_system' + print("✅ 使用默认系统播放器") + + else: + print("⚠ 未知语音库类型,语音功能可能不正常") + self.enabled = False + + except Exception as e: + print(f"⚠ 语音引擎初始化失败: {e}") + self.enabled = False + + def play_audio_file(self, audio_file): + """播放音频文件(根据可用库选择方法)""" + try: + if self.audio_method == 'gtts_pygame' and 'pygame' in self.speech_lib: + pygame = self.speech_lib['pygame'] + pygame.mixer.music.load(audio_file) + pygame.mixer.music.play() + + # 等待播放完成 + while pygame.mixer.music.get_busy(): + pygame.time.Clock().tick(10) + + elif self.audio_method == 'gtts_pydub' and 'pydub' in self.speech_lib: + AudioSegment = self.speech_lib['AudioSegment'] + play = self.speech_lib['play'] + + audio = AudioSegment.from_mp3(audio_file) + play(audio) + + elif self.audio_method == 'gtts_windows': + # Windows系统命令 + os.startfile(audio_file) + # 等待播放完成(简单等待) + time.sleep(2) + + elif self.audio_method == 'gtts_posix': + # Linux/Mac系统命令 + import subprocess + if sys.platform == 'darwin': # macOS + subprocess.call(['afplay', audio_file]) + else: # Linux + subprocess.call(['xdg-open', audio_file]) + + else: + # 通用方法:使用系统默认播放器 + import subprocess + if sys.platform == 'win32': + os.startfile(audio_file) + elif sys.platform == 'darwin': + subprocess.call(['open', audio_file]) + else: + subprocess.call(['xdg-open', audio_file]) + + return True + + except Exception as e: + print(f"⚠ 音频播放失败: {e}") + return False + + def speak(self, message_key, force=False): + """播放语音""" + if not self.enabled: + return + + # 检查是否在最小间隔内 + current_time = time.time() + if not force and message_key in self.last_speech_time: + if current_time - self.last_speech_time[message_key] < self.min_interval: + return + + # 获取消息文本 + if message_key in self.messages: + text = self.messages[message_key] + else: + text = message_key # 直接使用传入的文本 + + # 添加到语音队列 + self.speech_queue.append(text) + + # 如果没有在播放,启动播放线程 + if not self.is_speaking and self.queue_thread is None: + self.queue_thread = threading.Thread(target=self._process_speech_queue) + self.queue_thread.daemon = True + self.queue_thread.start() + + # 更新时间戳 + self.last_speech_time[message_key] = current_time + + def _process_speech_queue(self): + """处理语音队列""" + while self.speech_queue and self.enabled: + self.is_speaking = True + + text = self.speech_queue.pop(0) + + try: + if self.audio_method == 'pyttsx3': + # 清除之前的语音 + self.engine.stop() + # 播报新语音 + self.engine.say(text) + self.engine.runAndWait() + + elif self.audio_method.startswith('gtts'): + # gTTS需要网络连接 + tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') + + # 保存临时文件 + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: + temp_file = f.name + tts.save(temp_file) + + # 播放音频 + self.play_audio_file(temp_file) + + # 删除临时文件 + try: + os.unlink(temp_file) + except: + pass # 忽略删除错误 + + except Exception as e: + print(f"⚠ 语音播放失败: {e}") + + time.sleep(0.1) # 避免过快 + + self.is_speaking = False + self.queue_thread = None + + def speak_direct(self, text): + """直接播放文本(不通过消息映射)""" + if not self.enabled: + return + + # 在新线程中播放 + thread = threading.Thread(target=self._speak_thread, args=(text,)) + thread.daemon = True + thread.start() + + def _speak_thread(self, text): + """语音播放线程""" + try: + if self.audio_method == 'pyttsx3': + self.engine.say(text) + self.engine.runAndWait() + + elif self.audio_method.startswith('gtts'): + tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') + + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: + temp_file = f.name + tts.save(temp_file) + + self.play_audio_file(temp_file) + + try: + os.unlink(temp_file) + except: + pass + + except Exception as e: + print(f"⚠ 直接语音播放失败: {e}") + + def stop(self): + """停止所有语音""" + if hasattr(self, 'engine'): + self.engine.stop() + + self.speech_queue.clear() + self.is_speaking = False + + def set_enabled(self, enabled): + """启用/禁用语音""" + self.enabled = enabled + if not enabled: + self.stop() + + def toggle_enabled(self): + """切换语音启用状态""" + self.enabled = not self.enabled + status = "启用" if self.enabled else "禁用" + self.speak_direct(f"语音反馈已{status}") + return self.enabled + + def get_status(self): + """获取语音状态""" + return { + 'enabled': self.enabled, + 'engine': 'pyttsx3' if self.audio_method == 'pyttsx3' else + 'gTTS' if self.audio_method.startswith('gtts') else + 'None', + 'queue_size': len(self.speech_queue), + 'is_speaking': self.is_speaking, + 'audio_method': self.audio_method + } + + # ========== 配置管理器 ========== class ConfigManager: """配置管理器""" def __init__(self): - self.config_file = os.path.join(current_dir, 'gesture_config_v2.json') + self.config_file = os.path.join(current_dir, 'gesture_config_v3.json') # 更新版本号 self.default_config = { 'camera': { 'index': 0, @@ -141,7 +516,8 @@ def __init__(self): 'show_fingertips': True, 'show_palm_center': True, 'show_hand_direction': True, - 'show_debug_info': False + 'show_debug_info': False, + 'show_speech_status': True, # 新增:显示语音状态 }, 'performance': { 'target_fps': 30, @@ -152,6 +528,15 @@ def __init__(self): 'auto_calibrate_skin': True, 'skin_calibration_frames': 30, 'hand_size_calibration': True + }, + 'speech': { # 新增:语音配置 + 'enabled': True, + 'volume': 1.0, + 'rate': 150, + 'announce_gestures': True, # 是否播报手势 + 'announce_connections': True, # 是否播报连接状态 + 'announce_flight_events': True, # 是否播报飞行事件 + 'min_gesture_confidence': 0.7, # 播报手势的最小置信度 } } self.config = self.load_config() @@ -271,8 +656,9 @@ def calibrate_hand_size(self, hand_area): class ImprovedGestureRecognizer: """改进的手势识别器 - 纯OpenCV实现""" - def __init__(self): + def __init__(self, speech_manager=None): self.config = config.get('gesture') + self.speech_manager = speech_manager # 手势历史和平滑 self.history_size = self.config['history_size'] @@ -281,6 +667,11 @@ def __init__(self): self.current_gesture = "Waiting" self.current_confidence = 0.0 + # 记录上次播报的手势,避免重复播报 + self.last_announced_gesture = None + self.last_announced_time = 0 + self.gesture_announce_interval = 3.0 # 手势播报最小间隔 + # 手部跟踪和状态 self.last_hand_position = None self.hand_tracking = False @@ -305,6 +696,18 @@ def __init__(self): "Hover": (255, 255, 255) # 白色 } + # 手势到语音的映射 + self.gesture_speech_map = { + "Stop": "gesture_stop", + "Forward": "gesture_forward", + "Up": "gesture_up", + "Down": "gesture_down", + "Left": "gesture_left", + "Right": "gesture_right", + "Waiting": "gesture_waiting", + "Error": "gesture_error", + } + # 背景减除器 self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( history=100, varThreshold=25, detectShadows=True @@ -315,6 +718,10 @@ def __init__(self): print("✓ 改进的手势识别器已初始化 (纯OpenCV实现)") + # 语音提示初始化完成 + if self.speech_manager: + self.speech_manager.speak('gesture_detected') + def get_skin_mask(self, frame): """获取肤色掩码""" # 从配置获取肤色范围 @@ -860,6 +1267,22 @@ def recognize(self, frame): # 平滑手势 final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) + # 手势语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gestures') and + final_confidence >= config.get('speech', 'min_gesture_confidence')): + + current_time = time.time() + if (final_gesture != self.last_announced_gesture or + current_time - self.last_announced_time > self.gesture_announce_interval): + + if final_gesture in self.gesture_speech_map: + self.speech_manager.speak(self.gesture_speech_map[final_gesture]) + + self.last_announced_gesture = final_gesture + self.last_announced_time = current_time + # 可视化结果 if hand_data is not None: processed_frame = self.visualize_detection( @@ -891,16 +1314,26 @@ def set_simulated_gesture(self, gesture): self.current_gesture = gesture self.current_confidence = 0.9 + # 模拟手势也触发语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gestures') and + gesture in self.gesture_speech_map): + self.speech_manager.speak(self.gesture_speech_map[gesture]) + self.last_announced_gesture = gesture + self.last_announced_time = time.time() + # ========== 简单的无人机控制器 ========== class SimpleDroneController: """简单的无人机控制器""" - def __init__(self, airsim_module): + def __init__(self, airsim_module, speech_manager=None): self.airsim = airsim_module self.client = None self.connected = False self.flying = False + self.speech_manager = speech_manager # 控制参数 self.velocity = config.get('drone', 'velocity') @@ -912,6 +1345,11 @@ def __init__(self, airsim_module): self.last_control_time = 0 self.last_gesture = None + # 上次语音提示状态 + self.last_connection_announced = False + self.last_takeoff_announced = False + self.last_land_announced = False + print("✓ 简单的无人机控制器已初始化") def connect(self): @@ -919,8 +1357,21 @@ def connect(self): if self.connected: return True + # 语音提示:正在连接 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('connecting') + if self.airsim is None: print("⚠ AirSim不可用,使用模拟模式") + + # 语音提示:模拟模式 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('simulation_mode') + self.connected = True return True @@ -931,6 +1382,12 @@ def connect(self): self.client.confirmConnection() print("✅ 已连接AirSim!") + # 语音提示:连接成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('connected') + self.client.enableApiControl(True) print("✅ API控制已启用") @@ -942,11 +1399,25 @@ def connect(self): except Exception as e: print(f"❌ 连接失败: {e}") + + # 语音提示:连接失败 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('connection_failed') + print("\n使用模拟模式继续? (y/n)") choice = input().strip().lower() if choice == 'y': self.connected = True print("✅ 使用模拟模式") + + # 语音提示:模拟模式 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('simulation_mode') + return True return False @@ -956,10 +1427,26 @@ def takeoff(self): if not self.connected: return False + # 语音提示:正在起飞 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events') and + not self.last_takeoff_announced): + self.speech_manager.speak('taking_off') + self.last_takeoff_announced = True + self.last_land_announced = False + try: if self.airsim is None or self.client is None: print("✅ 模拟起飞") self.flying = True + + # 语音提示:起飞成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('takeoff_success') + return True print("起飞中...") @@ -971,9 +1458,23 @@ def takeoff(self): self.flying = True print("✅ 无人机成功起飞") + + # 语音提示:起飞成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('takeoff_success') + return True except Exception as e: print(f"❌ 起飞失败: {e}") + + # 语音提示:起飞失败 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('takeoff_failed') + return False def land(self): @@ -981,16 +1482,39 @@ def land(self): if not self.connected: return False + # 语音提示:正在降落 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events') and + not self.last_land_announced): + self.speech_manager.speak('landing') + self.last_land_announced = True + self.last_takeoff_announced = False + try: if self.airsim is None or self.client is None: print("✅ 模拟降落") self.flying = False + + # 语音提示:降落成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('land_success') + return True print("降落中...") self.client.landAsync().join() self.flying = False print("✅ 无人机已降落") + + # 语音提示:降落成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('land_success') + return True except Exception as e: print(f"降落失败: {e}") @@ -1009,6 +1533,12 @@ def move_by_gesture(self, gesture, confidence): # 检查置信度阈值 min_confidence = config.get('gesture', 'min_confidence') if confidence < min_confidence: + # 低置信度语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gestures') and + confidence < min_confidence * 0.8): # 如果置信度特别低 + self.speech_manager.speak('low_confidence') return False try: @@ -1057,6 +1587,13 @@ def emergency_stop(self): try: if self.flying and self.client is not None: print("紧急降落...") + + # 语音提示:紧急停止 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('emergency_stop') + self.land() if self.client is not None: self.client.armDisarm(False) @@ -1073,8 +1610,9 @@ def emergency_stop(self): class ChineseUIRenderer: """中文UI渲染器""" - def __init__(self): + def __init__(self, speech_manager=None): self.fonts = {} + self.speech_manager = speech_manager self.load_fonts() # 颜色定义 @@ -1086,7 +1624,9 @@ def __init__(self): 'landed': (255, 165, 0), # 橙色 'warning': (0, 165, 255), # 浅蓝色 'info': (255, 255, 255), # 白色 - 'help': (255, 200, 100) # 浅橙色 + 'help': (255, 200, 100), # 浅橙色 + 'speech_enabled': (0, 255, 0), # 绿色 + 'speech_disabled': (255, 0, 0), # 红色 } print("✓ 中文UI渲染器已初始化") @@ -1150,7 +1690,7 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 优化版" + title = "手势控制无人机系统 - 优化版 (带语音反馈)" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -1177,6 +1717,13 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) + # 语音状态 + if config.get('display', 'show_speech_status') and self.speech_manager: + speech_status = self.speech_manager.get_status() + speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] + speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" + frame = self.draw_text(frame, speech_text, (w // 2, 65), size=16, color=speech_color) + # 性能信息 if config.get('display', 'show_fps'): perf_text = f"帧率: {fps:.1f}" @@ -1199,16 +1746,17 @@ def draw_help_bar(self, frame): h, w = frame.shape[:2] # 绘制底部帮助栏 - cv2.rectangle(frame, (0, h - 60), (w, h), (0, 0, 0), -1) + cv2.rectangle(frame, (0, h - 80), (w, h), (0, 0, 0), -1) # 帮助文本 help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", - "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息" + "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", + "V:切换语音反馈 M:测试语音" ] for i, line in enumerate(help_lines): - y_pos = h - 45 + i * 20 + y_pos = h - 65 + i * 20 frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame @@ -1270,12 +1818,20 @@ def get_stats(self): # ========== 主程序 ========== def main(): """主函数""" + # 初始化语音管理器 + print("初始化语音反馈系统...") + speech_manager = SpeechFeedbackManager(libs['speech']) + + # 程序启动语音提示 + if speech_manager.enabled: + speech_manager.speak('program_start', force=True) + # 初始化组件 print("初始化组件...") - gesture_recognizer = ImprovedGestureRecognizer() - drone_controller = SimpleDroneController(libs['airsim']) - ui_renderer = ChineseUIRenderer() + gesture_recognizer = ImprovedGestureRecognizer(speech_manager) + drone_controller = SimpleDroneController(libs['airsim'], speech_manager) + ui_renderer = ChineseUIRenderer(speech_manager) performance_monitor = PerformanceMonitor() # 初始化摄像头 @@ -1297,9 +1853,19 @@ def main(): print(f" 帧率: {actual_fps}") else: print("❌ 摄像头不可用,使用模拟模式") + + # 摄像头错误语音提示 + if speech_manager.enabled: + speech_manager.speak('camera_error') + cap = None except Exception as e: print(f"⚠ 摄像头初始化失败: {e}") + + # 摄像头错误语音提示 + if speech_manager.enabled: + speech_manager.speak('camera_error') + cap = None # 显示欢迎信息 @@ -1309,6 +1875,7 @@ def main(): print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 改进的OpenCV算法") + print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) @@ -1326,7 +1893,9 @@ def main(): print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") print("5. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") - print("6. 按 [ESC] 安全退出") + print("6. 语音控制:") + print(" [V]切换语音反馈 [M]测试语音") + print("7. 按 [ESC] 安全退出") print("=" * 60) print("程序启动成功!") print("-" * 60) @@ -1405,7 +1974,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Optimized', frame) + cv2.imshow('Gesture Controlled Drone - Optimized with Speech', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -1432,23 +2001,58 @@ def main(): config.set('display', 'show_help', value=not current) print(f"帮助显示: {'开启' if not current else '关闭'}") + # 语音提示 + if speech_manager.enabled: + speech_manager.speak('help_toggled') + elif key == ord('r') or key == ord('R'): # 重置手势识别 print("重置手势识别...") - gesture_recognizer = ImprovedGestureRecognizer() + gesture_recognizer = ImprovedGestureRecognizer(speech_manager) print("✓ 手势识别已重置") + # 语音提示 + if speech_manager.enabled: + speech_manager.speak_direct("手势识别已重置") + elif key == ord('t') or key == ord('T'): # 切换显示模式 current_display_mode = (current_display_mode + 1) % len(display_modes) mode_name = display_modes[current_display_mode] print(f"显示模式: {mode_name}") + # 语音提示 + if speech_manager.enabled: + speech_manager.speak('display_mode_changed') + elif key == ord('d') or key == ord('D'): # 切换调试信息 current = config.get('display', 'show_debug_info') config.set('display', 'show_debug_info', value=not current) - print(f"调试信息: {'开启' if not current else '关闭'}") + status = '开启' if not current else '关闭' + print(f"调试信息: {status}") + + # 语音提示 + if speech_manager.enabled: + if not current: + speech_manager.speak('debug_mode_on') + else: + speech_manager.speak('debug_mode_off') + + elif key == ord('v') or key == ord('V'): + # 切换语音反馈 + new_status = speech_manager.toggle_enabled() + status = '启用' if new_status else '禁用' + print(f"语音反馈: {status}") + config.set('speech', 'enabled', value=new_status) + + elif key == ord('m') or key == ord('M'): + # 测试语音 + if speech_manager.enabled: + print("测试语音...") + speech_manager.speak_direct("语音反馈测试,这是一条测试消息") + else: + print("语音反馈已禁用,按V键启用") elif key in key_to_gesture: # 键盘控制 @@ -1477,6 +2081,12 @@ def main(): if cap: cap.release() cv2.destroyAllWindows() + + # 程序退出语音提示 + if speech_manager.enabled: + speech_manager.speak('program_exit', force=True) + time.sleep(1) # 确保语音播报完成 + drone_controller.emergency_stop() config.save_config() From fb866db00dc95facc02a5c036b92a0527e8c3592 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sat, 20 Dec 2025 11:09:56 +0800 Subject: [PATCH 08/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E5=B9=B3=E6=BB=91=E5=A4=84=E7=90=86=EF=BC=8C=E6=89=A9=E5=B1=95?= =?UTF-8?q?=E8=AF=AD=E9=9F=B3=E5=8F=8D=E9=A6=88=E7=B3=BB=E7=BB=9F=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=89=8B=E5=8A=BF=E7=8A=B6=E6=80=81=E8=BF=BD?= =?UTF-8?q?=E8=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 8 +- .../gesture_config.json | 168 +++-- .../main.py | 660 ++++++++++++------ 3 files changed, 555 insertions(+), 281 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index 0ad6fac5ef..b1c539fd44 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -10,4 +10,10 @@ __pycache__/ .idea/ # 忽略日志文件夹(临时日志文件,无需提交到仓库) -logs/ \ No newline at end of file +logs/ + +# 忽略手势识别配置文件(保留本地,不提交到仓库) +gesture_config.json +gesture_config (1).json +gesture_config_v2.json +gesture_config_v3.json \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index f07c4b5817..c60f9cec3d 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -1,85 +1,24 @@ { - "system": { - "version": "1.0", - "last_updated": "2025-12-19 20:51:44", - "debug_mode": false - }, "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30, "device_id": 0, "resolution": { "width": 640, "height": 480 }, - "fps": 30, "flip_horizontal": true, - "use_virtual_camera": false, - "index": 0, - "width": 640, - "height": 480 - }, - "gesture_detection": { - "skin_hsv_ranges": { - "h_low": 0, - "h_high": 25, - "s_low": 30, - "s_high": 255, - "v_low": 60, - "v_high": 255 - }, - "min_hand_area": 2000, - "max_hand_area": 30000, - "confidence_threshold": 0.5, - "smoothing_window": 5, - "detection_interval": 1 - }, - "gesture_mapping": { - "0": "stop", - "1": "forward", - "2": "backward", - "3": "up", - "4": "down", - "5": "left", - "6": "right", - "7": "hover" - }, - "drone_control": { - "base_speed": 2.0, - "altitude": -10.0, - "control_delay": 0.3, - "takeoff_height": 5.0, - "emergency_stop_enabled": true - }, - "display": { - "show_fps": true, - "show_confidence": true, - "show_gesture_info": true, - "show_control_help": true, - "show_detection_overlay": true, - "theme": "dark", - "show_help": true, - "show_contours": true, - "show_bbox": true, - "show_fingertips": true, - "show_palm_center": false, - "show_hand_direction": false, - "show_debug_info": false, - "show_direction": true, - "show_region_grid": true - }, - "performance": { - "target_fps": 30, - "enable_multithreading": false, - "enable_hardware_acceleration": true, - "resize_factor": 1.0, - "enable_multiprocessing": false + "use_virtual_camera": false }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 25, - "skin_lower_s": 30, - "skin_upper_s": 255, - "skin_lower_v": 60, - "skin_upper_v": 255, + "skin_upper_h": 178, + "skin_lower_s": 0, + "skin_upper_s": 71, + "skin_lower_v": 32, + "skin_upper_v": 85, "min_hand_area": 3000, "max_hand_area": 30000, "history_size": 10, @@ -90,6 +29,10 @@ "contour_simplify_epsilon": 0.02, "defect_distance_threshold": 20, "palm_circle_radius_ratio": 0.3, + "gesture_stability_threshold": 5, + "transition_threshold": 0.3, + "position_stability_weight": 0.4, + "gesture_cooldown": 0.5, "skin_lower": [ 0, 30, @@ -110,5 +53,88 @@ "duration": 0.3, "altitude": -10.0, "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": true, + "show_hand_direction": true, + "show_debug_info": false, + "show_speech_status": true, + "show_gesture_history": true, + "show_stability_indicator": true, + "show_gesture_info": true, + "show_control_help": true, + "show_detection_overlay": true, + "theme": "dark", + "show_direction": true, + "show_region_grid": true + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false, + "enable_multithreading": false, + "enable_hardware_acceleration": true + }, + "calibration": { + "auto_calibrate_skin": true, + "skin_calibration_frames": 30, + "hand_size_calibration": true + }, + "speech": { + "enabled": true, + "volume": 1.0, + "rate": 150, + "announce_gestures": true, + "announce_connections": true, + "announce_flight_events": true, + "announce_gesture_changes": true, + "announce_hand_status": true, + "announce_performance": true, + "min_gesture_confidence": 0.7, + "gesture_start_threshold": 3, + "gesture_end_threshold": 10 + }, + "system": { + "version": "1.0", + "last_updated": "2025-12-19 20:51:44", + "debug_mode": false + }, + "gesture_detection": { + "skin_hsv_ranges": { + "h_low": 0, + "h_high": 25, + "s_low": 30, + "s_high": 255, + "v_low": 60, + "v_high": 255 + }, + "min_hand_area": 2000, + "max_hand_area": 30000, + "confidence_threshold": 0.5, + "smoothing_window": 5, + "detection_interval": 1 + }, + "gesture_mapping": { + "0": "stop", + "1": "forward", + "2": "backward", + "3": "up", + "4": "down", + "5": "left", + "6": "right", + "7": "hover" + }, + "drone_control": { + "base_speed": 2.0, + "altitude": -10.0, + "control_delay": 0.3, + "takeoff_height": 5.0, + "emergency_stop_enabled": true } } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 4e33bea6dd..fe271047a5 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 手势识别优化版(添加语音反馈) -优化手势识别算法,解决位置敏感问题,添加语音反馈功能 +手势控制AirSim无人机 - 手势识别优化版(增强语音反馈和手势平滑) +优化手势识别算法,增强语音反馈,改进手势平滑处理 作者: xiaoshiyuan888 """ @@ -20,8 +20,8 @@ from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Gesture Recognition Optimized") -print("现在添加了语音反馈功能!") +print("Gesture Controlled Drone - Enhanced Gesture Recognition") +print("增强语音反馈和手势平滑处理!") print("=" * 60) # ========== 修复导入路径 ========== @@ -158,9 +158,9 @@ def safe_import(): Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -# ========== 语音反馈管理器 ========== -class SpeechFeedbackManager: - """语音反馈管理器""" +# ========== 增强语音反馈管理器 ========== +class EnhancedSpeechFeedbackManager: + """增强的语音反馈管理器""" def __init__(self, speech_lib): self.speech_lib = speech_lib @@ -169,7 +169,7 @@ def __init__(self, speech_lib): self.rate = 150 self.voice_id = None self.last_speech_time = {} - self.min_interval = 2.0 # 相同语音的最小间隔(秒) + self.min_interval = 1.5 # 缩短相同语音的最小间隔(秒) # 语音队列,避免语音重叠 self.speech_queue = [] @@ -179,7 +179,11 @@ def __init__(self, speech_lib): # 音频播放方法 self.audio_method = None - # 语音消息映射 + # 新增:手势状态追踪 + self.last_gesture_state = "none" # 记录上次手势状态 + self.gesture_active_time = 0 # 手势持续活跃时间 + + # 增强的语音消息映射 self.messages = { # 连接相关 'connecting': "正在连接无人机,请稍候", @@ -193,9 +197,12 @@ def __init__(self, speech_lib): 'landing': "无人机正在降落", 'land_success': "降落成功", 'emergency_stop': "紧急停止,无人机已降落", + 'hovering': "无人机悬停中", - # 手势相关 - 'gesture_detected': "手势识别就绪", + # 手势相关 - 增强 + 'gesture_detected': "手势识别就绪,请开始手势", + 'gesture_start': "开始识别手势", + 'gesture_end': "手势识别结束", 'gesture_stop': "停止", 'gesture_up': "向上", 'gesture_down': "向下", @@ -204,12 +211,17 @@ def __init__(self, speech_lib): 'gesture_forward': "向前", 'gesture_waiting': "等待手势", 'gesture_error': "手势识别错误", + 'gesture_stable': "手势稳定", + 'gesture_change': "手势变化", + 'gesture_low_confidence': "手势识别置信度低", + 'gesture_good_confidence': "手势识别置信度高", # 系统相关 'program_start': "手势控制无人机系统已启动", 'program_exit': "程序退出,感谢使用", 'camera_error': "摄像头错误,请检查连接", - 'low_confidence': "置信度过低,请调整手势", + 'camera_ready': "摄像头就绪", + 'system_ready': "系统准备就绪", # 模式相关 'simulation_mode': "进入模拟模式", @@ -217,6 +229,17 @@ def __init__(self, speech_lib): 'debug_mode_off': "调试模式已关闭", 'display_mode_changed': "显示模式已切换", 'help_toggled': "帮助信息已切换", + + # 新增:性能相关 + 'performance_good': "系统运行流畅", + 'performance_warning': "系统性能警告", + + # 新增:手势指导 + 'move_closer': "请将手靠近摄像头", + 'move_away': "请将手移远一些", + 'good_position': "手部位置良好", + 'hand_detected': "手部已检测到", + 'hand_lost': "手部丢失,请重新放置", } # 初始化语音引擎 @@ -240,7 +263,7 @@ def init_speech_engine(self): # 尝试寻找中文语音 for voice in voices: # 检查语音名称是否包含中文相关标识 - if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower(): + if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower() or 'zh_CN' in voice.name.lower(): self.engine.setProperty('voice', voice.id) self.voice_id = voice.id print(f"[Speech] 使用中文语音: {voice.name}") @@ -306,7 +329,7 @@ def play_audio_file(self, audio_file): # Windows系统命令 os.startfile(audio_file) # 等待播放完成(简单等待) - time.sleep(2) + time.sleep(1.5) elif self.audio_method == 'gtts_posix': # Linux/Mac系统命令 @@ -332,7 +355,7 @@ def play_audio_file(self, audio_file): print(f"⚠ 音频播放失败: {e}") return False - def speak(self, message_key, force=False): + def speak(self, message_key, force=False, immediate=False): """播放语音""" if not self.enabled: return @@ -349,14 +372,18 @@ def speak(self, message_key, force=False): else: text = message_key # 直接使用传入的文本 - # 添加到语音队列 - self.speech_queue.append(text) + # 如果是立即播放,直接在新线程中播放 + if immediate: + self.speak_direct(text) + else: + # 添加到语音队列 + self.speech_queue.append(text) - # 如果没有在播放,启动播放线程 - if not self.is_speaking and self.queue_thread is None: - self.queue_thread = threading.Thread(target=self._process_speech_queue) - self.queue_thread.daemon = True - self.queue_thread.start() + # 如果没有在播放,启动播放线程 + if not self.is_speaking and self.queue_thread is None: + self.queue_thread = threading.Thread(target=self._process_speech_queue) + self.queue_thread.daemon = True + self.queue_thread.start() # 更新时间戳 self.last_speech_time[message_key] = current_time @@ -397,7 +424,7 @@ def _process_speech_queue(self): except Exception as e: print(f"⚠ 语音播放失败: {e}") - time.sleep(0.1) # 避免过快 + time.sleep(0.05) # 减少等待时间,避免卡顿 self.is_speaking = False self.queue_thread = None @@ -475,7 +502,7 @@ class ConfigManager: """配置管理器""" def __init__(self): - self.config_file = os.path.join(current_dir, 'gesture_config_v3.json') # 更新版本号 + self.config_file = os.path.join(current_dir, 'gesture_config.json') self.default_config = { 'camera': { 'index': 0, @@ -492,14 +519,18 @@ def __init__(self): 'skin_upper_v': 255, 'min_hand_area': 2000, 'max_hand_area': 30000, - 'history_size': 10, - 'smooth_frames': 5, - 'min_confidence': 0.5, + 'history_size': 15, # 增加历史记录大小 + 'smooth_frames': 7, # 增加平滑帧数 + 'min_confidence': 0.6, # 提高最小置信度阈值 'detection_interval': 1, - 'hand_ratio_threshold': 1.5, # 手部长宽比阈值 + 'hand_ratio_threshold': 1.5, 'contour_simplify_epsilon': 0.02, - 'defect_distance_threshold': 20, # 凸缺陷距离阈值 - 'palm_circle_radius_ratio': 0.3, # 手掌半径与边界框的比例 + 'defect_distance_threshold': 20, + 'palm_circle_radius_ratio': 0.3, + 'gesture_stability_threshold': 5, # 新增:手势稳定性阈值 + 'transition_threshold': 0.3, # 新增:手势转换阈值 + 'position_stability_weight': 0.4, # 新增:位置稳定性权重 + 'gesture_cooldown': 0.5, # 新增:手势冷却时间 }, 'drone': { 'velocity': 2.5, @@ -517,7 +548,9 @@ def __init__(self): 'show_palm_center': True, 'show_hand_direction': True, 'show_debug_info': False, - 'show_speech_status': True, # 新增:显示语音状态 + 'show_speech_status': True, + 'show_gesture_history': True, # 新增:显示手势历史 + 'show_stability_indicator': True, # 新增:显示稳定性指示器 }, 'performance': { 'target_fps': 30, @@ -529,14 +562,19 @@ def __init__(self): 'skin_calibration_frames': 30, 'hand_size_calibration': True }, - 'speech': { # 新增:语音配置 + 'speech': { 'enabled': True, 'volume': 1.0, 'rate': 150, - 'announce_gestures': True, # 是否播报手势 - 'announce_connections': True, # 是否播报连接状态 - 'announce_flight_events': True, # 是否播报飞行事件 - 'min_gesture_confidence': 0.7, # 播报手势的最小置信度 + 'announce_gestures': True, + 'announce_connections': True, + 'announce_flight_events': True, + 'announce_gesture_changes': True, # 新增:播报手势变化 + 'announce_hand_status': True, # 新增:播报手部状态 + 'announce_performance': True, # 新增:播报性能状态 + 'min_gesture_confidence': 0.7, + 'gesture_start_threshold': 3, # 新增:手势开始识别阈值 + 'gesture_end_threshold': 10, # 新增:手势结束识别阈值 } } self.config = self.load_config() @@ -652,36 +690,48 @@ def calibrate_hand_size(self, hand_area): config = ConfigManager() -# ========== 改进的手势识别器 ========== -class ImprovedGestureRecognizer: - """改进的手势识别器 - 纯OpenCV实现""" +# ========== 改进的手势识别器(增强平滑处理) ========== +class EnhancedGestureRecognizer: + """增强的手势识别器 - 改进平滑处理和稳定性""" def __init__(self, speech_manager=None): self.config = config.get('gesture') self.speech_manager = speech_manager - # 手势历史和平滑 + # 增强的手势历史和平滑 self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) self.confidence_history = deque(maxlen=self.history_size) + self.position_history = deque(maxlen=self.history_size) # 新增:位置历史 self.current_gesture = "Waiting" self.current_confidence = 0.0 - # 记录上次播报的手势,避免重复播报 + # 新增:手势状态追踪 + self.gesture_state = "none" # none, starting, active, ending + self.gesture_stability_counter = 0 + self.last_stable_gesture = "Waiting" + self.gesture_active_frames = 0 + self.last_gesture_change_time = 0 + + # 记录上次播报的手势 self.last_announced_gesture = None self.last_announced_time = 0 - self.gesture_announce_interval = 3.0 # 手势播报最小间隔 + self.last_hand_status_time = 0 + self.gesture_announce_interval = 2.0 # 缩短手势播报间隔 # 手部跟踪和状态 self.last_hand_position = None self.hand_tracking = False self.track_window = None - self.hand_states = deque(maxlen=10) # 手部状态历史 + self.hand_states = deque(maxlen=15) + self.hand_detected_frames = 0 + self.hand_lost_frames = 0 # 性能统计 self.process_times = deque(maxlen=30) self.frame_counter = 0 self.detection_interval = self.config['detection_interval'] + self.last_performance_report = 0 # 手势颜色映射 self.gesture_colors = { @@ -708,6 +758,14 @@ def __init__(self, speech_manager=None): "Error": "gesture_error", } + # 手势状态颜色 + self.state_colors = { + "none": (100, 100, 100), # 灰色 + "starting": (255, 165, 0), # 橙色 + "active": (0, 255, 0), # 绿色 + "ending": (255, 0, 0), # 红色 + } + # 背景减除器 self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( history=100, varThreshold=25, detectShadows=True @@ -716,15 +774,15 @@ def __init__(self, speech_manager=None): # 形态学操作核 self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - print("✓ 改进的手势识别器已初始化 (纯OpenCV实现)") + # 新增:性能监控 + self.avg_process_time = 0 + self.frame_rate = 0 + self.last_fps_check = time.time() - # 语音提示初始化完成 - if self.speech_manager: - self.speech_manager.speak('gesture_detected') + print("✓ 增强的手势识别器已初始化 (改进平滑处理)") def get_skin_mask(self, frame): """获取肤色掩码""" - # 从配置获取肤色范围 h_low = config.get('gesture', 'skin_lower_h') h_high = config.get('gesture', 'skin_upper_h') s_low = config.get('gesture', 'skin_lower_s') @@ -732,60 +790,42 @@ def get_skin_mask(self, frame): v_low = config.get('gesture', 'skin_lower_v') v_high = config.get('gesture', 'skin_upper_v') - # 转换为HSV颜色空间 hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - - # 定义肤色范围 lower_skin = np.array([h_low, s_low, v_low], dtype=np.uint8) upper_skin = np.array([h_high, s_high, v_high], dtype=np.uint8) - - # 创建肤色掩码 skin_mask = cv2.inRange(hsv, lower_skin, upper_skin) return skin_mask, hsv def enhance_skin_detection(self, frame, skin_mask): """增强肤色检测""" - # 应用背景减除 fg_mask = self.bg_subtractor.apply(frame) - - # 结合肤色掩码和前景掩码 combined_mask = cv2.bitwise_and(skin_mask, fg_mask) - - # 形态学操作 combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - - # 高斯模糊 combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) return combined_mask def find_best_hand_contour(self, mask, frame): """找到最佳的手部轮廓""" - # 查找轮廓 contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: return None, 0.0 - # 按面积排序 contours = sorted(contours, key=cv2.contourArea, reverse=True) - best_contour = None best_score = 0.0 - min_area = config.get('gesture', 'min_hand_area') max_area = config.get('gesture', 'max_hand_area') - for contour in contours[:3]: # 只检查前3个最大轮廓 + for contour in contours[:3]: area = cv2.contourArea(contour) - # 面积过滤 if area < min_area or area > max_area: continue - # 计算轮廓评分 score = self.rate_contour(contour, frame.shape) if score > best_score: @@ -797,35 +837,27 @@ def find_best_hand_contour(self, mask, frame): def rate_contour(self, contour, frame_shape): """评估轮廓作为手部的可能性""" score = 0.0 - - # 面积评分 area = cv2.contourArea(contour) min_area = config.get('gesture', 'min_hand_area') max_area = config.get('gesture', 'max_hand_area') if min_area < area < max_area: - # 面积在合理范围内 area_ratio = min(area / max_area, 1.0) score += area_ratio * 0.3 - # 周长评分 perimeter = cv2.arcLength(contour, True) - if perimeter > 100: # 最小周长 + if perimeter > 100: score += 0.2 - # 紧凑度评分 (周长^2 / 面积) if area > 0: compactness = perimeter ** 2 / area - # 手的紧凑度通常在14-20之间 if 12 < compactness < 25: compactness_score = 1.0 - abs(compactness - 18) / 6 score += compactness_score * 0.3 - # 长宽比评分 x, y, w, h = cv2.boundingRect(contour) if h > 0: aspect_ratio = w / h - # 手的长宽比通常在0.5-2.0之间 if 0.4 < aspect_ratio < 2.5: aspect_score = 1.0 - abs(aspect_ratio - 1.0) / 1.5 score += aspect_score * 0.2 @@ -837,54 +869,37 @@ def analyze_hand_features(self, contour, frame_shape): if contour is None: return None, 0.0 - # 计算轮廓面积 area = cv2.contourArea(contour) - - # 计算轮廓中心 M = cv2.moments(contour) if M["m00"] == 0: return None, 0.0 cx = int(M["m10"] / M["m00"]) cy = int(M["m01"] / M["m00"]) - - # 计算边界框 x, y, w, h = cv2.boundingRect(contour) bbox_area = w * h - - # 计算手掌中心(假设是轮廓的中心) palm_center = (cx, cy) - - # 计算手掌半径(假设为边界框宽度的30%) palm_radius = int(w * config.get('gesture', 'palm_circle_radius_ratio')) - - # 分析手指 fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius) - - # 计算手部方向 direction = self.calculate_hand_direction(contour, cx, cy) - - # 计算手部位置(归一化) h_img, w_img = frame_shape[:2] norm_x = cx / w_img norm_y = cy / h_img - - # 计算手势置信度 confidence = self.calculate_confidence(area, len(fingers), len(contour), bbox_area) - # 返回结果 result = { 'contour': contour, 'center': (cx, cy), 'bbox': (x, y, x + w, y + h), - 'fingers': fingers, # 手指列表 - 'fingertips': fingertips, # 指尖点列表 - 'defects': defects, # 凸缺陷列表 + 'fingers': fingers, + 'fingertips': fingertips, + 'defects': defects, 'palm_center': palm_center, 'palm_radius': palm_radius, - 'direction': direction, # 手部方向(角度) + 'direction': direction, 'area': area, 'position': (norm_x, norm_y), + 'bbox_size': (w, h), 'confidence': confidence } @@ -892,19 +907,14 @@ def analyze_hand_features(self, contour, frame_shape): def analyze_fingers(self, contour, palm_center, palm_radius): """分析手指""" - # 简化轮廓 epsilon = config.get('gesture', 'contour_simplify_epsilon') * cv2.arcLength(contour, True) approx = cv2.approxPolyDP(contour, epsilon, True) - - # 计算凸包 hull = cv2.convexHull(approx, returnPoints=False) if hull is None or len(hull) < 3: return [], [], [] - # 计算凸缺陷 defects = cv2.convexityDefects(approx, hull) - fingers = [] fingertips = [] defect_points = [] @@ -916,17 +926,14 @@ def analyze_fingers(self, contour, palm_center, palm_radius): end = tuple(approx[e][0]) far = tuple(approx[f][0]) - # 计算距离 start_dist = np.linalg.norm(np.array(start) - np.array(palm_center)) end_dist = np.linalg.norm(np.array(end) - np.array(palm_center)) far_dist = np.linalg.norm(np.array(far) - np.array(palm_center)) - # 检查是否可能是手指 if (start_dist > palm_radius * 1.2 and end_dist > palm_radius * 1.2 and d > config.get('gesture', 'defect_distance_threshold') * 256): - # 检查角度 a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) @@ -935,9 +942,7 @@ def analyze_fingers(self, contour, palm_center, palm_radius): angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) angle_degrees = math.degrees(angle) - # 如果角度小于90度,可能是指缝 if angle_degrees < 90: - # 添加手指 finger = { 'start': start, 'end': end, @@ -947,77 +952,59 @@ def analyze_fingers(self, contour, palm_center, palm_radius): } fingers.append(finger) - # 添加指尖(开始和结束点都可能是指尖) for point in [start, end]: if point not in fingertips: - # 检查点是否在手掌半径之外 point_dist = np.linalg.norm(np.array(point) - np.array(palm_center)) if point_dist > palm_radius * 1.5: fingertips.append(point) defect_points.append((start, end, far, d)) - # 如果没有检测到手指,尝试使用凸包点作为指尖 if len(fingertips) == 0: hull_points = cv2.convexHull(approx, returnPoints=True) if len(hull_points) > 0: hull_points = hull_points.reshape(-1, 2) - # 过滤掉离手掌中心太近的点 for point in hull_points: point_tuple = tuple(point) point_dist = np.linalg.norm(point - np.array(palm_center)) if point_dist > palm_radius * 1.5 and point_tuple not in fingertips: fingertips.append(point_tuple) - # 限制指尖数量为5 fingertips = fingertips[:5] return fingers, fingertips, defect_points def calculate_hand_direction(self, contour, cx, cy): """计算手部方向""" - # 使用PCA(主成分分析)计算手部方向 if len(contour) < 5: return 0.0 - # 将轮廓转换为点集 points = contour.reshape(-1, 2).astype(np.float32) - - # 计算PCA mean = np.empty((0)) mean, eigenvectors, eigenvalues = cv2.PCACompute2(points, mean) - - # 计算方向角度(度) direction = math.degrees(math.atan2(eigenvectors[0, 1], eigenvectors[0, 0])) return direction def calculate_confidence(self, area, finger_count, contour_length, bbox_area): """计算手势置信度""" - confidence = 0.5 # 基础置信度 - - # 基于面积的置信度 + confidence = 0.5 min_area = config.get('gesture', 'min_hand_area') max_area = config.get('gesture', 'max_hand_area') if min_area < area < max_area: - # 面积在合理范围内 area_norm = (area - min_area) / (max_area - min_area) confidence += area_norm * 0.2 - # 基于手指数量的置信度 if 0 <= finger_count <= 5: confidence += 0.2 - # 基于轮廓复杂度的置信度 if contour_length > 200: confidence += 0.1 - # 基于填充度的置信度(轮廓面积/边界框面积) if bbox_area > 0: fill_ratio = area / bbox_area - # 手的填充度通常在0.3-0.7之间 if 0.2 < fill_ratio < 0.8: fill_score = 1.0 - abs(fill_ratio - 0.5) / 0.3 confidence += fill_score * 0.1 @@ -1034,116 +1021,219 @@ def recognize_gesture_improved(self, hand_data): norm_x, norm_y = hand_data['position'] direction = hand_data.get('direction', 0.0) confidence = hand_data['confidence'] + w, h = hand_data['bbox_size'] + aspect_ratio = w / h if h > 0 else 1.0 # 根据手指数量分类 if finger_count == 0: - # 握拳或没有手指 if len(fingertips) == 0: return "Stop", confidence * 0.9 else: - # 有指尖但不是手指,可能是部分握拳 return "Stop", confidence * 0.7 elif finger_count == 1: - # 单指手势 if len(fingertips) >= 1: - # 检查指尖方向 cx, cy = hand_data['center'] fingertip = fingertips[0] - - # 计算指尖相对于手中心的方向 dx = fingertip[0] - cx dy = fingertip[1] - cy - # 判断指尖方向 - if abs(dx) > abs(dy): # 主要水平方向 + if abs(dx) > abs(dy): if dx > 0: return "Right", confidence * 0.8 else: return "Left", confidence * 0.8 - else: # 主要垂直方向 - if dy < 0: # 指尖在手中心上方 + else: + if dy < 0: return "Up", confidence * 0.8 else: return "Forward", confidence * 0.8 return "Forward", confidence * 0.7 elif finger_count == 2: - # 双指手势 return "Forward", confidence * 0.7 elif finger_count == 3: - # 三指手势,根据手部方向判断 - if -45 <= direction <= 45: # 大致水平 + if -45 <= direction <= 45: if direction > 0: return "Right", confidence * 0.7 else: return "Left", confidence * 0.7 - else: # 大致垂直 + else: if direction > 0: return "Down", confidence * 0.7 else: return "Up", confidence * 0.7 elif finger_count >= 4: - # 多指手势(手掌张开) - # 根据手部位置判断 - if norm_x < 0.4: # 左侧 + if norm_x < 0.4: return "Left", confidence * 0.8 - elif norm_x > 0.6: # 右侧 + elif norm_x > 0.6: return "Right", confidence * 0.8 - elif norm_y < 0.4: # 上方 + elif norm_y < 0.4: return "Up", confidence * 0.8 - elif norm_y > 0.6: # 下方 + elif norm_y > 0.6: return "Down", confidence * 0.8 - else: # 中间 - # 根据手部方向判断 - if -45 <= direction <= 45: # 大致水平 + else: + if -45 <= direction <= 45: return "Forward", confidence * 0.7 - else: # 大致垂直 + else: return "Stop", confidence * 0.7 return "Waiting", confidence * 0.5 - def smooth_gesture(self, new_gesture, new_confidence): - """平滑手势输出""" + def smooth_gesture_enhanced(self, new_gesture, new_confidence, hand_data): + """增强的手势平滑处理""" + current_time = time.time() + + # 检查手势冷却时间 + if current_time - self.last_gesture_change_time < config.get('gesture', 'gesture_cooldown'): + return self.current_gesture, self.current_confidence + # 添加到历史 self.gesture_history.append(new_gesture) self.confidence_history.append(new_confidence) - # 如果历史记录不足,直接返回 - if len(self.gesture_history) < 3: - self.current_gesture = new_gesture - self.current_confidence = new_confidence - return new_gesture, new_confidence + if hand_data is not None: + self.position_history.append(hand_data['position']) + + # 计算手势稳定性 + if len(self.gesture_history) >= 3: + # 检查最近N个手势是否一致 + recent_gestures = list(self.gesture_history)[-3:] + gesture_counter = Counter(recent_gestures) + most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] + + # 计算位置稳定性 + position_stability = 1.0 + if len(self.position_history) >= 2 and hand_data is not None: + current_pos = hand_data['position'] + prev_pos = self.position_history[-2] if len(self.position_history) >= 2 else current_pos + position_diff = math.sqrt((current_pos[0] - prev_pos[0]) ** 2 + (current_pos[1] - prev_pos[1]) ** 2) + position_stability = max(0, 1.0 - position_diff * 5.0) + + # 增强的稳定性检查 + stability_threshold = config.get('gesture', 'gesture_stability_threshold') + transition_threshold = config.get('gesture', 'transition_threshold') + position_weight = config.get('gesture', 'position_stability_weight') + + # 计算综合稳定性得分 + gesture_stability = most_common_count / 3.0 + overall_stability = gesture_stability * (1.0 - position_weight) + position_stability * position_weight + + # 手势状态转换逻辑 + if overall_stability >= transition_threshold: + if most_common_gesture != self.last_stable_gesture: + self.gesture_stability_counter += 1 + else: + self.gesture_stability_counter = max(0, self.gesture_stability_counter - 1) + + if self.gesture_stability_counter >= stability_threshold: + # 手势稳定,更新当前手势 + self.current_gesture = most_common_gesture + self.current_confidence = np.mean(list(self.confidence_history)[-3:]) + self.last_stable_gesture = most_common_gesture + self.gesture_stability_counter = 0 + self.last_gesture_change_time = current_time + else: + # 手势不稳定,重置计数器 + self.gesture_stability_counter = max(0, self.gesture_stability_counter - 2) + + return self.current_gesture, self.current_confidence + + def update_gesture_state(self, hand_data, gesture, confidence): + """更新手势状态""" + current_time = time.time() - # 统计最常见的姿势 - gesture_counter = Counter(self.gesture_history) - most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] + if hand_data is None: + # 手部丢失 + self.hand_lost_frames += 1 + self.hand_detected_frames = max(0, self.hand_detected_frames - 1) - # 计算平均置信度 - avg_confidence = np.mean(list(self.confidence_history)) + if self.hand_lost_frames > 10 and self.gesture_state != "none": + self.gesture_state = "none" + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_hand_status') and + current_time - self.last_hand_status_time > 3.0): + self.speech_manager.speak('hand_lost', immediate=True) + self.last_hand_status_time = current_time + return - # 如果最常见的姿势出现次数足够多,使用它 - smooth_frames = self.config['smooth_frames'] - if most_common_count >= smooth_frames: - self.current_gesture = most_common_gesture - self.current_confidence = avg_confidence + # 手部检测到 + self.hand_detected_frames += 1 + self.hand_lost_frames = 0 - return self.current_gesture, self.current_confidence + # 检查手部大小和位置 + hand_area = hand_data['area'] + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') + + # 提供手部位置反馈 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_hand_status') and + current_time - self.last_hand_status_time > 5.0): + + if hand_area < min_area * 0.8: + self.speech_manager.speak('move_closer', immediate=True) + self.last_hand_status_time = current_time + elif hand_area > max_area * 1.2: + self.speech_manager.speak('move_away', immediate=True) + self.last_hand_status_time = current_time + elif self.hand_detected_frames == 5: # 首次稳定检测 + self.speech_manager.speak('hand_detected', immediate=True) + self.last_hand_status_time = current_time + + # 手势状态转换 + if self.gesture_state == "none" and gesture != "Waiting" and confidence > 0.6: + # 手势开始 + self.gesture_state = "starting" + self.gesture_active_frames = 0 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gesture_changes')): + self.speech_manager.speak('gesture_start', immediate=True) + + elif self.gesture_state == "starting": + self.gesture_active_frames += 1 + if self.gesture_active_frames >= config.get('speech', 'gesture_start_threshold'): + self.gesture_state = "active" + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gesture_changes')): + self.speech_manager.speak('gesture_stable', immediate=True) + + elif self.gesture_state == "active": + if gesture == "Waiting" or confidence < 0.5: + self.gesture_active_frames = max(0, self.gesture_active_frames - 2) + if self.gesture_active_frames <= 0: + self.gesture_state = "ending" + else: + self.gesture_active_frames = min(20, self.gesture_active_frames + 1) + + elif self.gesture_state == "ending": + self.gesture_active_frames -= 1 + if self.gesture_active_frames <= 0: + self.gesture_state = "none" + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gesture_changes')): + self.speech_manager.speak('gesture_end', immediate=True) def visualize_detection(self, frame, hand_data, gesture, confidence): """可视化检测结果""" if hand_data is None: return frame - # 获取显示配置 show_contours = config.get('display', 'show_contours') show_bbox = config.get('display', 'show_bbox') show_fingertips = config.get('display', 'show_fingertips') show_palm_center = config.get('display', 'show_palm_center') show_hand_direction = config.get('display', 'show_hand_direction') show_debug_info = config.get('display', 'show_debug_info') + show_gesture_history = config.get('display', 'show_gesture_history') + show_stability_indicator = config.get('display', 'show_stability_indicator') # 绘制轮廓 if show_contours and 'contour' in hand_data: @@ -1153,13 +1243,24 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): if show_bbox and 'bbox' in hand_data: x1, y1, x2, y2 = hand_data['bbox'] color = self.gesture_colors.get(gesture, (255, 255, 255)) + + # 根据手势状态调整边界框颜色 + state_color = self.state_colors.get(self.gesture_state, color) + if self.gesture_state != "none": + color = state_color + cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) - # 显示手势标签 - 使用英文避免字体问题 + # 显示手势标签 label = f"{gesture}" if config.get('display', 'show_confidence'): label += f" ({confidence:.0%})" + # 显示手势状态 + if self.gesture_state != "none": + state_text = {"starting": "开始", "active": "活跃", "ending": "结束"} + label += f" [{state_text.get(self.gesture_state, '')}]" + # 计算文本大小 (text_width, text_height), baseline = cv2.getTextSize( label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 @@ -1197,19 +1298,65 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): direction = hand_data['direction'] length = 50 - # 计算方向向量 dx = length * math.cos(math.radians(direction)) dy = length * math.sin(math.radians(direction)) - # 绘制方向线 end_point = (int(cx + dx), int(cy + dy)) cv2.arrowedLine(frame, (cx, cy), end_point, (255, 255, 0), 2) - # 显示方向角度 angle_text = f"Dir: {direction:.0f}°" cv2.putText(frame, angle_text, (cx, cy - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 1) + # 绘制稳定性指示器 + if show_stability_indicator: + h, w = frame.shape[:2] + indicator_x = w - 100 + indicator_y = 30 + + # 绘制稳定性背景 + cv2.rectangle(frame, (indicator_x, indicator_y), + (indicator_x + 80, indicator_y + 15), (50, 50, 50), -1) + + # 计算稳定性指示条长度 + stability_level = min(1.0, + self.gesture_stability_counter / config.get('gesture', 'gesture_stability_threshold')) + bar_length = int(70 * stability_level) + + # 根据稳定性级别选择颜色 + if stability_level > 0.7: + bar_color = (0, 255, 0) # 绿色 + elif stability_level > 0.4: + bar_color = (255, 165, 0) # 橙色 + else: + bar_color = (255, 0, 0) # 红色 + + # 绘制稳定性指示条 + cv2.rectangle(frame, (indicator_x + 5, indicator_y + 5), + (indicator_x + 5 + bar_length, indicator_y + 10), bar_color, -1) + + # 绘制稳定性文本 + cv2.putText(frame, "稳定度", (indicator_x, indicator_y - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 绘制手势历史(如果启用) + if show_gesture_history and len(self.gesture_history) > 0: + h, w = frame.shape[:2] + history_y = h - 50 + + # 绘制历史背景 + cv2.rectangle(frame, (10, history_y - 20), (200, history_y + 10), (0, 0, 0), -1) + cv2.putText(frame, "手势历史:", (15, history_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 显示最近几个手势 + recent_gestures = list(self.gesture_history)[-5:] if len(self.gesture_history) >= 5 else list( + self.gesture_history) + for i, gest in enumerate(recent_gestures): + color = self.gesture_colors.get(gest, (255, 255, 255)) + cv2.putText(frame, gest[0], (85 + i * 20, history_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + # 显示调试信息 if show_debug_info: # 显示手指数量 @@ -1223,6 +1370,16 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): cv2.putText(frame, pos_text, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + # 显示手势状态 + state_text = f"State: {self.gesture_state}" + cv2.putText(frame, state_text, (150, frame.shape[0] - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 显示稳定性计数器 + stability_text = f"Stability: {self.gesture_stability_counter}" + cv2.putText(frame, stability_text, (150, frame.shape[0] - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + return frame def recognize(self, frame): @@ -1261,20 +1418,36 @@ def recognize(self, frame): # 识别手势 gesture, raw_confidence = self.recognize_gesture_improved(hand_data) confidence = max(confidence, raw_confidence) + + # 更新手势状态 + self.update_gesture_state(hand_data, gesture, confidence) + + # 增强的手势平滑 + final_gesture, final_confidence = self.smooth_gesture_enhanced(gesture, confidence, hand_data) else: gesture, confidence = "Waiting", 0.3 - - # 平滑手势 - final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) + self.update_gesture_state(None, gesture, confidence) + final_gesture, final_confidence = gesture, confidence # 手势语音提示 if (self.speech_manager and config.get('speech', 'enabled') and - config.get('speech', 'announce_gestures') and - final_confidence >= config.get('speech', 'min_gesture_confidence')): + config.get('speech', 'announce_gestures')): current_time = time.time() - if (final_gesture != self.last_announced_gesture or + + # 根据置信度提供反馈 + if confidence >= 0.8 and current_time - self.last_hand_status_time > 5.0: + self.speech_manager.speak('gesture_good_confidence', immediate=True) + self.last_hand_status_time = current_time + elif confidence < 0.5 and current_time - self.last_hand_status_time > 5.0: + self.speech_manager.speak('gesture_low_confidence', immediate=True) + self.last_hand_status_time = current_time + + # 手势变化播报 + if (final_gesture != self.last_announced_gesture and + final_gesture not in ["Waiting", "Error"] and + final_confidence >= config.get('speech', 'min_gesture_confidence') and current_time - self.last_announced_time > self.gesture_announce_interval): if final_gesture in self.gesture_speech_map: @@ -1283,6 +1456,20 @@ def recognize(self, frame): self.last_announced_gesture = final_gesture self.last_announced_time = current_time + # 性能报告 + current_time = time.time() + if current_time - self.last_performance_report > 30.0: # 每30秒报告一次 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_performance')): + + if self.avg_process_time < 20: # 处理时间小于20ms + self.speech_manager.speak('performance_good', immediate=True) + elif self.avg_process_time > 50: # 处理时间大于50ms + self.speech_manager.speak('performance_warning', immediate=True) + + self.last_performance_report = current_time + # 可视化结果 if hand_data is not None: processed_frame = self.visualize_detection( @@ -1296,6 +1483,17 @@ def recognize(self, frame): process_time = (time.time() - start_time) * 1000 self.process_times.append(process_time) + # 更新平均处理时间 + if len(self.process_times) > 0: + self.avg_process_time = np.mean(list(self.process_times)) + + # 更新帧率 + current_time = time.time() + if current_time - self.last_fps_check >= 1.0: + self.frame_rate = self.frame_counter + self.frame_counter = 0 + self.last_fps_check = current_time + return final_gesture, final_confidence, processed_frame except Exception as e: @@ -1305,9 +1503,9 @@ def recognize(self, frame): def get_performance_stats(self): """获取性能统计""" if len(self.process_times) == 0: - return 0.0 + return 0.0, self.frame_rate - return np.mean(list(self.process_times)) + return np.mean(list(self.process_times)), self.frame_rate def set_simulated_gesture(self, gesture): """设置模拟的手势""" @@ -1538,7 +1736,7 @@ def move_by_gesture(self, gesture, confidence): config.get('speech', 'enabled') and config.get('speech', 'announce_gestures') and confidence < min_confidence * 0.8): # 如果置信度特别低 - self.speech_manager.speak('low_confidence') + self.speech_manager.speak('gesture_low_confidence') return False try: @@ -1568,9 +1766,18 @@ def move_by_gesture(self, gesture, confidence): elif gesture == "Stop": self.client.hoverAsync() success = True + # 悬停语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('hovering') elif gesture == "Hover": self.client.hoverAsync() success = True + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('hovering') if success: self.last_control_time = current_time @@ -1627,6 +1834,9 @@ def __init__(self, speech_manager=None): 'help': (255, 200, 100), # 浅橙色 'speech_enabled': (0, 255, 0), # 绿色 'speech_disabled': (255, 0, 0), # 红色 + 'performance_good': (0, 255, 0), # 绿色 + 'performance_warning': (255, 165, 0), # 橙色 + 'performance_bad': (255, 0, 0), # 红色 } print("✓ 中文UI渲染器已初始化") @@ -1690,7 +1900,7 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 优化版 (带语音反馈)" + title = "手势控制无人机系统 - 增强版" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -1730,10 +1940,20 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro if process_time > 0: perf_text += f" | 延迟: {process_time:.1f}ms" - frame = self.draw_text(frame, perf_text, (w - 200, 65), size=14, color=self.colors['info']) + # 根据处理时间选择颜色 + if process_time < 20: + perf_color = self.colors['performance_good'] + elif process_time < 50: + perf_color = self.colors['performance_warning'] + else: + perf_color = self.colors['performance_bad'] + else: + perf_color = self.colors['info'] + + frame = self.draw_text(frame, perf_text, (w - 200, 65), size=14, color=perf_color) # 控制提示 - control_text = "提示: 确保手部完全进入画面,光线充足" + control_text = "提示: 确保手部完全进入画面,保持稳定手势" frame = self.draw_text(frame, control_text, (10, 90), size=14, color=self.colors['info']) return frame @@ -1752,7 +1972,7 @@ def draw_help_bar(self, frame): help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音" + "V:切换语音反馈 M:测试语音 P:性能报告" ] for i, line in enumerate(help_lines): @@ -1820,16 +2040,17 @@ def main(): """主函数""" # 初始化语音管理器 print("初始化语音反馈系统...") - speech_manager = SpeechFeedbackManager(libs['speech']) + speech_manager = EnhancedSpeechFeedbackManager(libs['speech']) # 程序启动语音提示 if speech_manager.enabled: - speech_manager.speak('program_start', force=True) + speech_manager.speak('program_start', force=True, immediate=True) + speech_manager.speak('system_ready', immediate=True) # 初始化组件 print("初始化组件...") - gesture_recognizer = ImprovedGestureRecognizer(speech_manager) + gesture_recognizer = EnhancedGestureRecognizer(speech_manager) drone_controller = SimpleDroneController(libs['airsim'], speech_manager) ui_renderer = ChineseUIRenderer(speech_manager) performance_monitor = PerformanceMonitor() @@ -1851,12 +2072,16 @@ def main(): print(f"✓ 摄像头已初始化") print(f" 分辨率: {actual_width}x{actual_height}") print(f" 帧率: {actual_fps}") + + # 摄像头就绪语音提示 + if speech_manager.enabled: + speech_manager.speak('camera_ready', immediate=True) else: print("❌ 摄像头不可用,使用模拟模式") # 摄像头错误语音提示 if speech_manager.enabled: - speech_manager.speak('camera_error') + speech_manager.speak('camera_error', immediate=True) cap = None except Exception as e: @@ -1864,18 +2089,19 @@ def main(): # 摄像头错误语音提示 if speech_manager.enabled: - speech_manager.speak('camera_error') + speech_manager.speak('camera_error', immediate=True) cap = None # 显示欢迎信息 print("\n" + "=" * 60) - print("手势控制无人机系统 - 手势识别优化版") + print("手势控制无人机系统 - 增强版") print("=" * 60) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") - print(f" 手势识别: 改进的OpenCV算法") + print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") + print(f" 手势状态: 支持开始/活跃/结束状态追踪") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) @@ -1884,17 +2110,16 @@ def main(): print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") print("3. 手势控制改进:") - print(" - 握拳或握紧: Stop") - print(" - 单指指向: 根据指尖方向判断Up/Down/Left/Right") - print(" - 双指: Forward") - print(" - 手掌张开: 根据手的位置判断方向") - print(" * 手势识别置信度 > 50% 时才会执行") + print(" - 系统会自动检测手势开始、稳定和结束状态") + print(" - 手势稳定性越高,识别越准确") + print(" - 手部距离摄像头适中时效果最佳") + print(" * 手势识别置信度 > 60% 时才会执行") print("4. 键盘控制:") print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") print("5. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") print("6. 语音控制:") - print(" [V]切换语音反馈 [M]测试语音") + print(" [V]切换语音反馈 [M]测试语音 [P]性能报告") print("7. 按 [ESC] 安全退出") print("=" * 60) print("程序启动成功!") @@ -1939,13 +2164,15 @@ def main(): # 获取性能统计 perf_stats = performance_monitor.get_stats() - process_time = gesture_recognizer.get_performance_stats() + process_time, frame_rate = gesture_recognizer.get_performance_stats() # 根据显示模式调整显示选项 if display_modes[current_display_mode] == 'normal': config.set('display', 'show_contours', value=True) config.set('display', 'show_bbox', value=True) config.set('display', 'show_fingertips', value=True) + config.set('display', 'show_gesture_history', value=True) + config.set('display', 'show_stability_indicator', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -1953,11 +2180,15 @@ def main(): config.set('display', 'show_fingertips', value=True) config.set('display', 'show_palm_center', value=True) config.set('display', 'show_hand_direction', value=True) + config.set('display', 'show_gesture_history', value=True) + config.set('display', 'show_stability_indicator', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) config.set('display', 'show_bbox', value=True) config.set('display', 'show_fingertips', value=False) + config.set('display', 'show_gesture_history', value=False) + config.set('display', 'show_stability_indicator', value=False) config.set('display', 'show_debug_info', value=False) # 绘制UI @@ -1974,7 +2205,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Optimized with Speech', frame) + cv2.imshow('Gesture Controlled Drone - Enhanced', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -2003,12 +2234,12 @@ def main(): # 语音提示 if speech_manager.enabled: - speech_manager.speak('help_toggled') + speech_manager.speak('help_toggled', immediate=True) elif key == ord('r') or key == ord('R'): # 重置手势识别 print("重置手势识别...") - gesture_recognizer = ImprovedGestureRecognizer(speech_manager) + gesture_recognizer = EnhancedGestureRecognizer(speech_manager) print("✓ 手势识别已重置") # 语音提示 @@ -2023,7 +2254,7 @@ def main(): # 语音提示 if speech_manager.enabled: - speech_manager.speak('display_mode_changed') + speech_manager.speak('display_mode_changed', immediate=True) elif key == ord('d') or key == ord('D'): # 切换调试信息 @@ -2035,9 +2266,9 @@ def main(): # 语音提示 if speech_manager.enabled: if not current: - speech_manager.speak('debug_mode_on') + speech_manager.speak('debug_mode_on', immediate=True) else: - speech_manager.speak('debug_mode_off') + speech_manager.speak('debug_mode_off', immediate=True) elif key == ord('v') or key == ord('V'): # 切换语音反馈 @@ -2050,10 +2281,21 @@ def main(): # 测试语音 if speech_manager.enabled: print("测试语音...") - speech_manager.speak_direct("语音反馈测试,这是一条测试消息") + speech_manager.speak_direct("语音反馈测试,系统运行正常") else: print("语音反馈已禁用,按V键启用") + elif key == ord('p') or key == ord('P'): + # 性能报告 + if speech_manager.enabled: + print("生成性能报告...") + if process_time < 20: + speech_manager.speak_direct("系统性能优秀,运行流畅") + elif process_time < 50: + speech_manager.speak_direct("系统性能良好") + else: + speech_manager.speak_direct("系统性能警告,请检查") + elif key in key_to_gesture: # 键盘控制 simulated_gesture = key_to_gesture[key] @@ -2084,7 +2326,7 @@ def main(): # 程序退出语音提示 if speech_manager.enabled: - speech_manager.speak('program_exit', force=True) + speech_manager.speak('program_exit', force=True, immediate=True) time.sleep(1) # 确保语音播报完成 drone_controller.emergency_stop() From d831140f4fa47e437dc68e18637947b379c81eb0 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sat, 20 Dec 2025 20:30:10 +0800 Subject: [PATCH 09/11] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=88=91=E7=9A=84?= =?UTF-8?q?=E9=A1=B9=E7=9B=AEREADME.md=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../README.md | 40 +++++++++++++++++++ .../gesture_config.json | 10 +++++ 2 files changed, 50 insertions(+) diff --git a/src/autonomus_drone_hand_gesture_project/README.md b/src/autonomus_drone_hand_gesture_project/README.md index e69de29bb2..0b5c35f915 100644 --- a/src/autonomus_drone_hand_gesture_project/README.md +++ b/src/autonomus_drone_hand_gesture_project/README.md @@ -0,0 +1,40 @@ +# 手势控制AirSim无人机系统 + +[项目简介] +基于计算机视觉和语音反馈技术的手势控制无人机系统。使用纯OpenCV实现手部检测和手势识别,结合语音反馈系统,提供直观的无人机控制体验。系统支持AirSim仿真环境和真实无人机控制,具备手势平滑处理、状态追踪和多模式显示功能。 + +[核心功能] + +1. 手势识别与控制 + - 纯OpenCV实现:不依赖深度学习框架,使用肤色检测、轮廓分析和凸缺陷检测识别手势 + - 多种手势支持:支持停止、前进、后退、上升、下降、左转、右转等多种控制手势 + - 实时手势追踪:跟踪手部位置和状态,提供连续控制体验 + +2. 智能语音反馈系统 + - 多引擎支持:兼容pyttsx3(离线)和gTTS(在线)语音引擎 + - 丰富语音提示:提供连接状态、飞行状态、手势识别、系统状态等多种语音反馈 + - 智能语音队列:避免语音重叠,支持优先级播放和即时播放 + +3. 增强手势平滑处理 + - 手势状态追踪:实现手势状态机(none/starting/active/ending),跟踪手势生命周期 + - 稳定性检测:使用手势稳定性计数器和位置稳定性权重,减少误判 + - 冷却时间机制:防止手势切换过于频繁,提高控制精度 + +4. 多平台兼容性 + - 跨平台音频播放:支持pygame、pydub和系统默认播放器 + - 自动库检测:自动检测可用库并选择最优方案 + - 详细错误处理:完善的错误处理和降级方案 + +[环境依赖] +在运行代码之前,请确保已安装以下Python库: + +```bash +# 基础依赖 +pip install opencv-python pillow numpy + +# AirSim无人机控制(可选) +pip install airsim + +# 语音反馈系统(选择安装) +pip install pyttsx3 # 离线语音(推荐) +pip install gtts pygame # 在线语音(需要网络) \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index c60f9cec3d..145a26faac 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -67,6 +67,9 @@ "show_speech_status": true, "show_gesture_history": true, "show_stability_indicator": true, + "show_trajectory": true, + "show_trajectory_heatmap": false, + "trajectory_max_points": 100, "show_gesture_info": true, "show_control_help": true, "show_detection_overlay": true, @@ -96,10 +99,17 @@ "announce_gesture_changes": true, "announce_hand_status": true, "announce_performance": true, + "announce_recording": true, "min_gesture_confidence": 0.7, "gesture_start_threshold": 3, "gesture_end_threshold": 10 }, + "recording": { + "auto_save": false, + "default_filename": "gesture_recording.pkl", + "playback_speed": 1.0, + "loop_playback": false + }, "system": { "version": "1.0", "last_updated": "2025-12-19 20:51:44", From 15eb615ac71b54721929b2b980b5891c2094f45f Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sun, 21 Dec 2025 09:07:37 +0800 Subject: [PATCH 10/11] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E8=BD=A8=E8=BF=B9=E8=AE=B0=E5=BD=95=E4=B8=8E=E5=9B=9E=E6=94=BE?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=EF=BC=8C=E4=BC=98=E5=8C=96=E8=AF=AD=E9=9F=B3?= =?UTF-8?q?=E5=8F=8D=E9=A6=88=E7=B3=BB=E7=BB=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config.json | 152 +++-- .../main.py | 554 +++++++++++++++++- .../trajectory_20251221_085734.pkl | Bin 0 -> 5540 bytes .../trajectories/trajectory_temp.pkl | Bin 0 -> 4719 bytes 4 files changed, 633 insertions(+), 73 deletions(-) create mode 100644 src/autonomus_drone_hand_gesture_project/trajectories/trajectory_20251221_085734.pkl create mode 100644 src/autonomus_drone_hand_gesture_project/trajectories/trajectory_temp.pkl diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index 145a26faac..1ebcdda50e 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -4,21 +4,25 @@ "width": 640, "height": 480, "fps": 30, + "auto_exposure": true, + "brightness": 0, + "contrast": 1.0, "device_id": 0, + "exposure": 0, + "flip_horizontal": true, "resolution": { - "width": 640, - "height": 480 + "height": 480, + "width": 640 }, - "flip_horizontal": true, "use_virtual_camera": false }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 178, + "skin_upper_h": 125, "skin_lower_s": 0, - "skin_upper_s": 71, - "skin_lower_v": 32, - "skin_upper_v": 85, + "skin_upper_s": 31, + "skin_lower_v": 0, + "skin_upper_v": 74, "min_hand_area": 3000, "max_hand_area": 30000, "history_size": 10, @@ -33,6 +37,17 @@ "transition_threshold": 0.3, "position_stability_weight": 0.4, "gesture_cooldown": 0.5, + "adaptive_learning_rate": 0.1, + "enable_background_subtraction": true, + "enable_dynamic_threshold": true, + "enable_morphological_ops": true, + "face_exclusion_area": 8000, + "hand_region_threshold": 0.2, + "max_fingertips": 5, + "min_fingertip_distance": 10, + "noise_reduction": 0.5, + "palm_aspect_ratio": 1.3, + "single_finger_angle_threshold": 60, "skin_lower": [ 0, 30, @@ -42,17 +57,18 @@ 25, 255, 255 - ], - "hand_region_threshold": 0.2, - "face_exclusion_area": 8000, - "palm_aspect_ratio": 1.3, - "single_finger_angle_threshold": 60 + ] }, "drone": { "velocity": 2.5, "duration": 0.3, "altitude": -10.0, - "control_interval": 0.3 + "control_interval": 0.3, + "acceleration_limit": 2.0, + "auto_takeoff_delay": 2.0, + "emergency_land_delay": 3.0, + "emergency_land_timeout": 5.0, + "smooth_movement": true }, "display": { "show_fps": true, @@ -68,26 +84,43 @@ "show_gesture_history": true, "show_stability_indicator": true, "show_trajectory": true, - "show_trajectory_heatmap": false, - "trajectory_max_points": 100, - "show_gesture_info": true, + "show_recording_status": true, + "font_size_large": 20, + "font_size_medium": 16, + "font_size_small": 14, + "show_calibration_info": true, "show_control_help": true, + "show_control_hints": true, "show_detection_overlay": true, - "theme": "dark", "show_direction": true, - "show_region_grid": true + "show_gesture_info": true, + "show_performance_stats": true, + "show_region_grid": true, + "show_trajectory_heatmap": false, + "theme": "dark", + "trajectory_max_points": 100, + "ui_opacity": 0.7 }, "performance": { "target_fps": 30, "resize_factor": 1.0, "enable_multiprocessing": false, + "enable_gpu_acceleration": false, + "enable_hardware_acceleration": true, "enable_multithreading": false, - "enable_hardware_acceleration": true + "max_queue_size": 10, + "memory_limit_mb": 500, + "process_time_critical": 100.0, + "process_time_warning": 50.0 }, "calibration": { "auto_calibrate_skin": true, "skin_calibration_frames": 30, - "hand_size_calibration": true + "hand_size_calibration": true, + "calibration_interval": 300, + "calibration_timeout": 10.0, + "enable_adaptive_calibration": true, + "min_calibration_samples": 50 }, "speech": { "enabled": true, @@ -99,36 +132,50 @@ "announce_gesture_changes": true, "announce_hand_status": true, "announce_performance": true, - "announce_recording": true, + "announce_recording_events": true, "min_gesture_confidence": 0.7, "gesture_start_threshold": 3, - "gesture_end_threshold": 10 + "gesture_end_threshold": 10, + "announce_gesture_directions": true, + "announce_recording": true, + "enable_short_feedback": true, + "max_queue_size": 5, + "speech_cooldown": 1.0, + "speech_language": "zh-cn" }, "recording": { + "auto_save_interval": 5, + "max_trajectory_points": 1000, + "show_trajectory": true, + "trajectory_thickness": 2, + "trajectory_max_length": 100, + "default_save_dir": "trajectories", "auto_save": false, "default_filename": "gesture_recording.pkl", - "playback_speed": 1.0, - "loop_playback": false + "loop_playback": false, + "playback_speed": 1.0 }, - "system": { - "version": "1.0", - "last_updated": "2025-12-19 20:51:44", - "debug_mode": false + "drone_control": { + "altitude": -10.0, + "base_speed": 2.0, + "control_delay": 0.3, + "emergency_stop_enabled": true, + "takeoff_height": 5.0 }, "gesture_detection": { + "confidence_threshold": 0.5, + "detection_interval": 1, + "max_hand_area": 30000, + "min_hand_area": 2000, "skin_hsv_ranges": { - "h_low": 0, "h_high": 25, - "s_low": 30, + "h_low": 0, "s_high": 255, - "v_low": 60, - "v_high": 255 + "s_low": 30, + "v_high": 255, + "v_low": 60 }, - "min_hand_area": 2000, - "max_hand_area": 30000, - "confidence_threshold": 0.5, - "smoothing_window": 5, - "detection_interval": 1 + "smoothing_window": 5 }, "gesture_mapping": { "0": "stop", @@ -140,11 +187,30 @@ "6": "right", "7": "hover" }, - "drone_control": { - "base_speed": 2.0, - "altitude": -10.0, - "control_delay": 0.3, - "takeoff_height": 5.0, - "emergency_stop_enabled": true + "keyboard": { + "connect_key": "c", + "emergency_key": "q", + "enable_keyboard_control": true, + "key_repeat_delay": 0.2, + "land_key": "l", + "takeoff_key": " " + }, + "system": { + "debug_mode": false, + "last_updated": "2025-12-19 20:51:44", + "version": "1.0" + }, + "training": { + "auto_save_training": true, + "enable_training_mode": false, + "samples_per_gesture": 50, + "training_gestures": [ + "Stop", + "Forward", + "Up", + "Down", + "Left", + "Right" + ] } } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index fe271047a5..c7a24d5374 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 手势识别优化版(增强语音反馈和手势平滑) -优化手势识别算法,增强语音反馈,改进手势平滑处理 +手势控制AirSim无人机 - 手势轨迹记录与回放优化版 +修复手势语音播报问题,确保所有手势语音正常 作者: xiaoshiyuan888 """ @@ -13,6 +13,7 @@ import math import threading import tempfile +import pickle # 新增:用于数据序列化 from datetime import datetime from PIL import Image, ImageDraw, ImageFont import cv2 @@ -20,8 +21,8 @@ from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Enhanced Gesture Recognition") -print("增强语音反馈和手势平滑处理!") +print("Gesture Controlled Drone - Enhanced with Trajectory Recording") +print("手势轨迹记录与回放优化版!") print("=" * 60) # ========== 修复导入路径 ========== @@ -209,12 +210,14 @@ def __init__(self, speech_lib): 'gesture_left': "向左", 'gesture_right': "向右", 'gesture_forward': "向前", + 'gesture_backward': "向后", # 新增:向后手势 'gesture_waiting': "等待手势", 'gesture_error': "手势识别错误", 'gesture_stable': "手势稳定", 'gesture_change': "手势变化", 'gesture_low_confidence': "手势识别置信度低", 'gesture_good_confidence': "手势识别置信度高", + 'gesture_hover': "悬停", # 新增:悬停手势 # 系统相关 'program_start': "手势控制无人机系统已启动", @@ -240,6 +243,19 @@ def __init__(self, speech_lib): 'good_position': "手部位置良好", 'hand_detected': "手部已检测到", 'hand_lost': "手部丢失,请重新放置", + + # 新增:录制相关 + 'recording_start': "开始录制手势轨迹", + 'recording_stop': "停止录制", + 'recording_saved': "轨迹已保存", + 'recording_loaded': "轨迹已加载", + 'recording_playback_start': "开始回放手势轨迹", + 'recording_playback_stop': "回放结束", + 'recording_cleared': "轨迹已清除", + 'recording_paused': "回放已暂停", + 'recording_resumed': "回放继续", + 'recording_not_found': "未找到轨迹数据", + 'recording_frame_count': "轨迹帧数", } # 初始化语音引擎 @@ -497,6 +513,341 @@ def get_status(self): } +# ========== 手势轨迹记录器 ========== +class GestureTrajectoryRecorder: + """手势轨迹记录器 - 记录、保存、加载和回放手势轨迹""" + + def __init__(self, speech_manager=None): + self.speech_manager = speech_manager + self.trajectory_data = [] + self.is_recording = False + self.is_playing = False + self.playback_index = 0 + self.playback_paused = False + self.max_trajectory_points = 1000 # 最大轨迹点数 + self.recording_start_time = 0 + self.last_save_time = 0 + self.save_interval = 5 # 自动保存间隔(秒) + + # 轨迹文件路径 + self.trajectory_dir = os.path.join(current_dir, 'trajectories') + if not os.path.exists(self.trajectory_dir): + os.makedirs(self.trajectory_dir) + + # 默认轨迹文件名 + self.default_filename = os.path.join(self.trajectory_dir, + f'trajectory_{datetime.now().strftime("%Y%m%d_%H%M%S")}.pkl') + + # 轨迹可视化设置 + self.trajectory_colors = [ + (255, 0, 0), # 红色 - 起点 + (0, 255, 0), # 绿色 - 正常点 + (0, 0, 255), # 蓝色 - 终点 + (255, 255, 0), # 青色 - 特殊点 + (255, 0, 255) # 紫色 - 特殊点 + ] + + self.show_trajectory = True + self.trajectory_thickness = 2 + self.trajectory_max_length = 100 # 显示的最大轨迹长度 + + print("✓ 手势轨迹记录器已初始化") + + def start_recording(self): + """开始录制手势轨迹""" + if self.is_recording: + return False + + self.trajectory_data = [] + self.is_recording = True + self.recording_start_time = time.time() + self.last_save_time = time.time() + + print("🎬 开始录制手势轨迹") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_start', immediate=True) + + return True + + def stop_recording(self): + """停止录制手势轨迹""" + if not self.is_recording: + return False + + self.is_recording = False + recording_duration = time.time() - self.recording_start_time + + print(f"⏹️ 停止录制手势轨迹") + print(f" 录制时长: {recording_duration:.1f}秒") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_stop', immediate=True) + if len(self.trajectory_data) > 0: + self.speech_manager.speak_direct(f"录制了{len(self.trajectory_data)}个轨迹点") + + return True + + def add_trajectory_point(self, hand_data, gesture, confidence, frame_shape): + """添加轨迹点""" + if not self.is_recording or len(self.trajectory_data) >= self.max_trajectory_points: + return False + + if hand_data is None: + return False + + # 创建轨迹点数据 + trajectory_point = { + 'timestamp': time.time(), + 'hand_position': hand_data['position'] if 'position' in hand_data else (0.5, 0.5), + 'hand_center': hand_data['center'] if 'center' in hand_data else (0, 0), + 'gesture': gesture, + 'confidence': confidence, + 'fingertips': hand_data.get('fingertips', []), + 'frame_shape': frame_shape + } + + self.trajectory_data.append(trajectory_point) + + # 自动保存检查 + current_time = time.time() + if current_time - self.last_save_time >= self.save_interval and len(self.trajectory_data) > 10: + self.auto_save() + self.last_save_time = current_time + + return True + + def auto_save(self): + """自动保存轨迹(临时文件)""" + if len(self.trajectory_data) == 0: + return + + temp_file = os.path.join(self.trajectory_dir, 'trajectory_temp.pkl') + try: + with open(temp_file, 'wb') as f: + pickle.dump(self.trajectory_data, f) + print(f"💾 自动保存轨迹到临时文件 ({len(self.trajectory_data)}个点)") + except Exception as e: + print(f"⚠ 自动保存轨迹失败: {e}") + + def save_trajectory(self, filename=None): + """保存轨迹到文件""" + if len(self.trajectory_data) == 0: + print("⚠ 没有轨迹数据可保存") + return False + + if filename is None: + filename = self.default_filename + + try: + with open(filename, 'wb') as f: + pickle.dump(self.trajectory_data, f) + + print(f"💾 轨迹已保存到: {filename}") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_saved', immediate=True) + self.speech_manager.speak_direct(f"保存了{len(self.trajectory_data)}个轨迹点") + + return True + except Exception as e: + print(f"❌ 保存轨迹失败: {e}") + return False + + def load_trajectory(self, filename): + """从文件加载轨迹""" + try: + if not os.path.exists(filename): + print(f"❌ 轨迹文件不存在: {filename}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_not_found', immediate=True) + + return False + + with open(filename, 'rb') as f: + self.trajectory_data = pickle.load(f) + + print(f"📂 轨迹已从文件加载: {filename}") + print(f" 轨迹点数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_loaded', immediate=True) + self.speech_manager.speak_direct(f"加载了{len(self.trajectory_data)}个轨迹点") + + return True + except Exception as e: + print(f"❌ 加载轨迹失败: {e}") + return False + + def start_playback(self): + """开始回放轨迹""" + if len(self.trajectory_data) == 0: + print("⚠ 没有轨迹数据可回放") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_not_found', immediate=True) + + return False + + self.is_playing = True + self.playback_index = 0 + self.playback_paused = False + + print(f"▶️ 开始回放手势轨迹") + print(f" 总帧数: {len(self.trajectory_data)}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_playback_start', immediate=True) + + return True + + def stop_playback(self): + """停止回放轨迹""" + if not self.is_playing: + return False + + self.is_playing = False + self.playback_paused = False + + print("⏹️ 停止回放手势轨迹") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_playback_stop', immediate=True) + + return True + + def pause_playback(self): + """暂停/继续回放""" + self.playback_paused = not self.playback_paused + + status = "暂停" if self.playback_paused else "继续" + print(f"⏸️ 回放已{status}") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + if self.playback_paused: + self.speech_manager.speak('recording_paused', immediate=True) + else: + self.speech_manager.speak('recording_resumed', immediate=True) + + return self.playback_paused + + def get_next_playback_point(self): + """获取下一个回放点""" + if not self.is_playing or self.playback_paused or len(self.trajectory_data) == 0: + return None + + if self.playback_index >= len(self.trajectory_data): + self.stop_playback() + return None + + point = self.trajectory_data[self.playback_index] + self.playback_index += 1 + + # 检查是否到达末尾 + if self.playback_index >= len(self.trajectory_data): + self.stop_playback() + + return point + + def clear_trajectory(self): + """清除轨迹数据""" + self.trajectory_data = [] + self.is_recording = False + self.is_playing = False + self.playback_index = 0 + + print("🗑️ 轨迹数据已清除") + + # 语音提示 + if self.speech_manager and self.speech_manager.enabled: + self.speech_manager.speak('recording_cleared', immediate=True) + + return True + + def draw_trajectory(self, frame): + """在帧上绘制轨迹""" + if not self.show_trajectory or len(self.trajectory_data) == 0: + return frame + + h, w = frame.shape[:2] + + # 限制显示的轨迹点数 + display_points = min(len(self.trajectory_data), self.trajectory_max_length) + start_idx = max(0, len(self.trajectory_data) - display_points) + + # 绘制轨迹线 + for i in range(start_idx, len(self.trajectory_data) - 1): + point1 = self.trajectory_data[i] + point2 = self.trajectory_data[i + 1] + + # 获取手部中心位置(转换为图像坐标) + if 'hand_center' in point1 and 'hand_center' in point2: + x1, y1 = point1['hand_center'] + x2, y2 = point2['hand_center'] + + # 确保坐标在图像范围内 + x1 = max(0, min(w - 1, x1)) + y1 = max(0, min(h - 1, y1)) + x2 = max(0, min(w - 1, x2)) + y2 = max(0, min(h - 1, y2)) + + # 根据索引计算颜色(渐变色) + color_idx = int((i - start_idx) / display_points * (len(self.trajectory_colors) - 1)) + color = self.trajectory_colors[color_idx] + + # 绘制线条 + cv2.line(frame, (int(x1), int(y1)), (int(x2), int(y2)), + color, self.trajectory_thickness) + + # 绘制当前点(如果正在录制或回放) + if self.is_recording or self.is_playing: + current_idx = len(self.trajectory_data) - 1 if self.is_recording else self.playback_index - 1 + if 0 <= current_idx < len(self.trajectory_data): + point = self.trajectory_data[current_idx] + if 'hand_center' in point: + x, y = point['hand_center'] + x = max(0, min(w - 1, x)) + y = max(0, min(h - 1, y)) + + # 绘制当前点 + cv2.circle(frame, (int(x), int(y)), 8, (0, 255, 255), -1) + cv2.circle(frame, (int(x), int(y)), 8, (0, 0, 0), 2) + + return frame + + def get_status(self): + """获取录制状态""" + return { + 'is_recording': self.is_recording, + 'is_playing': self.is_playing, + 'playback_paused': self.playback_paused, + 'trajectory_points': len(self.trajectory_data), + 'playback_index': self.playback_index, + 'playback_total': len(self.trajectory_data), + 'recording_duration': time.time() - self.recording_start_time if self.is_recording else 0 + } + + def list_saved_trajectories(self): + """列出保存的轨迹文件""" + try: + files = [f for f in os.listdir(self.trajectory_dir) if f.endswith('.pkl')] + return sorted(files, reverse=True) + except: + return [] + + # ========== 配置管理器 ========== class ConfigManager: """配置管理器""" @@ -551,6 +902,8 @@ def __init__(self): 'show_speech_status': True, 'show_gesture_history': True, # 新增:显示手势历史 'show_stability_indicator': True, # 新增:显示稳定性指示器 + 'show_trajectory': True, # 新增:显示轨迹 + 'show_recording_status': True, # 新增:显示录制状态 }, 'performance': { 'target_fps': 30, @@ -572,9 +925,18 @@ def __init__(self): 'announce_gesture_changes': True, # 新增:播报手势变化 'announce_hand_status': True, # 新增:播报手部状态 'announce_performance': True, # 新增:播报性能状态 + 'announce_recording_events': True, # 新增:播报录制事件 'min_gesture_confidence': 0.7, 'gesture_start_threshold': 3, # 新增:手势开始识别阈值 'gesture_end_threshold': 10, # 新增:手势结束识别阈值 + }, + 'recording': { # 新增:录制配置 + 'auto_save_interval': 5, # 自动保存间隔(秒) + 'max_trajectory_points': 1000, # 最大轨迹点数 + 'show_trajectory': True, # 显示轨迹 + 'trajectory_thickness': 2, # 轨迹线粗细 + 'trajectory_max_length': 100, # 显示的最大轨迹长度 + 'default_save_dir': 'trajectories', # 默认保存目录 } } self.config = self.load_config() @@ -746,7 +1108,7 @@ def __init__(self, speech_manager=None): "Hover": (255, 255, 255) # 白色 } - # 手势到语音的映射 + # 手势到语音的映射 - 修复:确保所有手势都有对应语音 self.gesture_speech_map = { "Stop": "gesture_stop", "Forward": "gesture_forward", @@ -756,6 +1118,7 @@ def __init__(self, speech_manager=None): "Right": "gesture_right", "Waiting": "gesture_waiting", "Error": "gesture_error", + "Hover": "gesture_hover", # 新增悬停手势语音 } # 手势状态颜色 @@ -779,6 +1142,9 @@ def __init__(self, speech_manager=None): self.frame_rate = 0 self.last_fps_check = time.time() + # 新增:存储手部数据用于轨迹记录 + self.last_hand_data = None + print("✓ 增强的手势识别器已初始化 (改进平滑处理)") def get_skin_mask(self, frame): @@ -1407,6 +1773,9 @@ def recognize(self, frame): # 分析手部特征 hand_data, confidence = self.analyze_hand_features(hand_contour, processed_frame.shape) + # 保存手部数据用于轨迹记录 + self.last_hand_data = hand_data + # 识别手势 if hand_data is not None: # 校准肤色(如果需要) @@ -1429,7 +1798,8 @@ def recognize(self, frame): self.update_gesture_state(None, gesture, confidence) final_gesture, final_confidence = gesture, confidence - # 手势语音提示 + # ========== 修复手势语音播报 ========== + # 确保所有手势都有语音反馈 if (self.speech_manager and config.get('speech', 'enabled') and config.get('speech', 'announce_gestures')): @@ -1444,14 +1814,20 @@ def recognize(self, frame): self.speech_manager.speak('gesture_low_confidence', immediate=True) self.last_hand_status_time = current_time - # 手势变化播报 - if (final_gesture != self.last_announced_gesture and - final_gesture not in ["Waiting", "Error"] and + # 手势语音播报 - 修复:确保所有手势都能播报 + if (final_gesture != "Waiting" and + final_gesture != "Error" and + final_gesture != "摄像头错误" and final_confidence >= config.get('speech', 'min_gesture_confidence') and current_time - self.last_announced_time > self.gesture_announce_interval): + # 检查手势是否在语音映射中 if final_gesture in self.gesture_speech_map: - self.speech_manager.speak(self.gesture_speech_map[final_gesture]) + speech_key = self.gesture_speech_map[final_gesture] + self.speech_manager.speak(speech_key) + else: + # 如果手势不在映射中,直接播报手势名称 + self.speech_manager.speak_direct(f"手势{final_gesture}") self.last_announced_gesture = final_gesture self.last_announced_time = current_time @@ -1512,12 +1888,17 @@ def set_simulated_gesture(self, gesture): self.current_gesture = gesture self.current_confidence = 0.9 - # 模拟手势也触发语音提示 + # 模拟手势也触发语音提示 - 修复:确保所有模拟手势都有语音 if (self.speech_manager and config.get('speech', 'enabled') and - config.get('speech', 'announce_gestures') and - gesture in self.gesture_speech_map): - self.speech_manager.speak(self.gesture_speech_map[gesture]) + config.get('speech', 'announce_gestures')): + + if gesture in self.gesture_speech_map: + self.speech_manager.speak(self.gesture_speech_map[gesture]) + else: + # 如果手势不在映射中,直接播报手势名称 + self.speech_manager.speak_direct(f"手势{gesture}") + self.last_announced_gesture = gesture self.last_announced_time = time.time() @@ -1837,6 +2218,9 @@ def __init__(self, speech_manager=None): 'performance_good': (0, 255, 0), # 绿色 'performance_warning': (255, 165, 0), # 橙色 'performance_bad': (255, 0, 0), # 红色 + 'recording': (255, 50, 50), # 亮红色 + 'playback': (50, 50, 255), # 亮蓝色 + 'paused': (255, 255, 0), # 黄色 } print("✓ 中文UI渲染器已初始化") @@ -1890,7 +2274,8 @@ def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): size / 25, color, 1) return frame - def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time): + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, + trajectory_recorder=None): """绘制状态栏""" h, w = frame.shape[:2] @@ -1900,7 +2285,7 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 增强版" + title = "手势控制无人机系统 - 轨迹记录版" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -1927,12 +2312,32 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) + # 录制/回放状态 + if trajectory_recorder and config.get('display', 'show_recording_status'): + recorder_status = trajectory_recorder.get_status() + + if recorder_status['is_recording']: + status_color = self.colors['recording'] + status_text = f"录制中: {recorder_status['trajectory_points']}点" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=16, color=status_color) + elif recorder_status['is_playing']: + if recorder_status['playback_paused']: + status_color = self.colors['paused'] + status_text = f"回放暂停: {recorder_status['playback_index']}/{recorder_status['playback_total']}" + else: + status_color = self.colors['playback'] + status_text = f"回放中: {recorder_status['playback_index']}/{recorder_status['playback_total']}" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=16, color=status_color) + else: + status_text = f"轨迹点: {recorder_status['trajectory_points']}" + frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=self.colors['info']) + # 语音状态 if config.get('display', 'show_speech_status') and self.speech_manager: speech_status = self.speech_manager.get_status() speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" - frame = self.draw_text(frame, speech_text, (w // 2, 65), size=16, color=speech_color) + frame = self.draw_text(frame, speech_text, (w // 2, 90), size=14, color=speech_color) # 性能信息 if config.get('display', 'show_fps'): @@ -1966,17 +2371,18 @@ def draw_help_bar(self, frame): h, w = frame.shape[:2] # 绘制底部帮助栏 - cv2.rectangle(frame, (0, h - 80), (w, h), (0, 0, 0), -1) + cv2.rectangle(frame, (0, h - 100), (w, h), (0, 0, 0), -1) # 帮助文本 help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音 P:性能报告" + "V:切换语音反馈 M:测试语音 P:性能报告", + "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" ] for i, line in enumerate(help_lines): - y_pos = h - 65 + i * 20 + y_pos = h - 85 + i * 20 frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame @@ -2055,6 +2461,10 @@ def main(): ui_renderer = ChineseUIRenderer(speech_manager) performance_monitor = PerformanceMonitor() + # 新增:手势轨迹记录器 + print("初始化手势轨迹记录器...") + trajectory_recorder = GestureTrajectoryRecorder(speech_manager) + # 初始化摄像头 cap = None try: @@ -2095,12 +2505,13 @@ def main(): # 显示欢迎信息 print("\n" + "=" * 60) - print("手势控制无人机系统 - 增强版") + print("手势控制无人机系统 - 轨迹记录版") print("=" * 60) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") + print(f" 轨迹记录: 支持录制/回放功能") print(f" 手势状态: 支持开始/活跃/结束状态追踪") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) @@ -2109,18 +2520,23 @@ def main(): print("\n操作说明:") print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") - print("3. 手势控制改进:") - print(" - 系统会自动检测手势开始、稳定和结束状态") + print("3. 手势控制:") + print(" - 手势识别后会有语音提示: 向上、向下、向左、向右、向前、停止") print(" - 手势稳定性越高,识别越准确") print(" - 手部距离摄像头适中时效果最佳") print(" * 手势识别置信度 > 60% 时才会执行") - print("4. 键盘控制:") - print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") - print("5. 调试功能:") + print("4. 轨迹记录功能:") + print(" [1]开始录制 [2]停止录制 [3]保存轨迹 [4]回放轨迹 [5]清除轨迹 [6]暂停/继续") + print(" - 录制时系统会记录手部位置和手势") + print(" - 回放时可以查看录制的轨迹") + print(" - 轨迹数据自动保存到 trajectories/ 目录") + print("5. 键盘控制:") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print("6. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") - print("6. 语音控制:") + print("7. 语音控制:") print(" [V]切换语音反馈 [M]测试语音 [P]性能报告") - print("7. 按 [ESC] 安全退出") + print("8. 按 [ESC] 安全退出") print("=" * 60) print("程序启动成功!") print("-" * 60) @@ -2133,6 +2549,7 @@ def main(): ord('d'): "Right", ord('D'): "Right", ord('f'): "Forward", ord('F'): "Forward", ord('x'): "Stop", ord('X'): "Stop", + ord('h'): "Hover", ord('H'): "Hover", } # 显示模式 @@ -2173,6 +2590,8 @@ def main(): config.set('display', 'show_fingertips', value=True) config.set('display', 'show_gesture_history', value=True) config.set('display', 'show_stability_indicator', value=True) + config.set('display', 'show_trajectory', value=True) + config.set('display', 'show_recording_status', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -2182,6 +2601,8 @@ def main(): config.set('display', 'show_hand_direction', value=True) config.set('display', 'show_gesture_history', value=True) config.set('display', 'show_stability_indicator', value=True) + config.set('display', 'show_trajectory', value=True) + config.set('display', 'show_recording_status', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) @@ -2189,12 +2610,18 @@ def main(): config.set('display', 'show_fingertips', value=False) config.set('display', 'show_gesture_history', value=False) config.set('display', 'show_stability_indicator', value=False) + config.set('display', 'show_trajectory', value=True) + config.set('display', 'show_recording_status', value=True) config.set('display', 'show_debug_info', value=False) + # 绘制轨迹(如果启用) + if config.get('display', 'show_trajectory'): + frame = trajectory_recorder.draw_trajectory(frame) + # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, - perf_stats['fps'], process_time + perf_stats['fps'], process_time, trajectory_recorder ) frame = ui_renderer.draw_help_bar(frame) @@ -2205,7 +2632,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Enhanced', frame) + cv2.imshow('Gesture Controlled Drone - Trajectory Recording', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -2296,6 +2723,49 @@ def main(): else: speech_manager.speak_direct("系统性能警告,请检查") + # 新增:轨迹记录控制 + elif key == ord('1'): + # 开始录制轨迹 + if trajectory_recorder.start_recording(): + print("✅ 开始录制手势轨迹") + else: + print("⚠ 已经在录制中") + + elif key == ord('2'): + # 停止录制轨迹 + if trajectory_recorder.stop_recording(): + print("✅ 停止录制手势轨迹") + else: + print("⚠ 当前没有在录制") + + elif key == ord('3'): + # 保存轨迹 + if trajectory_recorder.save_trajectory(): + print("✅ 轨迹保存成功") + else: + print("⚠ 没有轨迹数据可保存") + + elif key == ord('4'): + # 回放轨迹 + if trajectory_recorder.start_playback(): + print("✅ 开始回放手势轨迹") + else: + print("⚠ 没有轨迹数据可回放") + + elif key == ord('5'): + # 清除轨迹 + if trajectory_recorder.clear_trajectory(): + print("✅ 轨迹数据已清除") + else: + print("⚠ 清除轨迹失败") + + elif key == ord('6'): + # 暂停/继续回放 + if trajectory_recorder.pause_playback(): + print("✅ 切换回放暂停状态") + else: + print("⚠ 当前没有在回放") + elif key in key_to_gesture: # 键盘控制 simulated_gesture = key_to_gesture[key] @@ -2312,6 +2782,24 @@ def main(): drone_controller.connected and drone_controller.flying): drone_controller.move_by_gesture(gesture, confidence) + # 处理轨迹记录 + if cap and ret: + # 如果正在录制,添加轨迹点 + if trajectory_recorder.is_recording: + # 获取手势识别的手部数据 + if hasattr(gesture_recognizer, 'last_hand_data'): + hand_data = gesture_recognizer.last_hand_data + trajectory_recorder.add_trajectory_point( + hand_data, gesture, confidence, frame.shape + ) + + # 如果正在回放,获取回放点 + if trajectory_recorder.is_playing and not trajectory_recorder.playback_paused: + playback_point = trajectory_recorder.get_next_playback_point() + if playback_point: + # 这里可以添加回放点的可视化或处理 + pass + except KeyboardInterrupt: print("\n程序被用户中断") except Exception as e: @@ -2330,6 +2818,12 @@ def main(): time.sleep(1) # 确保语音播报完成 drone_controller.emergency_stop() + + # 自动保存轨迹 + if trajectory_recorder and len(trajectory_recorder.trajectory_data) > 0: + print("自动保存轨迹数据...") + trajectory_recorder.save_trajectory() + config.save_config() print("程序安全退出") diff --git a/src/autonomus_drone_hand_gesture_project/trajectories/trajectory_20251221_085734.pkl b/src/autonomus_drone_hand_gesture_project/trajectories/trajectory_20251221_085734.pkl new file mode 100644 index 0000000000000000000000000000000000000000..2e15536f58fda247e254762d50bee730511e0186 GIT binary patch literal 5540 zcmchbYitx%6vsQ;F0>RSXm}KqheD{dHiTyb$bu+~U8jIh1VgDyyJdFU-IncEK!a32 zloD1{7*sGsi6$UaX%`rk+5z1fH>B2_0)BGYB(w6^?S-fZhxunoOOP|nf=uRgW=mWcc37zw4}@% z@|Fa2r{fqu$LY9Kg>7S{9!_S*_e!`_dC;SW6BdR_O7(D`{E|SS zx4;v~hp_>Ha)0UCjQo}hNbPR9?2uzd;w?XDAQsscZ5w&iWo!Bt+Skg#5( z0iGXWV98|HDBJdc>_s3u$GD-kSR%W|zCV-6el#k(?K-v_O!jB$Bi6Ei5oHw^vbfH6 zG4Jo5e@V4Dqwp7+h@Wql3`EzW*uU;9hiX4>FA)fL9?q7MRi+q zpCTa~b2NoCT_4Z@&mX0b@_T52=Nl-b{7xF+`PV6=d<_lod~XWnRa`SQ@kQ_@48FWz z@@2Ekm$^UN(3f^9ht_*o0|9pRu%c#I`|>3T;kqu+^pWnj#QZrUFP+b<3nxJLhoJi- zlkS5u-PvbrneIZ?;OR5USglvoG;7^wL|r%Oen-q-?t$+44GEz86zD!}(%m4_z3AW= z)ZIb4n~YUM{aSHd2M2FnHCjYjLntJ>zoG#?9kVH<{4pBf`G+Z_{9zj4`ELzuK2Rs- z4;p#tbcVO3A?MkY=sTko&VdGZw7ZU|>DJTp7aU=Ft*xiqlFln)evXls>e#H0)Z%@K zaHMAUC1Dq{*VhL_>vvwwmffS6ezsh|deOQU+dihhUY*=cKey>`QQotLJh$vNdi5HR zcQ=0T91iUu!w*d&yiTZI*(TT64(+v#ZnxclR0p27jnT`DGcBzMJ=)h*hPvs!21*u zz$y`7ypfjz^#5@>0OX)lyKgae2EgMON&p7m7$g96T*P*X01B-Eib)X%$fA${%0+;2 zMqUaq0BE02tZEg;Yd^ zbmp+eoV=>2erqSEiB9DjH{#ZAp#dJEKWuFzryb(?la0L8=`l-E!D+lOIK34+!)YBu zNvVvz7-3G+ORJOrV1S*98n6a@nG{I?W{3c$Qj!9sx19rk_c*|Q5dil}GJtX4=P-c2 zY>|mAI}G5mqOR%@fR}hAGqBLeO993=KLY@tqg1;OU(2Q$IF6xY1~MY?T23 z(+o68XCOs%c!eR5Pf5I(f2Di=T^itrV~GeQxSIxeekp}yP%rWNxSU4Mml`y+WjPr1 z9UKV;HDYI!=y?n!2FrGb{z1**FaV@xlOz z6JP>Nz)%AC)cPq1*xt@&9so@n$CHX$W?jOo$#SkY_DIqUrHcTjoud?B&C^`~FpL8X z7Xe1d0U~p%zF;#%FU%J4h1uU4V7(}?6XsVa!EX5nh?f92(Ev9fk3#vDCNlTbF#w#v z0Vj%plWqa1PI?0YKjvIcN5r&cbhz#XMX^$5UwYuf4OT0|;p&D{2;Xrebr6|1rUlf^ zK5EW-NJFr|AV@m%xX}V%hfFx)#cqn-I9t1RCl#8x*cpmXQKPr6q|axb#WBJ z6;1Zx_FgECfSAYsL85tCbw`%nf%X=mRJ*SjyV#`p{1{q4o{=X96(GQsaP$q+WZH~$ zDK7lJ61!0BBU$Wm)(8mWpAs`+;K_L~lN8}sTJ+F!R9(+ECtWf3we+81-ea~SR NlkIyRLo>f8;~#jwO6>pu literal 0 HcmV?d00001 diff --git a/src/autonomus_drone_hand_gesture_project/trajectories/trajectory_temp.pkl b/src/autonomus_drone_hand_gesture_project/trajectories/trajectory_temp.pkl new file mode 100644 index 0000000000000000000000000000000000000000..03d88104f34a56b85d410659b9b7749aaba9edb9 GIT binary patch literal 4719 zcmbW4Yitx%6vsQ;9cU?rpy91NjD$*SLqLH5vLMQ0*J)HJN+Ob_-7>qd+p@bA&>(y& zC5)(WtAZiXXaWLC3qDA&P!TWzYJ#W;$kV<=0zP7*(TL~VyQg>VGB*O@tvGLoo( zT)#wZyk^(_bbTn(@~nCJAWiY69ClIdtoip^(sZ@Z7< zi{3AvV&CEM3_lX``{9N=XZTBuOjyUlPC->N9FHtW*F48_S>f2!|rM4+4=SWd!RkY9_c;3`LEMZ#Qpb52SlP?tv6Fx$09n+1Aj)rmC@iz zkp!}fb<=Nb>W{9RJdUF^82Diyj!!|L89HJW8t})>w!BL^gvv`464o1(5cS~_ODDV0 zXxIj_=YZ^T`9gcHMs}mQYa*BZSVngHH5}Ku?9Z%6y=DKZ%BqyIgw77N9?!17qSgc1 z_4R7~K3Ugd7wo(O*ipcaxv&@OuqS-860zI&jKdKObkg2no7Qf_3VU-OBOwBFB!$$j zk0>GP2PvfbHcE(kD}_|wLJ3iSlR~ODQbN>wQm9|Wjg#_T245WTvg`&`o)L7 zbkK5WyPFRXU|RbPtHj%vFG+~db(Y#ky5CXjr(|8*&(gC;K=;R>`xBS$y*k}xC!4tL zDn8)XCzr9+Vpv7qx=*OOZqogpTEBD)x*u=#gYILX`?yPYt4{Zvy`xZfC+WT*cMa=n zBXkWMym{BStjZcpA=&*EB}6-BP)PMdlo0htD5Uy+N{ITm5?dsi)%spp*S6DXC_>JY zgED7E8{7i}-e`snSjFD$`5R`qj`z2s<~H2QzImmXDWidt<#Y z^gi<%H*CF!>*w1A>=$j@aqQswTdeU}`h`vZsPbNv^1`~;u${|6-d#f8P*vVAy}YKJ zqcc%n`@D?3=nM>$XYDYoWm)<%hjJS!5_(ebZ!un^ zzqpNSz?<@Z?}q?gs2{&6y6`wU-pT>48dlmH;4M-l1n!}b0G{zF3uuC@>lR2;!37Hd zrlC|bHXSb#;2{jX7f4f|iya(5=mr3>QaHfRhPBum;B8VQ06wIU0G6r%V`W_n(D$b) z08oxn&DdPLNPwp>)Bp^;K0pEJJcr{v0aSSdEFnb#pp-%as8s>R$hsC_@E>ge5JRbE zY%yLW0KQHKFmxS!0f6x>eCflU>H@fKSpO3ss{`Y10boSC5VATWq^q0{?&KB2ihDab zS#@f@e37tr10_U={_wq#oDQh>kC%0=)1#g)1gG)A;PghkNT(Twnx!)O+-~kP9a>%D z4-VL6SP5^yS4oiuV5$n>T1r}g;`UPj@PPo>r2^o$k`7?Z_vIX*H{WEsw;c{}$*`8* z5so-Z7heE?&rzxw!(-Xifx{STIxz7(KaM(oQDeon`j4T&=yML0@p<)%l_2sAqSpNJC02nF&hN%F<^#HqP)_=h} oLLE Date: Sun, 21 Dec 2025 12:15:47 +0800 Subject: [PATCH 11/11] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E9=80=89=E6=8B=A9=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config.json | 27 +- .../main.py | 534 +++++++++++++----- 2 files changed, 410 insertions(+), 151 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index 1ebcdda50e..40656680c9 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -18,25 +18,25 @@ }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 125, + "skin_upper_h": 165, "skin_lower_s": 0, - "skin_upper_s": 31, + "skin_upper_s": 36, "skin_lower_v": 0, - "skin_upper_v": 74, + "skin_upper_v": 69, "min_hand_area": 3000, "max_hand_area": 30000, - "history_size": 10, - "smooth_frames": 5, - "min_confidence": 0.6, - "detection_interval": 2, "hand_ratio_threshold": 1.5, - "contour_simplify_epsilon": 0.02, "defect_distance_threshold": 20, "palm_circle_radius_ratio": 0.3, - "gesture_stability_threshold": 5, "transition_threshold": 0.3, "position_stability_weight": 0.4, "gesture_cooldown": 0.5, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.6, + "detection_interval": 2, + "contour_simplify_epsilon": 0.02, + "gesture_stability_threshold": 5, "adaptive_learning_rate": 0.1, "enable_background_subtraction": true, "enable_dynamic_threshold": true, @@ -85,6 +85,7 @@ "show_stability_indicator": true, "show_trajectory": true, "show_recording_status": true, + "show_performance_mode": true, "font_size_large": 20, "font_size_medium": 16, "font_size_small": 14, @@ -105,6 +106,13 @@ "target_fps": 30, "resize_factor": 1.0, "enable_multiprocessing": false, + "mode": "accurate", + "current_mode_index": 2, + "modes": [ + "fast", + "balanced", + "accurate" + ], "enable_gpu_acceleration": false, "enable_hardware_acceleration": true, "enable_multithreading": false, @@ -133,6 +141,7 @@ "announce_hand_status": true, "announce_performance": true, "announce_recording_events": true, + "announce_performance_mode": true, "min_gesture_confidence": 0.7, "gesture_start_threshold": 3, "gesture_end_threshold": 10, diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index c7a24d5374..3076402e7a 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 手势轨迹记录与回放优化版 -修复手势语音播报问题,确保所有手势语音正常 +手势控制AirSim无人机 - 性能模式选择优化版 +新增fast(最快)、balanced(平衡)、accurate(最准)三种性能模式 作者: xiaoshiyuan888 """ @@ -21,8 +21,8 @@ from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Enhanced with Trajectory Recording") -print("手势轨迹记录与回放优化版!") +print("Gesture Controlled Drone - Performance Mode Selection") +print("性能模式选择优化版!") print("=" * 60) # ========== 修复导入路径 ========== @@ -210,14 +210,14 @@ def __init__(self, speech_lib): 'gesture_left': "向左", 'gesture_right': "向右", 'gesture_forward': "向前", - 'gesture_backward': "向后", # 新增:向后手势 + 'gesture_backward': "向后", 'gesture_waiting': "等待手势", 'gesture_error': "手势识别错误", 'gesture_stable': "手势稳定", 'gesture_change': "手势变化", 'gesture_low_confidence': "手势识别置信度低", 'gesture_good_confidence': "手势识别置信度高", - 'gesture_hover': "悬停", # 新增:悬停手势 + 'gesture_hover': "悬停", # 系统相关 'program_start': "手势控制无人机系统已启动", @@ -232,6 +232,9 @@ def __init__(self, speech_lib): 'debug_mode_off': "调试模式已关闭", 'display_mode_changed': "显示模式已切换", 'help_toggled': "帮助信息已切换", + 'performance_mode_fast': "切换到最快性能模式", + 'performance_mode_balanced': "切换到平衡性能模式", + 'performance_mode_accurate': "切换到最准确性能模式", # 新增:性能相关 'performance_good': "系统运行流畅", @@ -854,6 +857,53 @@ class ConfigManager: def __init__(self): self.config_file = os.path.join(current_dir, 'gesture_config.json') + + # 性能模式配置 + self.performance_modes = { + 'fast': { + 'name': '最快', + 'description': '性能优先,降低识别精度换取更高帧率', + 'detection_interval': 2, # 每2帧检测一次 + 'smooth_frames': 3, # 平滑帧数 + 'min_confidence': 0.5, # 最小置信度阈值 + 'resize_factor': 0.5, # 图像缩放因子 + 'skin_detection_enabled': True, + 'background_subtraction_enabled': False, + 'contour_simplify_epsilon': 0.03, + 'history_size': 10, + 'gesture_stability_threshold': 3, + 'color': (0, 255, 0), # 绿色 + }, + 'balanced': { + 'name': '平衡', + 'description': '平衡性能与精度,适用于大多数场景', + 'detection_interval': 1, # 每帧检测 + 'smooth_frames': 5, # 平滑帧数 + 'min_confidence': 0.6, # 最小置信度阈值 + 'resize_factor': 0.75, # 图像缩放因子 + 'skin_detection_enabled': True, + 'background_subtraction_enabled': True, + 'contour_simplify_epsilon': 0.02, + 'history_size': 15, + 'gesture_stability_threshold': 5, + 'color': (255, 165, 0), # 橙色 + }, + 'accurate': { + 'name': '最准', + 'description': '精度优先,提供最准确的手势识别', + 'detection_interval': 1, # 每帧检测 + 'smooth_frames': 7, # 平滑帧数 + 'min_confidence': 0.7, # 最小置信度阈值 + 'resize_factor': 1.0, # 不缩放 + 'skin_detection_enabled': True, + 'background_subtraction_enabled': True, + 'contour_simplify_epsilon': 0.01, + 'history_size': 20, + 'gesture_stability_threshold': 7, + 'color': (255, 0, 0), # 红色 + } + } + self.default_config = { 'camera': { 'index': 0, @@ -870,18 +920,12 @@ def __init__(self): 'skin_upper_v': 255, 'min_hand_area': 2000, 'max_hand_area': 30000, - 'history_size': 15, # 增加历史记录大小 - 'smooth_frames': 7, # 增加平滑帧数 - 'min_confidence': 0.6, # 提高最小置信度阈值 - 'detection_interval': 1, 'hand_ratio_threshold': 1.5, - 'contour_simplify_epsilon': 0.02, 'defect_distance_threshold': 20, 'palm_circle_radius_ratio': 0.3, - 'gesture_stability_threshold': 5, # 新增:手势稳定性阈值 - 'transition_threshold': 0.3, # 新增:手势转换阈值 - 'position_stability_weight': 0.4, # 新增:位置稳定性权重 - 'gesture_cooldown': 0.5, # 新增:手势冷却时间 + 'transition_threshold': 0.3, + 'position_stability_weight': 0.4, + 'gesture_cooldown': 0.5, }, 'drone': { 'velocity': 2.5, @@ -900,15 +944,19 @@ def __init__(self): 'show_hand_direction': True, 'show_debug_info': False, 'show_speech_status': True, - 'show_gesture_history': True, # 新增:显示手势历史 - 'show_stability_indicator': True, # 新增:显示稳定性指示器 - 'show_trajectory': True, # 新增:显示轨迹 - 'show_recording_status': True, # 新增:显示录制状态 + 'show_gesture_history': True, + 'show_stability_indicator': True, + 'show_trajectory': True, + 'show_recording_status': True, + 'show_performance_mode': True, # 新增:显示性能模式 }, 'performance': { 'target_fps': 30, 'resize_factor': 1.0, - 'enable_multiprocessing': False + 'enable_multiprocessing': False, + 'mode': 'balanced', # 新增:性能模式选择 + 'current_mode_index': 1, # 当前模式索引 + 'modes': ['fast', 'balanced', 'accurate'], # 可用模式列表 }, 'calibration': { 'auto_calibrate_skin': True, @@ -922,21 +970,22 @@ def __init__(self): 'announce_gestures': True, 'announce_connections': True, 'announce_flight_events': True, - 'announce_gesture_changes': True, # 新增:播报手势变化 - 'announce_hand_status': True, # 新增:播报手部状态 - 'announce_performance': True, # 新增:播报性能状态 - 'announce_recording_events': True, # 新增:播报录制事件 + 'announce_gesture_changes': True, + 'announce_hand_status': True, + 'announce_performance': True, + 'announce_recording_events': True, + 'announce_performance_mode': True, # 新增:播报性能模式切换 'min_gesture_confidence': 0.7, - 'gesture_start_threshold': 3, # 新增:手势开始识别阈值 - 'gesture_end_threshold': 10, # 新增:手势结束识别阈值 + 'gesture_start_threshold': 3, + 'gesture_end_threshold': 10, }, - 'recording': { # 新增:录制配置 - 'auto_save_interval': 5, # 自动保存间隔(秒) - 'max_trajectory_points': 1000, # 最大轨迹点数 - 'show_trajectory': True, # 显示轨迹 - 'trajectory_thickness': 2, # 轨迹线粗细 - 'trajectory_max_length': 100, # 显示的最大轨迹长度 - 'default_save_dir': 'trajectories', # 默认保存目录 + 'recording': { + 'auto_save_interval': 5, + 'max_trajectory_points': 1000, + 'show_trajectory': True, + 'trajectory_thickness': 2, + 'trajectory_max_length': 100, + 'default_save_dir': 'trajectories', } } self.config = self.load_config() @@ -1002,6 +1051,56 @@ def set(self, *keys, value): config[keys[-1]] = value self.save_config() + def get_performance_mode_config(self, mode=None): + """获取性能模式配置""" + if mode is None: + mode = self.get('performance', 'mode') + + if mode in self.performance_modes: + return self.performance_modes[mode] + else: + return self.performance_modes['balanced'] + + def get_current_performance_mode(self): + """获取当前性能模式""" + mode = self.get('performance', 'mode') + if mode in self.performance_modes: + return mode + return 'balanced' + + def set_performance_mode(self, mode): + """设置性能模式""" + if mode in self.performance_modes: + self.set('performance', 'mode', value=mode) + + # 更新当前模式索引 + modes = self.get('performance', 'modes') + if modes and mode in modes: + index = modes.index(mode) + self.set('performance', 'current_mode_index', value=index) + + print(f"✓ 性能模式设置为: {self.performance_modes[mode]['name']}") + return True + return False + + def cycle_performance_mode(self): + """循环切换性能模式""" + modes = self.get('performance', 'modes') + if not modes: + modes = ['fast', 'balanced', 'accurate'] + + current_index = self.get('performance', 'current_mode_index') + if current_index is None: + current_index = 0 + + # 计算下一个模式索引 + next_index = (current_index + 1) % len(modes) + next_mode = modes[next_index] + + # 设置新模式 + self.set('performance', 'current_mode_index', value=next_index) + return self.set_performance_mode(next_mode) + def calibrate_skin_color(self, frame, hand_mask): """自动校准肤色范围""" if not self.get('calibration', 'auto_calibrate_skin'): @@ -1052,24 +1151,33 @@ def calibrate_hand_size(self, hand_area): config = ConfigManager() -# ========== 改进的手势识别器(增强平滑处理) ========== +# ========== 改进的手势识别器(支持性能模式) ========== class EnhancedGestureRecognizer: - """增强的手势识别器 - 改进平滑处理和稳定性""" + """增强的手势识别器 - 支持性能模式选择""" def __init__(self, speech_manager=None): - self.config = config.get('gesture') self.speech_manager = speech_manager + # 加载性能模式配置 + self.performance_mode = config.get_current_performance_mode() + self.mode_config = config.get_performance_mode_config(self.performance_mode) + + # 根据性能模式初始化参数 + self.history_size = self.mode_config['history_size'] + self.detection_interval = self.mode_config['detection_interval'] + self.smooth_frames = self.mode_config['smooth_frames'] + self.min_confidence = self.mode_config['min_confidence'] + self.resize_factor = self.mode_config['resize_factor'] + # 增强的手势历史和平滑 - self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) self.confidence_history = deque(maxlen=self.history_size) - self.position_history = deque(maxlen=self.history_size) # 新增:位置历史 + self.position_history = deque(maxlen=self.history_size) self.current_gesture = "Waiting" self.current_confidence = 0.0 # 新增:手势状态追踪 - self.gesture_state = "none" # none, starting, active, ending + self.gesture_state = "none" self.gesture_stability_counter = 0 self.last_stable_gesture = "Waiting" self.gesture_active_frames = 0 @@ -1079,7 +1187,7 @@ def __init__(self, speech_manager=None): self.last_announced_gesture = None self.last_announced_time = 0 self.last_hand_status_time = 0 - self.gesture_announce_interval = 2.0 # 缩短手势播报间隔 + self.gesture_announce_interval = 2.0 # 手部跟踪和状态 self.last_hand_position = None @@ -1092,23 +1200,22 @@ def __init__(self, speech_manager=None): # 性能统计 self.process_times = deque(maxlen=30) self.frame_counter = 0 - self.detection_interval = self.config['detection_interval'] self.last_performance_report = 0 # 手势颜色映射 self.gesture_colors = { - "Stop": (0, 0, 255), # 红色 - "Forward": (0, 255, 0), # 绿色 - "Up": (255, 255, 0), # 青色 - "Down": (255, 0, 255), # 紫色 - "Left": (255, 165, 0), # 橙色 - "Right": (0, 165, 255), # 浅蓝色 - "Waiting": (200, 200, 200), # 灰色 - "Error": (255, 0, 0), # 蓝色 - "Hover": (255, 255, 255) # 白色 + "Stop": (0, 0, 255), + "Forward": (0, 255, 0), + "Up": (255, 255, 0), + "Down": (255, 0, 255), + "Left": (255, 165, 0), + "Right": (0, 165, 255), + "Waiting": (200, 200, 200), + "Error": (255, 0, 0), + "Hover": (255, 255, 255) } - # 手势到语音的映射 - 修复:确保所有手势都有对应语音 + # 手势到语音的映射 self.gesture_speech_map = { "Stop": "gesture_stop", "Forward": "gesture_forward", @@ -1118,34 +1225,71 @@ def __init__(self, speech_manager=None): "Right": "gesture_right", "Waiting": "gesture_waiting", "Error": "gesture_error", - "Hover": "gesture_hover", # 新增悬停手势语音 + "Hover": "gesture_hover", } # 手势状态颜色 self.state_colors = { - "none": (100, 100, 100), # 灰色 - "starting": (255, 165, 0), # 橙色 - "active": (0, 255, 0), # 绿色 - "ending": (255, 0, 0), # 红色 + "none": (100, 100, 100), + "starting": (255, 165, 0), + "active": (0, 255, 0), + "ending": (255, 0, 0), } - # 背景减除器 - self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( - history=100, varThreshold=25, detectShadows=True - ) + # 根据性能模式初始化背景减除器 + self.bg_subtractor = None + if self.mode_config['background_subtraction_enabled']: + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( + history=100, varThreshold=25, detectShadows=True + ) # 形态学操作核 self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - # 新增:性能监控 + # 性能监控 self.avg_process_time = 0 self.frame_rate = 0 self.last_fps_check = time.time() - # 新增:存储手部数据用于轨迹记录 + # 存储手部数据用于轨迹记录 self.last_hand_data = None - print("✓ 增强的手势识别器已初始化 (改进平滑处理)") + # 性能模式信息 + self.performance_mode_color = self.mode_config['color'] + self.performance_mode_name = self.mode_config['name'] + + print(f"✓ 增强的手势识别器已初始化 - 性能模式: {self.performance_mode_name}") + + def set_performance_mode(self, mode): + """设置性能模式""" + self.performance_mode = mode + self.mode_config = config.get_performance_mode_config(mode) + + # 更新参数 + self.history_size = self.mode_config['history_size'] + self.detection_interval = self.mode_config['detection_interval'] + self.smooth_frames = self.mode_config['smooth_frames'] + self.min_confidence = self.mode_config['min_confidence'] + self.resize_factor = self.mode_config['resize_factor'] + + # 更新背景减除器 + if self.mode_config['background_subtraction_enabled'] and self.bg_subtractor is None: + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( + history=100, varThreshold=25, detectShadows=True + ) + elif not self.mode_config['background_subtraction_enabled']: + self.bg_subtractor = None + + # 更新队列大小 + self.gesture_history = deque(maxlen=self.history_size) + self.confidence_history = deque(maxlen=self.history_size) + self.position_history = deque(maxlen=self.history_size) + + # 更新显示信息 + self.performance_mode_color = self.mode_config['color'] + self.performance_mode_name = self.mode_config['name'] + + print(f"✓ 切换到性能模式: {self.performance_mode_name}") def get_skin_mask(self, frame): """获取肤色掩码""" @@ -1165,14 +1309,36 @@ def get_skin_mask(self, frame): def enhance_skin_detection(self, frame, skin_mask): """增强肤色检测""" - fg_mask = self.bg_subtractor.apply(frame) - combined_mask = cv2.bitwise_and(skin_mask, fg_mask) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) + if not self.mode_config['skin_detection_enabled']: + return skin_mask + + if self.bg_subtractor is not None: + fg_mask = self.bg_subtractor.apply(frame) + combined_mask = cv2.bitwise_and(skin_mask, fg_mask) + else: + combined_mask = skin_mask + + # 根据性能模式决定形态学操作次数 + if self.performance_mode == 'accurate': + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) + combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) + elif self.performance_mode == 'balanced': + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=1) + combined_mask = cv2.GaussianBlur(combined_mask, (3, 3), 0) + # fast模式不进行额外的形态学操作 return combined_mask + def preprocess_frame(self, frame): + """预处理帧(根据性能模式调整大小)""" + if self.resize_factor != 1.0: + new_width = int(frame.shape[1] * self.resize_factor) + new_height = int(frame.shape[0] * self.resize_factor) + resized_frame = cv2.resize(frame, (new_width, new_height), interpolation=cv2.INTER_AREA) + return resized_frame + return frame + def find_best_hand_contour(self, mask, frame): """找到最佳的手部轮廓""" contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) @@ -1183,10 +1349,13 @@ def find_best_hand_contour(self, mask, frame): contours = sorted(contours, key=cv2.contourArea, reverse=True) best_contour = None best_score = 0.0 - min_area = config.get('gesture', 'min_hand_area') - max_area = config.get('gesture', 'max_hand_area') + min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) + + # 根据性能模式调整检查的轮廓数量 + max_contours = 3 if self.performance_mode != 'fast' else 1 - for contour in contours[:3]: + for contour in contours[:max_contours]: area = cv2.contourArea(contour) if area < min_area or area > max_area: @@ -1204,8 +1373,8 @@ def rate_contour(self, contour, frame_shape): """评估轮廓作为手部的可能性""" score = 0.0 area = cv2.contourArea(contour) - min_area = config.get('gesture', 'min_hand_area') - max_area = config.get('gesture', 'max_hand_area') + min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) if min_area < area < max_area: area_ratio = min(area / max_area, 1.0) @@ -1246,7 +1415,11 @@ def analyze_hand_features(self, contour, frame_shape): bbox_area = w * h palm_center = (cx, cy) palm_radius = int(w * config.get('gesture', 'palm_circle_radius_ratio')) - fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius) + + # 根据性能模式调整轮廓简化程度 + epsilon = self.mode_config['contour_simplify_epsilon'] * cv2.arcLength(contour, True) + fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius, epsilon) + direction = self.calculate_hand_direction(contour, cx, cy) h_img, w_img = frame_shape[:2] norm_x = cx / w_img @@ -1271,9 +1444,8 @@ def analyze_hand_features(self, contour, frame_shape): return result, confidence - def analyze_fingers(self, contour, palm_center, palm_radius): + def analyze_fingers(self, contour, palm_center, palm_radius, epsilon): """分析手指""" - epsilon = config.get('gesture', 'contour_simplify_epsilon') * cv2.arcLength(contour, True) approx = cv2.approxPolyDP(contour, epsilon, True) hull = cv2.convexHull(approx, returnPoints=False) @@ -1356,8 +1528,8 @@ def calculate_hand_direction(self, contour, cx, cy): def calculate_confidence(self, area, finger_count, contour_length, bbox_area): """计算手势置信度""" confidence = 0.5 - min_area = config.get('gesture', 'min_hand_area') - max_area = config.get('gesture', 'max_hand_area') + min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) if min_area < area < max_area: area_norm = (area - min_area) / (max_area - min_area) @@ -1479,7 +1651,7 @@ def smooth_gesture_enhanced(self, new_gesture, new_confidence, hand_data): position_stability = max(0, 1.0 - position_diff * 5.0) # 增强的稳定性检查 - stability_threshold = config.get('gesture', 'gesture_stability_threshold') + stability_threshold = self.mode_config['gesture_stability_threshold'] transition_threshold = config.get('gesture', 'transition_threshold') position_weight = config.get('gesture', 'position_stability_weight') @@ -1532,8 +1704,8 @@ def update_gesture_state(self, hand_data, gesture, confidence): # 检查手部大小和位置 hand_area = hand_data['area'] - min_area = config.get('gesture', 'min_hand_area') - max_area = config.get('gesture', 'max_hand_area') + min_area = config.get('gesture', 'min_hand_area') * (self.resize_factor ** 2) + max_area = config.get('gesture', 'max_hand_area') * (self.resize_factor ** 2) # 提供手部位置反馈 if (self.speech_manager and @@ -1686,7 +1858,7 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): # 计算稳定性指示条长度 stability_level = min(1.0, - self.gesture_stability_counter / config.get('gesture', 'gesture_stability_threshold')) + self.gesture_stability_counter / self.mode_config['gesture_stability_threshold']) bar_length = int(70 * stability_level) # 根据稳定性级别选择颜色 @@ -1756,10 +1928,15 @@ def recognize(self, frame): # 预处理帧 processed_frame = cv2.flip(frame, 1) + # 根据性能模式调整图像大小 + original_frame = processed_frame.copy() + if self.resize_factor != 1.0: + processed_frame = self.preprocess_frame(processed_frame) + # 每隔几帧检测一次以提高性能 if self.frame_counter % self.detection_interval != 0: self.frame_counter += 1 - return self.current_gesture, self.current_confidence, processed_frame + return self.current_gesture, self.current_confidence, original_frame # 获取肤色掩码 skin_mask, hsv = self.get_skin_mask(processed_frame) @@ -1798,8 +1975,7 @@ def recognize(self, frame): self.update_gesture_state(None, gesture, confidence) final_gesture, final_confidence = gesture, confidence - # ========== 修复手势语音播报 ========== - # 确保所有手势都有语音反馈 + # 手势语音播报 if (self.speech_manager and config.get('speech', 'enabled') and config.get('speech', 'announce_gestures')): @@ -1814,19 +1990,17 @@ def recognize(self, frame): self.speech_manager.speak('gesture_low_confidence', immediate=True) self.last_hand_status_time = current_time - # 手势语音播报 - 修复:确保所有手势都能播报 + # 手势语音播报 if (final_gesture != "Waiting" and final_gesture != "Error" and final_gesture != "摄像头错误" and - final_confidence >= config.get('speech', 'min_gesture_confidence') and + final_confidence >= self.min_confidence and current_time - self.last_announced_time > self.gesture_announce_interval): - # 检查手势是否在语音映射中 if final_gesture in self.gesture_speech_map: speech_key = self.gesture_speech_map[final_gesture] self.speech_manager.speak(speech_key) else: - # 如果手势不在映射中,直接播报手势名称 self.speech_manager.speak_direct(f"手势{final_gesture}") self.last_announced_gesture = final_gesture @@ -1834,22 +2008,44 @@ def recognize(self, frame): # 性能报告 current_time = time.time() - if current_time - self.last_performance_report > 30.0: # 每30秒报告一次 + if current_time - self.last_performance_report > 30.0: if (self.speech_manager and config.get('speech', 'enabled') and config.get('speech', 'announce_performance')): - if self.avg_process_time < 20: # 处理时间小于20ms + if self.avg_process_time < 20: self.speech_manager.speak('performance_good', immediate=True) - elif self.avg_process_time > 50: # 处理时间大于50ms + elif self.avg_process_time > 50: self.speech_manager.speak('performance_warning', immediate=True) self.last_performance_report = current_time - # 可视化结果 + # 可视化结果(在原图上) if hand_data is not None: - processed_frame = self.visualize_detection( - processed_frame, hand_data, final_gesture, final_confidence + # 需要将坐标转换回原始图像大小 + if self.resize_factor != 1.0: + # 缩放坐标 + scale_factor = 1.0 / self.resize_factor + if 'center' in hand_data: + hand_data['center'] = (int(hand_data['center'][0] * scale_factor), + int(hand_data['center'][1] * scale_factor)) + if 'bbox' in hand_data: + x1, y1, x2, y2 = hand_data['bbox'] + hand_data['bbox'] = (int(x1 * scale_factor), int(y1 * scale_factor), + int(x2 * scale_factor), int(y2 * scale_factor)) + if 'fingertips' in hand_data: + hand_data['fingertips'] = [(int(x * scale_factor), int(y * scale_factor)) + for (x, y) in hand_data['fingertips']] + if 'palm_center' in hand_data: + hand_data['palm_center'] = (int(hand_data['palm_center'][0] * scale_factor), + int(hand_data['palm_center'][1] * scale_factor)) + + original_frame = self.visualize_detection( + original_frame, hand_data, final_gesture, final_confidence + ) + else: + original_frame = self.visualize_detection( + original_frame, None, final_gesture, final_confidence ) # 更新计数器 @@ -1870,7 +2066,7 @@ def recognize(self, frame): self.frame_counter = 0 self.last_fps_check = current_time - return final_gesture, final_confidence, processed_frame + return final_gesture, final_confidence, original_frame except Exception as e: print(f"⚠ 手势识别错误: {e}") @@ -1883,12 +2079,24 @@ def get_performance_stats(self): return np.mean(list(self.process_times)), self.frame_rate + def get_performance_mode_info(self): + """获取性能模式信息""" + return { + 'name': self.performance_mode_name, + 'mode': self.performance_mode, + 'color': self.performance_mode_color, + 'detection_interval': self.detection_interval, + 'resize_factor': self.resize_factor, + 'smooth_frames': self.smooth_frames, + 'min_confidence': self.min_confidence + } + def set_simulated_gesture(self, gesture): """设置模拟的手势""" self.current_gesture = gesture self.current_confidence = 0.9 - # 模拟手势也触发语音提示 - 修复:确保所有模拟手势都有语音 + # 模拟手势也触发语音提示 if (self.speech_manager and config.get('speech', 'enabled') and config.get('speech', 'announce_gestures')): @@ -1896,7 +2104,6 @@ def set_simulated_gesture(self, gesture): if gesture in self.gesture_speech_map: self.speech_manager.speak(self.gesture_speech_map[gesture]) else: - # 如果手势不在映射中,直接播报手势名称 self.speech_manager.speak_direct(f"手势{gesture}") self.last_announced_gesture = gesture @@ -2116,7 +2323,7 @@ def move_by_gesture(self, gesture, confidence): if (self.speech_manager and config.get('speech', 'enabled') and config.get('speech', 'announce_gestures') and - confidence < min_confidence * 0.8): # 如果置信度特别低 + confidence < min_confidence * 0.8): self.speech_manager.speak('gesture_low_confidence') return False @@ -2205,22 +2412,25 @@ def __init__(self, speech_manager=None): # 颜色定义 self.colors = { - 'title': (0, 255, 255), # 青色 - 'connected': (0, 255, 0), # 绿色 - 'disconnected': (0, 0, 255), # 红色 - 'flying': (0, 255, 0), # 绿色 - 'landed': (255, 165, 0), # 橙色 - 'warning': (0, 165, 255), # 浅蓝色 - 'info': (255, 255, 255), # 白色 - 'help': (255, 200, 100), # 浅橙色 - 'speech_enabled': (0, 255, 0), # 绿色 - 'speech_disabled': (255, 0, 0), # 红色 - 'performance_good': (0, 255, 0), # 绿色 - 'performance_warning': (255, 165, 0), # 橙色 - 'performance_bad': (255, 0, 0), # 红色 - 'recording': (255, 50, 50), # 亮红色 - 'playback': (50, 50, 255), # 亮蓝色 - 'paused': (255, 255, 0), # 黄色 + 'title': (0, 255, 255), + 'connected': (0, 255, 0), + 'disconnected': (0, 0, 255), + 'flying': (0, 255, 0), + 'landed': (255, 165, 0), + 'warning': (0, 165, 255), + 'info': (255, 255, 255), + 'help': (255, 200, 100), + 'speech_enabled': (0, 255, 0), + 'speech_disabled': (255, 0, 0), + 'performance_good': (0, 255, 0), + 'performance_warning': (255, 165, 0), + 'performance_bad': (255, 0, 0), + 'recording': (255, 50, 50), + 'playback': (50, 50, 255), + 'paused': (255, 255, 0), + 'performance_fast': (0, 255, 0), # 绿色 + 'performance_balanced': (255, 165, 0), # 橙色 + 'performance_accurate': (255, 0, 0), # 红色 } print("✓ 中文UI渲染器已初始化") @@ -2274,8 +2484,8 @@ def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): size / 25, color, 1) return frame - def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, - trajectory_recorder=None): + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time, trajectory_recorder=None, + gesture_recognizer=None): """绘制状态栏""" h, w = frame.shape[:2] @@ -2285,7 +2495,7 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 轨迹记录版" + title = "手势控制无人机系统 - 性能模式版" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -2300,11 +2510,11 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro # 手势信息 if confidence > 0.7: - gesture_color = (0, 255, 0) # 绿色 + gesture_color = (0, 255, 0) elif confidence > 0.5: - gesture_color = (255, 165, 0) # 橙色 + gesture_color = (255, 165, 0) else: - gesture_color = (200, 200, 200) # 灰色 + gesture_color = (200, 200, 200) gesture_text = f"当前手势: {gesture}" if config.get('display', 'show_confidence'): @@ -2312,14 +2522,29 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) + # 性能模式显示 + if gesture_recognizer and config.get('display', 'show_performance_mode'): + mode_info = gesture_recognizer.get_performance_mode_info() + + # 根据模式选择颜色 + if mode_info['mode'] == 'fast': + mode_color = self.colors['performance_fast'] + elif mode_info['mode'] == 'balanced': + mode_color = self.colors['performance_balanced'] + else: # accurate + mode_color = self.colors['performance_accurate'] + + mode_text = f"性能模式: {mode_info['name']}" + frame = self.draw_text(frame, mode_text, (w // 2, 65), size=14, color=mode_color) + # 录制/回放状态 - if trajectory_recorder and config.get('display', 'show_recording_status'): + elif trajectory_recorder and config.get('display', 'show_recording_status'): recorder_status = trajectory_recorder.get_status() if recorder_status['is_recording']: status_color = self.colors['recording'] status_text = f"录制中: {recorder_status['trajectory_points']}点" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=16, color=status_color) + frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) elif recorder_status['is_playing']: if recorder_status['playback_paused']: status_color = self.colors['paused'] @@ -2327,17 +2552,17 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro else: status_color = self.colors['playback'] status_text = f"回放中: {recorder_status['playback_index']}/{recorder_status['playback_total']}" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=16, color=status_color) + frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=status_color) else: status_text = f"轨迹点: {recorder_status['trajectory_points']}" - frame = self.draw_text(frame, status_text, (w // 2, 65), size=14, color=self.colors['info']) + frame = self.draw_text(frame, status_text, (w // 2, 65), size=12, color=self.colors['info']) # 语音状态 if config.get('display', 'show_speech_status') and self.speech_manager: speech_status = self.speech_manager.get_status() speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" - frame = self.draw_text(frame, speech_text, (w // 2, 90), size=14, color=speech_color) + frame = self.draw_text(frame, speech_text, (w // 2, 90), size=12, color=speech_color) # 性能信息 if config.get('display', 'show_fps'): @@ -2355,11 +2580,11 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro else: perf_color = self.colors['info'] - frame = self.draw_text(frame, perf_text, (w - 200, 65), size=14, color=perf_color) + frame = self.draw_text(frame, perf_text, (w - 200, 65), size=12, color=perf_color) # 控制提示 control_text = "提示: 确保手部完全进入画面,保持稳定手势" - frame = self.draw_text(frame, control_text, (10, 90), size=14, color=self.colors['info']) + frame = self.draw_text(frame, control_text, (10, 90), size=12, color=self.colors['info']) return frame @@ -2377,7 +2602,7 @@ def draw_help_bar(self, frame): help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音 P:性能报告", + "V:切换语音反馈 M:测试语音 P:性能报告 O:切换性能模式", "1:开始录制 2:停止录制 3:保存轨迹 4:回放轨迹 5:清除轨迹 6:暂停/继续" ] @@ -2393,7 +2618,7 @@ def draw_warning(self, frame, message): # 在顶部绘制警告 warning_bg = np.zeros((40, w, 3), dtype=np.uint8) - warning_bg[:, :] = (0, 69, 255) # 橙色 + warning_bg[:, :] = (0, 69, 255) frame[120:160, 0:w] = cv2.addWeighted( frame[120:160, 0:w], 0.3, @@ -2461,7 +2686,7 @@ def main(): ui_renderer = ChineseUIRenderer(speech_manager) performance_monitor = PerformanceMonitor() - # 新增:手势轨迹记录器 + # 手势轨迹记录器 print("初始化手势轨迹记录器...") trajectory_recorder = GestureTrajectoryRecorder(speech_manager) @@ -2505,14 +2730,14 @@ def main(): # 显示欢迎信息 print("\n" + "=" * 60) - print("手势控制无人机系统 - 轨迹记录版") + print("手势控制无人机系统 - 性能模式版") print("=" * 60) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") + print(f" 性能模式: {gesture_recognizer.performance_mode_name}") print(f" 轨迹记录: 支持录制/回放功能") - print(f" 手势状态: 支持开始/活跃/结束状态追踪") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) @@ -2520,23 +2745,28 @@ def main(): print("\n操作说明:") print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") - print("3. 手势控制:") + print("3. 性能模式选择:") + print(" - 按 [O] 键循环切换性能模式: 最快(fast) → 平衡(balanced) → 最准(accurate)") + print(" - 最快模式: 性能优先,帧率最高,识别精度较低") + print(" - 平衡模式: 性能与精度平衡,适用于大多数场景") + print(" - 最准模式: 精度优先,提供最准确的手势识别") + print("4. 手势控制:") print(" - 手势识别后会有语音提示: 向上、向下、向左、向右、向前、停止") print(" - 手势稳定性越高,识别越准确") print(" - 手部距离摄像头适中时效果最佳") print(" * 手势识别置信度 > 60% 时才会执行") - print("4. 轨迹记录功能:") + print("5. 轨迹记录功能:") print(" [1]开始录制 [2]停止录制 [3]保存轨迹 [4]回放轨迹 [5]清除轨迹 [6]暂停/继续") print(" - 录制时系统会记录手部位置和手势") print(" - 回放时可以查看录制的轨迹") print(" - 轨迹数据自动保存到 trajectories/ 目录") - print("5. 键盘控制:") + print("6. 键盘控制:") print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") - print("6. 调试功能:") + print("7. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") - print("7. 语音控制:") + print("8. 语音控制:") print(" [V]切换语音反馈 [M]测试语音 [P]性能报告") - print("8. 按 [ESC] 安全退出") + print("9. 按 [ESC] 安全退出") print("=" * 60) print("程序启动成功!") print("-" * 60) @@ -2592,6 +2822,7 @@ def main(): config.set('display', 'show_stability_indicator', value=True) config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) + config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -2603,6 +2834,7 @@ def main(): config.set('display', 'show_stability_indicator', value=True) config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) + config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) @@ -2612,6 +2844,7 @@ def main(): config.set('display', 'show_stability_indicator', value=False) config.set('display', 'show_trajectory', value=True) config.set('display', 'show_recording_status', value=True) + config.set('display', 'show_performance_mode', value=True) config.set('display', 'show_debug_info', value=False) # 绘制轨迹(如果启用) @@ -2621,7 +2854,7 @@ def main(): # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, - perf_stats['fps'], process_time, trajectory_recorder + perf_stats['fps'], process_time, trajectory_recorder, gesture_recognizer ) frame = ui_renderer.draw_help_bar(frame) @@ -2632,7 +2865,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Trajectory Recording', frame) + cv2.imshow('Gesture Controlled Drone - Performance Mode', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -2723,6 +2956,23 @@ def main(): else: speech_manager.speak_direct("系统性能警告,请检查") + elif key == ord('o') or key == ord('O'): + # 切换性能模式 + if config.cycle_performance_mode(): + new_mode = config.get_current_performance_mode() + gesture_recognizer.set_performance_mode(new_mode) + + # 语音提示 + if speech_manager.enabled: + if new_mode == 'fast': + speech_manager.speak('performance_mode_fast', immediate=True) + elif new_mode == 'balanced': + speech_manager.speak('performance_mode_balanced', immediate=True) + else: # accurate + speech_manager.speak('performance_mode_accurate', immediate=True) + + print(f"✓ 已切换到性能模式: {gesture_recognizer.performance_mode_name}") + # 新增:轨迹记录控制 elif key == ord('1'): # 开始录制轨迹 @@ -2815,7 +3065,7 @@ def main(): # 程序退出语音提示 if speech_manager.enabled: speech_manager.speak('program_exit', force=True, immediate=True) - time.sleep(1) # 确保语音播报完成 + time.sleep(1) drone_controller.emergency_stop()