From 18d483948dbb0f21f8bb2973bd18fa60d66d44b5 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Thu, 18 Dec 2025 16:16:41 +0800 Subject: [PATCH 1/8] =?UTF-8?q?new=E5=88=86=E6=94=AF=EF=BC=9A1.=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0.gitignore=E5=BF=BD=E7=95=A5Lib=E7=9B=AE=E5=BD=95=202.?= =?UTF-8?q?=E4=BF=AE=E6=94=B9main.py=E4=BC=98=E5=8C=96=E6=97=A0=E4=BA=BA?= =?UTF-8?q?=E6=9C=BA=E6=8E=A7=E5=88=B6=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 6 + .../main.py | 546 ++++++++++++++++++ 2 files changed, 552 insertions(+) create mode 100644 src/autonomus_drone_hand_gesture_project/.gitignore diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore new file mode 100644 index 0000000000..ad298e3e2d --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -0,0 +1,6 @@ +# 忽略整个Lib目录(包含所有子文件/子目录) +Lib/ +# 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 +__pycache__/ +*.pyc +*.dll.a \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index e69de29bb2..b77d05ba12 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -0,0 +1,546 @@ +# -*- coding: utf-8 -*- +""" +手势控制AirSim无人机 - 修复连接问题版 +确保正确连接AirSim,移除模拟模式 +作者: xiaoshiyuan888 +""" + +import sys +import os +import time +import traceback +import subprocess +from PIL import Image, ImageDraw, ImageFont +import cv2 +import numpy as np + +print("=" * 60) +print("Gesture Controlled Drone - Fixed Connection Version") +print("=" * 60) + +# ========== 修复导入路径 ========== +# 添加当前目录到Python路径 +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + + +# ========== 核心模块导入与异常处理 ========== +def safe_import(): + """安全导入所有模块""" + modules_status = {} + + # 1. 导入PIL + try: + from PIL import Image, ImageDraw, ImageFont + modules_status['PIL'] = True + print("[PIL] ✓ Image processing library ready") + except Exception as e: + modules_status['PIL'] = False + print(f"[PIL] ✗ Import failed: {e}") + return None, modules_status + + # 2. 导入OpenCV + try: + import cv2 + import numpy as np + modules_status['OpenCV'] = True + print("[OpenCV] ✓ Computer vision library ready") + except Exception as e: + modules_status['OpenCV'] = False + print(f"[OpenCV] ✗ Import failed: {e}") + return None, modules_status + + # 3. 导入MediaPipe + mp = None + mp_hands = None + mp_drawing = None + try: + import mediapipe as mp + mp_hands = mp.solutions.hands + mp_drawing = mp.solutions.drawing_utils + modules_status['MediaPipe'] = True + print("[MediaPipe] ✓ Gesture recognition library ready") + except Exception as e: + modules_status['MediaPipe'] = False + print(f"[MediaPipe] ✗ Import failed: {e}") + print(" Please install: pip install mediapipe") + + # 4. 尝试导入AirSim - 关键修复部分 + airsim_module = None + airsim_client = None + + # 尝试多种导入方式 + import_methods = [ + lambda: __import__('airsim'), + lambda: __import__('AirSim'), + ] + + for method in import_methods: + try: + airsim_module = method() + modules_status['AirSim'] = True + print(f"[AirSim] ✓ Successfully imported airsim from {airsim_module.__file__}") + break + except ImportError: + continue + except Exception as e: + print(f"[AirSim] Import error: {e}") + continue + + if not modules_status.get('AirSim', False): + print("\n" + "!" * 60) + print("⚠ AirSim library NOT FOUND!") + print("!" * 60) + print("To install AirSim, run:") + print("1. First install: pip install msgpack-rpc-python") + print("2. Then install: pip install airsim") + print("\nOr from source:") + print(" pip install git+https://github.com/microsoft/AirSim.git") + print("!" * 60) + + # 询问是否自动安装 + print("\nDo you want to try automatic installation? (y/n)") + choice = input().strip().lower() + if choice == 'y': + try: + print("Installing msgpack-rpc-python...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "msgpack-rpc-python"]) + print("Installing AirSim...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "airsim"]) + print("\n✅ Installation complete! Please restart the program.") + except Exception as e: + print(f"Installation failed: {e}") + sys.exit(1) + + return { + 'cv2': cv2, + 'np': np, + 'mp': mp, + 'mp_hands': mp_hands, + 'mp_drawing': mp_drawing, + 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, + 'airsim': airsim_module + }, modules_status + + +# 执行导入 +libs, status = safe_import() +if not status.get('OpenCV', False) or not status.get('PIL', False): + print("\n❌ Core libraries missing, cannot start.") + input("Press Enter to exit...") + sys.exit(1) + +print("-" * 60) +print("✅ Environment check passed, initializing...") +print("-" * 60) + +# 解包库 +cv2, np, mp = libs['cv2'], libs['np'], libs['mp'] +Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] +mp_hands, mp_drawing = libs['mp_hands'], libs['mp_drawing'] + + +# ========== 中文显示模块 ========== +class ChineseTextRenderer: + """中文文本渲染器""" + + def __init__(self): + self.fonts = {} + self.load_fonts() + + def load_fonts(self): + """加载字体""" + font_paths = [ + 'simhei.ttf', + 'C:/Windows/Fonts/simhei.ttf', + 'C:/Windows/Fonts/msyh.ttc' + ] + for path in font_paths: + try: + self.fonts[20] = ImageFont.truetype(path, 20) + self.fonts[24] = ImageFont.truetype(path, 24) + self.fonts[30] = ImageFont.truetype(path, 30) + print(f"✓ Chinese fonts loaded: {path}") + return + except: + continue + print("⚠ No Chinese fonts found, using default") + + def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): + """在图像上绘制中文文本""" + try: + img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + pil_img = Image.fromarray(img_rgb) + draw = ImageDraw.Draw(pil_img) + + font = self.fonts.get(size, self.fonts.get(20)) + + # 绘制阴影 + shadow_color = (0, 0, 0) + shadow_pos = (pos[0] + 1, pos[1] + 1) + draw.text(shadow_pos, text, font=font, fill=shadow_color) + + # 绘制文字 + rgb_color = color[::-1] + draw.text(pos, text, font=font, fill=rgb_color) + + return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) + except: + cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, + size / 30, color, 2) + return frame + + +# 初始化中文渲染器 +chinese_renderer = ChineseTextRenderer() + + +# ========== 手势识别模块 ========== +class GestureRecognizer: + """手势识别器""" + + def __init__(self): + try: + self.hands = mp_hands.Hands( + max_num_hands=1, + min_detection_confidence=0.7, + min_tracking_confidence=0.5 + ) + print("✓ MediaPipe gesture detector initialized") + except Exception as e: + print(f"⚠ MediaPipe init failed: {e}") + self.hands = None + + self.current_gesture = "等待手势" + + def recognize(self, frame): + """识别手势""" + if self.hands is None: + return self.current_gesture + + try: + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + results = self.hands.process(rgb_frame) + + if results.multi_hand_landmarks: + for hand_landmarks in results.multi_hand_landmarks: + # 绘制手部关键点 + mp_drawing.draw_landmarks( + frame, hand_landmarks, mp_hands.HAND_CONNECTIONS, + mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2), + mp_drawing.DrawingSpec(color=(255, 0, 0), thickness=2) + ) + + # 简化的手势判断 + landmarks = hand_landmarks.landmark + if len(landmarks) >= 9: + wrist = landmarks[0] + index_tip = landmarks[8] + + # 手势判断逻辑 + if index_tip.y < wrist.y - 0.2: + self.current_gesture = "向上" + elif index_tip.y > wrist.y + 0.2: + self.current_gesture = "向下" + elif index_tip.x < wrist.x - 0.2: + self.current_gesture = "向左" + elif index_tip.x > wrist.x + 0.2: + self.current_gesture = "向右" + elif abs(index_tip.y - wrist.y) < 0.1: + self.current_gesture = "向前" + else: + self.current_gesture = "停止" + + return self.current_gesture + except Exception as e: + print(f"Gesture recognition error: {e}") + return "识别异常" + + def set_simulated_gesture(self, gesture): + """设置模拟的手势""" + self.current_gesture = gesture + + +# ========== 无人机控制模块 ========== +class DroneController: + """无人机控制器""" + + def __init__(self, airsim_module): + self.airsim = airsim_module + self.client = None + self.connected = False + self.flying = False + self.connection_attempted = False + + def connect(self): + """连接AirSim无人机""" + if self.connection_attempted: + return self.connected + + self.connection_attempted = True + print("Connecting to AirSim...") + + try: + # 创建客户端 + self.client = self.airsim.MultirotorClient() + self.client.confirmConnection() + print("✅ Connected to AirSim!") + + # 启用API控制 + self.client.enableApiControl(True) + print("✅ API control enabled") + + # 解锁无人机 + self.client.armDisarm(True) + print("✅ Drone armed") + + self.connected = True + return True + + except Exception as e: + print(f"❌ Connection failed: {e}") + print("\nPlease ensure:") + print("1. AirSim simulator is running") + print("2. In simulator: Settings → Computer Vision mode is OFF") + print("3. Drone is spawned in the world") + print("4. Press 'R' in simulator to reset drone if needed") + return False + + def takeoff(self): + """起飞""" + if not self.connected: + print("❌ Drone not connected") + return False + + try: + print("Taking off...") + # 起飞到5米高度 + self.client.takeoffAsync().join() + # 悬停1秒稳定 + time.sleep(1) + self.flying = True + print("✅ Drone took off successfully") + return True + except Exception as e: + print(f"❌ Takeoff failed: {e}") + return False + + def land(self): + """降落""" + if not self.connected: + return False + + try: + print("Landing...") + self.client.landAsync().join() + self.flying = False + print("✅ Drone landed") + return True + except Exception as e: + print(f"Landing failed: {e}") + return False + + def move_by_gesture(self, gesture): + """根据手势移动""" + if not self.connected or not self.flying: + return + + try: + # 设置速度参数(单位:米/秒) + velocity = 3 + duration = 0.5 # 持续0.5秒 + + if gesture == "向上": + self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() + elif gesture == "向下": + self.client.moveByVelocityZAsync(0, 0, velocity, duration).join() + elif gesture == "向左": + self.client.moveByVelocityAsync(-velocity, 0, 0, duration).join() + elif gesture == "向右": + self.client.moveByVelocityAsync(velocity, 0, 0, duration).join() + elif gesture == "向前": + self.client.moveByVelocityAsync(0, -velocity, 0, duration).join() + elif gesture == "停止": + self.client.hoverAsync().join() + except Exception as e: + print(f"Control command failed: {e}") + + def emergency_stop(self): + """紧急停止""" + if self.connected: + try: + if self.flying: + print("Emergency landing...") + self.land() + self.client.armDisarm(False) + self.client.enableApiControl(False) + print("✅ Emergency stop complete") + except: + pass + self.connected = False + self.flying = False + + +# ========== 主程序 ========== +def main(): + """主函数""" + # 初始化组件 + gesture_recognizer = GestureRecognizer() + drone_controller = DroneController(libs['airsim']) + + # 初始化摄像头 + cap = None + try: + cap = cv2.VideoCapture(0) + if not cap.isOpened(): + print("❌ Camera not available") + cap = None + else: + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + print("✓ Camera initialized") + except: + print("⚠ Camera init failed") + cap = None + + # 显示说明 + print("\n" + "=" * 60) + print("INSTRUCTIONS") + print("=" * 60) + print("1. Start AirSim simulator first") + print("2. Choose environment (e.g., Landscape Mountains)") + print("3. Press [C] in this window to connect drone") + print("4. Press [SPACE] to takeoff/land") + if cap: + print("5. Gesture control with hand in front of camera") + else: + print("5. Keyboard control:") + print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") + print("6. Press [ESC] to exit safely") + print("=" * 60) + print("Program started successfully!") + print("-" * 60) + + # 键盘手势映射 + key_to_gesture = { + ord('w'): "向上", ord('W'): "向上", + ord('s'): "向下", ord('S'): "向下", + ord('a'): "向左", ord('A'): "向左", + ord('d'): "向右", ord('D'): "向右", + ord('f'): "向前", ord('F'): "向前", + ord('x'): "停止", ord('X'): "停止", + } + + # 主循环 + while True: + frame = None + + # 读取摄像头画面 + if cap: + ret, frame = cap.read() + if ret: + frame = cv2.flip(frame, 1) + current_gesture = gesture_recognizer.recognize(frame) + else: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + current_gesture = "摄像头错误" + else: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + current_gesture = gesture_recognizer.current_gesture + + # ========== 绘制界面 ========== + # 标题栏 + overlay = frame.copy() + cv2.rectangle(overlay, (0, 0), (640, 100), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + + # 主标题 + frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), + size=30, color=(0, 255, 255)) + + # 状态信息 + status_color = (0, 255, 0) if drone_controller.connected else (0, 0, 255) + status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" + frame = chinese_renderer.put_text(frame, status_text, (10, 50), + size=24, color=status_color) + + flight_color = (0, 255, 0) if drone_controller.flying else (255, 255, 0) + flight_status = f"飞行: {'是' if drone_controller.flying else '否'}" + frame = chinese_renderer.put_text(frame, flight_status, (10, 80), + size=24, color=flight_color) + + # 手势信息 + gesture_color = (0, 255, 0) if current_gesture != "等待手势" else (200, 200, 200) + gesture_text = f"手势: {current_gesture}" + frame = chinese_renderer.put_text(frame, gesture_text, (10, 120), + size=24, color=gesture_color) + + # 操作提示 + help_text = "C:连接 空格:起飞/降落 ESC:退出" + if not cap: + help_text += " W/A/S/D/F/X:控制" + + frame = chinese_renderer.put_text(frame, help_text, (10, 450), + size=20, color=(255, 255, 255)) + + # AirSim连接状态 + if not drone_controller.connected: + warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" + frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + size=18, color=(0, 165, 255)) + + # 显示窗口 + cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) + + # ========== 键盘控制 ========== + key = cv2.waitKey(30) & 0xFF + + if key == 27: # ESC键 + print("\nExiting...") + break + + elif key == ord('c') or key == ord('C'): # 连接无人机 + if not drone_controller.connected: + drone_controller.connect() + + elif key == 32: # 空格键 + if drone_controller.connected: + if drone_controller.flying: + drone_controller.land() + else: + drone_controller.takeoff() + time.sleep(0.3) + + elif key in key_to_gesture: # 键盘控制 + gesture = key_to_gesture[key] + gesture_recognizer.set_simulated_gesture(gesture) + if drone_controller.connected and drone_controller.flying: + drone_controller.move_by_gesture(gesture) + + # 真实手势控制 + if (current_gesture and current_gesture != "等待手势" and + current_gesture != "摄像头错误" and + drone_controller.connected and drone_controller.flying): + drone_controller.move_by_gesture(current_gesture) + + # ========== 清理资源 ========== + print("\nCleaning up resources...") + if cap: + cap.release() + cv2.destroyAllWindows() + drone_controller.emergency_stop() + print("Program safely exited") + print("=" * 60) + + +# ========== 程序入口 ========== +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nProgram interrupted by user") + except Exception as e: + print(f"\nProgram error: {e}") + traceback.print_exc() + finally: + input("\nPress Enter to exit...") \ No newline at end of file From dd54e834a48d55daa8c8f6dd313c074efcea1d05 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Thu, 18 Dec 2025 22:05:02 +0800 Subject: [PATCH 2/8] =?UTF-8?q?=E6=94=B9=E8=BF=9B=E6=97=A0=E4=BA=BA?= =?UTF-8?q?=E6=9C=BA=E6=89=8B=E5=8A=BF=E6=8E=A7=E5=88=B6=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=EF=BC=8C=E4=BC=98=E5=8C=96=E6=91=84=E5=83=8F=E5=A4=B4=E5=A4=84?= =?UTF-8?q?=E7=90=86=E5=92=8C=E6=89=8B=E5=8A=BF=E7=A8=B3=E5=AE=9A=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 4 +- .../main.py | 362 ++++++++++++------ 2 files changed, 246 insertions(+), 120 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index ad298e3e2d..05dab76400 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -3,4 +3,6 @@ Lib/ # 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 __pycache__/ *.pyc -*.dll.a \ No newline at end of file +*.dll.a +# 忽略IDEA配置文件 +.idea/ \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index b77d05ba12..f2a7020806 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 修复连接问题版 -确保正确连接AirSim,移除模拟模式 +手势控制AirSim无人机 - 修复手势识别版 +优化了手势识别算法,增加更多手势判断逻辑 作者: xiaoshiyuan888 """ @@ -15,21 +15,19 @@ import numpy as np print("=" * 60) -print("Gesture Controlled Drone - Fixed Connection Version") +print("Gesture Controlled Drone - Enhanced Gesture Recognition") print("=" * 60) # ========== 修复导入路径 ========== -# 添加当前目录到Python路径 current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -# ========== 核心模块导入与异常处理 ========== +# ========== 核心模块导入 ========== def safe_import(): """安全导入所有模块""" modules_status = {} - # 1. 导入PIL try: from PIL import Image, ImageDraw, ImageFont modules_status['PIL'] = True @@ -39,7 +37,6 @@ def safe_import(): print(f"[PIL] ✗ Import failed: {e}") return None, modules_status - # 2. 导入OpenCV try: import cv2 import numpy as np @@ -50,7 +47,6 @@ def safe_import(): print(f"[OpenCV] ✗ Import failed: {e}") return None, modules_status - # 3. 导入MediaPipe mp = None mp_hands = None mp_drawing = None @@ -65,11 +61,7 @@ def safe_import(): print(f"[MediaPipe] ✗ Import failed: {e}") print(" Please install: pip install mediapipe") - # 4. 尝试导入AirSim - 关键修复部分 airsim_module = None - airsim_client = None - - # 尝试多种导入方式 import_methods = [ lambda: __import__('airsim'), lambda: __import__('AirSim'), @@ -79,7 +71,7 @@ def safe_import(): try: airsim_module = method() modules_status['AirSim'] = True - print(f"[AirSim] ✓ Successfully imported airsim from {airsim_module.__file__}") + print(f"[AirSim] ✓ Successfully imported") break except ImportError: continue @@ -98,7 +90,6 @@ def safe_import(): print(" pip install git+https://github.com/microsoft/AirSim.git") print("!" * 60) - # 询问是否自动安装 print("\nDo you want to try automatic installation? (y/n)") choice = input().strip().lower() if choice == 'y': @@ -153,7 +144,9 @@ def load_fonts(self): font_paths = [ 'simhei.ttf', 'C:/Windows/Fonts/simhei.ttf', - 'C:/Windows/Fonts/msyh.ttc' + 'C:/Windows/Fonts/msyh.ttc', + '/System/Library/Fonts/PingFang.ttc', + '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' ] for path in font_paths: try: @@ -177,7 +170,7 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): # 绘制阴影 shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 1, pos[1] + 1) + shadow_pos = (pos[0] + 2, pos[1] + 2) draw.text(shadow_pos, text, font=font, fill=shadow_color) # 绘制文字 @@ -186,70 +179,172 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) except: + # 备用方案:使用OpenCV绘制英文 cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, size / 30, color, 2) return frame -# 初始化中文渲染器 chinese_renderer = ChineseTextRenderer() -# ========== 手势识别模块 ========== -class GestureRecognizer: - """手势识别器""" +# ========== 简化的手势识别模块 ========== +class SimpleGestureRecognizer: + """简化的手势识别器,不依赖MediaPipe""" def __init__(self): - try: - self.hands = mp_hands.Hands( - max_num_hands=1, - min_detection_confidence=0.7, - min_tracking_confidence=0.5 - ) - print("✓ MediaPipe gesture detector initialized") - except Exception as e: - print(f"⚠ MediaPipe init failed: {e}") - self.hands = None - self.current_gesture = "等待手势" + self.last_gesture_time = time.time() + self.gesture_history = [] + self.max_history = 5 + print("✓ Simple gesture recognizer initialized (不使用MediaPipe)") + + # 加载OpenCV的预训练模型用于手部检测 + try: + # 尝试加载Haar级联分类器 + cascade_path = cv2.data.haarcascades + 'haarcascade_frontalface_default.xml' + # 我们使用人脸检测器作为替代,实际上应该用手部检测器 + self.face_cascade = cv2.CascadeClassifier(cascade_path) + print("✓ OpenCV cascade classifier loaded") + except: + print("⚠ OpenCV cascade classifier not available") + self.face_cascade = None + + def detect_hand_contour(self, frame): + """使用轮廓检测手部""" + # 转换为HSV颜色空间 + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + + # 定义肤色范围 + lower_skin = np.array([0, 20, 70], dtype=np.uint8) + upper_skin = np.array([20, 255, 255], dtype=np.uint8) + + # 创建肤色掩码 + mask = cv2.inRange(hsv, lower_skin, upper_skin) + + # 形态学操作 + kernel = np.ones((5, 5), np.uint8) + mask = cv2.dilate(mask, kernel, iterations=2) + mask = cv2.erode(mask, kernel, iterations=1) + mask = cv2.GaussianBlur(mask, (5, 5), 0) + + # 查找轮廓 + contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + if contours: + # 找到最大的轮廓(假设是手) + largest_contour = max(contours, key=cv2.contourArea) + + # 计算轮廓面积 + area = cv2.contourArea(largest_contour) + + if area > 1000: # 最小面积阈值 + # 获取边界矩形 + x, y, w, h = cv2.boundingRect(largest_contour) + + # 计算轮廓的中心点 + M = cv2.moments(largest_contour) + if M["m00"] != 0: + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + + # 计算凸包和凸缺陷 + hull = cv2.convexHull(largest_contour, returnPoints=False) + if len(hull) > 3: + defects = cv2.convexityDefects(largest_contour, hull) + + if defects is not None: + # 计算手指数量 + finger_count = 0 + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(largest_contour[s][0]) + end = tuple(largest_contour[e][0]) + far = tuple(largest_contour[f][0]) + + # 计算角度 + a = np.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = np.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = np.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + + angle = np.arccos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + + if angle <= np.pi / 2: + finger_count += 1 + + finger_count = min(finger_count + 1, 5) + + # 绘制轮廓和中心点 + cv2.drawContours(frame, [largest_contour], -1, (0, 255, 0), 2) + cv2.circle(frame, (cx, cy), 7, (255, 0, 0), -1) + cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) + + # 在边界框上方显示手指数量 + cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + + return cx, cy, w, h, finger_count, frame + + return None, None, None, None, 0, frame + + def recognize_simple_gesture(self, frame): + """简化的手势识别""" + # 检测手部轮廓 + cx, cy, w, h, finger_count, frame = self.detect_hand_contour(frame) + + if cx is None or cy is None: + return "等待手势", frame + + # 获取图像尺寸 + height, width = frame.shape[:2] + + # 定义区域 + center_x = width // 2 + center_y = height // 2 + + # 根据手指数量和位置判断手势 + if finger_count == 0: + return "停止", frame + elif finger_count == 1: + return "向前", frame + elif finger_count >= 4: + # 手掌张开,根据位置判断方向 + if cy < center_y - 50: + return "向上", frame + elif cy > center_y + 50: + return "向下", frame + elif cx < center_x - 50: + return "向左", frame + elif cx > center_x + 50: + return "向右", frame + else: + return "停止", frame + else: + return "等待手势", frame def recognize(self, frame): """识别手势""" - if self.hands is None: - return self.current_gesture - try: - rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - results = self.hands.process(rgb_frame) - - if results.multi_hand_landmarks: - for hand_landmarks in results.multi_hand_landmarks: - # 绘制手部关键点 - mp_drawing.draw_landmarks( - frame, hand_landmarks, mp_hands.HAND_CONNECTIONS, - mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2), - mp_drawing.DrawingSpec(color=(255, 0, 0), thickness=2) - ) - - # 简化的手势判断 - landmarks = hand_landmarks.landmark - if len(landmarks) >= 9: - wrist = landmarks[0] - index_tip = landmarks[8] - - # 手势判断逻辑 - if index_tip.y < wrist.y - 0.2: - self.current_gesture = "向上" - elif index_tip.y > wrist.y + 0.2: - self.current_gesture = "向下" - elif index_tip.x < wrist.x - 0.2: - self.current_gesture = "向左" - elif index_tip.x > wrist.x + 0.2: - self.current_gesture = "向右" - elif abs(index_tip.y - wrist.y) < 0.1: - self.current_gesture = "向前" - else: - self.current_gesture = "停止" + # 使用简化的手势识别 + gesture, processed_frame = self.recognize_simple_gesture(frame) + + # 手势历史记录和平滑处理 + self.gesture_history.append(gesture) + if len(self.gesture_history) > self.max_history: + self.gesture_history.pop(0) + + # 使用多数表决决定最终手势 + if self.gesture_history: + gesture_counts = {} + for g in self.gesture_history: + gesture_counts[g] = gesture_counts.get(g, 0) + 1 + + # 找到出现次数最多的手势 + max_gesture = max(gesture_counts, key=gesture_counts.get) + if gesture_counts[max_gesture] > len(self.gesture_history) // 2: + self.current_gesture = max_gesture + else: + self.current_gesture = gesture return self.current_gesture except Exception as e: @@ -281,16 +376,13 @@ def connect(self): print("Connecting to AirSim...") try: - # 创建客户端 self.client = self.airsim.MultirotorClient() self.client.confirmConnection() print("✅ Connected to AirSim!") - # 启用API控制 self.client.enableApiControl(True) print("✅ API control enabled") - # 解锁无人机 self.client.armDisarm(True) print("✅ Drone armed") @@ -299,11 +391,11 @@ def connect(self): except Exception as e: print(f"❌ Connection failed: {e}") - print("\nPlease ensure:") - print("1. AirSim simulator is running") - print("2. In simulator: Settings → Computer Vision mode is OFF") - print("3. Drone is spawned in the world") - print("4. Press 'R' in simulator to reset drone if needed") + print("\n请确保:") + print("1. AirSim模拟器正在运行") + print("2. 选择环境(如:Landscape Mountains)") + print("3. 无人机已在世界中生成") + print("4. 如果需要,按模拟器中的'R'键重置无人机") return False def takeoff(self): @@ -314,9 +406,7 @@ def takeoff(self): try: print("Taking off...") - # 起飞到5米高度 self.client.takeoffAsync().join() - # 悬停1秒稳定 time.sleep(1) self.flying = True print("✅ Drone took off successfully") @@ -346,9 +436,8 @@ def move_by_gesture(self, gesture): return try: - # 设置速度参数(单位:米/秒) - velocity = 3 - duration = 0.5 # 持续0.5秒 + velocity = 2.0 # 降低速度以获得更平滑的控制 + duration = 0.3 # 减少持续时间 if gesture == "向上": self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() @@ -384,8 +473,8 @@ def emergency_stop(self): # ========== 主程序 ========== def main(): """主函数""" - # 初始化组件 - gesture_recognizer = GestureRecognizer() + # 使用简化的手势识别器 + gesture_recognizer = SimpleGestureRecognizer() drone_controller = DroneController(libs['airsim']) # 初始化摄像头 @@ -393,32 +482,46 @@ def main(): try: cap = cv2.VideoCapture(0) if not cap.isOpened(): - print("❌ Camera not available") - cap = None + # 尝试其他摄像头索引 + for i in range(1, 5): + cap = cv2.VideoCapture(i) + if cap.isOpened(): + print(f"✓ Camera {i} initialized") + break + else: + print("❌ Camera not available, using keyboard control only") + cap = None else: + print("✓ Camera 0 initialized") + + if cap: cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - print("✓ Camera initialized") - except: - print("⚠ Camera init failed") + cap.set(cv2.CAP_PROP_FPS, 30) + except Exception as e: + print(f"⚠ Camera init failed: {e}") cap = None # 显示说明 print("\n" + "=" * 60) - print("INSTRUCTIONS") + print("操作说明") print("=" * 60) - print("1. Start AirSim simulator first") - print("2. Choose environment (e.g., Landscape Mountains)") - print("3. Press [C] in this window to connect drone") - print("4. Press [SPACE] to takeoff/land") + print("1. 首先启动AirSim模拟器") + print("2. 选择环境(例如:Landscape Mountains)") + print("3. 按 [C] 连接无人机") + print("4. 按 [空格键] 起飞/降落") if cap: - print("5. Gesture control with hand in front of camera") + print("5. 手势控制:") + print(" - 握拳(0个手指):停止") + print(" - 伸出1个手指:向前") + print(" - 张开手掌(4-5个手指):根据手的位置控制方向") + print(" * 确保手部在摄像头视野内,光线充足,背景简单") else: - print("5. Keyboard control:") - print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") - print("6. Press [ESC] to exit safely") + print("5. 键盘控制:") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print("6. 按 [ESC] 安全退出") print("=" * 60) - print("Program started successfully!") + print("程序启动成功!") print("-" * 60) # 键盘手势映射 @@ -431,14 +534,17 @@ def main(): ord('x'): "停止", ord('X'): "停止", } + last_control_time = time.time() + control_interval = 0.5 # 控制间隔,避免过于频繁 + # 主循环 while True: frame = None - # 读取摄像头画面 if cap: ret, frame = cap.read() if ret: + # 镜像图像以便更直观的控制 frame = cv2.flip(frame, 1) current_gesture = gesture_recognizer.recognize(frame) else: @@ -449,12 +555,16 @@ def main(): current_gesture = gesture_recognizer.current_gesture # ========== 绘制界面 ========== - # 标题栏 + # 创建半透明覆盖层 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (640, 100), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + cv2.rectangle(overlay, (0, 0), (640, 120), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.6, frame, 0.4, 0) - # 主标题 + # 绘制底部操作提示区域 + cv2.rectangle(frame, (0, 400), (640, 480), (0, 0, 0), -1) + frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) + + # 标题 frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), size=30, color=(0, 255, 255)) @@ -470,9 +580,15 @@ def main(): size=24, color=flight_color) # 手势信息 - gesture_color = (0, 255, 0) if current_gesture != "等待手势" else (200, 200, 200) + if current_gesture != "等待手势" and current_gesture != "摄像头错误": + gesture_color = (0, 255, 0) # 绿色 + # 添加手势识别置信度指示 + cv2.circle(frame, (600, 60), 10, gesture_color, -1) + else: + gesture_color = (200, 200, 200) # 灰色 + gesture_text = f"手势: {current_gesture}" - frame = chinese_renderer.put_text(frame, gesture_text, (10, 120), + frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), size=24, color=gesture_color) # 操作提示 @@ -480,26 +596,30 @@ def main(): if not cap: help_text += " W/A/S/D/F/X:控制" - frame = chinese_renderer.put_text(frame, help_text, (10, 450), + frame = chinese_renderer.put_text(frame, help_text, (10, 420), size=20, color=(255, 255, 255)) - # AirSim连接状态 + # 添加手势提示 + if cap: + gesture_hint = "提示: 握拳停止,食指向前,手掌张开移动" + frame = chinese_renderer.put_text(frame, gesture_hint, (10, 450), + size=18, color=(255, 200, 100)) + if not drone_controller.connected: warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" - frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + frame = chinese_renderer.put_text(frame, warning_text, (10, 360), size=18, color=(0, 165, 255)) - # 显示窗口 cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) # ========== 键盘控制 ========== - key = cv2.waitKey(30) & 0xFF + key = cv2.waitKey(1) & 0xFF if key == 27: # ESC键 - print("\nExiting...") + print("\n退出程序...") break - elif key == ord('c') or key == ord('C'): # 连接无人机 + elif key == ord('c') or key == ord('C'): if not drone_controller.connected: drone_controller.connect() @@ -511,25 +631,29 @@ def main(): drone_controller.takeoff() time.sleep(0.3) - elif key in key_to_gesture: # 键盘控制 + elif key in key_to_gesture: gesture = key_to_gesture[key] gesture_recognizer.set_simulated_gesture(gesture) + current_gesture = gesture if drone_controller.connected and drone_controller.flying: drone_controller.move_by_gesture(gesture) - # 真实手势控制 + # 真实手势控制(添加间隔控制避免过于频繁) + current_time = time.time() if (current_gesture and current_gesture != "等待手势" and - current_gesture != "摄像头错误" and - drone_controller.connected and drone_controller.flying): + current_gesture != "摄像头错误" and current_gesture != "识别异常" and + drone_controller.connected and drone_controller.flying and + current_time - last_control_time > control_interval): drone_controller.move_by_gesture(current_gesture) + last_control_time = current_time - # ========== 清理资源 ========== - print("\nCleaning up resources...") + # 清理资源 + print("\n清理资源...") if cap: cap.release() cv2.destroyAllWindows() drone_controller.emergency_stop() - print("Program safely exited") + print("程序安全退出") print("=" * 60) @@ -538,9 +662,9 @@ def main(): try: main() except KeyboardInterrupt: - print("\nProgram interrupted by user") + print("\n程序被用户中断") except Exception as e: - print(f"\nProgram error: {e}") + print(f"\n程序错误: {e}") traceback.print_exc() finally: - input("\nPress Enter to exit...") \ No newline at end of file + input("\n按回车键退出...") \ No newline at end of file From d1726c94f7739505e7cd5e035dd74dc285ac53ba Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 09:57:53 +0800 Subject: [PATCH 3/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E8=AF=86=E5=88=AB=E7=AE=97=E6=B3=95=EF=BC=8C=E6=8F=90=E5=8D=87?= =?UTF-8?q?=E7=B3=BB=E7=BB=9F=E7=A8=B3=E5=AE=9A=E6=80=A7=E5=92=8C=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E4=BD=93=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main.py | 857 ++++++++++++------ 1 file changed, 588 insertions(+), 269 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index f2a7020806..5325f35afa 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 修复手势识别版 -优化了手势识别算法,增加更多手势判断逻辑 +手势控制AirSim无人机 - 深度优化版 +优化了手势识别准确性和系统稳定性 作者: xiaoshiyuan888 """ @@ -10,12 +10,14 @@ import time import traceback import subprocess +import math from PIL import Image, ImageDraw, ImageFont import cv2 import numpy as np +from collections import deque print("=" * 60) -print("Gesture Controlled Drone - Enhanced Gesture Recognition") +print("Gesture Controlled Drone - Advanced Recognition v2.0") print("=" * 60) # ========== 修复导入路径 ========== @@ -47,6 +49,7 @@ def safe_import(): print(f"[OpenCV] ✗ Import failed: {e}") return None, modules_status + # 尝试导入mediapipe,如果失败则使用OpenCV方案 mp = None mp_hands = None mp_drawing = None @@ -58,28 +61,14 @@ def safe_import(): print("[MediaPipe] ✓ Gesture recognition library ready") except Exception as e: modules_status['MediaPipe'] = False - print(f"[MediaPipe] ✗ Import failed: {e}") - print(" Please install: pip install mediapipe") + print(f"[MediaPipe] ✗ Import failed, using OpenCV fallback: {e}") airsim_module = None - import_methods = [ - lambda: __import__('airsim'), - lambda: __import__('AirSim'), - ] - - for method in import_methods: - try: - airsim_module = method() - modules_status['AirSim'] = True - print(f"[AirSim] ✓ Successfully imported") - break - except ImportError: - continue - except Exception as e: - print(f"[AirSim] Import error: {e}") - continue - - if not modules_status.get('AirSim', False): + try: + airsim_module = __import__('airsim') + modules_status['AirSim'] = True + print(f"[AirSim] ✓ Successfully imported") + except ImportError: print("\n" + "!" * 60) print("⚠ AirSim library NOT FOUND!") print("!" * 60) @@ -90,18 +79,10 @@ def safe_import(): print(" pip install git+https://github.com/microsoft/AirSim.git") print("!" * 60) - print("\nDo you want to try automatic installation? (y/n)") + print("\nContinue without AirSim? (y/n)") choice = input().strip().lower() - if choice == 'y': - try: - print("Installing msgpack-rpc-python...") - subprocess.check_call([sys.executable, "-m", "pip", "install", "msgpack-rpc-python"]) - print("Installing AirSim...") - subprocess.check_call([sys.executable, "-m", "pip", "install", "airsim"]) - print("\n✅ Installation complete! Please restart the program.") - except Exception as e: - print(f"Installation failed: {e}") - sys.exit(1) + if choice != 'y': + sys.exit(1) return { 'cv2': cv2, @@ -126,9 +107,8 @@ def safe_import(): print("-" * 60) # 解包库 -cv2, np, mp = libs['cv2'], libs['np'], libs['mp'] +cv2, np = libs['cv2'], libs['np'] Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -mp_hands, mp_drawing = libs['mp_hands'], libs['mp_drawing'] # ========== 中文显示模块 ========== @@ -150,16 +130,18 @@ def load_fonts(self): ] for path in font_paths: try: + self.fonts[16] = ImageFont.truetype(path, 16) self.fonts[20] = ImageFont.truetype(path, 20) self.fonts[24] = ImageFont.truetype(path, 24) - self.fonts[30] = ImageFont.truetype(path, 30) + self.fonts[28] = ImageFont.truetype(path, 28) + self.fonts[32] = ImageFont.truetype(path, 32) print(f"✓ Chinese fonts loaded: {path}") return except: continue print("⚠ No Chinese fonts found, using default") - def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): + def put_text(self, frame, text, pos, size=20, color=(255, 255, 255), bg_color=None): """在图像上绘制中文文本""" try: img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) @@ -168,197 +150,421 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): font = self.fonts.get(size, self.fonts.get(20)) + # 如果有背景色,先绘制背景 + if bg_color: + bbox = draw.textbbox(pos, text, font=font) + padding = 2 + draw.rectangle( + [bbox[0] - padding, bbox[1] - padding, bbox[2] + padding, bbox[3] + padding], + fill=bg_color + ) + # 绘制阴影 shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 2, pos[1] + 2) + shadow_pos = (pos[0] + 1, pos[1] + 1) draw.text(shadow_pos, text, font=font, fill=shadow_color) # 绘制文字 - rgb_color = color[::-1] + rgb_color = color[::-1] # BGR to RGB draw.text(pos, text, font=font, fill=rgb_color) return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) - except: + except Exception as e: # 备用方案:使用OpenCV绘制英文 cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, - size / 30, color, 2) + size / 25, color, 2) return frame chinese_renderer = ChineseTextRenderer() -# ========== 简化的手势识别模块 ========== -class SimpleGestureRecognizer: - """简化的手势识别器,不依赖MediaPipe""" +# ========== 高级手势识别模块 ========== +class AdvancedGestureRecognizer: + """高级手势识别器,结合多种检测方法""" def __init__(self): + self.use_mediapipe = status.get('MediaPipe', False) + + # 初始化MediaPipe(如果可用) + if self.use_mediapipe and libs['mp'] is not None: + try: + self.hands = libs['mp_hands'].Hands( + max_num_hands=2, + min_detection_confidence=0.7, + min_tracking_confidence=0.5, + static_image_mode=False, + model_complexity=1 + ) + print("✓ Using MediaPipe for gesture recognition") + except: + self.use_mediapipe = False + print("⚠ MediaPipe initialization failed, using OpenCV fallback") + else: + self.use_mediapipe = False + + # OpenCV手势识别参数 + self.history_size = 10 + self.gesture_history = deque(maxlen=self.history_size) + self.finger_history = deque(maxlen=self.history_size) self.current_gesture = "等待手势" + self.confidence = 0.0 self.last_gesture_time = time.time() - self.gesture_history = [] - self.max_history = 5 - print("✓ Simple gesture recognizer initialized (不使用MediaPipe)") - # 加载OpenCV的预训练模型用于手部检测 - try: - # 尝试加载Haar级联分类器 - cascade_path = cv2.data.haarcascades + 'haarcascade_frontalface_default.xml' - # 我们使用人脸检测器作为替代,实际上应该用手部检测器 - self.face_cascade = cv2.CascadeClassifier(cascade_path) - print("✓ OpenCV cascade classifier loaded") - except: - print("⚠ OpenCV cascade classifier not available") - self.face_cascade = None - - def detect_hand_contour(self, frame): - """使用轮廓检测手部""" - # 转换为HSV颜色空间 - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + # 肤色检测参数(自适应) + self.skin_lower = np.array([0, 30, 60], dtype=np.uint8) + self.skin_upper = np.array([25, 255, 255], dtype=np.uint8) + + # 背景减除 + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2(history=500, varThreshold=16, detectShadows=True) + + # 手部跟踪 + self.track_window = None + self.track_box = None + self.tracking = False + + print("✓ Advanced gesture recognizer initialized") - # 定义肤色范围 - lower_skin = np.array([0, 20, 70], dtype=np.uint8) - upper_skin = np.array([20, 255, 255], dtype=np.uint8) + def adaptive_skin_detection(self, frame): + """自适应肤色检测""" + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - # 创建肤色掩码 - mask = cv2.inRange(hsv, lower_skin, upper_skin) + # 自适应调整肤色范围 + mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) # 形态学操作 - kernel = np.ones((5, 5), np.uint8) - mask = cv2.dilate(mask, kernel, iterations=2) + kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) mask = cv2.erode(mask, kernel, iterations=1) + mask = cv2.dilate(mask, kernel, iterations=2) mask = cv2.GaussianBlur(mask, (5, 5), 0) + return mask + + def find_hand_contours(self, frame): + """查找手部轮廓""" + # 使用背景减除 + fg_mask = self.bg_subtractor.apply(frame) + + # 肤色检测 + skin_mask = self.adaptive_skin_detection(frame) + + # 结合两种掩码 + combined_mask = cv2.bitwise_and(fg_mask, skin_mask) + + # 形态学操作优化 + kernel = np.ones((7, 7), np.uint8) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, kernel) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, kernel) + # 查找轮廓 - contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours, _ = cv2.findContours(combined_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + if not contours: + return None, combined_mask + + # 找到最大的轮廓(假设是手) + largest_contour = max(contours, key=cv2.contourArea) + + # 过滤掉太小的轮廓 + if cv2.contourArea(largest_contour) < 5000: + return None, combined_mask + + return largest_contour, combined_mask - if contours: - # 找到最大的轮廓(假设是手) - largest_contour = max(contours, key=cv2.contourArea) + def analyze_hand_contour(self, contour): + """分析手部轮廓特征""" + # 计算轮廓特征 + area = cv2.contourArea(contour) + perimeter = cv2.arcLength(contour, True) - # 计算轮廓面积 - area = cv2.contourArea(largest_contour) + # 计算凸包和凸缺陷 + hull = cv2.convexHull(contour, returnPoints=False) + if len(hull) < 3: + return 0, None, None - if area > 1000: # 最小面积阈值 - # 获取边界矩形 - x, y, w, h = cv2.boundingRect(largest_contour) + defects = cv2.convexityDefects(contour, hull) - # 计算轮廓的中心点 - M = cv2.moments(largest_contour) - if M["m00"] != 0: - cx = int(M["m10"] / M["m00"]) - cy = int(M["m01"] / M["m00"]) + if defects is None: + return 0, None, None - # 计算凸包和凸缺陷 - hull = cv2.convexHull(largest_contour, returnPoints=False) - if len(hull) > 3: - defects = cv2.convexityDefects(largest_contour, hull) + # 计算手指数量 + finger_count = 0 + finger_tips = [] - if defects is not None: - # 计算手指数量 - finger_count = 0 - for i in range(defects.shape[0]): - s, e, f, d = defects[i, 0] - start = tuple(largest_contour[s][0]) - end = tuple(largest_contour[e][0]) - far = tuple(largest_contour[f][0]) + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(contour[s][0]) + end = tuple(contour[e][0]) + far = tuple(contour[f][0]) - # 计算角度 - a = np.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) - b = np.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) - c = np.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + # 计算角度 + a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) - angle = np.arccos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) - if angle <= np.pi / 2: - finger_count += 1 + # 如果角度小于90度,可能是指缝 + if angle < math.pi / 2: + finger_count += 1 + finger_tips.append(end) - finger_count = min(finger_count + 1, 5) + # 手指数量加1(大拇指) + finger_count = min(finger_count + 1, 5) - # 绘制轮廓和中心点 - cv2.drawContours(frame, [largest_contour], -1, (0, 255, 0), 2) - cv2.circle(frame, (cx, cy), 7, (255, 0, 0), -1) - cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) + # 计算轮廓中心 + M = cv2.moments(contour) + if M["m00"] != 0: + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + center = (cx, cy) + else: + center = None + + return finger_count, center, finger_tips - # 在边界框上方显示手指数量 - cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + def detect_fingertips_with_contours(self, contour): + """使用轮廓检测指尖""" + # 简化轮廓 + epsilon = 0.02 * cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon, True) - return cx, cy, w, h, finger_count, frame + # 查找凸点(可能是指尖) + hull = cv2.convexHull(approx, returnPoints=True) - return None, None, None, None, 0, frame + fingertips = [] + for point in hull: + fingertips.append(tuple(point[0])) - def recognize_simple_gesture(self, frame): - """简化的手势识别""" - # 检测手部轮廓 - cx, cy, w, h, finger_count, frame = self.detect_hand_contour(frame) + return fingertips[:5] # 最多5个指尖 - if cx is None or cy is None: - return "等待手势", frame + def recognize_gesture_by_fingers(self, finger_count, center, frame_shape): + """根据手指数量识别手势""" + if center is None: + return "等待手势", 0.5 - # 获取图像尺寸 - height, width = frame.shape[:2] + h, w = frame_shape[:2] + cx, cy = center - # 定义区域 - center_x = width // 2 - center_y = height // 2 + # 计算手部位置(归一化) + norm_x = cx / w + norm_y = cy / h - # 根据手指数量和位置判断手势 if finger_count == 0: - return "停止", frame + return "停止", 0.9 elif finger_count == 1: - return "向前", frame + return "向前", 0.8 + elif finger_count == 2: + # V字手势,可能是向前或特殊命令 + return "向前", 0.7 + elif finger_count == 3: + # 三指手势,根据位置判断 + if norm_y < 0.4: + return "向上", 0.7 + elif norm_y > 0.6: + return "向下", 0.7 + else: + return "停止", 0.6 elif finger_count >= 4: # 手掌张开,根据位置判断方向 - if cy < center_y - 50: - return "向上", frame - elif cy > center_y + 50: - return "向下", frame - elif cx < center_x - 50: - return "向左", frame - elif cx > center_x + 50: - return "向右", frame + if norm_x < 0.3: + return "向左", 0.8 + elif norm_x > 0.7: + return "向右", 0.8 + elif norm_y < 0.4: + return "向上", 0.8 + elif norm_y > 0.6: + return "向下", 0.8 else: - return "停止", frame - else: - return "等待手势", frame + return "停止", 0.7 + + return "等待手势", 0.5 + + def recognize_with_mediapipe(self, frame): + """使用MediaPipe进行手势识别""" + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + results = self.hands.process(rgb_frame) + + gesture = "等待手势" + confidence = 0.5 + + if results.multi_hand_landmarks: + for hand_landmarks in results.multi_hand_landmarks: + # 绘制手部关键点 + libs['mp_drawing'].draw_landmarks( + frame, hand_landmarks, libs['mp_hands'].HAND_CONNECTIONS, + libs['mp_drawing'].DrawingSpec(color=(0, 255, 0), thickness=2), + libs['mp_drawing'].DrawingSpec(color=(255, 0, 0), thickness=2) + ) + + # 计算手指状态 + landmarks = hand_landmarks.landmark + + # 简单的手指状态检测 + finger_tips = [4, 8, 12, 16, 20] + finger_pips = [2, 6, 10, 14, 18] + + extended_fingers = 0 + for tip, pip in zip(finger_tips, finger_pips): + if landmarks[tip].y < landmarks[pip].y: + extended_fingers += 1 + + # 根据伸直的手指数量判断手势 + if extended_fingers == 0: + gesture = "停止" + confidence = 0.9 + elif extended_fingers == 1: + gesture = "向前" + confidence = 0.8 + elif extended_fingers >= 4: + # 根据手掌方向判断 + wrist = landmarks[0] + middle_mcp = landmarks[9] + + dx = middle_mcp.x - wrist.x + dy = middle_mcp.y - wrist.y + + if abs(dx) > abs(dy): + if dx > 0: + gesture = "向右" + else: + gesture = "向左" + else: + if dy > 0: + gesture = "向下" + else: + gesture = "向上" + confidence = 0.7 + + return gesture, confidence, frame + + def smooth_gesture(self, gesture, confidence): + """平滑手势输出""" + self.gesture_history.append((gesture, confidence)) + + if len(self.gesture_history) < 3: + return gesture, confidence + + # 统计最近的手势 + gesture_counts = {} + confidence_sum = {} + + for g, c in self.gesture_history: + if g not in gesture_counts: + gesture_counts[g] = 0 + confidence_sum[g] = 0 + gesture_counts[g] += 1 + confidence_sum[g] += c + + # 找到最频繁的手势 + max_count = 0 + best_gesture = gesture + best_confidence = confidence + + for g, count in gesture_counts.items(): + if count > max_count and g != "等待手势": + max_count = count + best_gesture = g + best_confidence = confidence_sum[g] / count + + # 如果手势稳定,更新当前手势 + if max_count >= len(self.gesture_history) // 2: + self.current_gesture = best_gesture + self.confidence = best_confidence + + return self.current_gesture, self.confidence + + def visualize_hand(self, frame, contour, center, finger_count, fingertips): + """可视化手部检测结果""" + if contour is not None: + # 绘制轮廓 + cv2.drawContours(frame, [contour], -1, (0, 255, 0), 2) + + # 绘制边界框 + x, y, w, h = cv2.boundingRect(contour) + cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 0), 2) + + # 绘制中心点 + if center: + cx, cy = center + cv2.circle(frame, (cx, cy), 8, (0, 0, 255), -1) + + # 绘制指尖 + for tip in fingertips[:finger_count]: + cv2.circle(frame, tip, 6, (255, 255, 0), -1) + + # 显示手指数量 + cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + + return frame def recognize(self, frame): """识别手势""" + start_time = time.time() + try: - # 使用简化的手势识别 - gesture, processed_frame = self.recognize_simple_gesture(frame) - - # 手势历史记录和平滑处理 - self.gesture_history.append(gesture) - if len(self.gesture_history) > self.max_history: - self.gesture_history.pop(0) - - # 使用多数表决决定最终手势 - if self.gesture_history: - gesture_counts = {} - for g in self.gesture_history: - gesture_counts[g] = gesture_counts.get(g, 0) + 1 - - # 找到出现次数最多的手势 - max_gesture = max(gesture_counts, key=gesture_counts.get) - if gesture_counts[max_gesture] > len(self.gesture_history) // 2: - self.current_gesture = max_gesture + # 镜像图像以便更直观的控制 + frame = cv2.flip(frame, 1) + + # 方法1:使用MediaPipe(如果可用) + if self.use_mediapipe: + gesture, confidence, processed_frame = self.recognize_with_mediapipe(frame) + frame = processed_frame + else: + # 方法2:使用OpenCV + contour, mask = self.find_hand_contours(frame) + + if contour is not None: + finger_count, center, finger_tips = self.analyze_hand_contour(contour) + + # 如果凸缺陷检测失败,使用轮廓检测指尖 + if finger_tips is None or len(finger_tips) < finger_count: + finger_tips = self.detect_fingertips_with_contours(contour) + + self.finger_history.append(finger_count) + + # 计算平均手指数量(平滑) + if len(self.finger_history) > 0: + avg_fingers = sum(self.finger_history) / len(self.finger_history) + finger_count = int(round(avg_fingers)) + + # 识别手势 + gesture, confidence = self.recognize_gesture_by_fingers( + finger_count, center, frame.shape + ) + + # 可视化结果 + frame = self.visualize_hand(frame, contour, center, finger_count, finger_tips) else: - self.current_gesture = gesture + gesture, confidence = "等待手势", 0.3 + + # 手势平滑处理 + final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) + + # 更新最后检测时间 + self.last_gesture_time = time.time() + + # 显示处理时间 + process_time = (time.time() - start_time) * 1000 + cv2.putText(frame, f'Process: {process_time:.1f}ms', (10, frame.shape[0] - 20), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + return final_gesture, final_confidence, frame - return self.current_gesture except Exception as e: print(f"Gesture recognition error: {e}") - return "识别异常" + return "识别异常", 0.0, frame def set_simulated_gesture(self, gesture): """设置模拟的手势""" self.current_gesture = gesture + self.confidence = 0.9 -# ========== 无人机控制模块 ========== -class DroneController: - """无人机控制器""" +# ========== 增强的无人机控制模块 ========== +class EnhancedDroneController: + """增强的无人机控制器""" def __init__(self, airsim_module): self.airsim = airsim_module @@ -367,12 +573,27 @@ def __init__(self, airsim_module): self.flying = False self.connection_attempted = False + # 控制参数 + self.velocity = 3.0 + self.duration = 0.2 + self.altitude = -10.0 # 初始高度(负值表示向上) + + # 控制历史 + self.control_history = deque(maxlen=10) + self.last_control_time = 0 + def connect(self): """连接AirSim无人机""" if self.connection_attempted: return self.connected self.connection_attempted = True + + if self.airsim is None: + print("⚠ AirSim not available, using simulation mode") + self.connected = True + return True + print("Connecting to AirSim...") try: @@ -396,6 +617,14 @@ def connect(self): print("2. 选择环境(如:Landscape Mountains)") print("3. 无人机已在世界中生成") print("4. 如果需要,按模拟器中的'R'键重置无人机") + + print("\n使用模拟模式继续?(y/n)") + choice = input().strip().lower() + if choice == 'y': + self.connected = True + print("✅ Using simulation mode") + return True + return False def takeoff(self): @@ -405,15 +634,25 @@ def takeoff(self): return False try: + if self.airsim is None or self.client is None: + print("✅ Simulated takeoff") + self.flying = True + return True + print("Taking off...") self.client.takeoffAsync().join() time.sleep(1) + + # 上升到指定高度 + self.client.moveToZAsync(self.altitude, 3).join() + self.flying = True print("✅ Drone took off successfully") return True except Exception as e: print(f"❌ Takeoff failed: {e}") - return False + self.flying = True # 模拟模式 + return True def land(self): """降落""" @@ -421,6 +660,11 @@ def land(self): return False try: + if self.airsim is None or self.client is None: + print("✅ Simulated landing") + self.flying = False + return True + print("Landing...") self.client.landAsync().join() self.flying = False @@ -428,101 +672,155 @@ def land(self): return True except Exception as e: print(f"Landing failed: {e}") + self.flying = False return False - def move_by_gesture(self, gesture): + def move_by_gesture(self, gesture, confidence=0.7): """根据手势移动""" if not self.connected or not self.flying: - return + return False + + # 检查控制间隔 + current_time = time.time() + if current_time - self.last_control_time < 0.1: # 最小控制间隔 + return False + + # 如果置信度太低,不执行动作 + if confidence < 0.5: + return False try: - velocity = 2.0 # 降低速度以获得更平滑的控制 - duration = 0.3 # 减少持续时间 + if self.airsim is None or self.client is None: + print(f"Simulated move: {gesture}") + self.control_history.append((gesture, current_time)) + self.last_control_time = current_time + return True + + success = False if gesture == "向上": - self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() + self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) + success = True elif gesture == "向下": - self.client.moveByVelocityZAsync(0, 0, velocity, duration).join() + self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) + success = True elif gesture == "向左": - self.client.moveByVelocityAsync(-velocity, 0, 0, duration).join() + self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) + success = True elif gesture == "向右": - self.client.moveByVelocityAsync(velocity, 0, 0, duration).join() + self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) + success = True elif gesture == "向前": - self.client.moveByVelocityAsync(0, -velocity, 0, duration).join() + self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) + success = True elif gesture == "停止": - self.client.hoverAsync().join() + self.client.hoverAsync() + success = True + + if success: + self.control_history.append((gesture, current_time)) + self.last_control_time = current_time + + return success except Exception as e: print(f"Control command failed: {e}") + return False def emergency_stop(self): """紧急停止""" if self.connected: try: - if self.flying: + if self.flying and self.client is not None: print("Emergency landing...") self.land() - self.client.armDisarm(False) - self.client.enableApiControl(False) - print("✅ Emergency stop complete") + if self.client is not None: + self.client.armDisarm(False) + self.client.enableApiControl(False) + print("✅ Emergency stop complete") except: pass self.connected = False self.flying = False +# ========== 性能监控模块 ========== +class PerformanceMonitor: + """性能监控器""" + + def __init__(self): + self.frame_times = deque(maxlen=60) + self.fps = 0 + self.last_update = time.time() + + def update(self): + """更新性能数据""" + current_time = time.time() + self.frame_times.append(current_time) + + # 每秒更新一次FPS + if current_time - self.last_update >= 1.0: + if len(self.frame_times) > 1: + self.fps = len(self.frame_times) / (self.frame_times[-1] - self.frame_times[0]) + self.last_update = current_time + + def get_stats(self): + """获取性能统计""" + return { + 'fps': self.fps, + 'frame_count': len(self.frame_times) + } + + # ========== 主程序 ========== def main(): """主函数""" - # 使用简化的手势识别器 - gesture_recognizer = SimpleGestureRecognizer() - drone_controller = DroneController(libs['airsim']) + # 初始化组件 + gesture_recognizer = AdvancedGestureRecognizer() + drone_controller = EnhancedDroneController(libs['airsim']) + performance_monitor = PerformanceMonitor() # 初始化摄像头 cap = None - try: - cap = cv2.VideoCapture(0) - if not cap.isOpened(): - # 尝试其他摄像头索引 - for i in range(1, 5): - cap = cv2.VideoCapture(i) - if cap.isOpened(): - print(f"✓ Camera {i} initialized") - break - else: - print("❌ Camera not available, using keyboard control only") - cap = None - else: - print("✓ Camera 0 initialized") - - if cap: - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - cap.set(cv2.CAP_PROP_FPS, 30) - except Exception as e: - print(f"⚠ Camera init failed: {e}") + available_cameras = [] + + # 检测可用摄像头 + for i in range(0, 4): + temp_cap = cv2.VideoCapture(i) + if temp_cap.isOpened(): + available_cameras.append(i) + temp_cap.release() + + if available_cameras: + print(f"✓ Available cameras: {available_cameras}") + cap = cv2.VideoCapture(available_cameras[0]) + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + cap.set(cv2.CAP_PROP_FPS, 30) + print(f"✓ Using camera {available_cameras[0]}") + else: + print("❌ No cameras available, using keyboard control only") cap = None # 显示说明 print("\n" + "=" * 60) - print("操作说明") + print("手势控制无人机系统 - 高级版") print("=" * 60) - print("1. 首先启动AirSim模拟器") - print("2. 选择环境(例如:Landscape Mountains)") - print("3. 按 [C] 连接无人机") - print("4. 按 [空格键] 起飞/降落") + print("操作说明:") + print("1. 首先启动AirSim模拟器(可选)") + print("2. 按 [C] 连接无人机") + print("3. 按 [空格键] 起飞/降落") if cap: - print("5. 手势控制:") - print(" - 握拳(0个手指):停止") - print(" - 伸出1个手指:向前") - print(" - 张开手掌(4-5个手指):根据手的位置控制方向") - print(" * 确保手部在摄像头视野内,光线充足,背景简单") + print("4. 手势控制:") + print(" - 握拳(0指):停止") + print(" - 食指(1指):向前") + print(" - 手掌张开(4-5指):根据手的位置控制方向") + print(" * 手势识别置信度 > 50% 时才会执行") else: - print("5. 键盘控制:") + print("4. 键盘控制:") print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") - print("6. 按 [ESC] 安全退出") + print("5. 按 [ESC] 安全退出") + print("6. 按 [R] 重置手势识别") print("=" * 60) - print("程序启动成功!") - print("-" * 60) # 键盘手势映射 key_to_gesture = { @@ -534,83 +832,91 @@ def main(): ord('x'): "停止", ord('X'): "停止", } - last_control_time = time.time() - control_interval = 0.5 # 控制间隔,避免过于频繁 - # 主循环 + last_gesture_time = time.time() + gesture_timeout = 2.0 # 手势超时时间 + while True: - frame = None + performance_monitor.update() + # 读取摄像头帧 if cap: ret, frame = cap.read() - if ret: - # 镜像图像以便更直观的控制 - frame = cv2.flip(frame, 1) - current_gesture = gesture_recognizer.recognize(frame) + if not ret: + # 如果读取失败,尝试重新初始化摄像头 + print("⚠ Camera read failed, trying to reinitialize...") + cap.release() + if available_cameras: + cap = cv2.VideoCapture(available_cameras[0]) + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + ret, frame = cap.read() + + if not ret: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + current_gesture = "摄像头错误" + confidence = 0.0 + else: + current_gesture, confidence, frame = gesture_recognizer.recognize(frame) else: - frame = np.zeros((480, 640, 3), dtype=np.uint8) - current_gesture = "摄像头错误" + current_gesture, confidence, frame = gesture_recognizer.recognize(frame) else: frame = np.zeros((480, 640, 3), dtype=np.uint8) current_gesture = gesture_recognizer.current_gesture + confidence = gesture_recognizer.confidence # ========== 绘制界面 ========== - # 创建半透明覆盖层 + # 创建信息覆盖层 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (640, 120), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.6, frame, 0.4, 0) - - # 绘制底部操作提示区域 - cv2.rectangle(frame, (0, 400), (640, 480), (0, 0, 0), -1) - frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) + cv2.rectangle(overlay, (0, 0), (640, 140), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) - # 标题 - frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), - size=30, color=(0, 255, 255)) + # 标题和版本 + title = "手势控制无人机系统 - 高级版" + frame = chinese_renderer.put_text(frame, title, (10, 10), size=28, color=(0, 255, 255)) # 状态信息 status_color = (0, 255, 0) if drone_controller.connected else (0, 0, 255) - status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" - frame = chinese_renderer.put_text(frame, status_text, (10, 50), - size=24, color=status_color) + status_text = f"状态: {'已连接' if drone_controller.connected else '未连接'}" + frame = chinese_renderer.put_text(frame, status_text, (10, 50), size=24, color=status_color) - flight_color = (0, 255, 0) if drone_controller.flying else (255, 255, 0) - flight_status = f"飞行: {'是' if drone_controller.flying else '否'}" - frame = chinese_renderer.put_text(frame, flight_status, (10, 80), - size=24, color=flight_color) + flight_color = (0, 255, 0) if drone_controller.flying else (255, 165, 0) + flight_text = f"飞行: {'是' if drone_controller.flying else '否'}" + frame = chinese_renderer.put_text(frame, flight_text, (10, 80), size=24, color=flight_color) # 手势信息 - if current_gesture != "等待手势" and current_gesture != "摄像头错误": - gesture_color = (0, 255, 0) # 绿色 - # 添加手势识别置信度指示 - cv2.circle(frame, (600, 60), 10, gesture_color, -1) - else: - gesture_color = (200, 200, 200) # 灰色 + gesture_color = (0, 255, 0) if confidence > 0.7 else (255, 165, 0) if confidence > 0.5 else (200, 200, 200) + gesture_text = f"手势: {current_gesture} ({confidence:.1%})" + frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), size=24, color=gesture_color) - gesture_text = f"手势: {current_gesture}" - frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), - size=24, color=gesture_color) + # 性能信息 + stats = performance_monitor.get_stats() + perf_text = f"FPS: {stats['fps']:.1f} | 延迟: {1000 / stats['fps']:.1f}ms" if stats['fps'] > 0 else "FPS: 0" + frame = chinese_renderer.put_text(frame, perf_text, (440, 50), size=20, color=(255, 255, 255)) # 操作提示 - help_text = "C:连接 空格:起飞/降落 ESC:退出" + help_text = "C:连接 空格:起飞/降落 R:重置 ESC:退出" if not cap: help_text += " W/A/S/D/F/X:控制" - frame = chinese_renderer.put_text(frame, help_text, (10, 420), - size=20, color=(255, 255, 255)) + # 绘制底部提示栏 + cv2.rectangle(frame, (0, 440), (640, 480), (0, 0, 0), -1) + frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) + frame = chinese_renderer.put_text(frame, help_text, (10, 450), size=18, color=(255, 255, 255)) - # 添加手势提示 + # 手势提示 if cap: - gesture_hint = "提示: 握拳停止,食指向前,手掌张开移动" - frame = chinese_renderer.put_text(frame, gesture_hint, (10, 450), - size=18, color=(255, 200, 100)) + hint_text = "提示: 保持手势稳定,确保良好光线" + frame = chinese_renderer.put_text(frame, hint_text, (10, 470), size=16, color=(255, 200, 100)) + # 显示连接提示 if not drone_controller.connected: - warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" - frame = chinese_renderer.put_text(frame, warning_text, (10, 360), - size=18, color=(0, 165, 255)) + warning_text = "⚠ 按C键连接无人机,或继续使用模拟模式" + frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + size=18, color=(0, 165, 255), bg_color=(0, 0, 0)) - cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) + # 显示图像 + cv2.imshow('Gesture Controlled Drone - Advanced Version', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -629,23 +935,35 @@ def main(): drone_controller.land() else: drone_controller.takeoff() - time.sleep(0.3) + time.sleep(0.5) + + elif key == ord('r') or key == ord('R'): + print("重置手势识别...") + gesture_recognizer = AdvancedGestureRecognizer() elif key in key_to_gesture: gesture = key_to_gesture[key] gesture_recognizer.set_simulated_gesture(gesture) current_gesture = gesture + confidence = 0.9 if drone_controller.connected and drone_controller.flying: - drone_controller.move_by_gesture(gesture) + drone_controller.move_by_gesture(gesture, confidence) - # 真实手势控制(添加间隔控制避免过于频繁) + # 真实手势控制 current_time = time.time() if (current_gesture and current_gesture != "等待手势" and current_gesture != "摄像头错误" and current_gesture != "识别异常" and drone_controller.connected and drone_controller.flying and - current_time - last_control_time > control_interval): - drone_controller.move_by_gesture(current_gesture) - last_control_time = current_time + confidence > 0.5 and # 只有置信度足够高才执行 + current_time - last_gesture_time > 0.2): # 控制频率限制 + + success = drone_controller.move_by_gesture(current_gesture, confidence) + if success: + last_gesture_time = current_time + + # 检查手势超时 + if current_time - gesture_recognizer.last_gesture_time > gesture_timeout: + gesture_recognizer.current_gesture = "等待手势" # 清理资源 print("\n清理资源...") @@ -667,4 +985,5 @@ def main(): print(f"\n程序错误: {e}") traceback.print_exc() finally: - input("\n按回车键退出...") \ No newline at end of file + print("\n按回车键退出...") + input() \ No newline at end of file From 4b5e52e8e77f1a51b3fadc2b457a45463264baca Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 14:30:43 +0800 Subject: [PATCH 4/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=8B=B1=E6=96=87?= =?UTF-8?q?=E7=95=8C=E9=9D=A2=EF=BC=8C=E6=94=B9=E8=BF=9B=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E8=AF=86=E5=88=AB=E7=AE=97=E6=B3=95=EF=BC=8C=E5=A2=9E=E5=BC=BA?= =?UTF-8?q?=E5=8D=95=E6=8C=87=E8=AF=86=E5=88=AB=E5=87=86=E7=A1=AE=E7=8E=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 9 +- .../config.json | 53 + .../gesture_config.json | 45 + .../main.py | 1409 ++++++++++------- 4 files changed, 906 insertions(+), 610 deletions(-) create mode 100644 src/autonomus_drone_hand_gesture_project/config.json create mode 100644 src/autonomus_drone_hand_gesture_project/gesture_config.json diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index 05dab76400..0ad6fac5ef 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -1,8 +1,13 @@ # 忽略整个Lib目录(包含所有子文件/子目录) Lib/ -# 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 + +# 忽略Python编译文件,避免无关文件干扰 __pycache__/ *.pyc *.dll.a + # 忽略IDEA配置文件 -.idea/ \ No newline at end of file +.idea/ + +# 忽略日志文件夹(临时日志文件,无需提交到仓库) +logs/ \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/config.json b/src/autonomus_drone_hand_gesture_project/config.json new file mode 100644 index 0000000000..1fa5f144eb --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/config.json @@ -0,0 +1,53 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "detection_confidence": 0.7, + "tracking_confidence": 0.5, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.6 + }, + "drone": { + "velocity": 3.0, + "duration": 0.2, + "altitude": -10.0, + "control_interval": 0.2 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_landmarks": true, + "show_help": true + }, + "performance": { + "target_fps": 30, + "resize_factor": 0.5, + "enable_threading": true + }, + "camera_width": 640, + "camera_height": 480, + "camera_fps": 30, + "camera_index": 0, + "mirror_camera": false, + "min_detection_confidence": 0.6, + "min_tracking_confidence": 0.5, + "gesture_smoothness": 5, + "gesture_stability_threshold": 3, + "control_velocity": 3.0, + "control_duration": 0.25, + "control_deadzone": 0.1, + "max_altitude": -20.0, + "min_altitude": -5.0, + "target_fps": 30, + "frame_timeout": 2.0, + "log_level": "INFO", + "ui_theme": "dark", + "show_fps": true, + "show_landmarks": true, + "show_confidence": true +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json new file mode 100644 index 0000000000..c6d931acbe --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -0,0 +1,45 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "skin_lower": [ + 0, + 30, + 60 + ], + "skin_upper": [ + 25, + 255, + 255 + ], + "min_hand_area": 3000, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.6, + "detection_interval": 2, + "single_finger_angle_threshold": 60 + }, + "drone": { + "velocity": 2.5, + "duration": 0.3, + "altitude": -10.0, + "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false + } +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 5325f35afa..4bd8903a38 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,33 +1,34 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 深度优化版 -优化了手势识别准确性和系统稳定性 -作者: xiaoshiyuan888 +Gesture Controlled AirSim Drone - Optimized Version +Pure OpenCV implementation with English interface +Author: xiaoshiyuan888 """ import sys import os import time import traceback -import subprocess +import json import math +from datetime import datetime from PIL import Image, ImageDraw, ImageFont import cv2 import numpy as np -from collections import deque +from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Advanced Recognition v2.0") +print("Gesture Controlled Drone - Optimized Version") print("=" * 60) -# ========== 修复导入路径 ========== +# ========== Fix import path ========== current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -# ========== 核心模块导入 ========== +# ========== Core module imports ========== def safe_import(): - """安全导入所有模块""" + """Safely import all modules""" modules_status = {} try: @@ -49,20 +50,6 @@ def safe_import(): print(f"[OpenCV] ✗ Import failed: {e}") return None, modules_status - # 尝试导入mediapipe,如果失败则使用OpenCV方案 - mp = None - mp_hands = None - mp_drawing = None - try: - import mediapipe as mp - mp_hands = mp.solutions.hands - mp_drawing = mp.solutions.drawing_utils - modules_status['MediaPipe'] = True - print("[MediaPipe] ✓ Gesture recognition library ready") - except Exception as e: - modules_status['MediaPipe'] = False - print(f"[MediaPipe] ✗ Import failed, using OpenCV fallback: {e}") - airsim_module = None try: airsim_module = __import__('airsim') @@ -87,15 +74,12 @@ def safe_import(): return { 'cv2': cv2, 'np': np, - 'mp': mp, - 'mp_hands': mp_hands, - 'mp_drawing': mp_drawing, 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, 'airsim': airsim_module }, modules_status -# 执行导入 +# Execute imports libs, status = safe_import() if not status.get('OpenCV', False) or not status.get('PIL', False): print("\n❌ Core libraries missing, cannot start.") @@ -106,488 +90,608 @@ def safe_import(): print("✅ Environment check passed, initializing...") print("-" * 60) -# 解包库 +# Unpack libraries cv2, np = libs['cv2'], libs['np'] Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -# ========== 中文显示模块 ========== -class ChineseTextRenderer: - """中文文本渲染器""" +# ========== Configuration Manager ========== +class ConfigManager: + """Configuration Manager""" def __init__(self): - self.fonts = {} - self.load_fonts() - - def load_fonts(self): - """加载字体""" - font_paths = [ - 'simhei.ttf', - 'C:/Windows/Fonts/simhei.ttf', - 'C:/Windows/Fonts/msyh.ttc', - '/System/Library/Fonts/PingFang.ttc', - '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' - ] - for path in font_paths: + self.config_file = os.path.join(current_dir, 'gesture_config.json') + self.default_config = { + 'camera': { + 'index': 0, + 'width': 640, + 'height': 480, + 'fps': 30 + }, + 'gesture': { + 'skin_lower': [0, 30, 60], + 'skin_upper': [25, 255, 255], + 'min_hand_area': 3000, + 'history_size': 10, + 'smooth_frames': 5, + 'min_confidence': 0.6, + 'detection_interval': 1, + 'single_finger_angle_threshold': 60 # 新增:单个手指角度阈值 + }, + 'drone': { + 'velocity': 2.5, + 'duration': 0.3, + 'altitude': -10.0, + 'control_interval': 0.3 + }, + 'display': { + 'show_fps': True, + 'show_confidence': True, + 'show_help': True, + 'show_contours': True, + 'show_bbox': True, + 'show_fingertips': True # 新增:显示指尖 + }, + 'performance': { + 'target_fps': 30, + 'resize_factor': 1.0, + 'enable_multiprocessing': False + } + } + self.config = self.load_config() + + def load_config(self): + """Load configuration""" + if os.path.exists(self.config_file): try: - self.fonts[16] = ImageFont.truetype(path, 16) - self.fonts[20] = ImageFont.truetype(path, 20) - self.fonts[24] = ImageFont.truetype(path, 24) - self.fonts[28] = ImageFont.truetype(path, 28) - self.fonts[32] = ImageFont.truetype(path, 32) - print(f"✓ Chinese fonts loaded: {path}") - return - except: - continue - print("⚠ No Chinese fonts found, using default") + with open(self.config_file, 'r', encoding='utf-8') as f: + loaded_config = json.load(f) + config = self.default_config.copy() + self._merge_config(config, loaded_config) + print("✓ Configuration loaded from file") + return config + except Exception as e: + print(f"⚠ Failed to load config: {e}, using defaults") + return self.default_config.copy() + else: + print("✓ Using default configuration") + return self.default_config.copy() + + def _merge_config(self, base, update): + """Recursively merge configurations""" + for key, value in update.items(): + if key in base and isinstance(base[key], dict) and isinstance(value, dict): + self._merge_config(base[key], value) + else: + base[key] = value - def put_text(self, frame, text, pos, size=20, color=(255, 255, 255), bg_color=None): - """在图像上绘制中文文本""" + def save_config(self): + """Save configuration""" try: - img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - pil_img = Image.fromarray(img_rgb) - draw = ImageDraw.Draw(pil_img) - - font = self.fonts.get(size, self.fonts.get(20)) - - # 如果有背景色,先绘制背景 - if bg_color: - bbox = draw.textbbox(pos, text, font=font) - padding = 2 - draw.rectangle( - [bbox[0] - padding, bbox[1] - padding, bbox[2] + padding, bbox[3] + padding], - fill=bg_color - ) + with open(self.config_file, 'w', encoding='utf-8') as f: + json.dump(self.config, f, indent=2, ensure_ascii=False) + print("✓ Configuration saved") + except Exception as e: + print(f"⚠ Failed to save config: {e}") + + def get(self, *keys): + """Get configuration value""" + value = self.config + for key in keys: + if isinstance(value, dict) and key in value: + value = value[key] + else: + return None + return value - # 绘制阴影 - shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 1, pos[1] + 1) - draw.text(shadow_pos, text, font=font, fill=shadow_color) + def set(self, *keys, value): + """Set configuration value""" + if len(keys) == 0: + return - # 绘制文字 - rgb_color = color[::-1] # BGR to RGB - draw.text(pos, text, font=font, fill=rgb_color) + config = self.config + for key in keys[:-1]: + if key not in config: + config[key] = {} + config = config[key] - return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) - except Exception as e: - # 备用方案:使用OpenCV绘制英文 - cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, - size / 25, color, 2) - return frame + config[keys[-1]] = value + self.save_config() -chinese_renderer = ChineseTextRenderer() +config = ConfigManager() -# ========== 高级手势识别模块 ========== -class AdvancedGestureRecognizer: - """高级手势识别器,结合多种检测方法""" +# ========== Robust Gesture Recognizer ========== +class RobustGestureRecognizer: + """Robust Gesture Recognizer - Pure OpenCV Implementation""" def __init__(self): - self.use_mediapipe = status.get('MediaPipe', False) - - # 初始化MediaPipe(如果可用) - if self.use_mediapipe and libs['mp'] is not None: - try: - self.hands = libs['mp_hands'].Hands( - max_num_hands=2, - min_detection_confidence=0.7, - min_tracking_confidence=0.5, - static_image_mode=False, - model_complexity=1 - ) - print("✓ Using MediaPipe for gesture recognition") - except: - self.use_mediapipe = False - print("⚠ MediaPipe initialization failed, using OpenCV fallback") - else: - self.use_mediapipe = False + self.config = config.get('gesture') - # OpenCV手势识别参数 - self.history_size = 10 + # Gesture history and smoothing + self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) - self.finger_history = deque(maxlen=self.history_size) - self.current_gesture = "等待手势" - self.confidence = 0.0 - self.last_gesture_time = time.time() + self.confidence_history = deque(maxlen=self.history_size) + self.current_gesture = "Waiting" + self.current_confidence = 0.0 - # 肤色检测参数(自适应) - self.skin_lower = np.array([0, 30, 60], dtype=np.uint8) - self.skin_upper = np.array([25, 255, 255], dtype=np.uint8) + # Hand tracking + self.last_hand_position = None + self.hand_tracking = False + self.track_window = None - # 背景减除 - self.bg_subtractor = cv2.createBackgroundSubtractorMOG2(history=500, varThreshold=16, detectShadows=True) + # Performance statistics + self.process_times = deque(maxlen=30) + self.frame_counter = 0 + self.detection_interval = self.config['detection_interval'] + + # Gesture color mapping + self.gesture_colors = { + "Stop": (0, 0, 255), # Red + "Forward": (0, 255, 0), # Green + "Up": (255, 255, 0), # Cyan + "Down": (255, 0, 255), # Purple + "Left": (255, 165, 0), # Orange + "Right": (0, 165, 255), # Light Blue + "Waiting": (200, 200, 200), # Gray + "Error": (255, 0, 0) # Blue + } - # 手部跟踪 - self.track_window = None - self.track_box = None - self.tracking = False + # Skin detection parameters + self.skin_lower = np.array(self.config['skin_lower'], dtype=np.uint8) + self.skin_upper = np.array(self.config['skin_upper'], dtype=np.uint8) - print("✓ Advanced gesture recognizer initialized") + # Background subtractor + self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( + history=100, varThreshold=25, detectShadows=True + ) - def adaptive_skin_detection(self, frame): - """自适应肤色检测""" - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + # Morphological operation kernels + self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) + + # Fingertip detection + self.prev_fingertips = None + self.fingertip_stability_threshold = 3 - # 自适应调整肤色范围 - mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) + print("✓ Robust gesture recognizer initialized (Pure OpenCV)") + + def preprocess_frame(self, frame): + """Preprocess frame""" + # Flip image horizontally for intuitive control + frame = cv2.flip(frame, 1) + + # Adjust size for performance + resize_factor = config.get('performance', 'resize_factor') + if resize_factor != 1.0: + new_width = int(frame.shape[1] * resize_factor) + new_height = int(frame.shape[0] * resize_factor) + frame = cv2.resize(frame, (new_width, new_height)) + + return frame - # 形态学操作 - kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - mask = cv2.erode(mask, kernel, iterations=1) - mask = cv2.dilate(mask, kernel, iterations=2) - mask = cv2.GaussianBlur(mask, (5, 5), 0) + def detect_skin(self, frame): + """Detect skin regions""" + # Convert to HSV color space + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - return mask + # Skin mask + skin_mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) - def find_hand_contours(self, frame): - """查找手部轮廓""" - # 使用背景减除 + # Apply background subtraction fg_mask = self.bg_subtractor.apply(frame) - # 肤色检测 - skin_mask = self.adaptive_skin_detection(frame) + # Combine skin mask and foreground mask + combined_mask = cv2.bitwise_and(skin_mask, fg_mask) + + # Morphological operations + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) + combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - # 结合两种掩码 - combined_mask = cv2.bitwise_and(fg_mask, skin_mask) + # Gaussian blur + combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) - # 形态学操作优化 - kernel = np.ones((7, 7), np.uint8) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, kernel) - combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, kernel) + return combined_mask - # 查找轮廓 - contours, _ = cv2.findContours(combined_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + def find_hand_contours(self, mask): + """Find hand contours""" + # Find contours + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: - return None, combined_mask + return None - # 找到最大的轮廓(假设是手) - largest_contour = max(contours, key=cv2.contourArea) + # Find the largest contour (assumed to be hand) + contours = sorted(contours, key=cv2.contourArea, reverse=True) - # 过滤掉太小的轮廓 - if cv2.contourArea(largest_contour) < 5000: - return None, combined_mask + for contour in contours: + area = cv2.contourArea(contour) - return largest_contour, combined_mask + # Filter out too small contours + if area < self.config['min_hand_area']: + continue - def analyze_hand_contour(self, contour): - """分析手部轮廓特征""" - # 计算轮廓特征 - area = cv2.contourArea(contour) - perimeter = cv2.arcLength(contour, True) + # Calculate contour perimeter + perimeter = cv2.arcLength(contour, True) + + # Filter out too simple contours + if perimeter < 100: + continue + + return contour + + return None + + def detect_fingertips_advanced(self, contour): + """Advanced fingertip detection""" + # Simplify contour + epsilon = 0.01 * cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon, True) + + # Find convex hull + hull = cv2.convexHull(approx, returnPoints=False) - # 计算凸包和凸缺陷 - hull = cv2.convexHull(contour, returnPoints=False) - if len(hull) < 3: - return 0, None, None + if hull is None or len(hull) < 3: + return [], 0 - defects = cv2.convexityDefects(contour, hull) + # Find convexity defects + defects = cv2.convexityDefects(approx, hull) if defects is None: - return 0, None, None + return [], 0 - # 计算手指数量 - finger_count = 0 - finger_tips = [] + fingertips = [] + defects_points = [] for i in range(defects.shape[0]): s, e, f, d = defects[i, 0] - start = tuple(contour[s][0]) - end = tuple(contour[e][0]) - far = tuple(contour[f][0]) + start = tuple(approx[s][0]) + end = tuple(approx[e][0]) + far = tuple(approx[f][0]) - # 计算角度 + defects_points.append((start, end, far)) + + # Calculate triangle sides a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) - angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + # Calculate angle + if b * c != 0: + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + angle_degrees = math.degrees(angle) + + # If angle is less than 90 degrees, it might be a finger + if angle_degrees < 90: + # Check if this is a deep defect (far from hull) + if d > 1000: # Adjust this threshold based on your needs + # Both start and end points could be fingertips + for point in [start, end]: + if point not in fingertips: + fingertips.append(point) + + # If we found few fingertips, try alternative method + if len(fingertips) <= 2: + # Use convex hull points as fingertips + hull_points = cv2.convexHull(approx, returnPoints=True) + + # Find extreme points in hull + if len(hull_points) > 0: + hull_points = hull_points.reshape(-1, 2) + + # Sort by y-coordinate (top to bottom) + hull_points = hull_points[hull_points[:, 1].argsort()] + + # Take top points as fingertips + num_fingertips = min(5, len(hull_points)) + for i in range(num_fingertips): + point = tuple(hull_points[i]) + if point not in fingertips: + fingertips.append(point) + + return fingertips, len(fingertips) + + def analyze_hand_contour(self, contour, frame_shape): + """Analyze hand contour""" + if contour is None: + return None, 0.0 + + # Calculate contour area + area = cv2.contourArea(contour) - # 如果角度小于90度,可能是指缝 - if angle < math.pi / 2: - finger_count += 1 - finger_tips.append(end) + # Calculate contour center + M = cv2.moments(contour) + if M["m00"] == 0: + return None, 0.0 + + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + + # Calculate bounding box + x, y, w, h = cv2.boundingRect(contour) + + # Advanced fingertip detection + fingertips, finger_count = self.detect_fingertips_advanced(contour) + + # Calculate hand position (normalized) + h_img, w_img = frame_shape[:2] + norm_x = cx / w_img + norm_y = cy / h_img + + # Calculate gesture confidence + confidence = self.calculate_confidence(area, finger_count, len(contour)) + + # Return result + result = { + 'contour': contour, + 'center': (cx, cy), + 'bbox': (x, y, x + w, y + h), + 'finger_count': finger_count, + 'fingertips': fingertips, + 'area': area, + 'position': (norm_x, norm_y), + 'confidence': confidence + } - # 手指数量加1(大拇指) - finger_count = min(finger_count + 1, 5) + return result, confidence - # 计算轮廓中心 - M = cv2.moments(contour) - if M["m00"] != 0: - cx = int(M["m10"] / M["m00"]) - cy = int(M["m01"] / M["m00"]) - center = (cx, cy) - else: - center = None + def calculate_confidence(self, area, finger_count, contour_length): + """Calculate gesture confidence""" + confidence = 0.5 # Base confidence - return finger_count, center, finger_tips + # Confidence based on area + if area > 5000: + confidence += 0.2 + elif area > 3000: + confidence += 0.1 - def detect_fingertips_with_contours(self, contour): - """使用轮廓检测指尖""" - # 简化轮廓 - epsilon = 0.02 * cv2.arcLength(contour, True) - approx = cv2.approxPolyDP(contour, epsilon, True) + # Confidence based on finger count + if 0 <= finger_count <= 5: + confidence += 0.2 - # 查找凸点(可能是指尖) - hull = cv2.convexHull(approx, returnPoints=True) + # Confidence based on contour complexity + if contour_length > 200: + confidence += 0.1 - fingertips = [] - for point in hull: - fingertips.append(tuple(point[0])) + return min(confidence, 1.0) + + def recognize_gesture(self, hand_data): + """Recognize gesture based on hand data""" + if hand_data is None: + return "Waiting", 0.3 - return fingertips[:5] # 最多5个指尖 + finger_count = hand_data['finger_count'] + norm_x, norm_y = hand_data['position'] + confidence = hand_data['confidence'] + fingertips = hand_data.get('fingertips', []) - def recognize_gesture_by_fingers(self, finger_count, center, frame_shape): - """根据手指数量识别手势""" - if center is None: - return "等待手势", 0.5 + # Special case: single finger detection + if finger_count == 1: + # Additional check for single finger gesture + if len(fingertips) == 1: + # Check if the fingertip is above the hand center (typical for pointing) + cx, cy = hand_data['center'] + fingertip = fingertips[0] - h, w = frame_shape[:2] - cx, cy = center + if fingertip[1] < cy - 20: # Fingertip is above hand center + return "Forward", confidence * 0.9 + else: + return "Stop", confidence * 0.7 + return "Forward", confidence * 0.8 - # 计算手部位置(归一化) - norm_x = cx / w - norm_y = cy / h + # No fingers (fist) + elif finger_count == 0: + return "Stop", confidence * 0.9 - if finger_count == 0: - return "停止", 0.9 - elif finger_count == 1: - return "向前", 0.8 + # Two fingers (V sign) elif finger_count == 2: - # V字手势,可能是向前或特殊命令 - return "向前", 0.7 + return "Forward", confidence * 0.7 + + # Three fingers elif finger_count == 3: - # 三指手势,根据位置判断 + # Judge direction based on position if norm_y < 0.4: - return "向上", 0.7 + return "Up", confidence * 0.7 elif norm_y > 0.6: - return "向下", 0.7 + return "Down", confidence * 0.7 else: - return "停止", 0.6 + return "Stop", confidence * 0.6 + + # Four or five fingers (open hand) elif finger_count >= 4: - # 手掌张开,根据位置判断方向 + # Judge direction based on hand position if norm_x < 0.3: - return "向左", 0.8 + return "Left", confidence * 0.8 elif norm_x > 0.7: - return "向右", 0.8 + return "Right", confidence * 0.8 elif norm_y < 0.4: - return "向上", 0.8 + return "Up", confidence * 0.8 elif norm_y > 0.6: - return "向下", 0.8 + return "Down", confidence * 0.8 else: - return "停止", 0.7 + return "Stop", confidence * 0.7 - return "等待手势", 0.5 + return "Waiting", confidence * 0.5 - def recognize_with_mediapipe(self, frame): - """使用MediaPipe进行手势识别""" - rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - results = self.hands.process(rgb_frame) + def smooth_gesture(self, new_gesture, new_confidence): + """Smooth gesture output""" + # Add to history + self.gesture_history.append(new_gesture) + self.confidence_history.append(new_confidence) - gesture = "等待手势" - confidence = 0.5 + # If not enough history, return directly + if len(self.gesture_history) < 3: + self.current_gesture = new_gesture + self.current_confidence = new_confidence + return new_gesture, new_confidence - if results.multi_hand_landmarks: - for hand_landmarks in results.multi_hand_landmarks: - # 绘制手部关键点 - libs['mp_drawing'].draw_landmarks( - frame, hand_landmarks, libs['mp_hands'].HAND_CONNECTIONS, - libs['mp_drawing'].DrawingSpec(color=(0, 255, 0), thickness=2), - libs['mp_drawing'].DrawingSpec(color=(255, 0, 0), thickness=2) - ) + # Count most common gesture + gesture_counter = Counter(self.gesture_history) + most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] - # 计算手指状态 - landmarks = hand_landmarks.landmark - - # 简单的手指状态检测 - finger_tips = [4, 8, 12, 16, 20] - finger_pips = [2, 6, 10, 14, 18] - - extended_fingers = 0 - for tip, pip in zip(finger_tips, finger_pips): - if landmarks[tip].y < landmarks[pip].y: - extended_fingers += 1 - - # 根据伸直的手指数量判断手势 - if extended_fingers == 0: - gesture = "停止" - confidence = 0.9 - elif extended_fingers == 1: - gesture = "向前" - confidence = 0.8 - elif extended_fingers >= 4: - # 根据手掌方向判断 - wrist = landmarks[0] - middle_mcp = landmarks[9] - - dx = middle_mcp.x - wrist.x - dy = middle_mcp.y - wrist.y - - if abs(dx) > abs(dy): - if dx > 0: - gesture = "向右" - else: - gesture = "向左" - else: - if dy > 0: - gesture = "向下" - else: - gesture = "向上" - confidence = 0.7 + # Calculate average confidence + avg_confidence = np.mean(list(self.confidence_history)) - return gesture, confidence, frame + # If the most common gesture appears enough times, use it + smooth_frames = self.config['smooth_frames'] + if most_common_count >= smooth_frames: + self.current_gesture = most_common_gesture + self.current_confidence = avg_confidence - def smooth_gesture(self, gesture, confidence): - """平滑手势输出""" - self.gesture_history.append((gesture, confidence)) + return self.current_gesture, self.current_confidence - if len(self.gesture_history) < 3: - return gesture, confidence - - # 统计最近的手势 - gesture_counts = {} - confidence_sum = {} - - for g, c in self.gesture_history: - if g not in gesture_counts: - gesture_counts[g] = 0 - confidence_sum[g] = 0 - gesture_counts[g] += 1 - confidence_sum[g] += c - - # 找到最频繁的手势 - max_count = 0 - best_gesture = gesture - best_confidence = confidence - - for g, count in gesture_counts.items(): - if count > max_count and g != "等待手势": - max_count = count - best_gesture = g - best_confidence = confidence_sum[g] / count - - # 如果手势稳定,更新当前手势 - if max_count >= len(self.gesture_history) // 2: - self.current_gesture = best_gesture - self.confidence = best_confidence - - return self.current_gesture, self.confidence - - def visualize_hand(self, frame, contour, center, finger_count, fingertips): - """可视化手部检测结果""" - if contour is not None: - # 绘制轮廓 - cv2.drawContours(frame, [contour], -1, (0, 255, 0), 2) - - # 绘制边界框 - x, y, w, h = cv2.boundingRect(contour) - cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 0), 2) - - # 绘制中心点 - if center: - cx, cy = center - cv2.circle(frame, (cx, cy), 8, (0, 0, 255), -1) - - # 绘制指尖 - for tip in fingertips[:finger_count]: - cv2.circle(frame, tip, 6, (255, 255, 0), -1) - - # 显示手指数量 - cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + def visualize_detection(self, frame, hand_data, gesture, confidence): + """Visualize detection results""" + if hand_data is None: + return frame + + # Get display configuration + show_contours = config.get('display', 'show_contours') + show_bbox = config.get('display', 'show_bbox') + show_fingertips = config.get('display', 'show_fingertips') + + # Draw contour + if show_contours and 'contour' in hand_data: + cv2.drawContours(frame, [hand_data['contour']], -1, (0, 255, 0), 2) + + # Draw bounding box + if show_bbox and 'bbox' in hand_data: + x1, y1, x2, y2 = hand_data['bbox'] + color = self.gesture_colors.get(gesture, (255, 255, 255)) + cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) + + # Show gesture label + label = f"{gesture}" + if config.get('display', 'show_confidence'): + label += f" ({confidence:.0%})" + + # Calculate text size + (text_width, text_height), baseline = cv2.getTextSize( + label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 + ) + + # Draw text background + cv2.rectangle(frame, + (x1, y1 - text_height - 10), + (x1 + text_width, y1), + color, -1) + + # Draw text + cv2.putText(frame, label, (x1, y1 - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) + + # Draw center point + if 'center' in hand_data: + cx, cy = hand_data['center'] + cv2.circle(frame, (cx, cy), 5, (0, 0, 255), -1) + cv2.putText(frame, "Center", (cx + 10, cy), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) + + # Draw fingertips + if show_fingertips and 'fingertips' in hand_data: + for i, point in enumerate(hand_data['fingertips']): + cv2.circle(frame, point, 4, (255, 0, 0), -1) + cv2.putText(frame, f"F{i + 1}", (point[0] + 5, point[1]), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1) + + # Show finger count + if 'finger_count' in hand_data: + finger_text = f"Fingers: {hand_data['finger_count']}" + cv2.putText(frame, finger_text, (10, frame.shape[0] - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) return frame def recognize(self, frame): - """识别手势""" + """Recognize gesture""" start_time = time.time() try: - # 镜像图像以便更直观的控制 - frame = cv2.flip(frame, 1) - - # 方法1:使用MediaPipe(如果可用) - if self.use_mediapipe: - gesture, confidence, processed_frame = self.recognize_with_mediapipe(frame) - frame = processed_frame - else: - # 方法2:使用OpenCV - contour, mask = self.find_hand_contours(frame) + # Preprocess frame + processed_frame = self.preprocess_frame(frame) - if contour is not None: - finger_count, center, finger_tips = self.analyze_hand_contour(contour) + # Detect every few frames to improve performance + if self.frame_counter % self.detection_interval != 0: + self.frame_counter += 1 + return self.current_gesture, self.current_confidence, processed_frame - # 如果凸缺陷检测失败,使用轮廓检测指尖 - if finger_tips is None or len(finger_tips) < finger_count: - finger_tips = self.detect_fingertips_with_contours(contour) + # Detect skin + skin_mask = self.detect_skin(processed_frame) - self.finger_history.append(finger_count) + # Find hand contour + hand_contour = self.find_hand_contours(skin_mask) - # 计算平均手指数量(平滑) - if len(self.finger_history) > 0: - avg_fingers = sum(self.finger_history) / len(self.finger_history) - finger_count = int(round(avg_fingers)) + # Analyze hand contour + hand_data, confidence = self.analyze_hand_contour(hand_contour, processed_frame.shape) - # 识别手势 - gesture, confidence = self.recognize_gesture_by_fingers( - finger_count, center, frame.shape - ) - - # 可视化结果 - frame = self.visualize_hand(frame, contour, center, finger_count, finger_tips) - else: - gesture, confidence = "等待手势", 0.3 + # Recognize gesture + if hand_data is not None: + gesture, raw_confidence = self.recognize_gesture(hand_data) + confidence = max(confidence, raw_confidence) + else: + gesture, confidence = "Waiting", 0.3 - # 手势平滑处理 + # Smooth gesture final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) - # 更新最后检测时间 - self.last_gesture_time = time.time() + # Visualize results + if hand_data is not None: + processed_frame = self.visualize_detection( + processed_frame, hand_data, final_gesture, final_confidence + ) + + # Update counter + self.frame_counter += 1 - # 显示处理时间 + # Calculate processing time process_time = (time.time() - start_time) * 1000 - cv2.putText(frame, f'Process: {process_time:.1f}ms', (10, frame.shape[0] - 20), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + self.process_times.append(process_time) - return final_gesture, final_confidence, frame + return final_gesture, final_confidence, processed_frame except Exception as e: - print(f"Gesture recognition error: {e}") - return "识别异常", 0.0, frame + print(f"⚠ Gesture recognition error: {e}") + return "Error", 0.0, frame + + def get_performance_stats(self): + """Get performance statistics""" + if len(self.process_times) == 0: + return 0.0 + + return np.mean(list(self.process_times)) def set_simulated_gesture(self, gesture): - """设置模拟的手势""" + """Set simulated gesture""" self.current_gesture = gesture - self.confidence = 0.9 + self.current_confidence = 0.9 -# ========== 增强的无人机控制模块 ========== -class EnhancedDroneController: - """增强的无人机控制器""" +# ========== Simple Drone Controller ========== +class SimpleDroneController: + """Simple Drone Controller""" def __init__(self, airsim_module): self.airsim = airsim_module self.client = None self.connected = False self.flying = False - self.connection_attempted = False - # 控制参数 - self.velocity = 3.0 - self.duration = 0.2 - self.altitude = -10.0 # 初始高度(负值表示向上) + # Control parameters + self.velocity = config.get('drone', 'velocity') + self.duration = config.get('drone', 'duration') + self.altitude = config.get('drone', 'altitude') + self.control_interval = config.get('drone', 'control_interval') - # 控制历史 - self.control_history = deque(maxlen=10) + # Control state self.last_control_time = 0 + self.last_gesture = None - def connect(self): - """连接AirSim无人机""" - if self.connection_attempted: - return self.connected + print("✓ Simple drone controller initialized") - self.connection_attempted = True + def connect(self): + """Connect to AirSim drone""" + if self.connected: + return True if self.airsim is None: print("⚠ AirSim not available, using simulation mode") @@ -612,13 +716,7 @@ def connect(self): except Exception as e: print(f"❌ Connection failed: {e}") - print("\n请确保:") - print("1. AirSim模拟器正在运行") - print("2. 选择环境(如:Landscape Mountains)") - print("3. 无人机已在世界中生成") - print("4. 如果需要,按模拟器中的'R'键重置无人机") - - print("\n使用模拟模式继续?(y/n)") + print("\nContinue with simulation mode? (y/n)") choice = input().strip().lower() if choice == 'y': self.connected = True @@ -628,9 +726,8 @@ def connect(self): return False def takeoff(self): - """起飞""" + """Take off""" if not self.connected: - print("❌ Drone not connected") return False try: @@ -643,7 +740,7 @@ def takeoff(self): self.client.takeoffAsync().join() time.sleep(1) - # 上升到指定高度 + # Ascend to specified altitude self.client.moveToZAsync(self.altitude, 3).join() self.flying = True @@ -651,11 +748,10 @@ def takeoff(self): return True except Exception as e: print(f"❌ Takeoff failed: {e}") - self.flying = True # 模拟模式 - return True + return False def land(self): - """降落""" + """Land""" if not self.connected: return False @@ -672,54 +768,54 @@ def land(self): return True except Exception as e: print(f"Landing failed: {e}") - self.flying = False return False - def move_by_gesture(self, gesture, confidence=0.7): - """根据手势移动""" + def move_by_gesture(self, gesture, confidence): + """Move based on gesture""" if not self.connected or not self.flying: return False - # 检查控制间隔 + # Check control interval current_time = time.time() - if current_time - self.last_control_time < 0.1: # 最小控制间隔 + if current_time - self.last_control_time < self.control_interval: return False - # 如果置信度太低,不执行动作 - if confidence < 0.5: + # Check confidence threshold + min_confidence = config.get('gesture', 'min_confidence') + if confidence < min_confidence: return False try: if self.airsim is None or self.client is None: print(f"Simulated move: {gesture}") - self.control_history.append((gesture, current_time)) self.last_control_time = current_time + self.last_gesture = gesture return True success = False - if gesture == "向上": + if gesture == "Up": self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) success = True - elif gesture == "向下": + elif gesture == "Down": self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) success = True - elif gesture == "向左": + elif gesture == "Left": self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) success = True - elif gesture == "向右": + elif gesture == "Right": self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) success = True - elif gesture == "向前": + elif gesture == "Forward": self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) success = True - elif gesture == "停止": + elif gesture == "Stop": self.client.hoverAsync() success = True if success: - self.control_history.append((gesture, current_time)) self.last_control_time = current_time + self.last_gesture = gesture return success except Exception as e: @@ -727,7 +823,7 @@ def move_by_gesture(self, gesture, confidence=0.7): return False def emergency_stop(self): - """紧急停止""" + """Emergency stop""" if self.connected: try: if self.flying and self.client is not None: @@ -739,251 +835,348 @@ def emergency_stop(self): print("✅ Emergency stop complete") except: pass + self.connected = False self.flying = False -# ========== 性能监控模块 ========== +# ========== Simple UI Renderer ========== +class SimpleUIRenderer: + """Simple UI Renderer""" + + def __init__(self): + # Color definitions + self.colors = { + 'title': (0, 255, 255), # Cyan + 'connected': (0, 255, 0), # Green + 'disconnected': (0, 0, 255), # Red + 'flying': (0, 255, 0), # Green + 'landed': (255, 165, 0), # Orange + 'warning': (0, 165, 255), # Light Blue + 'info': (255, 255, 255), # White + 'help': (255, 200, 100) # Light Orange + } + + print("✓ Simple UI renderer initialized") + + def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time): + """Draw status bar""" + h, w = frame.shape[:2] + + # Draw semi-transparent background + overlay = frame.copy() + cv2.rectangle(overlay, (0, 0), (w, 100), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + + # Title + title = "Gesture Controlled Drone" + cv2.putText(frame, title, (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, self.colors['title'], 2) + + # Connection status + status_color = self.colors['connected'] if drone_controller.connected else self.colors['disconnected'] + status_text = f"Drone: {'Connected' if drone_controller.connected else 'Disconnected'}" + cv2.putText(frame, status_text, (10, 60), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, status_color, 1) + + # Flight status + flight_color = self.colors['flying'] if drone_controller.flying else self.colors['landed'] + flight_text = f"Flight: {'Flying' if drone_controller.flying else 'Landed'}" + cv2.putText(frame, flight_text, (10, 85), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, flight_color, 1) + + # Gesture information + if confidence > 0.7: + gesture_color = (0, 255, 0) # Green + elif confidence > 0.5: + gesture_color = (255, 165, 0) # Orange + else: + gesture_color = (200, 200, 200) # Gray + + gesture_text = f"Gesture: {gesture}" + if config.get('display', 'show_confidence'): + gesture_text += f" ({confidence:.0%})" + + cv2.putText(frame, gesture_text, (w // 2, 60), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, gesture_color, 1) + + # Performance information + if config.get('display', 'show_fps'): + perf_text = f"FPS: {fps:.1f}" + if process_time > 0: + perf_text += f" | Delay: {process_time:.1f}ms" + + cv2.putText(frame, perf_text, (w - 200, 85), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['info'], 1) + + return frame + + def draw_help_bar(self, frame): + """Draw help bar""" + if not config.get('display', 'show_help'): + return frame + + h, w = frame.shape[:2] + + # Draw bottom help bar + cv2.rectangle(frame, (0, h - 50), (w, h), (0, 0, 0), -1) + + # Help text + help_text = "C:Connect Space:Takeoff/Land ESC:Exit W/A/S/D/F/X:Keyboard" + cv2.putText(frame, help_text, (10, h - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['help'], 1) + + return frame + + def draw_warning(self, frame, message): + """Draw warning message""" + h, w = frame.shape[:2] + + # Draw warning at top + warning_bg = np.zeros((40, w, 3), dtype=np.uint8) + warning_bg[:, :] = (0, 69, 255) # Orange + + frame[100:140, 0:w] = cv2.addWeighted( + frame[100:140, 0:w], 0.3, + warning_bg, 0.7, 0 + ) + + # Draw warning text + cv2.putText(frame, message, (10, 125), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, self.colors['warning'], 1) + + return frame + + +# ========== Performance Monitor ========== class PerformanceMonitor: - """性能监控器""" + """Performance Monitor""" def __init__(self): self.frame_times = deque(maxlen=60) - self.fps = 0 self.last_update = time.time() + self.fps = 0 + self.frame_count = 0 def update(self): - """更新性能数据""" + """Update performance data""" current_time = time.time() self.frame_times.append(current_time) + self.frame_count += 1 - # 每秒更新一次FPS + # Calculate FPS once per second if current_time - self.last_update >= 1.0: if len(self.frame_times) > 1: - self.fps = len(self.frame_times) / (self.frame_times[-1] - self.frame_times[0]) + time_diff = self.frame_times[-1] - self.frame_times[0] + if time_diff > 0: + self.fps = len(self.frame_times) / time_diff + else: + self.fps = 0 self.last_update = current_time def get_stats(self): - """获取性能统计""" + """Get performance statistics""" return { 'fps': self.fps, - 'frame_count': len(self.frame_times) + 'frame_count': self.frame_count } -# ========== 主程序 ========== +# ========== Main Program ========== def main(): - """主函数""" - # 初始化组件 - gesture_recognizer = AdvancedGestureRecognizer() - drone_controller = EnhancedDroneController(libs['airsim']) + """Main function""" + # Initialize components + print("Initializing components...") + + gesture_recognizer = RobustGestureRecognizer() + drone_controller = SimpleDroneController(libs['airsim']) + ui_renderer = SimpleUIRenderer() performance_monitor = PerformanceMonitor() - # 初始化摄像头 + # Initialize camera cap = None - available_cameras = [] - - # 检测可用摄像头 - for i in range(0, 4): - temp_cap = cv2.VideoCapture(i) - if temp_cap.isOpened(): - available_cameras.append(i) - temp_cap.release() - - if available_cameras: - print(f"✓ Available cameras: {available_cameras}") - cap = cv2.VideoCapture(available_cameras[0]) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - cap.set(cv2.CAP_PROP_FPS, 30) - print(f"✓ Using camera {available_cameras[0]}") - else: - print("❌ No cameras available, using keyboard control only") + try: + cap = cv2.VideoCapture(0) + if cap.isOpened(): + cap.set(cv2.CAP_PROP_FRAME_WIDTH, config.get('camera', 'width')) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, config.get('camera', 'height')) + cap.set(cv2.CAP_PROP_FPS, config.get('camera', 'fps')) + + # Get actual parameters + actual_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + actual_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + actual_fps = int(cap.get(cv2.CAP_PROP_FPS)) + + print(f"✓ Camera initialized") + print(f" Resolution: {actual_width}x{actual_height}") + print(f" FPS: {actual_fps}") + else: + print("❌ Camera not available, using simulation mode") + cap = None + except Exception as e: + print(f"⚠ Camera init failed: {e}") cap = None - # 显示说明 + # Display welcome message print("\n" + "=" * 60) - print("手势控制无人机系统 - 高级版") + print("Gesture Controlled Drone - Optimized Version") print("=" * 60) - print("操作说明:") - print("1. 首先启动AirSim模拟器(可选)") - print("2. 按 [C] 连接无人机") - print("3. 按 [空格键] 起飞/降落") - if cap: - print("4. 手势控制:") - print(" - 握拳(0指):停止") - print(" - 食指(1指):向前") - print(" - 手掌张开(4-5指):根据手的位置控制方向") - print(" * 手势识别置信度 > 50% 时才会执行") - else: - print("4. 键盘控制:") - print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") - print("5. 按 [ESC] 安全退出") - print("6. 按 [R] 重置手势识别") + print("System Status:") + print(f" Camera: {'Connected' if cap else 'Simulation Mode'}") + print(f" Gesture Recognition: Pure OpenCV") + print(f" AirSim: {'Available' if libs['airsim'] else 'Simulation Mode'}") print("=" * 60) - # 键盘手势映射 + # Display operation instructions + print("\nOperation Instructions:") + print("1. Press [C] to connect drone (AirSim simulator)") + print("2. Press [SPACE] to takeoff/land") + print("3. Gesture Control:") + print(" - Fist (0 fingers): Stop") + print(" - Index finger (1 finger): Forward") + print(" - Open hand (4-5 fingers): Control direction by hand position") + print(" * Gesture confidence > 60% to execute") + print("4. Keyboard Control:") + print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") + print("5. Press [H] to toggle help display") + print("6. Press [R] to reset gesture recognition") + print("7. Press [ESC] to exit safely") + print("=" * 60) + print("Program started successfully!") + print("-" * 60) + + # Keyboard gesture mapping key_to_gesture = { - ord('w'): "向上", ord('W'): "向上", - ord('s'): "向下", ord('S'): "向下", - ord('a'): "向左", ord('A'): "向左", - ord('d'): "向右", ord('D'): "向右", - ord('f'): "向前", ord('F'): "向前", - ord('x'): "停止", ord('X'): "停止", + ord('w'): "Up", ord('W'): "Up", + ord('s'): "Down", ord('S'): "Down", + ord('a'): "Left", ord('A'): "Left", + ord('d'): "Right", ord('D'): "Right", + ord('f'): "Forward", ord('F'): "Forward", + ord('x'): "Stop", ord('X'): "Stop", } - # 主循环 - last_gesture_time = time.time() - gesture_timeout = 2.0 # 手势超时时间 - - while True: - performance_monitor.update() + # Main loop + print("\nEntering main loop, press ESC to exit...") - # 读取摄像头帧 - if cap: - ret, frame = cap.read() - if not ret: - # 如果读取失败,尝试重新初始化摄像头 - print("⚠ Camera read failed, trying to reinitialize...") - cap.release() - if available_cameras: - cap = cv2.VideoCapture(available_cameras[0]) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - ret, frame = cap.read() + try: + while True: + # Update performance monitor + performance_monitor.update() + # Read camera frame + if cap: + ret, frame = cap.read() if not ret: + # Create blank frame frame = np.zeros((480, 640, 3), dtype=np.uint8) - current_gesture = "摄像头错误" - confidence = 0.0 + gesture, confidence = "Camera Error", 0.0 else: - current_gesture, confidence, frame = gesture_recognizer.recognize(frame) + # Gesture recognition + gesture, confidence, frame = gesture_recognizer.recognize(frame) else: - current_gesture, confidence, frame = gesture_recognizer.recognize(frame) - else: - frame = np.zeros((480, 640, 3), dtype=np.uint8) - current_gesture = gesture_recognizer.current_gesture - confidence = gesture_recognizer.confidence - - # ========== 绘制界面 ========== - # 创建信息覆盖层 - overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (640, 140), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) - - # 标题和版本 - title = "手势控制无人机系统 - 高级版" - frame = chinese_renderer.put_text(frame, title, (10, 10), size=28, color=(0, 255, 255)) + # Simulation mode + frame = np.zeros((480, 640, 3), dtype=np.uint8) + gesture, confidence = gesture_recognizer.current_gesture, gesture_recognizer.current_confidence - # 状态信息 - status_color = (0, 255, 0) if drone_controller.connected else (0, 0, 255) - status_text = f"状态: {'已连接' if drone_controller.connected else '未连接'}" - frame = chinese_renderer.put_text(frame, status_text, (10, 50), size=24, color=status_color) + # Get performance statistics + perf_stats = performance_monitor.get_stats() + process_time = gesture_recognizer.get_performance_stats() - flight_color = (0, 255, 0) if drone_controller.flying else (255, 165, 0) - flight_text = f"飞行: {'是' if drone_controller.flying else '否'}" - frame = chinese_renderer.put_text(frame, flight_text, (10, 80), size=24, color=flight_color) + # Draw UI + frame = ui_renderer.draw_status_bar( + frame, drone_controller, gesture, confidence, + perf_stats['fps'], process_time + ) - # 手势信息 - gesture_color = (0, 255, 0) if confidence > 0.7 else (255, 165, 0) if confidence > 0.5 else (200, 200, 200) - gesture_text = f"手势: {current_gesture} ({confidence:.1%})" - frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), size=24, color=gesture_color) + frame = ui_renderer.draw_help_bar(frame) - # 性能信息 - stats = performance_monitor.get_stats() - perf_text = f"FPS: {stats['fps']:.1f} | 延迟: {1000 / stats['fps']:.1f}ms" if stats['fps'] > 0 else "FPS: 0" - frame = chinese_renderer.put_text(frame, perf_text, (440, 50), size=20, color=(255, 255, 255)) - - # 操作提示 - help_text = "C:连接 空格:起飞/降落 R:重置 ESC:退出" - if not cap: - help_text += " W/A/S/D/F/X:控制" - - # 绘制底部提示栏 - cv2.rectangle(frame, (0, 440), (640, 480), (0, 0, 0), -1) - frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) - frame = chinese_renderer.put_text(frame, help_text, (10, 450), size=18, color=(255, 255, 255)) - - # 手势提示 - if cap: - hint_text = "提示: 保持手势稳定,确保良好光线" - frame = chinese_renderer.put_text(frame, hint_text, (10, 470), size=16, color=(255, 200, 100)) - - # 显示连接提示 - if not drone_controller.connected: - warning_text = "⚠ 按C键连接无人机,或继续使用模拟模式" - frame = chinese_renderer.put_text(frame, warning_text, (10, 400), - size=18, color=(0, 165, 255), bg_color=(0, 0, 0)) + # Show connection warning + if not drone_controller.connected: + warning_msg = "⚠ Press C to connect drone, or use simulation mode" + frame = ui_renderer.draw_warning(frame, warning_msg) - # 显示图像 - cv2.imshow('Gesture Controlled Drone - Advanced Version', frame) + # Show image + window_title = "Gesture Controlled Drone - Press ESC to exit" + cv2.imshow(window_title, frame) - # ========== 键盘控制 ========== - key = cv2.waitKey(1) & 0xFF + # ========== Keyboard Control ========== + key = cv2.waitKey(1) & 0xFF - if key == 27: # ESC键 - print("\n退出程序...") - break + if key == 27: # ESC key + print("\nExiting program...") + break - elif key == ord('c') or key == ord('C'): - if not drone_controller.connected: - drone_controller.connect() + elif key == ord('c') or key == ord('C'): + if not drone_controller.connected: + drone_controller.connect() - elif key == 32: # 空格键 - if drone_controller.connected: - if drone_controller.flying: - drone_controller.land() - else: - drone_controller.takeoff() - time.sleep(0.5) - - elif key == ord('r') or key == ord('R'): - print("重置手势识别...") - gesture_recognizer = AdvancedGestureRecognizer() - - elif key in key_to_gesture: - gesture = key_to_gesture[key] - gesture_recognizer.set_simulated_gesture(gesture) - current_gesture = gesture - confidence = 0.9 - if drone_controller.connected and drone_controller.flying: + elif key == 32: # Space key + if drone_controller.connected: + if drone_controller.flying: + drone_controller.land() + else: + drone_controller.takeoff() + time.sleep(0.5) + + elif key == ord('h') or key == ord('H'): + # Toggle help display + current = config.get('display', 'show_help') + config.set('display', 'show_help', value=not current) + print(f"Help display: {'ON' if not current else 'OFF'}") + + elif key == ord('f') or key == ord('F'): + # Toggle FPS display + current = config.get('display', 'show_fps') + config.set('display', 'show_fps', value=not current) + print(f"FPS display: {'ON' if not current else 'OFF'}") + + elif key == ord('r') or key == ord('R'): + # Reset gesture recognition + print("Resetting gesture recognition...") + gesture_recognizer = RobustGestureRecognizer() + + elif key == ord('t') or key == ord('T'): + # Toggle fingertip display + current = config.get('display', 'show_fingertips') + config.set('display', 'show_fingertips', value=not current) + print(f"Fingertip display: {'ON' if not current else 'OFF'}") + + elif key in key_to_gesture: + # Keyboard control + simulated_gesture = key_to_gesture[key] + gesture_recognizer.set_simulated_gesture(simulated_gesture) + gesture = simulated_gesture + confidence = 0.9 + if drone_controller.connected and drone_controller.flying: + drone_controller.move_by_gesture(gesture, confidence) + + # Real gesture control + current_time = time.time() + if (gesture and gesture != "Waiting" and + gesture != "Camera Error" and gesture != "Error" and + drone_controller.connected and drone_controller.flying): drone_controller.move_by_gesture(gesture, confidence) - # 真实手势控制 - current_time = time.time() - if (current_gesture and current_gesture != "等待手势" and - current_gesture != "摄像头错误" and current_gesture != "识别异常" and - drone_controller.connected and drone_controller.flying and - confidence > 0.5 and # 只有置信度足够高才执行 - current_time - last_gesture_time > 0.2): # 控制频率限制 - - success = drone_controller.move_by_gesture(current_gesture, confidence) - if success: - last_gesture_time = current_time - - # 检查手势超时 - if current_time - gesture_recognizer.last_gesture_time > gesture_timeout: - gesture_recognizer.current_gesture = "等待手势" - - # 清理资源 - print("\n清理资源...") - if cap: - cap.release() - cv2.destroyAllWindows() - drone_controller.emergency_stop() - print("程序安全退出") - print("=" * 60) - - -# ========== 程序入口 ========== -if __name__ == "__main__": - try: - main() except KeyboardInterrupt: - print("\n程序被用户中断") + print("\nProgram interrupted by user") except Exception as e: - print(f"\n程序错误: {e}") + print(f"\nProgram error: {e}") traceback.print_exc() finally: - print("\n按回车键退出...") - input() \ No newline at end of file + # Cleanup resources + print("\nCleaning up resources...") + if cap: + cap.release() + cv2.destroyAllWindows() + drone_controller.emergency_stop() + config.save_config() + + print("Program exited safely") + print("=" * 60) + print("\nThank you for using Gesture Controlled Drone System!") + input("Press Enter to exit...") + + +# ========== Program Entry Point ========== +if __name__ == "__main__": + main() \ No newline at end of file From f1897a2c8b60eee2305bb24fc22521495d7de8ab Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 15:46:37 +0800 Subject: [PATCH 5/8] =?UTF-8?q?=E6=8C=81=E7=BB=AD=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E6=89=8B=E5=8A=BF=E8=AF=86=E5=88=AB=E7=AE=97=E6=B3=95=EF=BC=8C?= =?UTF-8?q?=E8=A7=A3=E5=86=B3=E4=BD=8D=E7=BD=AE=E6=95=8F=E6=84=9F=E9=97=AE?= =?UTF-8?q?=E9=A2=98=EF=BC=8C=E5=A2=9E=E5=BC=BA=E5=85=A8=E5=B1=8F=E6=89=8B?= =?UTF-8?q?=E5=8A=BF=E6=A3=80=E6=B5=8B=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config.json | 7 +- .../gesture_config_v2.json | 53 + .../gesture_config_v3.json | 53 + .../main.py | 1241 ++++++++++------- 4 files changed, 887 insertions(+), 467 deletions(-) create mode 100644 src/autonomus_drone_hand_gesture_project/gesture_config_v2.json create mode 100644 src/autonomus_drone_hand_gesture_project/gesture_config_v3.json diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index c6d931acbe..d41f020e20 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -21,6 +21,9 @@ "smooth_frames": 5, "min_confidence": 0.6, "detection_interval": 2, + "hand_region_threshold": 0.2, + "face_exclusion_area": 8000, + "palm_aspect_ratio": 1.3, "single_finger_angle_threshold": 60 }, "drone": { @@ -35,7 +38,9 @@ "show_help": true, "show_contours": true, "show_bbox": true, - "show_fingertips": true + "show_fingertips": true, + "show_direction": true, + "show_region_grid": true }, "performance": { "target_fps": 30, diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json new file mode 100644 index 0000000000..b5479855de --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json @@ -0,0 +1,53 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "skin_lower_h": 0, + "skin_upper_h": 178, + "skin_lower_s": 0, + "skin_upper_s": 67, + "skin_lower_v": 36, + "skin_upper_v": 98, + "min_hand_area": 2000, + "max_hand_area": 30000, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.5, + "detection_interval": 1, + "hand_ratio_threshold": 1.5, + "contour_simplify_epsilon": 0.02, + "defect_distance_threshold": 20, + "palm_circle_radius_ratio": 0.3 + }, + "drone": { + "velocity": 2.5, + "duration": 0.3, + "altitude": -10.0, + "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": true, + "show_hand_direction": true, + "show_debug_info": false + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false + }, + "calibration": { + "auto_calibrate_skin": true, + "skin_calibration_frames": 30, + "hand_size_calibration": true + } +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json new file mode 100644 index 0000000000..7491f9cd30 --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json @@ -0,0 +1,53 @@ +{ + "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30 + }, + "gesture": { + "skin_lower_h": 0, + "skin_upper_h": 25, + "skin_lower_s": 30, + "skin_upper_s": 255, + "skin_lower_v": 60, + "skin_upper_v": 255, + "min_hand_area": 2000, + "max_hand_area": 30000, + "history_size": 15, + "smooth_frames": 10, + "min_confidence": 0.5, + "detection_interval": 1, + "gesture_lock_frames": 8, + "contour_simplify_epsilon": 0.02, + "defect_distance_threshold": 20, + "palm_circle_radius_ratio": 0.3 + }, + "drone": { + "velocity": 2.5, + "duration": 0.3, + "altitude": -10.0, + "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": false, + "show_hand_direction": false, + "show_debug_info": false + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false + }, + "calibration": { + "auto_calibrate_skin": true, + "skin_calibration_frames": 30, + "hand_size_calibration": true + } +} \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 4bd8903a38..f6f31be3fe 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- """ -Gesture Controlled AirSim Drone - Optimized Version -Pure OpenCV implementation with English interface -Author: xiaoshiyuan888 +手势控制AirSim无人机 - 手势识别优化版 +优化手势识别算法,解决位置敏感问题 +作者: xiaoshiyuan888 """ import sys @@ -18,55 +18,55 @@ from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Optimized Version") +print("Gesture Controlled Drone - Gesture Recognition Optimized") print("=" * 60) -# ========== Fix import path ========== +# ========== 修复导入路径 ========== current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -# ========== Core module imports ========== +# ========== 核心模块导入 ========== def safe_import(): - """Safely import all modules""" + """安全导入所有模块""" modules_status = {} try: from PIL import Image, ImageDraw, ImageFont modules_status['PIL'] = True - print("[PIL] ✓ Image processing library ready") + print("[PIL] ✓ 图像处理库就绪") except Exception as e: modules_status['PIL'] = False - print(f"[PIL] ✗ Import failed: {e}") + print(f"[PIL] ✗ 导入失败: {e}") return None, modules_status try: import cv2 import numpy as np modules_status['OpenCV'] = True - print("[OpenCV] ✓ Computer vision library ready") + print("[OpenCV] ✓ 计算机视觉库就绪") except Exception as e: modules_status['OpenCV'] = False - print(f"[OpenCV] ✗ Import failed: {e}") + print(f"[OpenCV] ✗ 导入失败: {e}") return None, modules_status airsim_module = None try: airsim_module = __import__('airsim') modules_status['AirSim'] = True - print(f"[AirSim] ✓ Successfully imported") + print(f"[AirSim] ✓ 成功导入") except ImportError: print("\n" + "!" * 60) - print("⚠ AirSim library NOT FOUND!") + print("⚠ AirSim库未找到!") print("!" * 60) - print("To install AirSim, run:") - print("1. First install: pip install msgpack-rpc-python") - print("2. Then install: pip install airsim") - print("\nOr from source:") + print("安装AirSim:") + print("1. 首先安装: pip install msgpack-rpc-python") + print("2. 然后安装: pip install airsim") + print("\n或从源码安装:") print(" pip install git+https://github.com/microsoft/AirSim.git") print("!" * 60) - print("\nContinue without AirSim? (y/n)") + print("\n无AirSim继续运行? (y/n)") choice = input().strip().lower() if choice != 'y': sys.exit(1) @@ -79,28 +79,28 @@ def safe_import(): }, modules_status -# Execute imports +# 执行导入 libs, status = safe_import() if not status.get('OpenCV', False) or not status.get('PIL', False): - print("\n❌ Core libraries missing, cannot start.") - input("Press Enter to exit...") + print("\n❌ 核心库缺失,无法启动。") + input("按回车键退出...") sys.exit(1) print("-" * 60) -print("✅ Environment check passed, initializing...") +print("✅ 环境检查通过,正在初始化...") print("-" * 60) -# Unpack libraries +# 解包库 cv2, np = libs['cv2'], libs['np'] Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -# ========== Configuration Manager ========== +# ========== 配置管理器 ========== class ConfigManager: - """Configuration Manager""" + """配置管理器""" def __init__(self): - self.config_file = os.path.join(current_dir, 'gesture_config.json') + self.config_file = os.path.join(current_dir, 'gesture_config_v2.json') self.default_config = { 'camera': { 'index': 0, @@ -109,14 +109,22 @@ def __init__(self): 'fps': 30 }, 'gesture': { - 'skin_lower': [0, 30, 60], - 'skin_upper': [25, 255, 255], - 'min_hand_area': 3000, + 'skin_lower_h': 0, + 'skin_upper_h': 25, + 'skin_lower_s': 30, + 'skin_upper_s': 255, + 'skin_lower_v': 60, + 'skin_upper_v': 255, + 'min_hand_area': 2000, + 'max_hand_area': 30000, 'history_size': 10, 'smooth_frames': 5, - 'min_confidence': 0.6, + 'min_confidence': 0.5, 'detection_interval': 1, - 'single_finger_angle_threshold': 60 # 新增:单个手指角度阈值 + 'hand_ratio_threshold': 1.5, # 手部长宽比阈值 + 'contour_simplify_epsilon': 0.02, + 'defect_distance_threshold': 20, # 凸缺陷距离阈值 + 'palm_circle_radius_ratio': 0.3, # 手掌半径与边界框的比例 }, 'drone': { 'velocity': 2.5, @@ -130,35 +138,46 @@ def __init__(self): 'show_help': True, 'show_contours': True, 'show_bbox': True, - 'show_fingertips': True # 新增:显示指尖 + 'show_fingertips': True, + 'show_palm_center': True, + 'show_hand_direction': True, + 'show_debug_info': False }, 'performance': { 'target_fps': 30, 'resize_factor': 1.0, 'enable_multiprocessing': False + }, + 'calibration': { + 'auto_calibrate_skin': True, + 'skin_calibration_frames': 30, + 'hand_size_calibration': True } } self.config = self.load_config() + self.skin_calibration_data = [] + self.hand_size_calibration_done = False + self.reference_hand_size = 0 def load_config(self): - """Load configuration""" + """加载配置""" if os.path.exists(self.config_file): try: with open(self.config_file, 'r', encoding='utf-8') as f: loaded_config = json.load(f) config = self.default_config.copy() self._merge_config(config, loaded_config) - print("✓ Configuration loaded from file") + print("✓ 从文件加载配置") return config except Exception as e: - print(f"⚠ Failed to load config: {e}, using defaults") + print(f"⚠ 加载配置失败: {e}, 使用默认配置") return self.default_config.copy() else: - print("✓ Using default configuration") + print("✓ 使用默认配置") return self.default_config.copy() def _merge_config(self, base, update): - """Recursively merge configurations""" + """递归合并配置""" for key, value in update.items(): if key in base and isinstance(base[key], dict) and isinstance(value, dict): self._merge_config(base[key], value) @@ -166,16 +185,16 @@ def _merge_config(self, base, update): base[key] = value def save_config(self): - """Save configuration""" + """保存配置""" try: with open(self.config_file, 'w', encoding='utf-8') as f: json.dump(self.config, f, indent=2, ensure_ascii=False) - print("✓ Configuration saved") + print("✓ 配置已保存") except Exception as e: - print(f"⚠ Failed to save config: {e}") + print(f"⚠ 保存配置失败: {e}") def get(self, *keys): - """Get configuration value""" + """获取配置值""" value = self.config for key in keys: if isinstance(value, dict) and key in value: @@ -185,7 +204,7 @@ def get(self, *keys): return value def set(self, *keys, value): - """Set configuration value""" + """设置配置值""" if len(keys) == 0: return @@ -198,208 +217,223 @@ def set(self, *keys, value): config[keys[-1]] = value self.save_config() + def calibrate_skin_color(self, frame, hand_mask): + """自动校准肤色范围""" + if not self.get('calibration', 'auto_calibrate_skin'): + return + + if len(self.skin_calibration_data) < self.get('calibration', 'skin_calibration_frames'): + # 转换到HSV + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + + # 获取肤色区域的HSV值 + skin_pixels = hsv[hand_mask > 0] + + if len(skin_pixels) > 100: # 确保有足够的像素 + self.skin_calibration_data.append(skin_pixels) + + if len(self.skin_calibration_data) == self.get('calibration', 'skin_calibration_frames'): + # 计算肤色范围 + all_skin_pixels = np.vstack(self.skin_calibration_data) + + h_min, h_max = np.percentile(all_skin_pixels[:, 0], [2, 98]) + s_min, s_max = np.percentile(all_skin_pixels[:, 1], [2, 98]) + v_min, v_max = np.percentile(all_skin_pixels[:, 2], [2, 98]) + + # 更新配置 + self.set('gesture', 'skin_lower_h', value=int(max(0, h_min - 5))) + self.set('gesture', 'skin_upper_h', value=int(min(180, h_max + 5))) + self.set('gesture', 'skin_lower_s', value=int(max(0, s_min - 10))) + self.set('gesture', 'skin_upper_s', value=int(min(255, s_max + 10))) + self.set('gesture', 'skin_lower_v', value=int(max(0, v_min - 10))) + self.set('gesture', 'skin_upper_v', value=int(min(255, v_max + 10))) + + print("✓ 肤色校准完成") + print(f" 肤色范围: H[{self.get('gesture', 'skin_lower_h')}-{self.get('gesture', 'skin_upper_h')}], " + f"S[{self.get('gesture', 'skin_lower_s')}-{self.get('gesture', 'skin_upper_s')}], " + f"V[{self.get('gesture', 'skin_lower_v')}-{self.get('gesture', 'skin_upper_v')}]") + + def calibrate_hand_size(self, hand_area): + """校准手部大小""" + if not self.get('calibration', 'hand_size_calibration') or self.hand_size_calibration_done: + return + + if hand_area > 0: + self.reference_hand_size = hand_area + self.hand_size_calibration_done = True + print(f"✓ 手部大小校准完成: {self.reference_hand_size:.0f} 像素") + config = ConfigManager() -# ========== Robust Gesture Recognizer ========== -class RobustGestureRecognizer: - """Robust Gesture Recognizer - Pure OpenCV Implementation""" +# ========== 改进的手势识别器 ========== +class ImprovedGestureRecognizer: + """改进的手势识别器 - 纯OpenCV实现""" def __init__(self): self.config = config.get('gesture') - # Gesture history and smoothing + # 手势历史和平滑 self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) self.confidence_history = deque(maxlen=self.history_size) - self.current_gesture = "Waiting" + self.current_gesture = "等待手势" self.current_confidence = 0.0 - # Hand tracking + # 手部跟踪和状态 self.last_hand_position = None self.hand_tracking = False self.track_window = None + self.hand_states = deque(maxlen=10) # 手部状态历史 - # Performance statistics + # 性能统计 self.process_times = deque(maxlen=30) self.frame_counter = 0 self.detection_interval = self.config['detection_interval'] - # Gesture color mapping + # 手势颜色映射 self.gesture_colors = { - "Stop": (0, 0, 255), # Red - "Forward": (0, 255, 0), # Green - "Up": (255, 255, 0), # Cyan - "Down": (255, 0, 255), # Purple - "Left": (255, 165, 0), # Orange - "Right": (0, 165, 255), # Light Blue - "Waiting": (200, 200, 200), # Gray - "Error": (255, 0, 0) # Blue + "停止": (0, 0, 255), # 红色 + "向前": (0, 255, 0), # 绿色 + "向上": (255, 255, 0), # 青色 + "向下": (255, 0, 255), # 紫色 + "向左": (255, 165, 0), # 橙色 + "向右": (0, 165, 255), # 浅蓝色 + "等待手势": (200, 200, 200), # 灰色 + "识别异常": (255, 0, 0), # 蓝色 + "悬停": (255, 255, 255) # 白色 } - # Skin detection parameters - self.skin_lower = np.array(self.config['skin_lower'], dtype=np.uint8) - self.skin_upper = np.array(self.config['skin_upper'], dtype=np.uint8) - - # Background subtractor + # 背景减除器 self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( history=100, varThreshold=25, detectShadows=True ) - # Morphological operation kernels + # 形态学操作核 self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - # Fingertip detection - self.prev_fingertips = None - self.fingertip_stability_threshold = 3 + print("✓ 改进的手势识别器已初始化 (纯OpenCV实现)") - print("✓ Robust gesture recognizer initialized (Pure OpenCV)") + def get_skin_mask(self, frame): + """获取肤色掩码""" + # 从配置获取肤色范围 + h_low = config.get('gesture', 'skin_lower_h') + h_high = config.get('gesture', 'skin_upper_h') + s_low = config.get('gesture', 'skin_lower_s') + s_high = config.get('gesture', 'skin_upper_s') + v_low = config.get('gesture', 'skin_lower_v') + v_high = config.get('gesture', 'skin_upper_v') - def preprocess_frame(self, frame): - """Preprocess frame""" - # Flip image horizontally for intuitive control - frame = cv2.flip(frame, 1) + # 转换为HSV颜色空间 + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - # Adjust size for performance - resize_factor = config.get('performance', 'resize_factor') - if resize_factor != 1.0: - new_width = int(frame.shape[1] * resize_factor) - new_height = int(frame.shape[0] * resize_factor) - frame = cv2.resize(frame, (new_width, new_height)) + # 定义肤色范围 + lower_skin = np.array([h_low, s_low, v_low], dtype=np.uint8) + upper_skin = np.array([h_high, s_high, v_high], dtype=np.uint8) - return frame + # 创建肤色掩码 + skin_mask = cv2.inRange(hsv, lower_skin, upper_skin) - def detect_skin(self, frame): - """Detect skin regions""" - # Convert to HSV color space - hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + return skin_mask, hsv - # Skin mask - skin_mask = cv2.inRange(hsv, self.skin_lower, self.skin_upper) - - # Apply background subtraction + def enhance_skin_detection(self, frame, skin_mask): + """增强肤色检测""" + # 应用背景减除 fg_mask = self.bg_subtractor.apply(frame) - # Combine skin mask and foreground mask + # 结合肤色掩码和前景掩码 combined_mask = cv2.bitwise_and(skin_mask, fg_mask) - # Morphological operations + # 形态学操作 combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - # Gaussian blur + # 高斯模糊 combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) return combined_mask - def find_hand_contours(self, mask): - """Find hand contours""" - # Find contours + def find_best_hand_contour(self, mask, frame): + """找到最佳的手部轮廓""" + # 查找轮廓 contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: - return None + return None, 0.0 - # Find the largest contour (assumed to be hand) + # 按面积排序 contours = sorted(contours, key=cv2.contourArea, reverse=True) - for contour in contours: - area = cv2.contourArea(contour) + best_contour = None + best_score = 0.0 - # Filter out too small contours - if area < self.config['min_hand_area']: - continue + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') - # Calculate contour perimeter - perimeter = cv2.arcLength(contour, True) + for contour in contours[:3]: # 只检查前3个最大轮廓 + area = cv2.contourArea(contour) - # Filter out too simple contours - if perimeter < 100: + # 面积过滤 + if area < min_area or area > max_area: continue - return contour - - return None - - def detect_fingertips_advanced(self, contour): - """Advanced fingertip detection""" - # Simplify contour - epsilon = 0.01 * cv2.arcLength(contour, True) - approx = cv2.approxPolyDP(contour, epsilon, True) - - # Find convex hull - hull = cv2.convexHull(approx, returnPoints=False) + # 计算轮廓评分 + score = self.rate_contour(contour, frame.shape) - if hull is None or len(hull) < 3: - return [], 0 + if score > best_score: + best_score = score + best_contour = contour - # Find convexity defects - defects = cv2.convexityDefects(approx, hull) + return best_contour, best_score - if defects is None: - return [], 0 + def rate_contour(self, contour, frame_shape): + """评估轮廓作为手部的可能性""" + score = 0.0 - fingertips = [] - defects_points = [] - - for i in range(defects.shape[0]): - s, e, f, d = defects[i, 0] - start = tuple(approx[s][0]) - end = tuple(approx[e][0]) - far = tuple(approx[f][0]) - - defects_points.append((start, end, far)) - - # Calculate triangle sides - a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) - b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) - c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) - - # Calculate angle - if b * c != 0: - angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) - angle_degrees = math.degrees(angle) - - # If angle is less than 90 degrees, it might be a finger - if angle_degrees < 90: - # Check if this is a deep defect (far from hull) - if d > 1000: # Adjust this threshold based on your needs - # Both start and end points could be fingertips - for point in [start, end]: - if point not in fingertips: - fingertips.append(point) - - # If we found few fingertips, try alternative method - if len(fingertips) <= 2: - # Use convex hull points as fingertips - hull_points = cv2.convexHull(approx, returnPoints=True) - - # Find extreme points in hull - if len(hull_points) > 0: - hull_points = hull_points.reshape(-1, 2) - - # Sort by y-coordinate (top to bottom) - hull_points = hull_points[hull_points[:, 1].argsort()] - - # Take top points as fingertips - num_fingertips = min(5, len(hull_points)) - for i in range(num_fingertips): - point = tuple(hull_points[i]) - if point not in fingertips: - fingertips.append(point) + # 面积评分 + area = cv2.contourArea(contour) + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') + + if min_area < area < max_area: + # 面积在合理范围内 + area_ratio = min(area / max_area, 1.0) + score += area_ratio * 0.3 + + # 周长评分 + perimeter = cv2.arcLength(contour, True) + if perimeter > 100: # 最小周长 + score += 0.2 + + # 紧凑度评分 (周长^2 / 面积) + if area > 0: + compactness = perimeter ** 2 / area + # 手的紧凑度通常在14-20之间 + if 12 < compactness < 25: + compactness_score = 1.0 - abs(compactness - 18) / 6 + score += compactness_score * 0.3 + + # 长宽比评分 + x, y, w, h = cv2.boundingRect(contour) + if h > 0: + aspect_ratio = w / h + # 手的长宽比通常在0.5-2.0之间 + if 0.4 < aspect_ratio < 2.5: + aspect_score = 1.0 - abs(aspect_ratio - 1.0) / 1.5 + score += aspect_score * 0.2 - return fingertips, len(fingertips) + return score - def analyze_hand_contour(self, contour, frame_shape): - """Analyze hand contour""" + def analyze_hand_features(self, contour, frame_shape): + """分析手部特征""" if contour is None: return None, 0.0 - # Calculate contour area + # 计算轮廓面积 area = cv2.contourArea(contour) - # Calculate contour center + # 计算轮廓中心 M = cv2.moments(contour) if M["m00"] == 0: return None, 0.0 @@ -407,27 +441,41 @@ def analyze_hand_contour(self, contour, frame_shape): cx = int(M["m10"] / M["m00"]) cy = int(M["m01"] / M["m00"]) - # Calculate bounding box + # 计算边界框 x, y, w, h = cv2.boundingRect(contour) + bbox_area = w * h + + # 计算手掌中心(假设是轮廓的中心) + palm_center = (cx, cy) - # Advanced fingertip detection - fingertips, finger_count = self.detect_fingertips_advanced(contour) + # 计算手掌半径(假设为边界框宽度的30%) + palm_radius = int(w * config.get('gesture', 'palm_circle_radius_ratio')) - # Calculate hand position (normalized) + # 分析手指 + fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius) + + # 计算手部方向 + direction = self.calculate_hand_direction(contour, cx, cy) + + # 计算手部位置(归一化) h_img, w_img = frame_shape[:2] norm_x = cx / w_img norm_y = cy / h_img - # Calculate gesture confidence - confidence = self.calculate_confidence(area, finger_count, len(contour)) + # 计算手势置信度 + confidence = self.calculate_confidence(area, len(fingers), len(contour), bbox_area) - # Return result + # 返回结果 result = { 'contour': contour, 'center': (cx, cy), 'bbox': (x, y, x + w, y + h), - 'finger_count': finger_count, - 'fingertips': fingertips, + 'fingers': fingers, # 手指列表 + 'fingertips': fingertips, # 指尖点列表 + 'defects': defects, # 凸缺陷列表 + 'palm_center': palm_center, + 'palm_radius': palm_radius, + 'direction': direction, # 手部方向(角度) 'area': area, 'position': (norm_x, norm_y), 'confidence': confidence @@ -435,104 +483,241 @@ def analyze_hand_contour(self, contour, frame_shape): return result, confidence - def calculate_confidence(self, area, finger_count, contour_length): - """Calculate gesture confidence""" - confidence = 0.5 # Base confidence + def analyze_fingers(self, contour, palm_center, palm_radius): + """分析手指""" + # 简化轮廓 + epsilon = config.get('gesture', 'contour_simplify_epsilon') * cv2.arcLength(contour, True) + approx = cv2.approxPolyDP(contour, epsilon, True) - # Confidence based on area - if area > 5000: - confidence += 0.2 - elif area > 3000: - confidence += 0.1 + # 计算凸包 + hull = cv2.convexHull(approx, returnPoints=False) - # Confidence based on finger count + if hull is None or len(hull) < 3: + return [], [], [] + + # 计算凸缺陷 + defects = cv2.convexityDefects(approx, hull) + + fingers = [] + fingertips = [] + defect_points = [] + + if defects is not None: + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(approx[s][0]) + end = tuple(approx[e][0]) + far = tuple(approx[f][0]) + + # 计算距离 + start_dist = np.linalg.norm(np.array(start) - np.array(palm_center)) + end_dist = np.linalg.norm(np.array(end) - np.array(palm_center)) + far_dist = np.linalg.norm(np.array(far) - np.array(palm_center)) + + # 检查是否可能是手指 + if (start_dist > palm_radius * 1.2 and + end_dist > palm_radius * 1.2 and + d > config.get('gesture', 'defect_distance_threshold') * 256): + + # 检查角度 + a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + + if b * c != 0: + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + angle_degrees = math.degrees(angle) + + # 如果角度小于90度,可能是指缝 + if angle_degrees < 90: + # 添加手指 + finger = { + 'start': start, + 'end': end, + 'far': far, + 'depth': d, + 'angle': angle_degrees + } + fingers.append(finger) + + # 添加指尖(开始和结束点都可能是指尖) + for point in [start, end]: + if point not in fingertips: + # 检查点是否在手掌半径之外 + point_dist = np.linalg.norm(np.array(point) - np.array(palm_center)) + if point_dist > palm_radius * 1.5: + fingertips.append(point) + + defect_points.append((start, end, far, d)) + + # 如果没有检测到手指,尝试使用凸包点作为指尖 + if len(fingertips) == 0: + hull_points = cv2.convexHull(approx, returnPoints=True) + if len(hull_points) > 0: + hull_points = hull_points.reshape(-1, 2) + + # 过滤掉离手掌中心太近的点 + for point in hull_points: + point_tuple = tuple(point) + point_dist = np.linalg.norm(point - np.array(palm_center)) + if point_dist > palm_radius * 1.5 and point_tuple not in fingertips: + fingertips.append(point_tuple) + + # 限制指尖数量为5 + fingertips = fingertips[:5] + + return fingers, fingertips, defect_points + + def calculate_hand_direction(self, contour, cx, cy): + """计算手部方向""" + # 使用PCA(主成分分析)计算手部方向 + if len(contour) < 5: + return 0.0 + + # 将轮廓转换为点集 + points = contour.reshape(-1, 2).astype(np.float32) + + # 计算PCA + mean = np.empty((0)) + mean, eigenvectors, eigenvalues = cv2.PCACompute2(points, mean) + + # 计算方向角度(度) + direction = math.degrees(math.atan2(eigenvectors[0, 1], eigenvectors[0, 0])) + + return direction + + def calculate_confidence(self, area, finger_count, contour_length, bbox_area): + """计算手势置信度""" + confidence = 0.5 # 基础置信度 + + # 基于面积的置信度 + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') + + if min_area < area < max_area: + # 面积在合理范围内 + area_norm = (area - min_area) / (max_area - min_area) + confidence += area_norm * 0.2 + + # 基于手指数量的置信度 if 0 <= finger_count <= 5: confidence += 0.2 - # Confidence based on contour complexity + # 基于轮廓复杂度的置信度 if contour_length > 200: confidence += 0.1 + # 基于填充度的置信度(轮廓面积/边界框面积) + if bbox_area > 0: + fill_ratio = area / bbox_area + # 手的填充度通常在0.3-0.7之间 + if 0.2 < fill_ratio < 0.8: + fill_score = 1.0 - abs(fill_ratio - 0.5) / 0.3 + confidence += fill_score * 0.1 + return min(confidence, 1.0) - def recognize_gesture(self, hand_data): - """Recognize gesture based on hand data""" + def recognize_gesture_improved(self, hand_data): + """改进的手势识别逻辑""" if hand_data is None: - return "Waiting", 0.3 + return "等待手势", 0.3 - finger_count = hand_data['finger_count'] + finger_count = len(hand_data.get('fingers', [])) + fingertips = hand_data.get('fingertips', []) norm_x, norm_y = hand_data['position'] + direction = hand_data.get('direction', 0.0) confidence = hand_data['confidence'] - fingertips = hand_data.get('fingertips', []) - # Special case: single finger detection - if finger_count == 1: - # Additional check for single finger gesture - if len(fingertips) == 1: - # Check if the fingertip is above the hand center (typical for pointing) + # 根据手指数量分类 + if finger_count == 0: + # 握拳或没有手指 + if len(fingertips) == 0: + return "停止", confidence * 0.9 + else: + # 有指尖但不是手指,可能是部分握拳 + return "停止", confidence * 0.7 + + elif finger_count == 1: + # 单指手势 + if len(fingertips) >= 1: + # 检查指尖方向 cx, cy = hand_data['center'] fingertip = fingertips[0] - if fingertip[1] < cy - 20: # Fingertip is above hand center - return "Forward", confidence * 0.9 - else: - return "Stop", confidence * 0.7 - return "Forward", confidence * 0.8 + # 计算指尖相对于手中心的方向 + dx = fingertip[0] - cx + dy = fingertip[1] - cy - # No fingers (fist) - elif finger_count == 0: - return "Stop", confidence * 0.9 + # 判断指尖方向 + if abs(dx) > abs(dy): # 主要水平方向 + if dx > 0: + return "向右", confidence * 0.8 + else: + return "向左", confidence * 0.8 + else: # 主要垂直方向 + if dy < 0: # 指尖在手中心上方 + return "向上", confidence * 0.8 + else: + return "向前", confidence * 0.8 + return "向前", confidence * 0.7 - # Two fingers (V sign) elif finger_count == 2: - return "Forward", confidence * 0.7 + # 双指手势 + return "向前", confidence * 0.7 - # Three fingers elif finger_count == 3: - # Judge direction based on position - if norm_y < 0.4: - return "Up", confidence * 0.7 - elif norm_y > 0.6: - return "Down", confidence * 0.7 - else: - return "Stop", confidence * 0.6 + # 三指手势,根据手部方向判断 + if -45 <= direction <= 45: # 大致水平 + if direction > 0: + return "向右", confidence * 0.7 + else: + return "向左", confidence * 0.7 + else: # 大致垂直 + if direction > 0: + return "向下", confidence * 0.7 + else: + return "向上", confidence * 0.7 - # Four or five fingers (open hand) elif finger_count >= 4: - # Judge direction based on hand position - if norm_x < 0.3: - return "Left", confidence * 0.8 - elif norm_x > 0.7: - return "Right", confidence * 0.8 - elif norm_y < 0.4: - return "Up", confidence * 0.8 - elif norm_y > 0.6: - return "Down", confidence * 0.8 - else: - return "Stop", confidence * 0.7 - - return "Waiting", confidence * 0.5 + # 多指手势(手掌张开) + # 根据手部位置判断 + if norm_x < 0.4: # 左侧 + return "向左", confidence * 0.8 + elif norm_x > 0.6: # 右侧 + return "向右", confidence * 0.8 + elif norm_y < 0.4: # 上方 + return "向上", confidence * 0.8 + elif norm_y > 0.6: # 下方 + return "向下", confidence * 0.8 + else: # 中间 + # 根据手部方向判断 + if -45 <= direction <= 45: # 大致水平 + return "向前", confidence * 0.7 + else: # 大致垂直 + return "停止", confidence * 0.7 + + return "等待手势", confidence * 0.5 def smooth_gesture(self, new_gesture, new_confidence): - """Smooth gesture output""" - # Add to history + """平滑手势输出""" + # 添加到历史 self.gesture_history.append(new_gesture) self.confidence_history.append(new_confidence) - # If not enough history, return directly + # 如果历史记录不足,直接返回 if len(self.gesture_history) < 3: self.current_gesture = new_gesture self.current_confidence = new_confidence return new_gesture, new_confidence - # Count most common gesture + # 统计最常见的姿势 gesture_counter = Counter(self.gesture_history) most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] - # Calculate average confidence + # 计算平均置信度 avg_confidence = np.mean(list(self.confidence_history)) - # If the most common gesture appears enough times, use it + # 如果最常见的姿势出现次数足够多,使用它 smooth_frames = self.config['smooth_frames'] if most_common_count >= smooth_frames: self.current_gesture = most_common_gesture @@ -541,134 +726,175 @@ def smooth_gesture(self, new_gesture, new_confidence): return self.current_gesture, self.current_confidence def visualize_detection(self, frame, hand_data, gesture, confidence): - """Visualize detection results""" + """可视化检测结果""" if hand_data is None: return frame - # Get display configuration + # 获取显示配置 show_contours = config.get('display', 'show_contours') show_bbox = config.get('display', 'show_bbox') show_fingertips = config.get('display', 'show_fingertips') + show_palm_center = config.get('display', 'show_palm_center') + show_hand_direction = config.get('display', 'show_hand_direction') + show_debug_info = config.get('display', 'show_debug_info') - # Draw contour + # 绘制轮廓 if show_contours and 'contour' in hand_data: cv2.drawContours(frame, [hand_data['contour']], -1, (0, 255, 0), 2) - # Draw bounding box + # 绘制边界框 if show_bbox and 'bbox' in hand_data: x1, y1, x2, y2 = hand_data['bbox'] color = self.gesture_colors.get(gesture, (255, 255, 255)) cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) - # Show gesture label + # 显示手势标签 label = f"{gesture}" if config.get('display', 'show_confidence'): label += f" ({confidence:.0%})" - # Calculate text size + # 计算文本大小 (text_width, text_height), baseline = cv2.getTextSize( label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 ) - # Draw text background + # 绘制文本背景 cv2.rectangle(frame, (x1, y1 - text_height - 10), (x1 + text_width, y1), color, -1) - # Draw text + # 绘制文本 cv2.putText(frame, label, (x1, y1 - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 2) - # Draw center point - if 'center' in hand_data: - cx, cy = hand_data['center'] + # 绘制手掌中心 + if show_palm_center and 'palm_center' in hand_data: + cx, cy = hand_data['palm_center'] + palm_radius = hand_data.get('palm_radius', 20) cv2.circle(frame, (cx, cy), 5, (0, 0, 255), -1) - cv2.putText(frame, "Center", (cx + 10, cy), + cv2.circle(frame, (cx, cy), palm_radius, (0, 0, 255), 1) + cv2.putText(frame, "Palm", (cx + 10, cy), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) - # Draw fingertips + # 绘制指尖 if show_fingertips and 'fingertips' in hand_data: for i, point in enumerate(hand_data['fingertips']): cv2.circle(frame, point, 4, (255, 0, 0), -1) cv2.putText(frame, f"F{i + 1}", (point[0] + 5, point[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1) - # Show finger count - if 'finger_count' in hand_data: - finger_text = f"Fingers: {hand_data['finger_count']}" - cv2.putText(frame, finger_text, (10, frame.shape[0] - 10), + # 绘制手部方向 + if show_hand_direction and 'direction' in hand_data and 'center' in hand_data: + cx, cy = hand_data['center'] + direction = hand_data['direction'] + length = 50 + + # 计算方向向量 + dx = length * math.cos(math.radians(direction)) + dy = length * math.sin(math.radians(direction)) + + # 绘制方向线 + end_point = (int(cx + dx), int(cy + dy)) + cv2.arrowedLine(frame, (cx, cy), end_point, (255, 255, 0), 2) + + # 显示方向角度 + angle_text = f"Dir: {direction:.0f}°" + cv2.putText(frame, angle_text, (cx, cy - 20), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 1) + + # 显示调试信息 + if show_debug_info: + # 显示手指数量 + finger_count = len(hand_data.get('fingers', [])) + finger_text = f"Fingers: {finger_count}" + cv2.putText(frame, finger_text, (10, frame.shape[0] - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 显示手部位置 + pos_text = f"Pos: ({hand_data['position'][0]:.2f}, {hand_data['position'][1]:.2f})" + cv2.putText(frame, pos_text, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) return frame def recognize(self, frame): - """Recognize gesture""" + """识别手势""" start_time = time.time() try: - # Preprocess frame - processed_frame = self.preprocess_frame(frame) + # 预处理帧 + processed_frame = cv2.flip(frame, 1) - # Detect every few frames to improve performance + # 每隔几帧检测一次以提高性能 if self.frame_counter % self.detection_interval != 0: self.frame_counter += 1 return self.current_gesture, self.current_confidence, processed_frame - # Detect skin - skin_mask = self.detect_skin(processed_frame) + # 获取肤色掩码 + skin_mask, hsv = self.get_skin_mask(processed_frame) + + # 增强肤色检测 + enhanced_mask = self.enhance_skin_detection(processed_frame, skin_mask) - # Find hand contour - hand_contour = self.find_hand_contours(skin_mask) + # 找到最佳的手部轮廓 + hand_contour, contour_score = self.find_best_hand_contour(enhanced_mask, processed_frame) - # Analyze hand contour - hand_data, confidence = self.analyze_hand_contour(hand_contour, processed_frame.shape) + # 分析手部特征 + hand_data, confidence = self.analyze_hand_features(hand_contour, processed_frame.shape) - # Recognize gesture + # 识别手势 if hand_data is not None: - gesture, raw_confidence = self.recognize_gesture(hand_data) + # 校准肤色(如果需要) + config.calibrate_skin_color(processed_frame, enhanced_mask) + + # 校准手部大小(如果需要) + config.calibrate_hand_size(hand_data['area']) + + # 识别手势 + gesture, raw_confidence = self.recognize_gesture_improved(hand_data) confidence = max(confidence, raw_confidence) else: - gesture, confidence = "Waiting", 0.3 + gesture, confidence = "等待手势", 0.3 - # Smooth gesture + # 平滑手势 final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) - # Visualize results + # 可视化结果 if hand_data is not None: processed_frame = self.visualize_detection( processed_frame, hand_data, final_gesture, final_confidence ) - # Update counter + # 更新计数器 self.frame_counter += 1 - # Calculate processing time + # 计算处理时间 process_time = (time.time() - start_time) * 1000 self.process_times.append(process_time) return final_gesture, final_confidence, processed_frame except Exception as e: - print(f"⚠ Gesture recognition error: {e}") - return "Error", 0.0, frame + print(f"⚠ 手势识别错误: {e}") + return "识别异常", 0.0, frame def get_performance_stats(self): - """Get performance statistics""" + """获取性能统计""" if len(self.process_times) == 0: return 0.0 return np.mean(list(self.process_times)) def set_simulated_gesture(self, gesture): - """Set simulated gesture""" + """设置模拟的手势""" self.current_gesture = gesture self.current_confidence = 0.9 -# ========== Simple Drone Controller ========== +# ========== 简单的无人机控制器 ========== class SimpleDroneController: - """Simple Drone Controller""" + """简单的无人机控制器""" def __init__(self, airsim_module): self.airsim = airsim_module @@ -676,140 +902,143 @@ def __init__(self, airsim_module): self.connected = False self.flying = False - # Control parameters + # 控制参数 self.velocity = config.get('drone', 'velocity') self.duration = config.get('drone', 'duration') self.altitude = config.get('drone', 'altitude') self.control_interval = config.get('drone', 'control_interval') - # Control state + # 控制状态 self.last_control_time = 0 self.last_gesture = None - print("✓ Simple drone controller initialized") + print("✓ 简单的无人机控制器已初始化") def connect(self): - """Connect to AirSim drone""" + """连接AirSim无人机""" if self.connected: return True if self.airsim is None: - print("⚠ AirSim not available, using simulation mode") + print("⚠ AirSim不可用,使用模拟模式") self.connected = True return True - print("Connecting to AirSim...") + print("连接AirSim...") try: self.client = self.airsim.MultirotorClient() self.client.confirmConnection() - print("✅ Connected to AirSim!") + print("✅ 已连接AirSim!") self.client.enableApiControl(True) - print("✅ API control enabled") + print("✅ API控制已启用") self.client.armDisarm(True) - print("✅ Drone armed") + print("✅ 无人机已武装") self.connected = True return True except Exception as e: - print(f"❌ Connection failed: {e}") - print("\nContinue with simulation mode? (y/n)") + print(f"❌ 连接失败: {e}") + print("\n使用模拟模式继续? (y/n)") choice = input().strip().lower() if choice == 'y': self.connected = True - print("✅ Using simulation mode") + print("✅ 使用模拟模式") return True return False def takeoff(self): - """Take off""" + """起飞""" if not self.connected: return False try: if self.airsim is None or self.client is None: - print("✅ Simulated takeoff") + print("✅ 模拟起飞") self.flying = True return True - print("Taking off...") + print("起飞中...") self.client.takeoffAsync().join() time.sleep(1) - # Ascend to specified altitude + # 上升到指定高度 self.client.moveToZAsync(self.altitude, 3).join() self.flying = True - print("✅ Drone took off successfully") + print("✅ 无人机成功起飞") return True except Exception as e: - print(f"❌ Takeoff failed: {e}") + print(f"❌ 起飞失败: {e}") return False def land(self): - """Land""" + """降落""" if not self.connected: return False try: if self.airsim is None or self.client is None: - print("✅ Simulated landing") + print("✅ 模拟降落") self.flying = False return True - print("Landing...") + print("降落中...") self.client.landAsync().join() self.flying = False - print("✅ Drone landed") + print("✅ 无人机已降落") return True except Exception as e: - print(f"Landing failed: {e}") + print(f"降落失败: {e}") return False def move_by_gesture(self, gesture, confidence): - """Move based on gesture""" + """根据手势移动""" if not self.connected or not self.flying: return False - # Check control interval + # 检查控制间隔 current_time = time.time() if current_time - self.last_control_time < self.control_interval: return False - # Check confidence threshold + # 检查置信度阈值 min_confidence = config.get('gesture', 'min_confidence') if confidence < min_confidence: return False try: if self.airsim is None or self.client is None: - print(f"Simulated move: {gesture}") + print(f"模拟移动: {gesture}") self.last_control_time = current_time self.last_gesture = gesture return True success = False - if gesture == "Up": + if gesture == "向上": self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) success = True - elif gesture == "Down": + elif gesture == "向下": self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) success = True - elif gesture == "Left": + elif gesture == "向左": self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) success = True - elif gesture == "Right": + elif gesture == "向右": self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) success = True - elif gesture == "Forward": + elif gesture == "向前": self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) success = True - elif gesture == "Stop": + elif gesture == "停止": + self.client.hoverAsync() + success = True + elif gesture == "悬停": self.client.hoverAsync() success = True @@ -819,20 +1048,20 @@ def move_by_gesture(self, gesture, confidence): return success except Exception as e: - print(f"Control command failed: {e}") + print(f"控制命令失败: {e}") return False def emergency_stop(self): - """Emergency stop""" + """紧急停止""" if self.connected: try: if self.flying and self.client is not None: - print("Emergency landing...") + print("紧急降落...") self.land() if self.client is not None: self.client.armDisarm(False) self.client.enableApiControl(False) - print("✅ Emergency stop complete") + print("✅ 紧急停止完成") except: pass @@ -840,117 +1069,173 @@ def emergency_stop(self): self.flying = False -# ========== Simple UI Renderer ========== -class SimpleUIRenderer: - """Simple UI Renderer""" +# ========== 中文UI渲染器 ========== +class ChineseUIRenderer: + """中文UI渲染器""" def __init__(self): - # Color definitions + self.fonts = {} + self.load_fonts() + + # 颜色定义 self.colors = { - 'title': (0, 255, 255), # Cyan - 'connected': (0, 255, 0), # Green - 'disconnected': (0, 0, 255), # Red - 'flying': (0, 255, 0), # Green - 'landed': (255, 165, 0), # Orange - 'warning': (0, 165, 255), # Light Blue - 'info': (255, 255, 255), # White - 'help': (255, 200, 100) # Light Orange + 'title': (0, 255, 255), # 青色 + 'connected': (0, 255, 0), # 绿色 + 'disconnected': (0, 0, 255), # 红色 + 'flying': (0, 255, 0), # 绿色 + 'landed': (255, 165, 0), # 橙色 + 'warning': (0, 165, 255), # 浅蓝色 + 'info': (255, 255, 255), # 白色 + 'help': (255, 200, 100) # 浅橙色 } - print("✓ Simple UI renderer initialized") + print("✓ 中文UI渲染器已初始化") + + def load_fonts(self): + """加载字体""" + font_paths = [ + 'simhei.ttf', + 'C:/Windows/Fonts/simhei.ttf', + 'C:/Windows/Fonts/msyh.ttc', + '/System/Library/Fonts/PingFang.ttc', + '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' + ] + + for path in font_paths: + try: + self.fonts[14] = ImageFont.truetype(path, 14) + self.fonts[16] = ImageFont.truetype(path, 16) + self.fonts[18] = ImageFont.truetype(path, 18) + self.fonts[20] = ImageFont.truetype(path, 20) + self.fonts[24] = ImageFont.truetype(path, 24) + print(f"✓ 字体已加载: {path}") + return + except: + continue + + print("⚠ 未找到字体,使用默认") + + def draw_text(self, frame, text, pos, size=16, color=(255, 255, 255)): + """在图像上绘制文本""" + try: + img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + pil_img = Image.fromarray(img_rgb) + draw = ImageDraw.Draw(pil_img) + + font = self.fonts.get(size, self.fonts.get(16)) + + # 绘制阴影 + shadow_color = (0, 0, 0) + shadow_pos = (pos[0] + 1, pos[1] + 1) + draw.text(shadow_pos, text, font=font, fill=shadow_color) + + # 绘制文字 + rgb_color = color[::-1] # BGR to RGB + draw.text(pos, text, font=font, fill=rgb_color) + + return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) + except: + # 备用方案:使用OpenCV绘制英文 + cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, + size / 25, color, 1) + return frame def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, process_time): - """Draw status bar""" + """绘制状态栏""" h, w = frame.shape[:2] - # Draw semi-transparent background + # 绘制半透明背景 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (w, 100), (0, 0, 0), -1) + cv2.rectangle(overlay, (0, 0), (w, 120), (0, 0, 0), -1) frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) - # Title - title = "Gesture Controlled Drone" - cv2.putText(frame, title, (10, 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, self.colors['title'], 2) + # 标题 + title = "手势控制无人机系统 - 优化版" + frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) - # Connection status + # 连接状态 status_color = self.colors['connected'] if drone_controller.connected else self.colors['disconnected'] - status_text = f"Drone: {'Connected' if drone_controller.connected else 'Disconnected'}" - cv2.putText(frame, status_text, (10, 60), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, status_color, 1) + status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" + frame = self.draw_text(frame, status_text, (10, 40), size=16, color=status_color) - # Flight status + # 飞行状态 flight_color = self.colors['flying'] if drone_controller.flying else self.colors['landed'] - flight_text = f"Flight: {'Flying' if drone_controller.flying else 'Landed'}" - cv2.putText(frame, flight_text, (10, 85), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, flight_color, 1) + flight_text = f"飞行状态: {'飞行中' if drone_controller.flying else '已降落'}" + frame = self.draw_text(frame, flight_text, (10, 65), size=16, color=flight_color) - # Gesture information + # 手势信息 if confidence > 0.7: - gesture_color = (0, 255, 0) # Green + gesture_color = (0, 255, 0) # 绿色 elif confidence > 0.5: - gesture_color = (255, 165, 0) # Orange + gesture_color = (255, 165, 0) # 橙色 else: - gesture_color = (200, 200, 200) # Gray + gesture_color = (200, 200, 200) # 灰色 - gesture_text = f"Gesture: {gesture}" + gesture_text = f"当前手势: {gesture}" if config.get('display', 'show_confidence'): gesture_text += f" ({confidence:.0%})" - cv2.putText(frame, gesture_text, (w // 2, 60), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, gesture_color, 1) + frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) - # Performance information + # 性能信息 if config.get('display', 'show_fps'): - perf_text = f"FPS: {fps:.1f}" + perf_text = f"帧率: {fps:.1f}" if process_time > 0: - perf_text += f" | Delay: {process_time:.1f}ms" + perf_text += f" | 延迟: {process_time:.1f}ms" + + frame = self.draw_text(frame, perf_text, (w - 200, 65), size=14, color=self.colors['info']) - cv2.putText(frame, perf_text, (w - 200, 85), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['info'], 1) + # 控制提示 + control_text = "提示: 确保手部完全进入画面,光线充足" + frame = self.draw_text(frame, control_text, (10, 90), size=14, color=self.colors['info']) return frame def draw_help_bar(self, frame): - """Draw help bar""" + """绘制帮助栏""" if not config.get('display', 'show_help'): return frame h, w = frame.shape[:2] - # Draw bottom help bar - cv2.rectangle(frame, (0, h - 50), (w, h), (0, 0, 0), -1) + # 绘制底部帮助栏 + cv2.rectangle(frame, (0, h - 60), (w, h), (0, 0, 0), -1) + + # 帮助文本 + help_lines = [ + "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", + "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息" + ] - # Help text - help_text = "C:Connect Space:Takeoff/Land ESC:Exit W/A/S/D/F/X:Keyboard" - cv2.putText(frame, help_text, (10, h - 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, self.colors['help'], 1) + for i, line in enumerate(help_lines): + y_pos = h - 45 + i * 20 + frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame def draw_warning(self, frame, message): - """Draw warning message""" + """绘制警告信息""" h, w = frame.shape[:2] - # Draw warning at top + # 在顶部绘制警告 warning_bg = np.zeros((40, w, 3), dtype=np.uint8) - warning_bg[:, :] = (0, 69, 255) # Orange + warning_bg[:, :] = (0, 69, 255) # 橙色 - frame[100:140, 0:w] = cv2.addWeighted( - frame[100:140, 0:w], 0.3, + frame[120:160, 0:w] = cv2.addWeighted( + frame[120:160, 0:w], 0.3, warning_bg, 0.7, 0 ) - # Draw warning text - cv2.putText(frame, message, (10, 125), - cv2.FONT_HERSHEY_SIMPLEX, 0.6, self.colors['warning'], 1) + # 绘制警告文本 + frame = self.draw_text(frame, message, (10, 135), + size=16, color=self.colors['warning']) return frame -# ========== Performance Monitor ========== +# ========== 性能监控器 ========== class PerformanceMonitor: - """Performance Monitor""" + """性能监控器""" def __init__(self): self.frame_times = deque(maxlen=60) @@ -959,12 +1244,12 @@ def __init__(self): self.frame_count = 0 def update(self): - """Update performance data""" + """更新性能数据""" current_time = time.time() self.frame_times.append(current_time) self.frame_count += 1 - # Calculate FPS once per second + # 每秒计算一次FPS if current_time - self.last_update >= 1.0: if len(self.frame_times) > 1: time_diff = self.frame_times[-1] - self.frame_times[0] @@ -975,25 +1260,25 @@ def update(self): self.last_update = current_time def get_stats(self): - """Get performance statistics""" + """获取性能统计""" return { 'fps': self.fps, 'frame_count': self.frame_count } -# ========== Main Program ========== +# ========== 主程序 ========== def main(): - """Main function""" - # Initialize components - print("Initializing components...") + """主函数""" + # 初始化组件 + print("初始化组件...") - gesture_recognizer = RobustGestureRecognizer() + gesture_recognizer = ImprovedGestureRecognizer() drone_controller = SimpleDroneController(libs['airsim']) - ui_renderer = SimpleUIRenderer() + ui_renderer = ChineseUIRenderer() performance_monitor = PerformanceMonitor() - # Initialize camera + # 初始化摄像头 cap = None try: cap = cv2.VideoCapture(0) @@ -1002,87 +1287,111 @@ def main(): cap.set(cv2.CAP_PROP_FRAME_HEIGHT, config.get('camera', 'height')) cap.set(cv2.CAP_PROP_FPS, config.get('camera', 'fps')) - # Get actual parameters + # 获取实际参数 actual_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) actual_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) actual_fps = int(cap.get(cv2.CAP_PROP_FPS)) - print(f"✓ Camera initialized") - print(f" Resolution: {actual_width}x{actual_height}") - print(f" FPS: {actual_fps}") + print(f"✓ 摄像头已初始化") + print(f" 分辨率: {actual_width}x{actual_height}") + print(f" 帧率: {actual_fps}") else: - print("❌ Camera not available, using simulation mode") + print("❌ 摄像头不可用,使用模拟模式") cap = None except Exception as e: - print(f"⚠ Camera init failed: {e}") + print(f"⚠ 摄像头初始化失败: {e}") cap = None - # Display welcome message + # 显示欢迎信息 print("\n" + "=" * 60) - print("Gesture Controlled Drone - Optimized Version") + print("手势控制无人机系统 - 手势识别优化版") print("=" * 60) - print("System Status:") - print(f" Camera: {'Connected' if cap else 'Simulation Mode'}") - print(f" Gesture Recognition: Pure OpenCV") - print(f" AirSim: {'Available' if libs['airsim'] else 'Simulation Mode'}") + print("系统状态:") + print(f" 摄像头: {'已连接' if cap else '模拟模式'}") + print(f" 手势识别: 改进的OpenCV算法") + print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) - # Display operation instructions - print("\nOperation Instructions:") - print("1. Press [C] to connect drone (AirSim simulator)") - print("2. Press [SPACE] to takeoff/land") - print("3. Gesture Control:") - print(" - Fist (0 fingers): Stop") - print(" - Index finger (1 finger): Forward") - print(" - Open hand (4-5 fingers): Control direction by hand position") - print(" * Gesture confidence > 60% to execute") - print("4. Keyboard Control:") - print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") - print("5. Press [H] to toggle help display") - print("6. Press [R] to reset gesture recognition") - print("7. Press [ESC] to exit safely") + # 显示操作说明 + print("\n操作说明:") + print("1. 按 [C] 连接无人机 (AirSim模拟器)") + print("2. 按 [空格键] 起飞/降落") + print("3. 手势控制改进:") + print(" - 握拳或握紧: 停止") + print(" - 单指指向: 根据指尖方向判断上下左右") + print(" - 双指: 向前") + print(" - 手掌张开: 根据手的位置判断方向") + print(" * 手势识别置信度 > 50% 时才会执行") + print("4. 键盘控制:") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print("5. 调试功能:") + print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") + print("6. 按 [ESC] 安全退出") print("=" * 60) - print("Program started successfully!") + print("程序启动成功!") print("-" * 60) - # Keyboard gesture mapping + # 键盘手势映射 key_to_gesture = { - ord('w'): "Up", ord('W'): "Up", - ord('s'): "Down", ord('S'): "Down", - ord('a'): "Left", ord('A'): "Left", - ord('d'): "Right", ord('D'): "Right", - ord('f'): "Forward", ord('F'): "Forward", - ord('x'): "Stop", ord('X'): "Stop", + ord('w'): "向上", ord('W'): "向上", + ord('s'): "向下", ord('S'): "向下", + ord('a'): "向左", ord('A'): "向左", + ord('d'): "向右", ord('D'): "向右", + ord('f'): "向前", ord('F'): "向前", + ord('x'): "停止", ord('X'): "停止", } - # Main loop - print("\nEntering main loop, press ESC to exit...") + # 显示模式 + display_modes = ['normal', 'detailed', 'minimal'] + current_display_mode = 0 + + # 主循环 + print("\n进入主循环,按ESC退出...") try: while True: - # Update performance monitor + # 更新性能监控 performance_monitor.update() - # Read camera frame + # 读取摄像头帧 if cap: ret, frame = cap.read() if not ret: - # Create blank frame + # 创建空白帧 frame = np.zeros((480, 640, 3), dtype=np.uint8) - gesture, confidence = "Camera Error", 0.0 + gesture, confidence = "摄像头错误", 0.0 else: - # Gesture recognition + # 手势识别 gesture, confidence, frame = gesture_recognizer.recognize(frame) else: - # Simulation mode + # 模拟模式 frame = np.zeros((480, 640, 3), dtype=np.uint8) gesture, confidence = gesture_recognizer.current_gesture, gesture_recognizer.current_confidence - # Get performance statistics + # 获取性能统计 perf_stats = performance_monitor.get_stats() process_time = gesture_recognizer.get_performance_stats() - # Draw UI + # 根据显示模式调整显示选项 + if display_modes[current_display_mode] == 'normal': + config.set('display', 'show_contours', value=True) + config.set('display', 'show_bbox', value=True) + config.set('display', 'show_fingertips', value=True) + config.set('display', 'show_debug_info', value=False) + elif display_modes[current_display_mode] == 'detailed': + config.set('display', 'show_contours', value=True) + config.set('display', 'show_bbox', value=True) + config.set('display', 'show_fingertips', value=True) + config.set('display', 'show_palm_center', value=True) + config.set('display', 'show_hand_direction', value=True) + config.set('display', 'show_debug_info', value=True) + elif display_modes[current_display_mode] == 'minimal': + config.set('display', 'show_contours', value=False) + config.set('display', 'show_bbox', value=True) + config.set('display', 'show_fingertips', value=False) + config.set('display', 'show_debug_info', value=False) + + # 绘制UI frame = ui_renderer.draw_status_bar( frame, drone_controller, gesture, confidence, perf_stats['fps'], process_time @@ -1090,27 +1399,26 @@ def main(): frame = ui_renderer.draw_help_bar(frame) - # Show connection warning + # 显示连接提示 if not drone_controller.connected: - warning_msg = "⚠ Press C to connect drone, or use simulation mode" + warning_msg = "⚠ 按C键连接无人机,或使用模拟模式" frame = ui_renderer.draw_warning(frame, warning_msg) - # Show image - window_title = "Gesture Controlled Drone - Press ESC to exit" - cv2.imshow(window_title, frame) + # 显示图像(窗口标题用英文) + cv2.imshow('Gesture Controlled Drone - Optimized', frame) - # ========== Keyboard Control ========== + # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF - if key == 27: # ESC key - print("\nExiting program...") + if key == 27: # ESC键 + print("\n退出程序...") break elif key == ord('c') or key == ord('C'): if not drone_controller.connected: drone_controller.connect() - elif key == 32: # Space key + elif key == 32: # 空格键 if drone_controller.connected: if drone_controller.flying: drone_controller.land() @@ -1119,30 +1427,31 @@ def main(): time.sleep(0.5) elif key == ord('h') or key == ord('H'): - # Toggle help display + # 切换帮助显示 current = config.get('display', 'show_help') config.set('display', 'show_help', value=not current) - print(f"Help display: {'ON' if not current else 'OFF'}") - - elif key == ord('f') or key == ord('F'): - # Toggle FPS display - current = config.get('display', 'show_fps') - config.set('display', 'show_fps', value=not current) - print(f"FPS display: {'ON' if not current else 'OFF'}") + print(f"帮助显示: {'开启' if not current else '关闭'}") elif key == ord('r') or key == ord('R'): - # Reset gesture recognition - print("Resetting gesture recognition...") - gesture_recognizer = RobustGestureRecognizer() + # 重置手势识别 + print("重置手势识别...") + gesture_recognizer = ImprovedGestureRecognizer() + print("✓ 手势识别已重置") elif key == ord('t') or key == ord('T'): - # Toggle fingertip display - current = config.get('display', 'show_fingertips') - config.set('display', 'show_fingertips', value=not current) - print(f"Fingertip display: {'ON' if not current else 'OFF'}") + # 切换显示模式 + current_display_mode = (current_display_mode + 1) % len(display_modes) + mode_name = display_modes[current_display_mode] + print(f"显示模式: {mode_name}") + + elif key == ord('d') or key == ord('D'): + # 切换调试信息 + current = config.get('display', 'show_debug_info') + config.set('display', 'show_debug_info', value=not current) + print(f"调试信息: {'开启' if not current else '关闭'}") elif key in key_to_gesture: - # Keyboard control + # 键盘控制 simulated_gesture = key_to_gesture[key] gesture_recognizer.set_simulated_gesture(simulated_gesture) gesture = simulated_gesture @@ -1150,33 +1459,33 @@ def main(): if drone_controller.connected and drone_controller.flying: drone_controller.move_by_gesture(gesture, confidence) - # Real gesture control + # 真实手势控制 current_time = time.time() - if (gesture and gesture != "Waiting" and - gesture != "Camera Error" and gesture != "Error" and + if (gesture and gesture != "等待手势" and + gesture != "摄像头错误" and gesture != "识别异常" and drone_controller.connected and drone_controller.flying): drone_controller.move_by_gesture(gesture, confidence) except KeyboardInterrupt: - print("\nProgram interrupted by user") + print("\n程序被用户中断") except Exception as e: - print(f"\nProgram error: {e}") + print(f"\n程序错误: {e}") traceback.print_exc() finally: - # Cleanup resources - print("\nCleaning up resources...") + # 清理资源 + print("\n清理资源...") if cap: cap.release() cv2.destroyAllWindows() drone_controller.emergency_stop() config.save_config() - print("Program exited safely") + print("程序安全退出") print("=" * 60) - print("\nThank you for using Gesture Controlled Drone System!") - input("Press Enter to exit...") + print("\n感谢使用手势控制无人机系统!") + input("按回车键退出...") -# ========== Program Entry Point ========== +# ========== 程序入口 ========== if __name__ == "__main__": main() \ No newline at end of file From 1c7a07d2a683967d9c6e7563bd8b1eb75a071e31 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 17:15:10 +0800 Subject: [PATCH 6/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E6=A0=87=E7=AD=BE=E6=98=BE=E7=A4=BA=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=B8=AD=E6=96=87=E5=AD=97=E4=BD=93=E7=BC=BA=E5=A4=B1=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=9A=84=E9=97=AE=E5=8F=B7=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config_v2.json | 6 +- .../main.py | 104 +++++++++--------- 2 files changed, 55 insertions(+), 55 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json index b5479855de..57205fa642 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json @@ -9,9 +9,9 @@ "skin_lower_h": 0, "skin_upper_h": 178, "skin_lower_s": 0, - "skin_upper_s": 67, - "skin_lower_v": 36, - "skin_upper_v": 98, + "skin_upper_s": 255, + "skin_lower_v": 0, + "skin_upper_v": 77, "min_hand_area": 2000, "max_hand_area": 30000, "history_size": 10, diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index f6f31be3fe..bfa1b35eef 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -278,7 +278,7 @@ def __init__(self): self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) self.confidence_history = deque(maxlen=self.history_size) - self.current_gesture = "等待手势" + self.current_gesture = "Waiting" self.current_confidence = 0.0 # 手部跟踪和状态 @@ -294,15 +294,15 @@ def __init__(self): # 手势颜色映射 self.gesture_colors = { - "停止": (0, 0, 255), # 红色 - "向前": (0, 255, 0), # 绿色 - "向上": (255, 255, 0), # 青色 - "向下": (255, 0, 255), # 紫色 - "向左": (255, 165, 0), # 橙色 - "向右": (0, 165, 255), # 浅蓝色 - "等待手势": (200, 200, 200), # 灰色 - "识别异常": (255, 0, 0), # 蓝色 - "悬停": (255, 255, 255) # 白色 + "Stop": (0, 0, 255), # 红色 + "Forward": (0, 255, 0), # 绿色 + "Up": (255, 255, 0), # 青色 + "Down": (255, 0, 255), # 紫色 + "Left": (255, 165, 0), # 橙色 + "Right": (0, 165, 255), # 浅蓝色 + "Waiting": (200, 200, 200), # 灰色 + "Error": (255, 0, 0), # 蓝色 + "Hover": (255, 255, 255) # 白色 } # 背景减除器 @@ -620,7 +620,7 @@ def calculate_confidence(self, area, finger_count, contour_length, bbox_area): def recognize_gesture_improved(self, hand_data): """改进的手势识别逻辑""" if hand_data is None: - return "等待手势", 0.3 + return "Waiting", 0.3 finger_count = len(hand_data.get('fingers', [])) fingertips = hand_data.get('fingertips', []) @@ -632,10 +632,10 @@ def recognize_gesture_improved(self, hand_data): if finger_count == 0: # 握拳或没有手指 if len(fingertips) == 0: - return "停止", confidence * 0.9 + return "Stop", confidence * 0.9 else: # 有指尖但不是手指,可能是部分握拳 - return "停止", confidence * 0.7 + return "Stop", confidence * 0.7 elif finger_count == 1: # 单指手势 @@ -651,52 +651,52 @@ def recognize_gesture_improved(self, hand_data): # 判断指尖方向 if abs(dx) > abs(dy): # 主要水平方向 if dx > 0: - return "向右", confidence * 0.8 + return "Right", confidence * 0.8 else: - return "向左", confidence * 0.8 + return "Left", confidence * 0.8 else: # 主要垂直方向 if dy < 0: # 指尖在手中心上方 - return "向上", confidence * 0.8 + return "Up", confidence * 0.8 else: - return "向前", confidence * 0.8 - return "向前", confidence * 0.7 + return "Forward", confidence * 0.8 + return "Forward", confidence * 0.7 elif finger_count == 2: # 双指手势 - return "向前", confidence * 0.7 + return "Forward", confidence * 0.7 elif finger_count == 3: # 三指手势,根据手部方向判断 if -45 <= direction <= 45: # 大致水平 if direction > 0: - return "向右", confidence * 0.7 + return "Right", confidence * 0.7 else: - return "向左", confidence * 0.7 + return "Left", confidence * 0.7 else: # 大致垂直 if direction > 0: - return "向下", confidence * 0.7 + return "Down", confidence * 0.7 else: - return "向上", confidence * 0.7 + return "Up", confidence * 0.7 elif finger_count >= 4: # 多指手势(手掌张开) # 根据手部位置判断 if norm_x < 0.4: # 左侧 - return "向左", confidence * 0.8 + return "Left", confidence * 0.8 elif norm_x > 0.6: # 右侧 - return "向右", confidence * 0.8 + return "Right", confidence * 0.8 elif norm_y < 0.4: # 上方 - return "向上", confidence * 0.8 + return "Up", confidence * 0.8 elif norm_y > 0.6: # 下方 - return "向下", confidence * 0.8 + return "Down", confidence * 0.8 else: # 中间 # 根据手部方向判断 if -45 <= direction <= 45: # 大致水平 - return "向前", confidence * 0.7 + return "Forward", confidence * 0.7 else: # 大致垂直 - return "停止", confidence * 0.7 + return "Stop", confidence * 0.7 - return "等待手势", confidence * 0.5 + return "Waiting", confidence * 0.5 def smooth_gesture(self, new_gesture, new_confidence): """平滑手势输出""" @@ -748,7 +748,7 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): color = self.gesture_colors.get(gesture, (255, 255, 255)) cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) - # 显示手势标签 + # 显示手势标签 - 使用英文避免字体问题 label = f"{gesture}" if config.get('display', 'show_confidence'): label += f" ({confidence:.0%})" @@ -855,7 +855,7 @@ def recognize(self, frame): gesture, raw_confidence = self.recognize_gesture_improved(hand_data) confidence = max(confidence, raw_confidence) else: - gesture, confidence = "等待手势", 0.3 + gesture, confidence = "Waiting", 0.3 # 平滑手势 final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) @@ -877,7 +877,7 @@ def recognize(self, frame): except Exception as e: print(f"⚠ 手势识别错误: {e}") - return "识别异常", 0.0, frame + return "Error", 0.0, frame def get_performance_stats(self): """获取性能统计""" @@ -1020,25 +1020,25 @@ def move_by_gesture(self, gesture, confidence): success = False - if gesture == "向上": + if gesture == "Up": self.client.moveByVelocityZAsync(0, 0, -self.velocity, self.duration) success = True - elif gesture == "向下": + elif gesture == "Down": self.client.moveByVelocityZAsync(0, 0, self.velocity, self.duration) success = True - elif gesture == "向左": + elif gesture == "Left": self.client.moveByVelocityAsync(-self.velocity, 0, 0, self.duration) success = True - elif gesture == "向右": + elif gesture == "Right": self.client.moveByVelocityAsync(self.velocity, 0, 0, self.duration) success = True - elif gesture == "向前": + elif gesture == "Forward": self.client.moveByVelocityAsync(0, -self.velocity, 0, self.duration) success = True - elif gesture == "停止": + elif gesture == "Stop": self.client.hoverAsync() success = True - elif gesture == "悬停": + elif gesture == "Hover": self.client.hoverAsync() success = True @@ -1317,13 +1317,13 @@ def main(): print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") print("3. 手势控制改进:") - print(" - 握拳或握紧: 停止") - print(" - 单指指向: 根据指尖方向判断上下左右") - print(" - 双指: 向前") + print(" - 握拳或握紧: Stop") + print(" - 单指指向: 根据指尖方向判断Up/Down/Left/Right") + print(" - 双指: Forward") print(" - 手掌张开: 根据手的位置判断方向") print(" * 手势识别置信度 > 50% 时才会执行") print("4. 键盘控制:") - print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") print("5. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") print("6. 按 [ESC] 安全退出") @@ -1333,12 +1333,12 @@ def main(): # 键盘手势映射 key_to_gesture = { - ord('w'): "向上", ord('W'): "向上", - ord('s'): "向下", ord('S'): "向下", - ord('a'): "向左", ord('A'): "向左", - ord('d'): "向右", ord('D'): "向右", - ord('f'): "向前", ord('F'): "向前", - ord('x'): "停止", ord('X'): "停止", + ord('w'): "Up", ord('W'): "Up", + ord('s'): "Down", ord('S'): "Down", + ord('a'): "Left", ord('A'): "Left", + ord('d'): "Right", ord('D'): "Right", + ord('f'): "Forward", ord('F'): "Forward", + ord('x'): "Stop", ord('X'): "Stop", } # 显示模式 @@ -1461,8 +1461,8 @@ def main(): # 真实手势控制 current_time = time.time() - if (gesture and gesture != "等待手势" and - gesture != "摄像头错误" and gesture != "识别异常" and + if (gesture and gesture != "Waiting" and + gesture != "摄像头错误" and gesture != "Error" and drone_controller.connected and drone_controller.flying): drone_controller.move_by_gesture(gesture, confidence) From 36f5403394d01f3d3811cc6c2e3e994da5b6e6a7 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Fri, 19 Dec 2025 21:57:39 +0800 Subject: [PATCH 7/8] =?UTF-8?q?=E6=96=B0=E5=A2=9E=EF=BC=9A=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E5=8F=8D=E9=A6=88=E7=B3=BB=E7=BB=9F=EF=BC=8C=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=A4=9A=E5=B9=B3=E5=8F=B0=E9=9F=B3=E9=A2=91=E6=92=AD?= =?UTF-8?q?=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gesture_config.json | 108 ++- .../gesture_config_v2.json | 20 +- .../gesture_config_v3.json | 83 ++- .../main.py | 650 +++++++++++++++++- 4 files changed, 800 insertions(+), 61 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index d41f020e20..f07c4b5817 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -1,11 +1,95 @@ { + "system": { + "version": "1.0", + "last_updated": "2025-12-19 20:51:44", + "debug_mode": false + }, "camera": { + "device_id": 0, + "resolution": { + "width": 640, + "height": 480 + }, + "fps": 30, + "flip_horizontal": true, + "use_virtual_camera": false, "index": 0, "width": 640, - "height": 480, - "fps": 30 + "height": 480 + }, + "gesture_detection": { + "skin_hsv_ranges": { + "h_low": 0, + "h_high": 25, + "s_low": 30, + "s_high": 255, + "v_low": 60, + "v_high": 255 + }, + "min_hand_area": 2000, + "max_hand_area": 30000, + "confidence_threshold": 0.5, + "smoothing_window": 5, + "detection_interval": 1 + }, + "gesture_mapping": { + "0": "stop", + "1": "forward", + "2": "backward", + "3": "up", + "4": "down", + "5": "left", + "6": "right", + "7": "hover" + }, + "drone_control": { + "base_speed": 2.0, + "altitude": -10.0, + "control_delay": 0.3, + "takeoff_height": 5.0, + "emergency_stop_enabled": true + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_gesture_info": true, + "show_control_help": true, + "show_detection_overlay": true, + "theme": "dark", + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": false, + "show_hand_direction": false, + "show_debug_info": false, + "show_direction": true, + "show_region_grid": true + }, + "performance": { + "target_fps": 30, + "enable_multithreading": false, + "enable_hardware_acceleration": true, + "resize_factor": 1.0, + "enable_multiprocessing": false }, "gesture": { + "skin_lower_h": 0, + "skin_upper_h": 25, + "skin_lower_s": 30, + "skin_upper_s": 255, + "skin_lower_v": 60, + "skin_upper_v": 255, + "min_hand_area": 3000, + "max_hand_area": 30000, + "history_size": 10, + "smooth_frames": 5, + "min_confidence": 0.6, + "detection_interval": 2, + "hand_ratio_threshold": 1.5, + "contour_simplify_epsilon": 0.02, + "defect_distance_threshold": 20, + "palm_circle_radius_ratio": 0.3, "skin_lower": [ 0, 30, @@ -16,11 +100,6 @@ 255, 255 ], - "min_hand_area": 3000, - "history_size": 10, - "smooth_frames": 5, - "min_confidence": 0.6, - "detection_interval": 2, "hand_region_threshold": 0.2, "face_exclusion_area": 8000, "palm_aspect_ratio": 1.3, @@ -31,20 +110,5 @@ "duration": 0.3, "altitude": -10.0, "control_interval": 0.3 - }, - "display": { - "show_fps": true, - "show_confidence": true, - "show_help": true, - "show_contours": true, - "show_bbox": true, - "show_fingertips": true, - "show_direction": true, - "show_region_grid": true - }, - "performance": { - "target_fps": 30, - "resize_factor": 1.0, - "enable_multiprocessing": false } } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json index 57205fa642..fca5800e11 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v2.json @@ -7,11 +7,11 @@ }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 178, + "skin_upper_h": 175, "skin_lower_s": 0, - "skin_upper_s": 255, + "skin_upper_s": 57, "skin_lower_v": 0, - "skin_upper_v": 77, + "skin_upper_v": 53, "min_hand_area": 2000, "max_hand_area": 30000, "history_size": 10, @@ -21,7 +21,15 @@ "hand_ratio_threshold": 1.5, "contour_simplify_epsilon": 0.02, "defect_distance_threshold": 20, - "palm_circle_radius_ratio": 0.3 + "palm_circle_radius_ratio": 0.3, + "validation_buffer_size": 5, + "validation_threshold": 0.8, + "min_hold_frames": 8, + "false_trigger_cooldown": 0.5, + "min_gesture_duration": 0.3, + "stability_threshold": 3, + "position_change_threshold": 0.15, + "area_change_threshold": 0.3 }, "drone": { "velocity": 2.5, @@ -38,7 +46,9 @@ "show_fingertips": true, "show_palm_center": true, "show_hand_direction": true, - "show_debug_info": false + "show_debug_info": false, + "show_validation_info": true, + "show_stability_indicator": true }, "performance": { "target_fps": 30, diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json index 7491f9cd30..0f240d3668 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config_v3.json @@ -3,31 +3,56 @@ "index": 0, "width": 640, "height": 480, - "fps": 30 + "fps": 30, + "flip_horizontal": true, + "auto_exposure": true }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 25, - "skin_lower_s": 30, - "skin_upper_s": 255, - "skin_lower_v": 60, - "skin_upper_v": 255, + "skin_upper_h": 176, + "skin_lower_s": 0, + "skin_upper_s": 72, + "skin_lower_v": 1, + "skin_upper_v": 96, "min_hand_area": 2000, "max_hand_area": 30000, "history_size": 15, "smooth_frames": 10, "min_confidence": 0.5, "detection_interval": 1, - "gesture_lock_frames": 8, + "hand_ratio_threshold": 1.5, "contour_simplify_epsilon": 0.02, "defect_distance_threshold": 20, - "palm_circle_radius_ratio": 0.3 + "palm_circle_radius_ratio": 0.3, + "validation_buffer_size": 8, + "validation_threshold": 0.75, + "min_hold_frames": 10, + "false_trigger_cooldown": 0.8, + "min_gesture_duration": 0.5, + "stability_threshold": 5, + "position_change_threshold": 0.1, + "area_change_threshold": 0.25, + "use_background_subtraction": true, + "bg_history": 50, + "bg_var_threshold": 16, + "morph_close_iterations": 2, + "morph_open_iterations": 2, + "use_roi": true, + "roi_margin": 50, + "finger_min_length_ratio": 0.15, + "finger_max_length_ratio": 0.8, + "enable_multi_hand": false, + "hand_tracking_smoothness": 0.7, + "gesture_lock_frames": 8 }, "drone": { "velocity": 2.5, "duration": 0.3, "altitude": -10.0, - "control_interval": 0.3 + "control_interval": 0.3, + "angular_velocity": 45.0, + "smooth_movement": true, + "emergency_landing_timeout": 10.0 }, "display": { "show_fps": true, @@ -36,18 +61,48 @@ "show_contours": true, "show_bbox": true, "show_fingertips": true, - "show_palm_center": false, - "show_hand_direction": false, - "show_debug_info": false + "show_palm_center": true, + "show_hand_direction": true, + "show_debug_info": false, + "show_speech_status": true, + "show_validation_info": true, + "show_stability_indicator": true, + "use_english_text": true, + "show_skin_mask": false, + "show_trajectory": false, + "show_gesture_history": false, + "ui_theme": "dark", + "font_size": 16 }, "performance": { "target_fps": 30, "resize_factor": 1.0, - "enable_multiprocessing": false + "enable_multiprocessing": false, + "optimize_for_low_end": false, + "max_frame_skip": 2 }, "calibration": { "auto_calibrate_skin": true, "skin_calibration_frames": 30, - "hand_size_calibration": true + "hand_size_calibration": true, + "calibration_done": true, + "environment_adaptation": true, + "adaptive_threshold": true + }, + "speech": { + "enabled": true, + "volume": 1.0, + "rate": 150, + "announce_gestures": true, + "announce_connections": true, + "announce_flight_events": true, + "min_gesture_confidence": 0.7 + }, + "advanced": { + "use_ml_backup": false, + "enable_gesture_learning": false, + "save_gesture_samples": false, + "log_level": "INFO", + "auto_save_config": true } } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index bfa1b35eef..4e33bea6dd 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 手势识别优化版 -优化手势识别算法,解决位置敏感问题 +手势控制AirSim无人机 - 手势识别优化版(添加语音反馈) +优化手势识别算法,解决位置敏感问题,添加语音反馈功能 作者: xiaoshiyuan888 """ @@ -11,6 +11,8 @@ import traceback import json import math +import threading +import tempfile from datetime import datetime from PIL import Image, ImageDraw, ImageFont import cv2 @@ -19,6 +21,7 @@ print("=" * 60) print("Gesture Controlled Drone - Gesture Recognition Optimized") +print("现在添加了语音反馈功能!") print("=" * 60) # ========== 修复导入路径 ========== @@ -71,11 +74,71 @@ def safe_import(): if choice != 'y': sys.exit(1) + # 尝试导入语音合成库 + speech_module = None + try: + # 尝试导入pyttsx3(离线TTS) + import pyttsx3 + speech_module = pyttsx3 + modules_status['Speech'] = True + print("[Speech] ✓ pyttsx3语音库就绪 (离线)") + except ImportError: + print("\n" + "!" * 60) + print("⚠ pyttsx3语音库未找到!") + print("!" * 60) + print("安装语音库 (使用清华大学镜像源):") + print("1. 安装离线语音库: pip install pyttsx3 -i https://pypi.tuna.tsinghua.edu.cn/simple") + print("2. 或者安装在线语音库: pip install gtts pygame -i https://pypi.tuna.tsinghua.edu.cn/simple") + print("!" * 60) + + # 尝试其他语音库 + try: + # 尝试使用gTTS(需要网络) + from gtts import gTTS + speech_module = {'gTTS': gTTS, 'type': 'gtts'} + modules_status['Speech'] = True + print("[Speech] ✓ gTTS语音库就绪 (需要网络连接)") + + # 尝试导入音频播放库 + try: + import pygame + pygame.mixer.init() + speech_module['pygame'] = pygame + print("[Speech] ✓ pygame音频播放库就绪") + except ImportError: + # 尝试其他播放方式 + try: + import pydub + from pydub import AudioSegment + from pydub.playback import play + speech_module['pydub'] = pydub + speech_module['AudioSegment'] = AudioSegment + speech_module['play'] = play + print("[Speech] ✓ pydub音频播放库就绪") + except ImportError: + # 最后尝试使用系统命令 + if os.name == 'nt': # Windows + speech_module['play_method'] = 'windows' + print("[Speech] ✓ 使用Windows系统命令播放音频") + elif os.name == 'posix': # Linux/Mac + speech_module['play_method'] = 'posix' + print("[Speech] ✓ 使用系统命令播放音频") + else: + print("[Speech] ✗ 所有音频播放库导入失败,语音功能将不可用") + speech_module = None + modules_status['Speech'] = False + + except ImportError: + print("[Speech] ✗ 所有语音库导入失败,语音功能将不可用") + speech_module = None + modules_status['Speech'] = False + return { 'cv2': cv2, 'np': np, 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, - 'airsim': airsim_module + 'airsim': airsim_module, + 'speech': speech_module }, modules_status @@ -95,12 +158,324 @@ def safe_import(): Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] +# ========== 语音反馈管理器 ========== +class SpeechFeedbackManager: + """语音反馈管理器""" + + def __init__(self, speech_lib): + self.speech_lib = speech_lib + self.enabled = True + self.volume = 1.0 + self.rate = 150 + self.voice_id = None + self.last_speech_time = {} + self.min_interval = 2.0 # 相同语音的最小间隔(秒) + + # 语音队列,避免语音重叠 + self.speech_queue = [] + self.is_speaking = False + self.queue_thread = None + + # 音频播放方法 + self.audio_method = None + + # 语音消息映射 + self.messages = { + # 连接相关 + 'connecting': "正在连接无人机,请稍候", + 'connected': "无人机连接成功", + 'connection_failed': "无人机连接失败,进入模拟模式", + + # 飞行相关 + 'taking_off': "无人机正在起飞", + 'takeoff_success': "起飞成功", + 'takeoff_failed': "起飞失败", + 'landing': "无人机正在降落", + 'land_success': "降落成功", + 'emergency_stop': "紧急停止,无人机已降落", + + # 手势相关 + 'gesture_detected': "手势识别就绪", + 'gesture_stop': "停止", + 'gesture_up': "向上", + 'gesture_down': "向下", + 'gesture_left': "向左", + 'gesture_right': "向右", + 'gesture_forward': "向前", + 'gesture_waiting': "等待手势", + 'gesture_error': "手势识别错误", + + # 系统相关 + 'program_start': "手势控制无人机系统已启动", + 'program_exit': "程序退出,感谢使用", + 'camera_error': "摄像头错误,请检查连接", + 'low_confidence': "置信度过低,请调整手势", + + # 模式相关 + 'simulation_mode': "进入模拟模式", + 'debug_mode_on': "调试模式已开启", + 'debug_mode_off': "调试模式已关闭", + 'display_mode_changed': "显示模式已切换", + 'help_toggled': "帮助信息已切换", + } + + # 初始化语音引擎 + self.init_speech_engine() + + def init_speech_engine(self): + """初始化语音引擎""" + if self.speech_lib is None: + print("⚠ 语音库未找到,语音功能禁用") + self.enabled = False + return + + try: + if hasattr(self.speech_lib, 'init'): # pyttsx3 + self.engine = self.speech_lib.init() + self.audio_method = 'pyttsx3' + + # 设置语音参数 + voices = self.engine.getProperty('voices') + + # 尝试寻找中文语音 + for voice in voices: + # 检查语音名称是否包含中文相关标识 + if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower(): + self.engine.setProperty('voice', voice.id) + self.voice_id = voice.id + print(f"[Speech] 使用中文语音: {voice.name}") + break + + # 如果没找到中文语音,使用第一个可用语音 + if self.voice_id is None and len(voices) > 0: + self.engine.setProperty('voice', voices[0].id) + print(f"[Speech] 使用默认语音: {voices[0].name}") + + # 设置语速和音量 + self.engine.setProperty('rate', self.rate) + self.engine.setProperty('volume', self.volume) + + print("✅ 语音引擎初始化成功 (pyttsx3)") + + elif isinstance(self.speech_lib, dict) and self.speech_lib.get('type') == 'gtts': + print("✅ 语音引擎初始化成功 (gTTS,需要网络连接)") + self.audio_method = 'gtts' + + # 确定播放方法 + if 'pygame' in self.speech_lib: + self.audio_method = 'gtts_pygame' + print("✅ 使用pygame播放音频") + elif 'pydub' in self.speech_lib: + self.audio_method = 'gtts_pydub' + print("✅ 使用pydub播放音频") + elif 'play_method' in self.speech_lib: + self.audio_method = f"gtts_{self.speech_lib['play_method']}" + print(f"✅ 使用系统命令播放音频") + else: + self.audio_method = 'gtts_system' + print("✅ 使用默认系统播放器") + + else: + print("⚠ 未知语音库类型,语音功能可能不正常") + self.enabled = False + + except Exception as e: + print(f"⚠ 语音引擎初始化失败: {e}") + self.enabled = False + + def play_audio_file(self, audio_file): + """播放音频文件(根据可用库选择方法)""" + try: + if self.audio_method == 'gtts_pygame' and 'pygame' in self.speech_lib: + pygame = self.speech_lib['pygame'] + pygame.mixer.music.load(audio_file) + pygame.mixer.music.play() + + # 等待播放完成 + while pygame.mixer.music.get_busy(): + pygame.time.Clock().tick(10) + + elif self.audio_method == 'gtts_pydub' and 'pydub' in self.speech_lib: + AudioSegment = self.speech_lib['AudioSegment'] + play = self.speech_lib['play'] + + audio = AudioSegment.from_mp3(audio_file) + play(audio) + + elif self.audio_method == 'gtts_windows': + # Windows系统命令 + os.startfile(audio_file) + # 等待播放完成(简单等待) + time.sleep(2) + + elif self.audio_method == 'gtts_posix': + # Linux/Mac系统命令 + import subprocess + if sys.platform == 'darwin': # macOS + subprocess.call(['afplay', audio_file]) + else: # Linux + subprocess.call(['xdg-open', audio_file]) + + else: + # 通用方法:使用系统默认播放器 + import subprocess + if sys.platform == 'win32': + os.startfile(audio_file) + elif sys.platform == 'darwin': + subprocess.call(['open', audio_file]) + else: + subprocess.call(['xdg-open', audio_file]) + + return True + + except Exception as e: + print(f"⚠ 音频播放失败: {e}") + return False + + def speak(self, message_key, force=False): + """播放语音""" + if not self.enabled: + return + + # 检查是否在最小间隔内 + current_time = time.time() + if not force and message_key in self.last_speech_time: + if current_time - self.last_speech_time[message_key] < self.min_interval: + return + + # 获取消息文本 + if message_key in self.messages: + text = self.messages[message_key] + else: + text = message_key # 直接使用传入的文本 + + # 添加到语音队列 + self.speech_queue.append(text) + + # 如果没有在播放,启动播放线程 + if not self.is_speaking and self.queue_thread is None: + self.queue_thread = threading.Thread(target=self._process_speech_queue) + self.queue_thread.daemon = True + self.queue_thread.start() + + # 更新时间戳 + self.last_speech_time[message_key] = current_time + + def _process_speech_queue(self): + """处理语音队列""" + while self.speech_queue and self.enabled: + self.is_speaking = True + + text = self.speech_queue.pop(0) + + try: + if self.audio_method == 'pyttsx3': + # 清除之前的语音 + self.engine.stop() + # 播报新语音 + self.engine.say(text) + self.engine.runAndWait() + + elif self.audio_method.startswith('gtts'): + # gTTS需要网络连接 + tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') + + # 保存临时文件 + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: + temp_file = f.name + tts.save(temp_file) + + # 播放音频 + self.play_audio_file(temp_file) + + # 删除临时文件 + try: + os.unlink(temp_file) + except: + pass # 忽略删除错误 + + except Exception as e: + print(f"⚠ 语音播放失败: {e}") + + time.sleep(0.1) # 避免过快 + + self.is_speaking = False + self.queue_thread = None + + def speak_direct(self, text): + """直接播放文本(不通过消息映射)""" + if not self.enabled: + return + + # 在新线程中播放 + thread = threading.Thread(target=self._speak_thread, args=(text,)) + thread.daemon = True + thread.start() + + def _speak_thread(self, text): + """语音播放线程""" + try: + if self.audio_method == 'pyttsx3': + self.engine.say(text) + self.engine.runAndWait() + + elif self.audio_method.startswith('gtts'): + tts = self.speech_lib['gTTS'](text=text, lang='zh-cn') + + with tempfile.NamedTemporaryFile(suffix='.mp3', delete=False) as f: + temp_file = f.name + tts.save(temp_file) + + self.play_audio_file(temp_file) + + try: + os.unlink(temp_file) + except: + pass + + except Exception as e: + print(f"⚠ 直接语音播放失败: {e}") + + def stop(self): + """停止所有语音""" + if hasattr(self, 'engine'): + self.engine.stop() + + self.speech_queue.clear() + self.is_speaking = False + + def set_enabled(self, enabled): + """启用/禁用语音""" + self.enabled = enabled + if not enabled: + self.stop() + + def toggle_enabled(self): + """切换语音启用状态""" + self.enabled = not self.enabled + status = "启用" if self.enabled else "禁用" + self.speak_direct(f"语音反馈已{status}") + return self.enabled + + def get_status(self): + """获取语音状态""" + return { + 'enabled': self.enabled, + 'engine': 'pyttsx3' if self.audio_method == 'pyttsx3' else + 'gTTS' if self.audio_method.startswith('gtts') else + 'None', + 'queue_size': len(self.speech_queue), + 'is_speaking': self.is_speaking, + 'audio_method': self.audio_method + } + + # ========== 配置管理器 ========== class ConfigManager: """配置管理器""" def __init__(self): - self.config_file = os.path.join(current_dir, 'gesture_config_v2.json') + self.config_file = os.path.join(current_dir, 'gesture_config_v3.json') # 更新版本号 self.default_config = { 'camera': { 'index': 0, @@ -141,7 +516,8 @@ def __init__(self): 'show_fingertips': True, 'show_palm_center': True, 'show_hand_direction': True, - 'show_debug_info': False + 'show_debug_info': False, + 'show_speech_status': True, # 新增:显示语音状态 }, 'performance': { 'target_fps': 30, @@ -152,6 +528,15 @@ def __init__(self): 'auto_calibrate_skin': True, 'skin_calibration_frames': 30, 'hand_size_calibration': True + }, + 'speech': { # 新增:语音配置 + 'enabled': True, + 'volume': 1.0, + 'rate': 150, + 'announce_gestures': True, # 是否播报手势 + 'announce_connections': True, # 是否播报连接状态 + 'announce_flight_events': True, # 是否播报飞行事件 + 'min_gesture_confidence': 0.7, # 播报手势的最小置信度 } } self.config = self.load_config() @@ -271,8 +656,9 @@ def calibrate_hand_size(self, hand_area): class ImprovedGestureRecognizer: """改进的手势识别器 - 纯OpenCV实现""" - def __init__(self): + def __init__(self, speech_manager=None): self.config = config.get('gesture') + self.speech_manager = speech_manager # 手势历史和平滑 self.history_size = self.config['history_size'] @@ -281,6 +667,11 @@ def __init__(self): self.current_gesture = "Waiting" self.current_confidence = 0.0 + # 记录上次播报的手势,避免重复播报 + self.last_announced_gesture = None + self.last_announced_time = 0 + self.gesture_announce_interval = 3.0 # 手势播报最小间隔 + # 手部跟踪和状态 self.last_hand_position = None self.hand_tracking = False @@ -305,6 +696,18 @@ def __init__(self): "Hover": (255, 255, 255) # 白色 } + # 手势到语音的映射 + self.gesture_speech_map = { + "Stop": "gesture_stop", + "Forward": "gesture_forward", + "Up": "gesture_up", + "Down": "gesture_down", + "Left": "gesture_left", + "Right": "gesture_right", + "Waiting": "gesture_waiting", + "Error": "gesture_error", + } + # 背景减除器 self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( history=100, varThreshold=25, detectShadows=True @@ -315,6 +718,10 @@ def __init__(self): print("✓ 改进的手势识别器已初始化 (纯OpenCV实现)") + # 语音提示初始化完成 + if self.speech_manager: + self.speech_manager.speak('gesture_detected') + def get_skin_mask(self, frame): """获取肤色掩码""" # 从配置获取肤色范围 @@ -860,6 +1267,22 @@ def recognize(self, frame): # 平滑手势 final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) + # 手势语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gestures') and + final_confidence >= config.get('speech', 'min_gesture_confidence')): + + current_time = time.time() + if (final_gesture != self.last_announced_gesture or + current_time - self.last_announced_time > self.gesture_announce_interval): + + if final_gesture in self.gesture_speech_map: + self.speech_manager.speak(self.gesture_speech_map[final_gesture]) + + self.last_announced_gesture = final_gesture + self.last_announced_time = current_time + # 可视化结果 if hand_data is not None: processed_frame = self.visualize_detection( @@ -891,16 +1314,26 @@ def set_simulated_gesture(self, gesture): self.current_gesture = gesture self.current_confidence = 0.9 + # 模拟手势也触发语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gestures') and + gesture in self.gesture_speech_map): + self.speech_manager.speak(self.gesture_speech_map[gesture]) + self.last_announced_gesture = gesture + self.last_announced_time = time.time() + # ========== 简单的无人机控制器 ========== class SimpleDroneController: """简单的无人机控制器""" - def __init__(self, airsim_module): + def __init__(self, airsim_module, speech_manager=None): self.airsim = airsim_module self.client = None self.connected = False self.flying = False + self.speech_manager = speech_manager # 控制参数 self.velocity = config.get('drone', 'velocity') @@ -912,6 +1345,11 @@ def __init__(self, airsim_module): self.last_control_time = 0 self.last_gesture = None + # 上次语音提示状态 + self.last_connection_announced = False + self.last_takeoff_announced = False + self.last_land_announced = False + print("✓ 简单的无人机控制器已初始化") def connect(self): @@ -919,8 +1357,21 @@ def connect(self): if self.connected: return True + # 语音提示:正在连接 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('connecting') + if self.airsim is None: print("⚠ AirSim不可用,使用模拟模式") + + # 语音提示:模拟模式 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('simulation_mode') + self.connected = True return True @@ -931,6 +1382,12 @@ def connect(self): self.client.confirmConnection() print("✅ 已连接AirSim!") + # 语音提示:连接成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('connected') + self.client.enableApiControl(True) print("✅ API控制已启用") @@ -942,11 +1399,25 @@ def connect(self): except Exception as e: print(f"❌ 连接失败: {e}") + + # 语音提示:连接失败 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('connection_failed') + print("\n使用模拟模式继续? (y/n)") choice = input().strip().lower() if choice == 'y': self.connected = True print("✅ 使用模拟模式") + + # 语音提示:模拟模式 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_connections')): + self.speech_manager.speak('simulation_mode') + return True return False @@ -956,10 +1427,26 @@ def takeoff(self): if not self.connected: return False + # 语音提示:正在起飞 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events') and + not self.last_takeoff_announced): + self.speech_manager.speak('taking_off') + self.last_takeoff_announced = True + self.last_land_announced = False + try: if self.airsim is None or self.client is None: print("✅ 模拟起飞") self.flying = True + + # 语音提示:起飞成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('takeoff_success') + return True print("起飞中...") @@ -971,9 +1458,23 @@ def takeoff(self): self.flying = True print("✅ 无人机成功起飞") + + # 语音提示:起飞成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('takeoff_success') + return True except Exception as e: print(f"❌ 起飞失败: {e}") + + # 语音提示:起飞失败 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('takeoff_failed') + return False def land(self): @@ -981,16 +1482,39 @@ def land(self): if not self.connected: return False + # 语音提示:正在降落 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events') and + not self.last_land_announced): + self.speech_manager.speak('landing') + self.last_land_announced = True + self.last_takeoff_announced = False + try: if self.airsim is None or self.client is None: print("✅ 模拟降落") self.flying = False + + # 语音提示:降落成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('land_success') + return True print("降落中...") self.client.landAsync().join() self.flying = False print("✅ 无人机已降落") + + # 语音提示:降落成功 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('land_success') + return True except Exception as e: print(f"降落失败: {e}") @@ -1009,6 +1533,12 @@ def move_by_gesture(self, gesture, confidence): # 检查置信度阈值 min_confidence = config.get('gesture', 'min_confidence') if confidence < min_confidence: + # 低置信度语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gestures') and + confidence < min_confidence * 0.8): # 如果置信度特别低 + self.speech_manager.speak('low_confidence') return False try: @@ -1057,6 +1587,13 @@ def emergency_stop(self): try: if self.flying and self.client is not None: print("紧急降落...") + + # 语音提示:紧急停止 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('emergency_stop') + self.land() if self.client is not None: self.client.armDisarm(False) @@ -1073,8 +1610,9 @@ def emergency_stop(self): class ChineseUIRenderer: """中文UI渲染器""" - def __init__(self): + def __init__(self, speech_manager=None): self.fonts = {} + self.speech_manager = speech_manager self.load_fonts() # 颜色定义 @@ -1086,7 +1624,9 @@ def __init__(self): 'landed': (255, 165, 0), # 橙色 'warning': (0, 165, 255), # 浅蓝色 'info': (255, 255, 255), # 白色 - 'help': (255, 200, 100) # 浅橙色 + 'help': (255, 200, 100), # 浅橙色 + 'speech_enabled': (0, 255, 0), # 绿色 + 'speech_disabled': (255, 0, 0), # 红色 } print("✓ 中文UI渲染器已初始化") @@ -1150,7 +1690,7 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 优化版" + title = "手势控制无人机系统 - 优化版 (带语音反馈)" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -1177,6 +1717,13 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = self.draw_text(frame, gesture_text, (w // 2, 40), size=16, color=gesture_color) + # 语音状态 + if config.get('display', 'show_speech_status') and self.speech_manager: + speech_status = self.speech_manager.get_status() + speech_color = self.colors['speech_enabled'] if speech_status['enabled'] else self.colors['speech_disabled'] + speech_text = f"语音: {'启用' if speech_status['enabled'] else '禁用'}" + frame = self.draw_text(frame, speech_text, (w // 2, 65), size=16, color=speech_color) + # 性能信息 if config.get('display', 'show_fps'): perf_text = f"帧率: {fps:.1f}" @@ -1199,16 +1746,17 @@ def draw_help_bar(self, frame): h, w = frame.shape[:2] # 绘制底部帮助栏 - cv2.rectangle(frame, (0, h - 60), (w, h), (0, 0, 0), -1) + cv2.rectangle(frame, (0, h - 80), (w, h), (0, 0, 0), -1) # 帮助文本 help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", - "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息" + "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", + "V:切换语音反馈 M:测试语音" ] for i, line in enumerate(help_lines): - y_pos = h - 45 + i * 20 + y_pos = h - 65 + i * 20 frame = self.draw_text(frame, line, (10, y_pos), size=14, color=self.colors['help']) return frame @@ -1270,12 +1818,20 @@ def get_stats(self): # ========== 主程序 ========== def main(): """主函数""" + # 初始化语音管理器 + print("初始化语音反馈系统...") + speech_manager = SpeechFeedbackManager(libs['speech']) + + # 程序启动语音提示 + if speech_manager.enabled: + speech_manager.speak('program_start', force=True) + # 初始化组件 print("初始化组件...") - gesture_recognizer = ImprovedGestureRecognizer() - drone_controller = SimpleDroneController(libs['airsim']) - ui_renderer = ChineseUIRenderer() + gesture_recognizer = ImprovedGestureRecognizer(speech_manager) + drone_controller = SimpleDroneController(libs['airsim'], speech_manager) + ui_renderer = ChineseUIRenderer(speech_manager) performance_monitor = PerformanceMonitor() # 初始化摄像头 @@ -1297,9 +1853,19 @@ def main(): print(f" 帧率: {actual_fps}") else: print("❌ 摄像头不可用,使用模拟模式") + + # 摄像头错误语音提示 + if speech_manager.enabled: + speech_manager.speak('camera_error') + cap = None except Exception as e: print(f"⚠ 摄像头初始化失败: {e}") + + # 摄像头错误语音提示 + if speech_manager.enabled: + speech_manager.speak('camera_error') + cap = None # 显示欢迎信息 @@ -1309,6 +1875,7 @@ def main(): print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") print(f" 手势识别: 改进的OpenCV算法") + print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) @@ -1326,7 +1893,9 @@ def main(): print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") print("5. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") - print("6. 按 [ESC] 安全退出") + print("6. 语音控制:") + print(" [V]切换语音反馈 [M]测试语音") + print("7. 按 [ESC] 安全退出") print("=" * 60) print("程序启动成功!") print("-" * 60) @@ -1405,7 +1974,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Optimized', frame) + cv2.imshow('Gesture Controlled Drone - Optimized with Speech', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -1432,23 +2001,58 @@ def main(): config.set('display', 'show_help', value=not current) print(f"帮助显示: {'开启' if not current else '关闭'}") + # 语音提示 + if speech_manager.enabled: + speech_manager.speak('help_toggled') + elif key == ord('r') or key == ord('R'): # 重置手势识别 print("重置手势识别...") - gesture_recognizer = ImprovedGestureRecognizer() + gesture_recognizer = ImprovedGestureRecognizer(speech_manager) print("✓ 手势识别已重置") + # 语音提示 + if speech_manager.enabled: + speech_manager.speak_direct("手势识别已重置") + elif key == ord('t') or key == ord('T'): # 切换显示模式 current_display_mode = (current_display_mode + 1) % len(display_modes) mode_name = display_modes[current_display_mode] print(f"显示模式: {mode_name}") + # 语音提示 + if speech_manager.enabled: + speech_manager.speak('display_mode_changed') + elif key == ord('d') or key == ord('D'): # 切换调试信息 current = config.get('display', 'show_debug_info') config.set('display', 'show_debug_info', value=not current) - print(f"调试信息: {'开启' if not current else '关闭'}") + status = '开启' if not current else '关闭' + print(f"调试信息: {status}") + + # 语音提示 + if speech_manager.enabled: + if not current: + speech_manager.speak('debug_mode_on') + else: + speech_manager.speak('debug_mode_off') + + elif key == ord('v') or key == ord('V'): + # 切换语音反馈 + new_status = speech_manager.toggle_enabled() + status = '启用' if new_status else '禁用' + print(f"语音反馈: {status}") + config.set('speech', 'enabled', value=new_status) + + elif key == ord('m') or key == ord('M'): + # 测试语音 + if speech_manager.enabled: + print("测试语音...") + speech_manager.speak_direct("语音反馈测试,这是一条测试消息") + else: + print("语音反馈已禁用,按V键启用") elif key in key_to_gesture: # 键盘控制 @@ -1477,6 +2081,12 @@ def main(): if cap: cap.release() cv2.destroyAllWindows() + + # 程序退出语音提示 + if speech_manager.enabled: + speech_manager.speak('program_exit', force=True) + time.sleep(1) # 确保语音播报完成 + drone_controller.emergency_stop() config.save_config() From fb866db00dc95facc02a5c036b92a0527e8c3592 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Sat, 20 Dec 2025 11:09:56 +0800 Subject: [PATCH 8/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E5=B9=B3=E6=BB=91=E5=A4=84=E7=90=86=EF=BC=8C=E6=89=A9=E5=B1=95?= =?UTF-8?q?=E8=AF=AD=E9=9F=B3=E5=8F=8D=E9=A6=88=E7=B3=BB=E7=BB=9F=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=89=8B=E5=8A=BF=E7=8A=B6=E6=80=81=E8=BF=BD?= =?UTF-8?q?=E8=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 8 +- .../gesture_config.json | 168 +++-- .../main.py | 660 ++++++++++++------ 3 files changed, 555 insertions(+), 281 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index 0ad6fac5ef..b1c539fd44 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -10,4 +10,10 @@ __pycache__/ .idea/ # 忽略日志文件夹(临时日志文件,无需提交到仓库) -logs/ \ No newline at end of file +logs/ + +# 忽略手势识别配置文件(保留本地,不提交到仓库) +gesture_config.json +gesture_config (1).json +gesture_config_v2.json +gesture_config_v3.json \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/gesture_config.json b/src/autonomus_drone_hand_gesture_project/gesture_config.json index f07c4b5817..c60f9cec3d 100644 --- a/src/autonomus_drone_hand_gesture_project/gesture_config.json +++ b/src/autonomus_drone_hand_gesture_project/gesture_config.json @@ -1,85 +1,24 @@ { - "system": { - "version": "1.0", - "last_updated": "2025-12-19 20:51:44", - "debug_mode": false - }, "camera": { + "index": 0, + "width": 640, + "height": 480, + "fps": 30, "device_id": 0, "resolution": { "width": 640, "height": 480 }, - "fps": 30, "flip_horizontal": true, - "use_virtual_camera": false, - "index": 0, - "width": 640, - "height": 480 - }, - "gesture_detection": { - "skin_hsv_ranges": { - "h_low": 0, - "h_high": 25, - "s_low": 30, - "s_high": 255, - "v_low": 60, - "v_high": 255 - }, - "min_hand_area": 2000, - "max_hand_area": 30000, - "confidence_threshold": 0.5, - "smoothing_window": 5, - "detection_interval": 1 - }, - "gesture_mapping": { - "0": "stop", - "1": "forward", - "2": "backward", - "3": "up", - "4": "down", - "5": "left", - "6": "right", - "7": "hover" - }, - "drone_control": { - "base_speed": 2.0, - "altitude": -10.0, - "control_delay": 0.3, - "takeoff_height": 5.0, - "emergency_stop_enabled": true - }, - "display": { - "show_fps": true, - "show_confidence": true, - "show_gesture_info": true, - "show_control_help": true, - "show_detection_overlay": true, - "theme": "dark", - "show_help": true, - "show_contours": true, - "show_bbox": true, - "show_fingertips": true, - "show_palm_center": false, - "show_hand_direction": false, - "show_debug_info": false, - "show_direction": true, - "show_region_grid": true - }, - "performance": { - "target_fps": 30, - "enable_multithreading": false, - "enable_hardware_acceleration": true, - "resize_factor": 1.0, - "enable_multiprocessing": false + "use_virtual_camera": false }, "gesture": { "skin_lower_h": 0, - "skin_upper_h": 25, - "skin_lower_s": 30, - "skin_upper_s": 255, - "skin_lower_v": 60, - "skin_upper_v": 255, + "skin_upper_h": 178, + "skin_lower_s": 0, + "skin_upper_s": 71, + "skin_lower_v": 32, + "skin_upper_v": 85, "min_hand_area": 3000, "max_hand_area": 30000, "history_size": 10, @@ -90,6 +29,10 @@ "contour_simplify_epsilon": 0.02, "defect_distance_threshold": 20, "palm_circle_radius_ratio": 0.3, + "gesture_stability_threshold": 5, + "transition_threshold": 0.3, + "position_stability_weight": 0.4, + "gesture_cooldown": 0.5, "skin_lower": [ 0, 30, @@ -110,5 +53,88 @@ "duration": 0.3, "altitude": -10.0, "control_interval": 0.3 + }, + "display": { + "show_fps": true, + "show_confidence": true, + "show_help": true, + "show_contours": true, + "show_bbox": true, + "show_fingertips": true, + "show_palm_center": true, + "show_hand_direction": true, + "show_debug_info": false, + "show_speech_status": true, + "show_gesture_history": true, + "show_stability_indicator": true, + "show_gesture_info": true, + "show_control_help": true, + "show_detection_overlay": true, + "theme": "dark", + "show_direction": true, + "show_region_grid": true + }, + "performance": { + "target_fps": 30, + "resize_factor": 1.0, + "enable_multiprocessing": false, + "enable_multithreading": false, + "enable_hardware_acceleration": true + }, + "calibration": { + "auto_calibrate_skin": true, + "skin_calibration_frames": 30, + "hand_size_calibration": true + }, + "speech": { + "enabled": true, + "volume": 1.0, + "rate": 150, + "announce_gestures": true, + "announce_connections": true, + "announce_flight_events": true, + "announce_gesture_changes": true, + "announce_hand_status": true, + "announce_performance": true, + "min_gesture_confidence": 0.7, + "gesture_start_threshold": 3, + "gesture_end_threshold": 10 + }, + "system": { + "version": "1.0", + "last_updated": "2025-12-19 20:51:44", + "debug_mode": false + }, + "gesture_detection": { + "skin_hsv_ranges": { + "h_low": 0, + "h_high": 25, + "s_low": 30, + "s_high": 255, + "v_low": 60, + "v_high": 255 + }, + "min_hand_area": 2000, + "max_hand_area": 30000, + "confidence_threshold": 0.5, + "smoothing_window": 5, + "detection_interval": 1 + }, + "gesture_mapping": { + "0": "stop", + "1": "forward", + "2": "backward", + "3": "up", + "4": "down", + "5": "left", + "6": "right", + "7": "hover" + }, + "drone_control": { + "base_speed": 2.0, + "altitude": -10.0, + "control_delay": 0.3, + "takeoff_height": 5.0, + "emergency_stop_enabled": true } } \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index 4e33bea6dd..fe271047a5 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 手势识别优化版(添加语音反馈) -优化手势识别算法,解决位置敏感问题,添加语音反馈功能 +手势控制AirSim无人机 - 手势识别优化版(增强语音反馈和手势平滑) +优化手势识别算法,增强语音反馈,改进手势平滑处理 作者: xiaoshiyuan888 """ @@ -20,8 +20,8 @@ from collections import deque, Counter print("=" * 60) -print("Gesture Controlled Drone - Gesture Recognition Optimized") -print("现在添加了语音反馈功能!") +print("Gesture Controlled Drone - Enhanced Gesture Recognition") +print("增强语音反馈和手势平滑处理!") print("=" * 60) # ========== 修复导入路径 ========== @@ -158,9 +158,9 @@ def safe_import(): Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] -# ========== 语音反馈管理器 ========== -class SpeechFeedbackManager: - """语音反馈管理器""" +# ========== 增强语音反馈管理器 ========== +class EnhancedSpeechFeedbackManager: + """增强的语音反馈管理器""" def __init__(self, speech_lib): self.speech_lib = speech_lib @@ -169,7 +169,7 @@ def __init__(self, speech_lib): self.rate = 150 self.voice_id = None self.last_speech_time = {} - self.min_interval = 2.0 # 相同语音的最小间隔(秒) + self.min_interval = 1.5 # 缩短相同语音的最小间隔(秒) # 语音队列,避免语音重叠 self.speech_queue = [] @@ -179,7 +179,11 @@ def __init__(self, speech_lib): # 音频播放方法 self.audio_method = None - # 语音消息映射 + # 新增:手势状态追踪 + self.last_gesture_state = "none" # 记录上次手势状态 + self.gesture_active_time = 0 # 手势持续活跃时间 + + # 增强的语音消息映射 self.messages = { # 连接相关 'connecting': "正在连接无人机,请稍候", @@ -193,9 +197,12 @@ def __init__(self, speech_lib): 'landing': "无人机正在降落", 'land_success': "降落成功", 'emergency_stop': "紧急停止,无人机已降落", + 'hovering': "无人机悬停中", - # 手势相关 - 'gesture_detected': "手势识别就绪", + # 手势相关 - 增强 + 'gesture_detected': "手势识别就绪,请开始手势", + 'gesture_start': "开始识别手势", + 'gesture_end': "手势识别结束", 'gesture_stop': "停止", 'gesture_up': "向上", 'gesture_down': "向下", @@ -204,12 +211,17 @@ def __init__(self, speech_lib): 'gesture_forward': "向前", 'gesture_waiting': "等待手势", 'gesture_error': "手势识别错误", + 'gesture_stable': "手势稳定", + 'gesture_change': "手势变化", + 'gesture_low_confidence': "手势识别置信度低", + 'gesture_good_confidence': "手势识别置信度高", # 系统相关 'program_start': "手势控制无人机系统已启动", 'program_exit': "程序退出,感谢使用", 'camera_error': "摄像头错误,请检查连接", - 'low_confidence': "置信度过低,请调整手势", + 'camera_ready': "摄像头就绪", + 'system_ready': "系统准备就绪", # 模式相关 'simulation_mode': "进入模拟模式", @@ -217,6 +229,17 @@ def __init__(self, speech_lib): 'debug_mode_off': "调试模式已关闭", 'display_mode_changed': "显示模式已切换", 'help_toggled': "帮助信息已切换", + + # 新增:性能相关 + 'performance_good': "系统运行流畅", + 'performance_warning': "系统性能警告", + + # 新增:手势指导 + 'move_closer': "请将手靠近摄像头", + 'move_away': "请将手移远一些", + 'good_position': "手部位置良好", + 'hand_detected': "手部已检测到", + 'hand_lost': "手部丢失,请重新放置", } # 初始化语音引擎 @@ -240,7 +263,7 @@ def init_speech_engine(self): # 尝试寻找中文语音 for voice in voices: # 检查语音名称是否包含中文相关标识 - if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower(): + if 'chinese' in voice.name.lower() or 'zh' in voice.name.lower() or 'zh_CN' in voice.name.lower(): self.engine.setProperty('voice', voice.id) self.voice_id = voice.id print(f"[Speech] 使用中文语音: {voice.name}") @@ -306,7 +329,7 @@ def play_audio_file(self, audio_file): # Windows系统命令 os.startfile(audio_file) # 等待播放完成(简单等待) - time.sleep(2) + time.sleep(1.5) elif self.audio_method == 'gtts_posix': # Linux/Mac系统命令 @@ -332,7 +355,7 @@ def play_audio_file(self, audio_file): print(f"⚠ 音频播放失败: {e}") return False - def speak(self, message_key, force=False): + def speak(self, message_key, force=False, immediate=False): """播放语音""" if not self.enabled: return @@ -349,14 +372,18 @@ def speak(self, message_key, force=False): else: text = message_key # 直接使用传入的文本 - # 添加到语音队列 - self.speech_queue.append(text) + # 如果是立即播放,直接在新线程中播放 + if immediate: + self.speak_direct(text) + else: + # 添加到语音队列 + self.speech_queue.append(text) - # 如果没有在播放,启动播放线程 - if not self.is_speaking and self.queue_thread is None: - self.queue_thread = threading.Thread(target=self._process_speech_queue) - self.queue_thread.daemon = True - self.queue_thread.start() + # 如果没有在播放,启动播放线程 + if not self.is_speaking and self.queue_thread is None: + self.queue_thread = threading.Thread(target=self._process_speech_queue) + self.queue_thread.daemon = True + self.queue_thread.start() # 更新时间戳 self.last_speech_time[message_key] = current_time @@ -397,7 +424,7 @@ def _process_speech_queue(self): except Exception as e: print(f"⚠ 语音播放失败: {e}") - time.sleep(0.1) # 避免过快 + time.sleep(0.05) # 减少等待时间,避免卡顿 self.is_speaking = False self.queue_thread = None @@ -475,7 +502,7 @@ class ConfigManager: """配置管理器""" def __init__(self): - self.config_file = os.path.join(current_dir, 'gesture_config_v3.json') # 更新版本号 + self.config_file = os.path.join(current_dir, 'gesture_config.json') self.default_config = { 'camera': { 'index': 0, @@ -492,14 +519,18 @@ def __init__(self): 'skin_upper_v': 255, 'min_hand_area': 2000, 'max_hand_area': 30000, - 'history_size': 10, - 'smooth_frames': 5, - 'min_confidence': 0.5, + 'history_size': 15, # 增加历史记录大小 + 'smooth_frames': 7, # 增加平滑帧数 + 'min_confidence': 0.6, # 提高最小置信度阈值 'detection_interval': 1, - 'hand_ratio_threshold': 1.5, # 手部长宽比阈值 + 'hand_ratio_threshold': 1.5, 'contour_simplify_epsilon': 0.02, - 'defect_distance_threshold': 20, # 凸缺陷距离阈值 - 'palm_circle_radius_ratio': 0.3, # 手掌半径与边界框的比例 + 'defect_distance_threshold': 20, + 'palm_circle_radius_ratio': 0.3, + 'gesture_stability_threshold': 5, # 新增:手势稳定性阈值 + 'transition_threshold': 0.3, # 新增:手势转换阈值 + 'position_stability_weight': 0.4, # 新增:位置稳定性权重 + 'gesture_cooldown': 0.5, # 新增:手势冷却时间 }, 'drone': { 'velocity': 2.5, @@ -517,7 +548,9 @@ def __init__(self): 'show_palm_center': True, 'show_hand_direction': True, 'show_debug_info': False, - 'show_speech_status': True, # 新增:显示语音状态 + 'show_speech_status': True, + 'show_gesture_history': True, # 新增:显示手势历史 + 'show_stability_indicator': True, # 新增:显示稳定性指示器 }, 'performance': { 'target_fps': 30, @@ -529,14 +562,19 @@ def __init__(self): 'skin_calibration_frames': 30, 'hand_size_calibration': True }, - 'speech': { # 新增:语音配置 + 'speech': { 'enabled': True, 'volume': 1.0, 'rate': 150, - 'announce_gestures': True, # 是否播报手势 - 'announce_connections': True, # 是否播报连接状态 - 'announce_flight_events': True, # 是否播报飞行事件 - 'min_gesture_confidence': 0.7, # 播报手势的最小置信度 + 'announce_gestures': True, + 'announce_connections': True, + 'announce_flight_events': True, + 'announce_gesture_changes': True, # 新增:播报手势变化 + 'announce_hand_status': True, # 新增:播报手部状态 + 'announce_performance': True, # 新增:播报性能状态 + 'min_gesture_confidence': 0.7, + 'gesture_start_threshold': 3, # 新增:手势开始识别阈值 + 'gesture_end_threshold': 10, # 新增:手势结束识别阈值 } } self.config = self.load_config() @@ -652,36 +690,48 @@ def calibrate_hand_size(self, hand_area): config = ConfigManager() -# ========== 改进的手势识别器 ========== -class ImprovedGestureRecognizer: - """改进的手势识别器 - 纯OpenCV实现""" +# ========== 改进的手势识别器(增强平滑处理) ========== +class EnhancedGestureRecognizer: + """增强的手势识别器 - 改进平滑处理和稳定性""" def __init__(self, speech_manager=None): self.config = config.get('gesture') self.speech_manager = speech_manager - # 手势历史和平滑 + # 增强的手势历史和平滑 self.history_size = self.config['history_size'] self.gesture_history = deque(maxlen=self.history_size) self.confidence_history = deque(maxlen=self.history_size) + self.position_history = deque(maxlen=self.history_size) # 新增:位置历史 self.current_gesture = "Waiting" self.current_confidence = 0.0 - # 记录上次播报的手势,避免重复播报 + # 新增:手势状态追踪 + self.gesture_state = "none" # none, starting, active, ending + self.gesture_stability_counter = 0 + self.last_stable_gesture = "Waiting" + self.gesture_active_frames = 0 + self.last_gesture_change_time = 0 + + # 记录上次播报的手势 self.last_announced_gesture = None self.last_announced_time = 0 - self.gesture_announce_interval = 3.0 # 手势播报最小间隔 + self.last_hand_status_time = 0 + self.gesture_announce_interval = 2.0 # 缩短手势播报间隔 # 手部跟踪和状态 self.last_hand_position = None self.hand_tracking = False self.track_window = None - self.hand_states = deque(maxlen=10) # 手部状态历史 + self.hand_states = deque(maxlen=15) + self.hand_detected_frames = 0 + self.hand_lost_frames = 0 # 性能统计 self.process_times = deque(maxlen=30) self.frame_counter = 0 self.detection_interval = self.config['detection_interval'] + self.last_performance_report = 0 # 手势颜色映射 self.gesture_colors = { @@ -708,6 +758,14 @@ def __init__(self, speech_manager=None): "Error": "gesture_error", } + # 手势状态颜色 + self.state_colors = { + "none": (100, 100, 100), # 灰色 + "starting": (255, 165, 0), # 橙色 + "active": (0, 255, 0), # 绿色 + "ending": (255, 0, 0), # 红色 + } + # 背景减除器 self.bg_subtractor = cv2.createBackgroundSubtractorMOG2( history=100, varThreshold=25, detectShadows=True @@ -716,15 +774,15 @@ def __init__(self, speech_manager=None): # 形态学操作核 self.kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) - print("✓ 改进的手势识别器已初始化 (纯OpenCV实现)") + # 新增:性能监控 + self.avg_process_time = 0 + self.frame_rate = 0 + self.last_fps_check = time.time() - # 语音提示初始化完成 - if self.speech_manager: - self.speech_manager.speak('gesture_detected') + print("✓ 增强的手势识别器已初始化 (改进平滑处理)") def get_skin_mask(self, frame): """获取肤色掩码""" - # 从配置获取肤色范围 h_low = config.get('gesture', 'skin_lower_h') h_high = config.get('gesture', 'skin_upper_h') s_low = config.get('gesture', 'skin_lower_s') @@ -732,60 +790,42 @@ def get_skin_mask(self, frame): v_low = config.get('gesture', 'skin_lower_v') v_high = config.get('gesture', 'skin_upper_v') - # 转换为HSV颜色空间 hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) - - # 定义肤色范围 lower_skin = np.array([h_low, s_low, v_low], dtype=np.uint8) upper_skin = np.array([h_high, s_high, v_high], dtype=np.uint8) - - # 创建肤色掩码 skin_mask = cv2.inRange(hsv, lower_skin, upper_skin) return skin_mask, hsv def enhance_skin_detection(self, frame, skin_mask): """增强肤色检测""" - # 应用背景减除 fg_mask = self.bg_subtractor.apply(frame) - - # 结合肤色掩码和前景掩码 combined_mask = cv2.bitwise_and(skin_mask, fg_mask) - - # 形态学操作 combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_CLOSE, self.kernel, iterations=2) combined_mask = cv2.morphologyEx(combined_mask, cv2.MORPH_OPEN, self.kernel, iterations=1) - - # 高斯模糊 combined_mask = cv2.GaussianBlur(combined_mask, (5, 5), 0) return combined_mask def find_best_hand_contour(self, mask, frame): """找到最佳的手部轮廓""" - # 查找轮廓 contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if not contours: return None, 0.0 - # 按面积排序 contours = sorted(contours, key=cv2.contourArea, reverse=True) - best_contour = None best_score = 0.0 - min_area = config.get('gesture', 'min_hand_area') max_area = config.get('gesture', 'max_hand_area') - for contour in contours[:3]: # 只检查前3个最大轮廓 + for contour in contours[:3]: area = cv2.contourArea(contour) - # 面积过滤 if area < min_area or area > max_area: continue - # 计算轮廓评分 score = self.rate_contour(contour, frame.shape) if score > best_score: @@ -797,35 +837,27 @@ def find_best_hand_contour(self, mask, frame): def rate_contour(self, contour, frame_shape): """评估轮廓作为手部的可能性""" score = 0.0 - - # 面积评分 area = cv2.contourArea(contour) min_area = config.get('gesture', 'min_hand_area') max_area = config.get('gesture', 'max_hand_area') if min_area < area < max_area: - # 面积在合理范围内 area_ratio = min(area / max_area, 1.0) score += area_ratio * 0.3 - # 周长评分 perimeter = cv2.arcLength(contour, True) - if perimeter > 100: # 最小周长 + if perimeter > 100: score += 0.2 - # 紧凑度评分 (周长^2 / 面积) if area > 0: compactness = perimeter ** 2 / area - # 手的紧凑度通常在14-20之间 if 12 < compactness < 25: compactness_score = 1.0 - abs(compactness - 18) / 6 score += compactness_score * 0.3 - # 长宽比评分 x, y, w, h = cv2.boundingRect(contour) if h > 0: aspect_ratio = w / h - # 手的长宽比通常在0.5-2.0之间 if 0.4 < aspect_ratio < 2.5: aspect_score = 1.0 - abs(aspect_ratio - 1.0) / 1.5 score += aspect_score * 0.2 @@ -837,54 +869,37 @@ def analyze_hand_features(self, contour, frame_shape): if contour is None: return None, 0.0 - # 计算轮廓面积 area = cv2.contourArea(contour) - - # 计算轮廓中心 M = cv2.moments(contour) if M["m00"] == 0: return None, 0.0 cx = int(M["m10"] / M["m00"]) cy = int(M["m01"] / M["m00"]) - - # 计算边界框 x, y, w, h = cv2.boundingRect(contour) bbox_area = w * h - - # 计算手掌中心(假设是轮廓的中心) palm_center = (cx, cy) - - # 计算手掌半径(假设为边界框宽度的30%) palm_radius = int(w * config.get('gesture', 'palm_circle_radius_ratio')) - - # 分析手指 fingers, fingertips, defects = self.analyze_fingers(contour, palm_center, palm_radius) - - # 计算手部方向 direction = self.calculate_hand_direction(contour, cx, cy) - - # 计算手部位置(归一化) h_img, w_img = frame_shape[:2] norm_x = cx / w_img norm_y = cy / h_img - - # 计算手势置信度 confidence = self.calculate_confidence(area, len(fingers), len(contour), bbox_area) - # 返回结果 result = { 'contour': contour, 'center': (cx, cy), 'bbox': (x, y, x + w, y + h), - 'fingers': fingers, # 手指列表 - 'fingertips': fingertips, # 指尖点列表 - 'defects': defects, # 凸缺陷列表 + 'fingers': fingers, + 'fingertips': fingertips, + 'defects': defects, 'palm_center': palm_center, 'palm_radius': palm_radius, - 'direction': direction, # 手部方向(角度) + 'direction': direction, 'area': area, 'position': (norm_x, norm_y), + 'bbox_size': (w, h), 'confidence': confidence } @@ -892,19 +907,14 @@ def analyze_hand_features(self, contour, frame_shape): def analyze_fingers(self, contour, palm_center, palm_radius): """分析手指""" - # 简化轮廓 epsilon = config.get('gesture', 'contour_simplify_epsilon') * cv2.arcLength(contour, True) approx = cv2.approxPolyDP(contour, epsilon, True) - - # 计算凸包 hull = cv2.convexHull(approx, returnPoints=False) if hull is None or len(hull) < 3: return [], [], [] - # 计算凸缺陷 defects = cv2.convexityDefects(approx, hull) - fingers = [] fingertips = [] defect_points = [] @@ -916,17 +926,14 @@ def analyze_fingers(self, contour, palm_center, palm_radius): end = tuple(approx[e][0]) far = tuple(approx[f][0]) - # 计算距离 start_dist = np.linalg.norm(np.array(start) - np.array(palm_center)) end_dist = np.linalg.norm(np.array(end) - np.array(palm_center)) far_dist = np.linalg.norm(np.array(far) - np.array(palm_center)) - # 检查是否可能是手指 if (start_dist > palm_radius * 1.2 and end_dist > palm_radius * 1.2 and d > config.get('gesture', 'defect_distance_threshold') * 256): - # 检查角度 a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) @@ -935,9 +942,7 @@ def analyze_fingers(self, contour, palm_center, palm_radius): angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) angle_degrees = math.degrees(angle) - # 如果角度小于90度,可能是指缝 if angle_degrees < 90: - # 添加手指 finger = { 'start': start, 'end': end, @@ -947,77 +952,59 @@ def analyze_fingers(self, contour, palm_center, palm_radius): } fingers.append(finger) - # 添加指尖(开始和结束点都可能是指尖) for point in [start, end]: if point not in fingertips: - # 检查点是否在手掌半径之外 point_dist = np.linalg.norm(np.array(point) - np.array(palm_center)) if point_dist > palm_radius * 1.5: fingertips.append(point) defect_points.append((start, end, far, d)) - # 如果没有检测到手指,尝试使用凸包点作为指尖 if len(fingertips) == 0: hull_points = cv2.convexHull(approx, returnPoints=True) if len(hull_points) > 0: hull_points = hull_points.reshape(-1, 2) - # 过滤掉离手掌中心太近的点 for point in hull_points: point_tuple = tuple(point) point_dist = np.linalg.norm(point - np.array(palm_center)) if point_dist > palm_radius * 1.5 and point_tuple not in fingertips: fingertips.append(point_tuple) - # 限制指尖数量为5 fingertips = fingertips[:5] return fingers, fingertips, defect_points def calculate_hand_direction(self, contour, cx, cy): """计算手部方向""" - # 使用PCA(主成分分析)计算手部方向 if len(contour) < 5: return 0.0 - # 将轮廓转换为点集 points = contour.reshape(-1, 2).astype(np.float32) - - # 计算PCA mean = np.empty((0)) mean, eigenvectors, eigenvalues = cv2.PCACompute2(points, mean) - - # 计算方向角度(度) direction = math.degrees(math.atan2(eigenvectors[0, 1], eigenvectors[0, 0])) return direction def calculate_confidence(self, area, finger_count, contour_length, bbox_area): """计算手势置信度""" - confidence = 0.5 # 基础置信度 - - # 基于面积的置信度 + confidence = 0.5 min_area = config.get('gesture', 'min_hand_area') max_area = config.get('gesture', 'max_hand_area') if min_area < area < max_area: - # 面积在合理范围内 area_norm = (area - min_area) / (max_area - min_area) confidence += area_norm * 0.2 - # 基于手指数量的置信度 if 0 <= finger_count <= 5: confidence += 0.2 - # 基于轮廓复杂度的置信度 if contour_length > 200: confidence += 0.1 - # 基于填充度的置信度(轮廓面积/边界框面积) if bbox_area > 0: fill_ratio = area / bbox_area - # 手的填充度通常在0.3-0.7之间 if 0.2 < fill_ratio < 0.8: fill_score = 1.0 - abs(fill_ratio - 0.5) / 0.3 confidence += fill_score * 0.1 @@ -1034,116 +1021,219 @@ def recognize_gesture_improved(self, hand_data): norm_x, norm_y = hand_data['position'] direction = hand_data.get('direction', 0.0) confidence = hand_data['confidence'] + w, h = hand_data['bbox_size'] + aspect_ratio = w / h if h > 0 else 1.0 # 根据手指数量分类 if finger_count == 0: - # 握拳或没有手指 if len(fingertips) == 0: return "Stop", confidence * 0.9 else: - # 有指尖但不是手指,可能是部分握拳 return "Stop", confidence * 0.7 elif finger_count == 1: - # 单指手势 if len(fingertips) >= 1: - # 检查指尖方向 cx, cy = hand_data['center'] fingertip = fingertips[0] - - # 计算指尖相对于手中心的方向 dx = fingertip[0] - cx dy = fingertip[1] - cy - # 判断指尖方向 - if abs(dx) > abs(dy): # 主要水平方向 + if abs(dx) > abs(dy): if dx > 0: return "Right", confidence * 0.8 else: return "Left", confidence * 0.8 - else: # 主要垂直方向 - if dy < 0: # 指尖在手中心上方 + else: + if dy < 0: return "Up", confidence * 0.8 else: return "Forward", confidence * 0.8 return "Forward", confidence * 0.7 elif finger_count == 2: - # 双指手势 return "Forward", confidence * 0.7 elif finger_count == 3: - # 三指手势,根据手部方向判断 - if -45 <= direction <= 45: # 大致水平 + if -45 <= direction <= 45: if direction > 0: return "Right", confidence * 0.7 else: return "Left", confidence * 0.7 - else: # 大致垂直 + else: if direction > 0: return "Down", confidence * 0.7 else: return "Up", confidence * 0.7 elif finger_count >= 4: - # 多指手势(手掌张开) - # 根据手部位置判断 - if norm_x < 0.4: # 左侧 + if norm_x < 0.4: return "Left", confidence * 0.8 - elif norm_x > 0.6: # 右侧 + elif norm_x > 0.6: return "Right", confidence * 0.8 - elif norm_y < 0.4: # 上方 + elif norm_y < 0.4: return "Up", confidence * 0.8 - elif norm_y > 0.6: # 下方 + elif norm_y > 0.6: return "Down", confidence * 0.8 - else: # 中间 - # 根据手部方向判断 - if -45 <= direction <= 45: # 大致水平 + else: + if -45 <= direction <= 45: return "Forward", confidence * 0.7 - else: # 大致垂直 + else: return "Stop", confidence * 0.7 return "Waiting", confidence * 0.5 - def smooth_gesture(self, new_gesture, new_confidence): - """平滑手势输出""" + def smooth_gesture_enhanced(self, new_gesture, new_confidence, hand_data): + """增强的手势平滑处理""" + current_time = time.time() + + # 检查手势冷却时间 + if current_time - self.last_gesture_change_time < config.get('gesture', 'gesture_cooldown'): + return self.current_gesture, self.current_confidence + # 添加到历史 self.gesture_history.append(new_gesture) self.confidence_history.append(new_confidence) - # 如果历史记录不足,直接返回 - if len(self.gesture_history) < 3: - self.current_gesture = new_gesture - self.current_confidence = new_confidence - return new_gesture, new_confidence + if hand_data is not None: + self.position_history.append(hand_data['position']) + + # 计算手势稳定性 + if len(self.gesture_history) >= 3: + # 检查最近N个手势是否一致 + recent_gestures = list(self.gesture_history)[-3:] + gesture_counter = Counter(recent_gestures) + most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] + + # 计算位置稳定性 + position_stability = 1.0 + if len(self.position_history) >= 2 and hand_data is not None: + current_pos = hand_data['position'] + prev_pos = self.position_history[-2] if len(self.position_history) >= 2 else current_pos + position_diff = math.sqrt((current_pos[0] - prev_pos[0]) ** 2 + (current_pos[1] - prev_pos[1]) ** 2) + position_stability = max(0, 1.0 - position_diff * 5.0) + + # 增强的稳定性检查 + stability_threshold = config.get('gesture', 'gesture_stability_threshold') + transition_threshold = config.get('gesture', 'transition_threshold') + position_weight = config.get('gesture', 'position_stability_weight') + + # 计算综合稳定性得分 + gesture_stability = most_common_count / 3.0 + overall_stability = gesture_stability * (1.0 - position_weight) + position_stability * position_weight + + # 手势状态转换逻辑 + if overall_stability >= transition_threshold: + if most_common_gesture != self.last_stable_gesture: + self.gesture_stability_counter += 1 + else: + self.gesture_stability_counter = max(0, self.gesture_stability_counter - 1) + + if self.gesture_stability_counter >= stability_threshold: + # 手势稳定,更新当前手势 + self.current_gesture = most_common_gesture + self.current_confidence = np.mean(list(self.confidence_history)[-3:]) + self.last_stable_gesture = most_common_gesture + self.gesture_stability_counter = 0 + self.last_gesture_change_time = current_time + else: + # 手势不稳定,重置计数器 + self.gesture_stability_counter = max(0, self.gesture_stability_counter - 2) + + return self.current_gesture, self.current_confidence + + def update_gesture_state(self, hand_data, gesture, confidence): + """更新手势状态""" + current_time = time.time() - # 统计最常见的姿势 - gesture_counter = Counter(self.gesture_history) - most_common_gesture, most_common_count = gesture_counter.most_common(1)[0] + if hand_data is None: + # 手部丢失 + self.hand_lost_frames += 1 + self.hand_detected_frames = max(0, self.hand_detected_frames - 1) - # 计算平均置信度 - avg_confidence = np.mean(list(self.confidence_history)) + if self.hand_lost_frames > 10 and self.gesture_state != "none": + self.gesture_state = "none" + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_hand_status') and + current_time - self.last_hand_status_time > 3.0): + self.speech_manager.speak('hand_lost', immediate=True) + self.last_hand_status_time = current_time + return - # 如果最常见的姿势出现次数足够多,使用它 - smooth_frames = self.config['smooth_frames'] - if most_common_count >= smooth_frames: - self.current_gesture = most_common_gesture - self.current_confidence = avg_confidence + # 手部检测到 + self.hand_detected_frames += 1 + self.hand_lost_frames = 0 - return self.current_gesture, self.current_confidence + # 检查手部大小和位置 + hand_area = hand_data['area'] + min_area = config.get('gesture', 'min_hand_area') + max_area = config.get('gesture', 'max_hand_area') + + # 提供手部位置反馈 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_hand_status') and + current_time - self.last_hand_status_time > 5.0): + + if hand_area < min_area * 0.8: + self.speech_manager.speak('move_closer', immediate=True) + self.last_hand_status_time = current_time + elif hand_area > max_area * 1.2: + self.speech_manager.speak('move_away', immediate=True) + self.last_hand_status_time = current_time + elif self.hand_detected_frames == 5: # 首次稳定检测 + self.speech_manager.speak('hand_detected', immediate=True) + self.last_hand_status_time = current_time + + # 手势状态转换 + if self.gesture_state == "none" and gesture != "Waiting" and confidence > 0.6: + # 手势开始 + self.gesture_state = "starting" + self.gesture_active_frames = 0 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gesture_changes')): + self.speech_manager.speak('gesture_start', immediate=True) + + elif self.gesture_state == "starting": + self.gesture_active_frames += 1 + if self.gesture_active_frames >= config.get('speech', 'gesture_start_threshold'): + self.gesture_state = "active" + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gesture_changes')): + self.speech_manager.speak('gesture_stable', immediate=True) + + elif self.gesture_state == "active": + if gesture == "Waiting" or confidence < 0.5: + self.gesture_active_frames = max(0, self.gesture_active_frames - 2) + if self.gesture_active_frames <= 0: + self.gesture_state = "ending" + else: + self.gesture_active_frames = min(20, self.gesture_active_frames + 1) + + elif self.gesture_state == "ending": + self.gesture_active_frames -= 1 + if self.gesture_active_frames <= 0: + self.gesture_state = "none" + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_gesture_changes')): + self.speech_manager.speak('gesture_end', immediate=True) def visualize_detection(self, frame, hand_data, gesture, confidence): """可视化检测结果""" if hand_data is None: return frame - # 获取显示配置 show_contours = config.get('display', 'show_contours') show_bbox = config.get('display', 'show_bbox') show_fingertips = config.get('display', 'show_fingertips') show_palm_center = config.get('display', 'show_palm_center') show_hand_direction = config.get('display', 'show_hand_direction') show_debug_info = config.get('display', 'show_debug_info') + show_gesture_history = config.get('display', 'show_gesture_history') + show_stability_indicator = config.get('display', 'show_stability_indicator') # 绘制轮廓 if show_contours and 'contour' in hand_data: @@ -1153,13 +1243,24 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): if show_bbox and 'bbox' in hand_data: x1, y1, x2, y2 = hand_data['bbox'] color = self.gesture_colors.get(gesture, (255, 255, 255)) + + # 根据手势状态调整边界框颜色 + state_color = self.state_colors.get(self.gesture_state, color) + if self.gesture_state != "none": + color = state_color + cv2.rectangle(frame, (x1, y1), (x2, y2), color, 2) - # 显示手势标签 - 使用英文避免字体问题 + # 显示手势标签 label = f"{gesture}" if config.get('display', 'show_confidence'): label += f" ({confidence:.0%})" + # 显示手势状态 + if self.gesture_state != "none": + state_text = {"starting": "开始", "active": "活跃", "ending": "结束"} + label += f" [{state_text.get(self.gesture_state, '')}]" + # 计算文本大小 (text_width, text_height), baseline = cv2.getTextSize( label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2 @@ -1197,19 +1298,65 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): direction = hand_data['direction'] length = 50 - # 计算方向向量 dx = length * math.cos(math.radians(direction)) dy = length * math.sin(math.radians(direction)) - # 绘制方向线 end_point = (int(cx + dx), int(cy + dy)) cv2.arrowedLine(frame, (cx, cy), end_point, (255, 255, 0), 2) - # 显示方向角度 angle_text = f"Dir: {direction:.0f}°" cv2.putText(frame, angle_text, (cx, cy - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 1) + # 绘制稳定性指示器 + if show_stability_indicator: + h, w = frame.shape[:2] + indicator_x = w - 100 + indicator_y = 30 + + # 绘制稳定性背景 + cv2.rectangle(frame, (indicator_x, indicator_y), + (indicator_x + 80, indicator_y + 15), (50, 50, 50), -1) + + # 计算稳定性指示条长度 + stability_level = min(1.0, + self.gesture_stability_counter / config.get('gesture', 'gesture_stability_threshold')) + bar_length = int(70 * stability_level) + + # 根据稳定性级别选择颜色 + if stability_level > 0.7: + bar_color = (0, 255, 0) # 绿色 + elif stability_level > 0.4: + bar_color = (255, 165, 0) # 橙色 + else: + bar_color = (255, 0, 0) # 红色 + + # 绘制稳定性指示条 + cv2.rectangle(frame, (indicator_x + 5, indicator_y + 5), + (indicator_x + 5 + bar_length, indicator_y + 10), bar_color, -1) + + # 绘制稳定性文本 + cv2.putText(frame, "稳定度", (indicator_x, indicator_y - 5), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 绘制手势历史(如果启用) + if show_gesture_history and len(self.gesture_history) > 0: + h, w = frame.shape[:2] + history_y = h - 50 + + # 绘制历史背景 + cv2.rectangle(frame, (10, history_y - 20), (200, history_y + 10), (0, 0, 0), -1) + cv2.putText(frame, "手势历史:", (15, history_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 显示最近几个手势 + recent_gestures = list(self.gesture_history)[-5:] if len(self.gesture_history) >= 5 else list( + self.gesture_history) + for i, gest in enumerate(recent_gestures): + color = self.gesture_colors.get(gest, (255, 255, 255)) + cv2.putText(frame, gest[0], (85 + i * 20, history_y), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + # 显示调试信息 if show_debug_info: # 显示手指数量 @@ -1223,6 +1370,16 @@ def visualize_detection(self, frame, hand_data, gesture, confidence): cv2.putText(frame, pos_text, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + # 显示手势状态 + state_text = f"State: {self.gesture_state}" + cv2.putText(frame, state_text, (150, frame.shape[0] - 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + + # 显示稳定性计数器 + stability_text = f"Stability: {self.gesture_stability_counter}" + cv2.putText(frame, stability_text, (150, frame.shape[0] - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + return frame def recognize(self, frame): @@ -1261,20 +1418,36 @@ def recognize(self, frame): # 识别手势 gesture, raw_confidence = self.recognize_gesture_improved(hand_data) confidence = max(confidence, raw_confidence) + + # 更新手势状态 + self.update_gesture_state(hand_data, gesture, confidence) + + # 增强的手势平滑 + final_gesture, final_confidence = self.smooth_gesture_enhanced(gesture, confidence, hand_data) else: gesture, confidence = "Waiting", 0.3 - - # 平滑手势 - final_gesture, final_confidence = self.smooth_gesture(gesture, confidence) + self.update_gesture_state(None, gesture, confidence) + final_gesture, final_confidence = gesture, confidence # 手势语音提示 if (self.speech_manager and config.get('speech', 'enabled') and - config.get('speech', 'announce_gestures') and - final_confidence >= config.get('speech', 'min_gesture_confidence')): + config.get('speech', 'announce_gestures')): current_time = time.time() - if (final_gesture != self.last_announced_gesture or + + # 根据置信度提供反馈 + if confidence >= 0.8 and current_time - self.last_hand_status_time > 5.0: + self.speech_manager.speak('gesture_good_confidence', immediate=True) + self.last_hand_status_time = current_time + elif confidence < 0.5 and current_time - self.last_hand_status_time > 5.0: + self.speech_manager.speak('gesture_low_confidence', immediate=True) + self.last_hand_status_time = current_time + + # 手势变化播报 + if (final_gesture != self.last_announced_gesture and + final_gesture not in ["Waiting", "Error"] and + final_confidence >= config.get('speech', 'min_gesture_confidence') and current_time - self.last_announced_time > self.gesture_announce_interval): if final_gesture in self.gesture_speech_map: @@ -1283,6 +1456,20 @@ def recognize(self, frame): self.last_announced_gesture = final_gesture self.last_announced_time = current_time + # 性能报告 + current_time = time.time() + if current_time - self.last_performance_report > 30.0: # 每30秒报告一次 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_performance')): + + if self.avg_process_time < 20: # 处理时间小于20ms + self.speech_manager.speak('performance_good', immediate=True) + elif self.avg_process_time > 50: # 处理时间大于50ms + self.speech_manager.speak('performance_warning', immediate=True) + + self.last_performance_report = current_time + # 可视化结果 if hand_data is not None: processed_frame = self.visualize_detection( @@ -1296,6 +1483,17 @@ def recognize(self, frame): process_time = (time.time() - start_time) * 1000 self.process_times.append(process_time) + # 更新平均处理时间 + if len(self.process_times) > 0: + self.avg_process_time = np.mean(list(self.process_times)) + + # 更新帧率 + current_time = time.time() + if current_time - self.last_fps_check >= 1.0: + self.frame_rate = self.frame_counter + self.frame_counter = 0 + self.last_fps_check = current_time + return final_gesture, final_confidence, processed_frame except Exception as e: @@ -1305,9 +1503,9 @@ def recognize(self, frame): def get_performance_stats(self): """获取性能统计""" if len(self.process_times) == 0: - return 0.0 + return 0.0, self.frame_rate - return np.mean(list(self.process_times)) + return np.mean(list(self.process_times)), self.frame_rate def set_simulated_gesture(self, gesture): """设置模拟的手势""" @@ -1538,7 +1736,7 @@ def move_by_gesture(self, gesture, confidence): config.get('speech', 'enabled') and config.get('speech', 'announce_gestures') and confidence < min_confidence * 0.8): # 如果置信度特别低 - self.speech_manager.speak('low_confidence') + self.speech_manager.speak('gesture_low_confidence') return False try: @@ -1568,9 +1766,18 @@ def move_by_gesture(self, gesture, confidence): elif gesture == "Stop": self.client.hoverAsync() success = True + # 悬停语音提示 + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('hovering') elif gesture == "Hover": self.client.hoverAsync() success = True + if (self.speech_manager and + config.get('speech', 'enabled') and + config.get('speech', 'announce_flight_events')): + self.speech_manager.speak('hovering') if success: self.last_control_time = current_time @@ -1627,6 +1834,9 @@ def __init__(self, speech_manager=None): 'help': (255, 200, 100), # 浅橙色 'speech_enabled': (0, 255, 0), # 绿色 'speech_disabled': (255, 0, 0), # 红色 + 'performance_good': (0, 255, 0), # 绿色 + 'performance_warning': (255, 165, 0), # 橙色 + 'performance_bad': (255, 0, 0), # 红色 } print("✓ 中文UI渲染器已初始化") @@ -1690,7 +1900,7 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) # 标题 - title = "手势控制无人机系统 - 优化版 (带语音反馈)" + title = "手势控制无人机系统 - 增强版" frame = self.draw_text(frame, title, (10, 10), size=20, color=self.colors['title']) # 连接状态 @@ -1730,10 +1940,20 @@ def draw_status_bar(self, frame, drone_controller, gesture, confidence, fps, pro if process_time > 0: perf_text += f" | 延迟: {process_time:.1f}ms" - frame = self.draw_text(frame, perf_text, (w - 200, 65), size=14, color=self.colors['info']) + # 根据处理时间选择颜色 + if process_time < 20: + perf_color = self.colors['performance_good'] + elif process_time < 50: + perf_color = self.colors['performance_warning'] + else: + perf_color = self.colors['performance_bad'] + else: + perf_color = self.colors['info'] + + frame = self.draw_text(frame, perf_text, (w - 200, 65), size=14, color=perf_color) # 控制提示 - control_text = "提示: 确保手部完全进入画面,光线充足" + control_text = "提示: 确保手部完全进入画面,保持稳定手势" frame = self.draw_text(frame, control_text, (10, 90), size=14, color=self.colors['info']) return frame @@ -1752,7 +1972,7 @@ def draw_help_bar(self, frame): help_lines = [ "C:连接 空格:起飞/降落 ESC:退出 W/A/S/D/F/X:键盘控制", "H:切换帮助 R:重置识别 T:切换显示模式 D:调试信息", - "V:切换语音反馈 M:测试语音" + "V:切换语音反馈 M:测试语音 P:性能报告" ] for i, line in enumerate(help_lines): @@ -1820,16 +2040,17 @@ def main(): """主函数""" # 初始化语音管理器 print("初始化语音反馈系统...") - speech_manager = SpeechFeedbackManager(libs['speech']) + speech_manager = EnhancedSpeechFeedbackManager(libs['speech']) # 程序启动语音提示 if speech_manager.enabled: - speech_manager.speak('program_start', force=True) + speech_manager.speak('program_start', force=True, immediate=True) + speech_manager.speak('system_ready', immediate=True) # 初始化组件 print("初始化组件...") - gesture_recognizer = ImprovedGestureRecognizer(speech_manager) + gesture_recognizer = EnhancedGestureRecognizer(speech_manager) drone_controller = SimpleDroneController(libs['airsim'], speech_manager) ui_renderer = ChineseUIRenderer(speech_manager) performance_monitor = PerformanceMonitor() @@ -1851,12 +2072,16 @@ def main(): print(f"✓ 摄像头已初始化") print(f" 分辨率: {actual_width}x{actual_height}") print(f" 帧率: {actual_fps}") + + # 摄像头就绪语音提示 + if speech_manager.enabled: + speech_manager.speak('camera_ready', immediate=True) else: print("❌ 摄像头不可用,使用模拟模式") # 摄像头错误语音提示 if speech_manager.enabled: - speech_manager.speak('camera_error') + speech_manager.speak('camera_error', immediate=True) cap = None except Exception as e: @@ -1864,18 +2089,19 @@ def main(): # 摄像头错误语音提示 if speech_manager.enabled: - speech_manager.speak('camera_error') + speech_manager.speak('camera_error', immediate=True) cap = None # 显示欢迎信息 print("\n" + "=" * 60) - print("手势控制无人机系统 - 手势识别优化版") + print("手势控制无人机系统 - 增强版") print("=" * 60) print("系统状态:") print(f" 摄像头: {'已连接' if cap else '模拟模式'}") - print(f" 手势识别: 改进的OpenCV算法") + print(f" 手势识别: 增强的平滑处理算法") print(f" 语音反馈: {'已启用' if speech_manager.enabled else '已禁用'}") + print(f" 手势状态: 支持开始/活跃/结束状态追踪") print(f" AirSim: {'可用' if libs['airsim'] else '模拟模式'}") print("=" * 60) @@ -1884,17 +2110,16 @@ def main(): print("1. 按 [C] 连接无人机 (AirSim模拟器)") print("2. 按 [空格键] 起飞/降落") print("3. 手势控制改进:") - print(" - 握拳或握紧: Stop") - print(" - 单指指向: 根据指尖方向判断Up/Down/Left/Right") - print(" - 双指: Forward") - print(" - 手掌张开: 根据手的位置判断方向") - print(" * 手势识别置信度 > 50% 时才会执行") + print(" - 系统会自动检测手势开始、稳定和结束状态") + print(" - 手势稳定性越高,识别越准确") + print(" - 手部距离摄像头适中时效果最佳") + print(" * 手势识别置信度 > 60% 时才会执行") print("4. 键盘控制:") print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") print("5. 调试功能:") print(" [H]切换帮助显示 [R]重置手势识别 [T]切换显示模式 [D]调试信息") print("6. 语音控制:") - print(" [V]切换语音反馈 [M]测试语音") + print(" [V]切换语音反馈 [M]测试语音 [P]性能报告") print("7. 按 [ESC] 安全退出") print("=" * 60) print("程序启动成功!") @@ -1939,13 +2164,15 @@ def main(): # 获取性能统计 perf_stats = performance_monitor.get_stats() - process_time = gesture_recognizer.get_performance_stats() + process_time, frame_rate = gesture_recognizer.get_performance_stats() # 根据显示模式调整显示选项 if display_modes[current_display_mode] == 'normal': config.set('display', 'show_contours', value=True) config.set('display', 'show_bbox', value=True) config.set('display', 'show_fingertips', value=True) + config.set('display', 'show_gesture_history', value=True) + config.set('display', 'show_stability_indicator', value=True) config.set('display', 'show_debug_info', value=False) elif display_modes[current_display_mode] == 'detailed': config.set('display', 'show_contours', value=True) @@ -1953,11 +2180,15 @@ def main(): config.set('display', 'show_fingertips', value=True) config.set('display', 'show_palm_center', value=True) config.set('display', 'show_hand_direction', value=True) + config.set('display', 'show_gesture_history', value=True) + config.set('display', 'show_stability_indicator', value=True) config.set('display', 'show_debug_info', value=True) elif display_modes[current_display_mode] == 'minimal': config.set('display', 'show_contours', value=False) config.set('display', 'show_bbox', value=True) config.set('display', 'show_fingertips', value=False) + config.set('display', 'show_gesture_history', value=False) + config.set('display', 'show_stability_indicator', value=False) config.set('display', 'show_debug_info', value=False) # 绘制UI @@ -1974,7 +2205,7 @@ def main(): frame = ui_renderer.draw_warning(frame, warning_msg) # 显示图像(窗口标题用英文) - cv2.imshow('Gesture Controlled Drone - Optimized with Speech', frame) + cv2.imshow('Gesture Controlled Drone - Enhanced', frame) # ========== 键盘控制 ========== key = cv2.waitKey(1) & 0xFF @@ -2003,12 +2234,12 @@ def main(): # 语音提示 if speech_manager.enabled: - speech_manager.speak('help_toggled') + speech_manager.speak('help_toggled', immediate=True) elif key == ord('r') or key == ord('R'): # 重置手势识别 print("重置手势识别...") - gesture_recognizer = ImprovedGestureRecognizer(speech_manager) + gesture_recognizer = EnhancedGestureRecognizer(speech_manager) print("✓ 手势识别已重置") # 语音提示 @@ -2023,7 +2254,7 @@ def main(): # 语音提示 if speech_manager.enabled: - speech_manager.speak('display_mode_changed') + speech_manager.speak('display_mode_changed', immediate=True) elif key == ord('d') or key == ord('D'): # 切换调试信息 @@ -2035,9 +2266,9 @@ def main(): # 语音提示 if speech_manager.enabled: if not current: - speech_manager.speak('debug_mode_on') + speech_manager.speak('debug_mode_on', immediate=True) else: - speech_manager.speak('debug_mode_off') + speech_manager.speak('debug_mode_off', immediate=True) elif key == ord('v') or key == ord('V'): # 切换语音反馈 @@ -2050,10 +2281,21 @@ def main(): # 测试语音 if speech_manager.enabled: print("测试语音...") - speech_manager.speak_direct("语音反馈测试,这是一条测试消息") + speech_manager.speak_direct("语音反馈测试,系统运行正常") else: print("语音反馈已禁用,按V键启用") + elif key == ord('p') or key == ord('P'): + # 性能报告 + if speech_manager.enabled: + print("生成性能报告...") + if process_time < 20: + speech_manager.speak_direct("系统性能优秀,运行流畅") + elif process_time < 50: + speech_manager.speak_direct("系统性能良好") + else: + speech_manager.speak_direct("系统性能警告,请检查") + elif key in key_to_gesture: # 键盘控制 simulated_gesture = key_to_gesture[key] @@ -2084,7 +2326,7 @@ def main(): # 程序退出语音提示 if speech_manager.enabled: - speech_manager.speak('program_exit', force=True) + speech_manager.speak('program_exit', force=True, immediate=True) time.sleep(1) # 确保语音播报完成 drone_controller.emergency_stop()