From 18d483948dbb0f21f8bb2973bd18fa60d66d44b5 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Thu, 18 Dec 2025 16:16:41 +0800 Subject: [PATCH 1/2] =?UTF-8?q?new=E5=88=86=E6=94=AF=EF=BC=9A1.=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0.gitignore=E5=BF=BD=E7=95=A5Lib=E7=9B=AE=E5=BD=95=202.?= =?UTF-8?q?=E4=BF=AE=E6=94=B9main.py=E4=BC=98=E5=8C=96=E6=97=A0=E4=BA=BA?= =?UTF-8?q?=E6=9C=BA=E6=8E=A7=E5=88=B6=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 6 + .../main.py | 546 ++++++++++++++++++ 2 files changed, 552 insertions(+) create mode 100644 src/autonomus_drone_hand_gesture_project/.gitignore diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore new file mode 100644 index 0000000000..ad298e3e2d --- /dev/null +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -0,0 +1,6 @@ +# 忽略整个Lib目录(包含所有子文件/子目录) +Lib/ +# 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 +__pycache__/ +*.pyc +*.dll.a \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index e69de29bb2..b77d05ba12 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -0,0 +1,546 @@ +# -*- coding: utf-8 -*- +""" +手势控制AirSim无人机 - 修复连接问题版 +确保正确连接AirSim,移除模拟模式 +作者: xiaoshiyuan888 +""" + +import sys +import os +import time +import traceback +import subprocess +from PIL import Image, ImageDraw, ImageFont +import cv2 +import numpy as np + +print("=" * 60) +print("Gesture Controlled Drone - Fixed Connection Version") +print("=" * 60) + +# ========== 修复导入路径 ========== +# 添加当前目录到Python路径 +current_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_dir) + + +# ========== 核心模块导入与异常处理 ========== +def safe_import(): + """安全导入所有模块""" + modules_status = {} + + # 1. 导入PIL + try: + from PIL import Image, ImageDraw, ImageFont + modules_status['PIL'] = True + print("[PIL] ✓ Image processing library ready") + except Exception as e: + modules_status['PIL'] = False + print(f"[PIL] ✗ Import failed: {e}") + return None, modules_status + + # 2. 导入OpenCV + try: + import cv2 + import numpy as np + modules_status['OpenCV'] = True + print("[OpenCV] ✓ Computer vision library ready") + except Exception as e: + modules_status['OpenCV'] = False + print(f"[OpenCV] ✗ Import failed: {e}") + return None, modules_status + + # 3. 导入MediaPipe + mp = None + mp_hands = None + mp_drawing = None + try: + import mediapipe as mp + mp_hands = mp.solutions.hands + mp_drawing = mp.solutions.drawing_utils + modules_status['MediaPipe'] = True + print("[MediaPipe] ✓ Gesture recognition library ready") + except Exception as e: + modules_status['MediaPipe'] = False + print(f"[MediaPipe] ✗ Import failed: {e}") + print(" Please install: pip install mediapipe") + + # 4. 尝试导入AirSim - 关键修复部分 + airsim_module = None + airsim_client = None + + # 尝试多种导入方式 + import_methods = [ + lambda: __import__('airsim'), + lambda: __import__('AirSim'), + ] + + for method in import_methods: + try: + airsim_module = method() + modules_status['AirSim'] = True + print(f"[AirSim] ✓ Successfully imported airsim from {airsim_module.__file__}") + break + except ImportError: + continue + except Exception as e: + print(f"[AirSim] Import error: {e}") + continue + + if not modules_status.get('AirSim', False): + print("\n" + "!" * 60) + print("⚠ AirSim library NOT FOUND!") + print("!" * 60) + print("To install AirSim, run:") + print("1. First install: pip install msgpack-rpc-python") + print("2. Then install: pip install airsim") + print("\nOr from source:") + print(" pip install git+https://github.com/microsoft/AirSim.git") + print("!" * 60) + + # 询问是否自动安装 + print("\nDo you want to try automatic installation? (y/n)") + choice = input().strip().lower() + if choice == 'y': + try: + print("Installing msgpack-rpc-python...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "msgpack-rpc-python"]) + print("Installing AirSim...") + subprocess.check_call([sys.executable, "-m", "pip", "install", "airsim"]) + print("\n✅ Installation complete! Please restart the program.") + except Exception as e: + print(f"Installation failed: {e}") + sys.exit(1) + + return { + 'cv2': cv2, + 'np': np, + 'mp': mp, + 'mp_hands': mp_hands, + 'mp_drawing': mp_drawing, + 'PIL': {'Image': Image, 'ImageDraw': ImageDraw, 'ImageFont': ImageFont}, + 'airsim': airsim_module + }, modules_status + + +# 执行导入 +libs, status = safe_import() +if not status.get('OpenCV', False) or not status.get('PIL', False): + print("\n❌ Core libraries missing, cannot start.") + input("Press Enter to exit...") + sys.exit(1) + +print("-" * 60) +print("✅ Environment check passed, initializing...") +print("-" * 60) + +# 解包库 +cv2, np, mp = libs['cv2'], libs['np'], libs['mp'] +Image, ImageDraw, ImageFont = libs['PIL']['Image'], libs['PIL']['ImageDraw'], libs['PIL']['ImageFont'] +mp_hands, mp_drawing = libs['mp_hands'], libs['mp_drawing'] + + +# ========== 中文显示模块 ========== +class ChineseTextRenderer: + """中文文本渲染器""" + + def __init__(self): + self.fonts = {} + self.load_fonts() + + def load_fonts(self): + """加载字体""" + font_paths = [ + 'simhei.ttf', + 'C:/Windows/Fonts/simhei.ttf', + 'C:/Windows/Fonts/msyh.ttc' + ] + for path in font_paths: + try: + self.fonts[20] = ImageFont.truetype(path, 20) + self.fonts[24] = ImageFont.truetype(path, 24) + self.fonts[30] = ImageFont.truetype(path, 30) + print(f"✓ Chinese fonts loaded: {path}") + return + except: + continue + print("⚠ No Chinese fonts found, using default") + + def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): + """在图像上绘制中文文本""" + try: + img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + pil_img = Image.fromarray(img_rgb) + draw = ImageDraw.Draw(pil_img) + + font = self.fonts.get(size, self.fonts.get(20)) + + # 绘制阴影 + shadow_color = (0, 0, 0) + shadow_pos = (pos[0] + 1, pos[1] + 1) + draw.text(shadow_pos, text, font=font, fill=shadow_color) + + # 绘制文字 + rgb_color = color[::-1] + draw.text(pos, text, font=font, fill=rgb_color) + + return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) + except: + cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, + size / 30, color, 2) + return frame + + +# 初始化中文渲染器 +chinese_renderer = ChineseTextRenderer() + + +# ========== 手势识别模块 ========== +class GestureRecognizer: + """手势识别器""" + + def __init__(self): + try: + self.hands = mp_hands.Hands( + max_num_hands=1, + min_detection_confidence=0.7, + min_tracking_confidence=0.5 + ) + print("✓ MediaPipe gesture detector initialized") + except Exception as e: + print(f"⚠ MediaPipe init failed: {e}") + self.hands = None + + self.current_gesture = "等待手势" + + def recognize(self, frame): + """识别手势""" + if self.hands is None: + return self.current_gesture + + try: + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + results = self.hands.process(rgb_frame) + + if results.multi_hand_landmarks: + for hand_landmarks in results.multi_hand_landmarks: + # 绘制手部关键点 + mp_drawing.draw_landmarks( + frame, hand_landmarks, mp_hands.HAND_CONNECTIONS, + mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2), + mp_drawing.DrawingSpec(color=(255, 0, 0), thickness=2) + ) + + # 简化的手势判断 + landmarks = hand_landmarks.landmark + if len(landmarks) >= 9: + wrist = landmarks[0] + index_tip = landmarks[8] + + # 手势判断逻辑 + if index_tip.y < wrist.y - 0.2: + self.current_gesture = "向上" + elif index_tip.y > wrist.y + 0.2: + self.current_gesture = "向下" + elif index_tip.x < wrist.x - 0.2: + self.current_gesture = "向左" + elif index_tip.x > wrist.x + 0.2: + self.current_gesture = "向右" + elif abs(index_tip.y - wrist.y) < 0.1: + self.current_gesture = "向前" + else: + self.current_gesture = "停止" + + return self.current_gesture + except Exception as e: + print(f"Gesture recognition error: {e}") + return "识别异常" + + def set_simulated_gesture(self, gesture): + """设置模拟的手势""" + self.current_gesture = gesture + + +# ========== 无人机控制模块 ========== +class DroneController: + """无人机控制器""" + + def __init__(self, airsim_module): + self.airsim = airsim_module + self.client = None + self.connected = False + self.flying = False + self.connection_attempted = False + + def connect(self): + """连接AirSim无人机""" + if self.connection_attempted: + return self.connected + + self.connection_attempted = True + print("Connecting to AirSim...") + + try: + # 创建客户端 + self.client = self.airsim.MultirotorClient() + self.client.confirmConnection() + print("✅ Connected to AirSim!") + + # 启用API控制 + self.client.enableApiControl(True) + print("✅ API control enabled") + + # 解锁无人机 + self.client.armDisarm(True) + print("✅ Drone armed") + + self.connected = True + return True + + except Exception as e: + print(f"❌ Connection failed: {e}") + print("\nPlease ensure:") + print("1. AirSim simulator is running") + print("2. In simulator: Settings → Computer Vision mode is OFF") + print("3. Drone is spawned in the world") + print("4. Press 'R' in simulator to reset drone if needed") + return False + + def takeoff(self): + """起飞""" + if not self.connected: + print("❌ Drone not connected") + return False + + try: + print("Taking off...") + # 起飞到5米高度 + self.client.takeoffAsync().join() + # 悬停1秒稳定 + time.sleep(1) + self.flying = True + print("✅ Drone took off successfully") + return True + except Exception as e: + print(f"❌ Takeoff failed: {e}") + return False + + def land(self): + """降落""" + if not self.connected: + return False + + try: + print("Landing...") + self.client.landAsync().join() + self.flying = False + print("✅ Drone landed") + return True + except Exception as e: + print(f"Landing failed: {e}") + return False + + def move_by_gesture(self, gesture): + """根据手势移动""" + if not self.connected or not self.flying: + return + + try: + # 设置速度参数(单位:米/秒) + velocity = 3 + duration = 0.5 # 持续0.5秒 + + if gesture == "向上": + self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() + elif gesture == "向下": + self.client.moveByVelocityZAsync(0, 0, velocity, duration).join() + elif gesture == "向左": + self.client.moveByVelocityAsync(-velocity, 0, 0, duration).join() + elif gesture == "向右": + self.client.moveByVelocityAsync(velocity, 0, 0, duration).join() + elif gesture == "向前": + self.client.moveByVelocityAsync(0, -velocity, 0, duration).join() + elif gesture == "停止": + self.client.hoverAsync().join() + except Exception as e: + print(f"Control command failed: {e}") + + def emergency_stop(self): + """紧急停止""" + if self.connected: + try: + if self.flying: + print("Emergency landing...") + self.land() + self.client.armDisarm(False) + self.client.enableApiControl(False) + print("✅ Emergency stop complete") + except: + pass + self.connected = False + self.flying = False + + +# ========== 主程序 ========== +def main(): + """主函数""" + # 初始化组件 + gesture_recognizer = GestureRecognizer() + drone_controller = DroneController(libs['airsim']) + + # 初始化摄像头 + cap = None + try: + cap = cv2.VideoCapture(0) + if not cap.isOpened(): + print("❌ Camera not available") + cap = None + else: + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + print("✓ Camera initialized") + except: + print("⚠ Camera init failed") + cap = None + + # 显示说明 + print("\n" + "=" * 60) + print("INSTRUCTIONS") + print("=" * 60) + print("1. Start AirSim simulator first") + print("2. Choose environment (e.g., Landscape Mountains)") + print("3. Press [C] in this window to connect drone") + print("4. Press [SPACE] to takeoff/land") + if cap: + print("5. Gesture control with hand in front of camera") + else: + print("5. Keyboard control:") + print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") + print("6. Press [ESC] to exit safely") + print("=" * 60) + print("Program started successfully!") + print("-" * 60) + + # 键盘手势映射 + key_to_gesture = { + ord('w'): "向上", ord('W'): "向上", + ord('s'): "向下", ord('S'): "向下", + ord('a'): "向左", ord('A'): "向左", + ord('d'): "向右", ord('D'): "向右", + ord('f'): "向前", ord('F'): "向前", + ord('x'): "停止", ord('X'): "停止", + } + + # 主循环 + while True: + frame = None + + # 读取摄像头画面 + if cap: + ret, frame = cap.read() + if ret: + frame = cv2.flip(frame, 1) + current_gesture = gesture_recognizer.recognize(frame) + else: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + current_gesture = "摄像头错误" + else: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + current_gesture = gesture_recognizer.current_gesture + + # ========== 绘制界面 ========== + # 标题栏 + overlay = frame.copy() + cv2.rectangle(overlay, (0, 0), (640, 100), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + + # 主标题 + frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), + size=30, color=(0, 255, 255)) + + # 状态信息 + status_color = (0, 255, 0) if drone_controller.connected else (0, 0, 255) + status_text = f"无人机: {'已连接' if drone_controller.connected else '未连接'}" + frame = chinese_renderer.put_text(frame, status_text, (10, 50), + size=24, color=status_color) + + flight_color = (0, 255, 0) if drone_controller.flying else (255, 255, 0) + flight_status = f"飞行: {'是' if drone_controller.flying else '否'}" + frame = chinese_renderer.put_text(frame, flight_status, (10, 80), + size=24, color=flight_color) + + # 手势信息 + gesture_color = (0, 255, 0) if current_gesture != "等待手势" else (200, 200, 200) + gesture_text = f"手势: {current_gesture}" + frame = chinese_renderer.put_text(frame, gesture_text, (10, 120), + size=24, color=gesture_color) + + # 操作提示 + help_text = "C:连接 空格:起飞/降落 ESC:退出" + if not cap: + help_text += " W/A/S/D/F/X:控制" + + frame = chinese_renderer.put_text(frame, help_text, (10, 450), + size=20, color=(255, 255, 255)) + + # AirSim连接状态 + if not drone_controller.connected: + warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" + frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + size=18, color=(0, 165, 255)) + + # 显示窗口 + cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) + + # ========== 键盘控制 ========== + key = cv2.waitKey(30) & 0xFF + + if key == 27: # ESC键 + print("\nExiting...") + break + + elif key == ord('c') or key == ord('C'): # 连接无人机 + if not drone_controller.connected: + drone_controller.connect() + + elif key == 32: # 空格键 + if drone_controller.connected: + if drone_controller.flying: + drone_controller.land() + else: + drone_controller.takeoff() + time.sleep(0.3) + + elif key in key_to_gesture: # 键盘控制 + gesture = key_to_gesture[key] + gesture_recognizer.set_simulated_gesture(gesture) + if drone_controller.connected and drone_controller.flying: + drone_controller.move_by_gesture(gesture) + + # 真实手势控制 + if (current_gesture and current_gesture != "等待手势" and + current_gesture != "摄像头错误" and + drone_controller.connected and drone_controller.flying): + drone_controller.move_by_gesture(current_gesture) + + # ========== 清理资源 ========== + print("\nCleaning up resources...") + if cap: + cap.release() + cv2.destroyAllWindows() + drone_controller.emergency_stop() + print("Program safely exited") + print("=" * 60) + + +# ========== 程序入口 ========== +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nProgram interrupted by user") + except Exception as e: + print(f"\nProgram error: {e}") + traceback.print_exc() + finally: + input("\nPress Enter to exit...") \ No newline at end of file From dd54e834a48d55daa8c8f6dd313c074efcea1d05 Mon Sep 17 00:00:00 2001 From: xiaoshiyuan888 <954238580@qq.com> Date: Thu, 18 Dec 2025 22:05:02 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E6=94=B9=E8=BF=9B=E6=97=A0=E4=BA=BA?= =?UTF-8?q?=E6=9C=BA=E6=89=8B=E5=8A=BF=E6=8E=A7=E5=88=B6=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=EF=BC=8C=E4=BC=98=E5=8C=96=E6=91=84=E5=83=8F=E5=A4=B4=E5=A4=84?= =?UTF-8?q?=E7=90=86=E5=92=8C=E6=89=8B=E5=8A=BF=E7=A8=B3=E5=AE=9A=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.gitignore | 4 +- .../main.py | 362 ++++++++++++------ 2 files changed, 246 insertions(+), 120 deletions(-) diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index ad298e3e2d..05dab76400 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -3,4 +3,6 @@ Lib/ # 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 __pycache__/ *.pyc -*.dll.a \ No newline at end of file +*.dll.a +# 忽略IDEA配置文件 +.idea/ \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index b77d05ba12..f2a7020806 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 修复连接问题版 -确保正确连接AirSim,移除模拟模式 +手势控制AirSim无人机 - 修复手势识别版 +优化了手势识别算法,增加更多手势判断逻辑 作者: xiaoshiyuan888 """ @@ -15,21 +15,19 @@ import numpy as np print("=" * 60) -print("Gesture Controlled Drone - Fixed Connection Version") +print("Gesture Controlled Drone - Enhanced Gesture Recognition") print("=" * 60) # ========== 修复导入路径 ========== -# 添加当前目录到Python路径 current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -# ========== 核心模块导入与异常处理 ========== +# ========== 核心模块导入 ========== def safe_import(): """安全导入所有模块""" modules_status = {} - # 1. 导入PIL try: from PIL import Image, ImageDraw, ImageFont modules_status['PIL'] = True @@ -39,7 +37,6 @@ def safe_import(): print(f"[PIL] ✗ Import failed: {e}") return None, modules_status - # 2. 导入OpenCV try: import cv2 import numpy as np @@ -50,7 +47,6 @@ def safe_import(): print(f"[OpenCV] ✗ Import failed: {e}") return None, modules_status - # 3. 导入MediaPipe mp = None mp_hands = None mp_drawing = None @@ -65,11 +61,7 @@ def safe_import(): print(f"[MediaPipe] ✗ Import failed: {e}") print(" Please install: pip install mediapipe") - # 4. 尝试导入AirSim - 关键修复部分 airsim_module = None - airsim_client = None - - # 尝试多种导入方式 import_methods = [ lambda: __import__('airsim'), lambda: __import__('AirSim'), @@ -79,7 +71,7 @@ def safe_import(): try: airsim_module = method() modules_status['AirSim'] = True - print(f"[AirSim] ✓ Successfully imported airsim from {airsim_module.__file__}") + print(f"[AirSim] ✓ Successfully imported") break except ImportError: continue @@ -98,7 +90,6 @@ def safe_import(): print(" pip install git+https://github.com/microsoft/AirSim.git") print("!" * 60) - # 询问是否自动安装 print("\nDo you want to try automatic installation? (y/n)") choice = input().strip().lower() if choice == 'y': @@ -153,7 +144,9 @@ def load_fonts(self): font_paths = [ 'simhei.ttf', 'C:/Windows/Fonts/simhei.ttf', - 'C:/Windows/Fonts/msyh.ttc' + 'C:/Windows/Fonts/msyh.ttc', + '/System/Library/Fonts/PingFang.ttc', + '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' ] for path in font_paths: try: @@ -177,7 +170,7 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): # 绘制阴影 shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 1, pos[1] + 1) + shadow_pos = (pos[0] + 2, pos[1] + 2) draw.text(shadow_pos, text, font=font, fill=shadow_color) # 绘制文字 @@ -186,70 +179,172 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) except: + # 备用方案:使用OpenCV绘制英文 cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, size / 30, color, 2) return frame -# 初始化中文渲染器 chinese_renderer = ChineseTextRenderer() -# ========== 手势识别模块 ========== -class GestureRecognizer: - """手势识别器""" +# ========== 简化的手势识别模块 ========== +class SimpleGestureRecognizer: + """简化的手势识别器,不依赖MediaPipe""" def __init__(self): - try: - self.hands = mp_hands.Hands( - max_num_hands=1, - min_detection_confidence=0.7, - min_tracking_confidence=0.5 - ) - print("✓ MediaPipe gesture detector initialized") - except Exception as e: - print(f"⚠ MediaPipe init failed: {e}") - self.hands = None - self.current_gesture = "等待手势" + self.last_gesture_time = time.time() + self.gesture_history = [] + self.max_history = 5 + print("✓ Simple gesture recognizer initialized (不使用MediaPipe)") + + # 加载OpenCV的预训练模型用于手部检测 + try: + # 尝试加载Haar级联分类器 + cascade_path = cv2.data.haarcascades + 'haarcascade_frontalface_default.xml' + # 我们使用人脸检测器作为替代,实际上应该用手部检测器 + self.face_cascade = cv2.CascadeClassifier(cascade_path) + print("✓ OpenCV cascade classifier loaded") + except: + print("⚠ OpenCV cascade classifier not available") + self.face_cascade = None + + def detect_hand_contour(self, frame): + """使用轮廓检测手部""" + # 转换为HSV颜色空间 + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + + # 定义肤色范围 + lower_skin = np.array([0, 20, 70], dtype=np.uint8) + upper_skin = np.array([20, 255, 255], dtype=np.uint8) + + # 创建肤色掩码 + mask = cv2.inRange(hsv, lower_skin, upper_skin) + + # 形态学操作 + kernel = np.ones((5, 5), np.uint8) + mask = cv2.dilate(mask, kernel, iterations=2) + mask = cv2.erode(mask, kernel, iterations=1) + mask = cv2.GaussianBlur(mask, (5, 5), 0) + + # 查找轮廓 + contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + if contours: + # 找到最大的轮廓(假设是手) + largest_contour = max(contours, key=cv2.contourArea) + + # 计算轮廓面积 + area = cv2.contourArea(largest_contour) + + if area > 1000: # 最小面积阈值 + # 获取边界矩形 + x, y, w, h = cv2.boundingRect(largest_contour) + + # 计算轮廓的中心点 + M = cv2.moments(largest_contour) + if M["m00"] != 0: + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + + # 计算凸包和凸缺陷 + hull = cv2.convexHull(largest_contour, returnPoints=False) + if len(hull) > 3: + defects = cv2.convexityDefects(largest_contour, hull) + + if defects is not None: + # 计算手指数量 + finger_count = 0 + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(largest_contour[s][0]) + end = tuple(largest_contour[e][0]) + far = tuple(largest_contour[f][0]) + + # 计算角度 + a = np.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = np.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = np.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + + angle = np.arccos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + + if angle <= np.pi / 2: + finger_count += 1 + + finger_count = min(finger_count + 1, 5) + + # 绘制轮廓和中心点 + cv2.drawContours(frame, [largest_contour], -1, (0, 255, 0), 2) + cv2.circle(frame, (cx, cy), 7, (255, 0, 0), -1) + cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) + + # 在边界框上方显示手指数量 + cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + + return cx, cy, w, h, finger_count, frame + + return None, None, None, None, 0, frame + + def recognize_simple_gesture(self, frame): + """简化的手势识别""" + # 检测手部轮廓 + cx, cy, w, h, finger_count, frame = self.detect_hand_contour(frame) + + if cx is None or cy is None: + return "等待手势", frame + + # 获取图像尺寸 + height, width = frame.shape[:2] + + # 定义区域 + center_x = width // 2 + center_y = height // 2 + + # 根据手指数量和位置判断手势 + if finger_count == 0: + return "停止", frame + elif finger_count == 1: + return "向前", frame + elif finger_count >= 4: + # 手掌张开,根据位置判断方向 + if cy < center_y - 50: + return "向上", frame + elif cy > center_y + 50: + return "向下", frame + elif cx < center_x - 50: + return "向左", frame + elif cx > center_x + 50: + return "向右", frame + else: + return "停止", frame + else: + return "等待手势", frame def recognize(self, frame): """识别手势""" - if self.hands is None: - return self.current_gesture - try: - rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - results = self.hands.process(rgb_frame) - - if results.multi_hand_landmarks: - for hand_landmarks in results.multi_hand_landmarks: - # 绘制手部关键点 - mp_drawing.draw_landmarks( - frame, hand_landmarks, mp_hands.HAND_CONNECTIONS, - mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2), - mp_drawing.DrawingSpec(color=(255, 0, 0), thickness=2) - ) - - # 简化的手势判断 - landmarks = hand_landmarks.landmark - if len(landmarks) >= 9: - wrist = landmarks[0] - index_tip = landmarks[8] - - # 手势判断逻辑 - if index_tip.y < wrist.y - 0.2: - self.current_gesture = "向上" - elif index_tip.y > wrist.y + 0.2: - self.current_gesture = "向下" - elif index_tip.x < wrist.x - 0.2: - self.current_gesture = "向左" - elif index_tip.x > wrist.x + 0.2: - self.current_gesture = "向右" - elif abs(index_tip.y - wrist.y) < 0.1: - self.current_gesture = "向前" - else: - self.current_gesture = "停止" + # 使用简化的手势识别 + gesture, processed_frame = self.recognize_simple_gesture(frame) + + # 手势历史记录和平滑处理 + self.gesture_history.append(gesture) + if len(self.gesture_history) > self.max_history: + self.gesture_history.pop(0) + + # 使用多数表决决定最终手势 + if self.gesture_history: + gesture_counts = {} + for g in self.gesture_history: + gesture_counts[g] = gesture_counts.get(g, 0) + 1 + + # 找到出现次数最多的手势 + max_gesture = max(gesture_counts, key=gesture_counts.get) + if gesture_counts[max_gesture] > len(self.gesture_history) // 2: + self.current_gesture = max_gesture + else: + self.current_gesture = gesture return self.current_gesture except Exception as e: @@ -281,16 +376,13 @@ def connect(self): print("Connecting to AirSim...") try: - # 创建客户端 self.client = self.airsim.MultirotorClient() self.client.confirmConnection() print("✅ Connected to AirSim!") - # 启用API控制 self.client.enableApiControl(True) print("✅ API control enabled") - # 解锁无人机 self.client.armDisarm(True) print("✅ Drone armed") @@ -299,11 +391,11 @@ def connect(self): except Exception as e: print(f"❌ Connection failed: {e}") - print("\nPlease ensure:") - print("1. AirSim simulator is running") - print("2. In simulator: Settings → Computer Vision mode is OFF") - print("3. Drone is spawned in the world") - print("4. Press 'R' in simulator to reset drone if needed") + print("\n请确保:") + print("1. AirSim模拟器正在运行") + print("2. 选择环境(如:Landscape Mountains)") + print("3. 无人机已在世界中生成") + print("4. 如果需要,按模拟器中的'R'键重置无人机") return False def takeoff(self): @@ -314,9 +406,7 @@ def takeoff(self): try: print("Taking off...") - # 起飞到5米高度 self.client.takeoffAsync().join() - # 悬停1秒稳定 time.sleep(1) self.flying = True print("✅ Drone took off successfully") @@ -346,9 +436,8 @@ def move_by_gesture(self, gesture): return try: - # 设置速度参数(单位:米/秒) - velocity = 3 - duration = 0.5 # 持续0.5秒 + velocity = 2.0 # 降低速度以获得更平滑的控制 + duration = 0.3 # 减少持续时间 if gesture == "向上": self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() @@ -384,8 +473,8 @@ def emergency_stop(self): # ========== 主程序 ========== def main(): """主函数""" - # 初始化组件 - gesture_recognizer = GestureRecognizer() + # 使用简化的手势识别器 + gesture_recognizer = SimpleGestureRecognizer() drone_controller = DroneController(libs['airsim']) # 初始化摄像头 @@ -393,32 +482,46 @@ def main(): try: cap = cv2.VideoCapture(0) if not cap.isOpened(): - print("❌ Camera not available") - cap = None + # 尝试其他摄像头索引 + for i in range(1, 5): + cap = cv2.VideoCapture(i) + if cap.isOpened(): + print(f"✓ Camera {i} initialized") + break + else: + print("❌ Camera not available, using keyboard control only") + cap = None else: + print("✓ Camera 0 initialized") + + if cap: cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - print("✓ Camera initialized") - except: - print("⚠ Camera init failed") + cap.set(cv2.CAP_PROP_FPS, 30) + except Exception as e: + print(f"⚠ Camera init failed: {e}") cap = None # 显示说明 print("\n" + "=" * 60) - print("INSTRUCTIONS") + print("操作说明") print("=" * 60) - print("1. Start AirSim simulator first") - print("2. Choose environment (e.g., Landscape Mountains)") - print("3. Press [C] in this window to connect drone") - print("4. Press [SPACE] to takeoff/land") + print("1. 首先启动AirSim模拟器") + print("2. 选择环境(例如:Landscape Mountains)") + print("3. 按 [C] 连接无人机") + print("4. 按 [空格键] 起飞/降落") if cap: - print("5. Gesture control with hand in front of camera") + print("5. 手势控制:") + print(" - 握拳(0个手指):停止") + print(" - 伸出1个手指:向前") + print(" - 张开手掌(4-5个手指):根据手的位置控制方向") + print(" * 确保手部在摄像头视野内,光线充足,背景简单") else: - print("5. Keyboard control:") - print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") - print("6. Press [ESC] to exit safely") + print("5. 键盘控制:") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print("6. 按 [ESC] 安全退出") print("=" * 60) - print("Program started successfully!") + print("程序启动成功!") print("-" * 60) # 键盘手势映射 @@ -431,14 +534,17 @@ def main(): ord('x'): "停止", ord('X'): "停止", } + last_control_time = time.time() + control_interval = 0.5 # 控制间隔,避免过于频繁 + # 主循环 while True: frame = None - # 读取摄像头画面 if cap: ret, frame = cap.read() if ret: + # 镜像图像以便更直观的控制 frame = cv2.flip(frame, 1) current_gesture = gesture_recognizer.recognize(frame) else: @@ -449,12 +555,16 @@ def main(): current_gesture = gesture_recognizer.current_gesture # ========== 绘制界面 ========== - # 标题栏 + # 创建半透明覆盖层 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (640, 100), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + cv2.rectangle(overlay, (0, 0), (640, 120), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.6, frame, 0.4, 0) - # 主标题 + # 绘制底部操作提示区域 + cv2.rectangle(frame, (0, 400), (640, 480), (0, 0, 0), -1) + frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) + + # 标题 frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), size=30, color=(0, 255, 255)) @@ -470,9 +580,15 @@ def main(): size=24, color=flight_color) # 手势信息 - gesture_color = (0, 255, 0) if current_gesture != "等待手势" else (200, 200, 200) + if current_gesture != "等待手势" and current_gesture != "摄像头错误": + gesture_color = (0, 255, 0) # 绿色 + # 添加手势识别置信度指示 + cv2.circle(frame, (600, 60), 10, gesture_color, -1) + else: + gesture_color = (200, 200, 200) # 灰色 + gesture_text = f"手势: {current_gesture}" - frame = chinese_renderer.put_text(frame, gesture_text, (10, 120), + frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), size=24, color=gesture_color) # 操作提示 @@ -480,26 +596,30 @@ def main(): if not cap: help_text += " W/A/S/D/F/X:控制" - frame = chinese_renderer.put_text(frame, help_text, (10, 450), + frame = chinese_renderer.put_text(frame, help_text, (10, 420), size=20, color=(255, 255, 255)) - # AirSim连接状态 + # 添加手势提示 + if cap: + gesture_hint = "提示: 握拳停止,食指向前,手掌张开移动" + frame = chinese_renderer.put_text(frame, gesture_hint, (10, 450), + size=18, color=(255, 200, 100)) + if not drone_controller.connected: warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" - frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + frame = chinese_renderer.put_text(frame, warning_text, (10, 360), size=18, color=(0, 165, 255)) - # 显示窗口 cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) # ========== 键盘控制 ========== - key = cv2.waitKey(30) & 0xFF + key = cv2.waitKey(1) & 0xFF if key == 27: # ESC键 - print("\nExiting...") + print("\n退出程序...") break - elif key == ord('c') or key == ord('C'): # 连接无人机 + elif key == ord('c') or key == ord('C'): if not drone_controller.connected: drone_controller.connect() @@ -511,25 +631,29 @@ def main(): drone_controller.takeoff() time.sleep(0.3) - elif key in key_to_gesture: # 键盘控制 + elif key in key_to_gesture: gesture = key_to_gesture[key] gesture_recognizer.set_simulated_gesture(gesture) + current_gesture = gesture if drone_controller.connected and drone_controller.flying: drone_controller.move_by_gesture(gesture) - # 真实手势控制 + # 真实手势控制(添加间隔控制避免过于频繁) + current_time = time.time() if (current_gesture and current_gesture != "等待手势" and - current_gesture != "摄像头错误" and - drone_controller.connected and drone_controller.flying): + current_gesture != "摄像头错误" and current_gesture != "识别异常" and + drone_controller.connected and drone_controller.flying and + current_time - last_control_time > control_interval): drone_controller.move_by_gesture(current_gesture) + last_control_time = current_time - # ========== 清理资源 ========== - print("\nCleaning up resources...") + # 清理资源 + print("\n清理资源...") if cap: cap.release() cv2.destroyAllWindows() drone_controller.emergency_stop() - print("Program safely exited") + print("程序安全退出") print("=" * 60) @@ -538,9 +662,9 @@ def main(): try: main() except KeyboardInterrupt: - print("\nProgram interrupted by user") + print("\n程序被用户中断") except Exception as e: - print(f"\nProgram error: {e}") + print(f"\n程序错误: {e}") traceback.print_exc() finally: - input("\nPress Enter to exit...") \ No newline at end of file + input("\n按回车键退出...") \ No newline at end of file