diff --git a/src/autonomus_drone_hand_gesture_project/.gitignore b/src/autonomus_drone_hand_gesture_project/.gitignore index ad298e3e2d..05dab76400 100644 --- a/src/autonomus_drone_hand_gesture_project/.gitignore +++ b/src/autonomus_drone_hand_gesture_project/.gitignore @@ -3,4 +3,6 @@ Lib/ # 额外补充:忽略Python编译文件,避免后续其他无关文件干扰 __pycache__/ *.pyc -*.dll.a \ No newline at end of file +*.dll.a +# 忽略IDEA配置文件 +.idea/ \ No newline at end of file diff --git a/src/autonomus_drone_hand_gesture_project/main.py b/src/autonomus_drone_hand_gesture_project/main.py index b77d05ba12..f2a7020806 100644 --- a/src/autonomus_drone_hand_gesture_project/main.py +++ b/src/autonomus_drone_hand_gesture_project/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -手势控制AirSim无人机 - 修复连接问题版 -确保正确连接AirSim,移除模拟模式 +手势控制AirSim无人机 - 修复手势识别版 +优化了手势识别算法,增加更多手势判断逻辑 作者: xiaoshiyuan888 """ @@ -15,21 +15,19 @@ import numpy as np print("=" * 60) -print("Gesture Controlled Drone - Fixed Connection Version") +print("Gesture Controlled Drone - Enhanced Gesture Recognition") print("=" * 60) # ========== 修复导入路径 ========== -# 添加当前目录到Python路径 current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_dir) -# ========== 核心模块导入与异常处理 ========== +# ========== 核心模块导入 ========== def safe_import(): """安全导入所有模块""" modules_status = {} - # 1. 导入PIL try: from PIL import Image, ImageDraw, ImageFont modules_status['PIL'] = True @@ -39,7 +37,6 @@ def safe_import(): print(f"[PIL] ✗ Import failed: {e}") return None, modules_status - # 2. 导入OpenCV try: import cv2 import numpy as np @@ -50,7 +47,6 @@ def safe_import(): print(f"[OpenCV] ✗ Import failed: {e}") return None, modules_status - # 3. 导入MediaPipe mp = None mp_hands = None mp_drawing = None @@ -65,11 +61,7 @@ def safe_import(): print(f"[MediaPipe] ✗ Import failed: {e}") print(" Please install: pip install mediapipe") - # 4. 尝试导入AirSim - 关键修复部分 airsim_module = None - airsim_client = None - - # 尝试多种导入方式 import_methods = [ lambda: __import__('airsim'), lambda: __import__('AirSim'), @@ -79,7 +71,7 @@ def safe_import(): try: airsim_module = method() modules_status['AirSim'] = True - print(f"[AirSim] ✓ Successfully imported airsim from {airsim_module.__file__}") + print(f"[AirSim] ✓ Successfully imported") break except ImportError: continue @@ -98,7 +90,6 @@ def safe_import(): print(" pip install git+https://github.com/microsoft/AirSim.git") print("!" * 60) - # 询问是否自动安装 print("\nDo you want to try automatic installation? (y/n)") choice = input().strip().lower() if choice == 'y': @@ -153,7 +144,9 @@ def load_fonts(self): font_paths = [ 'simhei.ttf', 'C:/Windows/Fonts/simhei.ttf', - 'C:/Windows/Fonts/msyh.ttc' + 'C:/Windows/Fonts/msyh.ttc', + '/System/Library/Fonts/PingFang.ttc', + '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf' ] for path in font_paths: try: @@ -177,7 +170,7 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): # 绘制阴影 shadow_color = (0, 0, 0) - shadow_pos = (pos[0] + 1, pos[1] + 1) + shadow_pos = (pos[0] + 2, pos[1] + 2) draw.text(shadow_pos, text, font=font, fill=shadow_color) # 绘制文字 @@ -186,70 +179,172 @@ def put_text(self, frame, text, pos, size=20, color=(255, 255, 255)): return cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) except: + # 备用方案:使用OpenCV绘制英文 cv2.putText(frame, text, pos, cv2.FONT_HERSHEY_SIMPLEX, size / 30, color, 2) return frame -# 初始化中文渲染器 chinese_renderer = ChineseTextRenderer() -# ========== 手势识别模块 ========== -class GestureRecognizer: - """手势识别器""" +# ========== 简化的手势识别模块 ========== +class SimpleGestureRecognizer: + """简化的手势识别器,不依赖MediaPipe""" def __init__(self): - try: - self.hands = mp_hands.Hands( - max_num_hands=1, - min_detection_confidence=0.7, - min_tracking_confidence=0.5 - ) - print("✓ MediaPipe gesture detector initialized") - except Exception as e: - print(f"⚠ MediaPipe init failed: {e}") - self.hands = None - self.current_gesture = "等待手势" + self.last_gesture_time = time.time() + self.gesture_history = [] + self.max_history = 5 + print("✓ Simple gesture recognizer initialized (不使用MediaPipe)") + + # 加载OpenCV的预训练模型用于手部检测 + try: + # 尝试加载Haar级联分类器 + cascade_path = cv2.data.haarcascades + 'haarcascade_frontalface_default.xml' + # 我们使用人脸检测器作为替代,实际上应该用手部检测器 + self.face_cascade = cv2.CascadeClassifier(cascade_path) + print("✓ OpenCV cascade classifier loaded") + except: + print("⚠ OpenCV cascade classifier not available") + self.face_cascade = None + + def detect_hand_contour(self, frame): + """使用轮廓检测手部""" + # 转换为HSV颜色空间 + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + + # 定义肤色范围 + lower_skin = np.array([0, 20, 70], dtype=np.uint8) + upper_skin = np.array([20, 255, 255], dtype=np.uint8) + + # 创建肤色掩码 + mask = cv2.inRange(hsv, lower_skin, upper_skin) + + # 形态学操作 + kernel = np.ones((5, 5), np.uint8) + mask = cv2.dilate(mask, kernel, iterations=2) + mask = cv2.erode(mask, kernel, iterations=1) + mask = cv2.GaussianBlur(mask, (5, 5), 0) + + # 查找轮廓 + contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + if contours: + # 找到最大的轮廓(假设是手) + largest_contour = max(contours, key=cv2.contourArea) + + # 计算轮廓面积 + area = cv2.contourArea(largest_contour) + + if area > 1000: # 最小面积阈值 + # 获取边界矩形 + x, y, w, h = cv2.boundingRect(largest_contour) + + # 计算轮廓的中心点 + M = cv2.moments(largest_contour) + if M["m00"] != 0: + cx = int(M["m10"] / M["m00"]) + cy = int(M["m01"] / M["m00"]) + + # 计算凸包和凸缺陷 + hull = cv2.convexHull(largest_contour, returnPoints=False) + if len(hull) > 3: + defects = cv2.convexityDefects(largest_contour, hull) + + if defects is not None: + # 计算手指数量 + finger_count = 0 + for i in range(defects.shape[0]): + s, e, f, d = defects[i, 0] + start = tuple(largest_contour[s][0]) + end = tuple(largest_contour[e][0]) + far = tuple(largest_contour[f][0]) + + # 计算角度 + a = np.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = np.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = np.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + + angle = np.arccos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) + + if angle <= np.pi / 2: + finger_count += 1 + + finger_count = min(finger_count + 1, 5) + + # 绘制轮廓和中心点 + cv2.drawContours(frame, [largest_contour], -1, (0, 255, 0), 2) + cv2.circle(frame, (cx, cy), 7, (255, 0, 0), -1) + cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) + + # 在边界框上方显示手指数量 + cv2.putText(frame, f'Fingers: {finger_count}', (x, y - 10), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + + return cx, cy, w, h, finger_count, frame + + return None, None, None, None, 0, frame + + def recognize_simple_gesture(self, frame): + """简化的手势识别""" + # 检测手部轮廓 + cx, cy, w, h, finger_count, frame = self.detect_hand_contour(frame) + + if cx is None or cy is None: + return "等待手势", frame + + # 获取图像尺寸 + height, width = frame.shape[:2] + + # 定义区域 + center_x = width // 2 + center_y = height // 2 + + # 根据手指数量和位置判断手势 + if finger_count == 0: + return "停止", frame + elif finger_count == 1: + return "向前", frame + elif finger_count >= 4: + # 手掌张开,根据位置判断方向 + if cy < center_y - 50: + return "向上", frame + elif cy > center_y + 50: + return "向下", frame + elif cx < center_x - 50: + return "向左", frame + elif cx > center_x + 50: + return "向右", frame + else: + return "停止", frame + else: + return "等待手势", frame def recognize(self, frame): """识别手势""" - if self.hands is None: - return self.current_gesture - try: - rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - results = self.hands.process(rgb_frame) - - if results.multi_hand_landmarks: - for hand_landmarks in results.multi_hand_landmarks: - # 绘制手部关键点 - mp_drawing.draw_landmarks( - frame, hand_landmarks, mp_hands.HAND_CONNECTIONS, - mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2), - mp_drawing.DrawingSpec(color=(255, 0, 0), thickness=2) - ) - - # 简化的手势判断 - landmarks = hand_landmarks.landmark - if len(landmarks) >= 9: - wrist = landmarks[0] - index_tip = landmarks[8] - - # 手势判断逻辑 - if index_tip.y < wrist.y - 0.2: - self.current_gesture = "向上" - elif index_tip.y > wrist.y + 0.2: - self.current_gesture = "向下" - elif index_tip.x < wrist.x - 0.2: - self.current_gesture = "向左" - elif index_tip.x > wrist.x + 0.2: - self.current_gesture = "向右" - elif abs(index_tip.y - wrist.y) < 0.1: - self.current_gesture = "向前" - else: - self.current_gesture = "停止" + # 使用简化的手势识别 + gesture, processed_frame = self.recognize_simple_gesture(frame) + + # 手势历史记录和平滑处理 + self.gesture_history.append(gesture) + if len(self.gesture_history) > self.max_history: + self.gesture_history.pop(0) + + # 使用多数表决决定最终手势 + if self.gesture_history: + gesture_counts = {} + for g in self.gesture_history: + gesture_counts[g] = gesture_counts.get(g, 0) + 1 + + # 找到出现次数最多的手势 + max_gesture = max(gesture_counts, key=gesture_counts.get) + if gesture_counts[max_gesture] > len(self.gesture_history) // 2: + self.current_gesture = max_gesture + else: + self.current_gesture = gesture return self.current_gesture except Exception as e: @@ -281,16 +376,13 @@ def connect(self): print("Connecting to AirSim...") try: - # 创建客户端 self.client = self.airsim.MultirotorClient() self.client.confirmConnection() print("✅ Connected to AirSim!") - # 启用API控制 self.client.enableApiControl(True) print("✅ API control enabled") - # 解锁无人机 self.client.armDisarm(True) print("✅ Drone armed") @@ -299,11 +391,11 @@ def connect(self): except Exception as e: print(f"❌ Connection failed: {e}") - print("\nPlease ensure:") - print("1. AirSim simulator is running") - print("2. In simulator: Settings → Computer Vision mode is OFF") - print("3. Drone is spawned in the world") - print("4. Press 'R' in simulator to reset drone if needed") + print("\n请确保:") + print("1. AirSim模拟器正在运行") + print("2. 选择环境(如:Landscape Mountains)") + print("3. 无人机已在世界中生成") + print("4. 如果需要,按模拟器中的'R'键重置无人机") return False def takeoff(self): @@ -314,9 +406,7 @@ def takeoff(self): try: print("Taking off...") - # 起飞到5米高度 self.client.takeoffAsync().join() - # 悬停1秒稳定 time.sleep(1) self.flying = True print("✅ Drone took off successfully") @@ -346,9 +436,8 @@ def move_by_gesture(self, gesture): return try: - # 设置速度参数(单位:米/秒) - velocity = 3 - duration = 0.5 # 持续0.5秒 + velocity = 2.0 # 降低速度以获得更平滑的控制 + duration = 0.3 # 减少持续时间 if gesture == "向上": self.client.moveByVelocityZAsync(0, 0, -velocity, duration).join() @@ -384,8 +473,8 @@ def emergency_stop(self): # ========== 主程序 ========== def main(): """主函数""" - # 初始化组件 - gesture_recognizer = GestureRecognizer() + # 使用简化的手势识别器 + gesture_recognizer = SimpleGestureRecognizer() drone_controller = DroneController(libs['airsim']) # 初始化摄像头 @@ -393,32 +482,46 @@ def main(): try: cap = cv2.VideoCapture(0) if not cap.isOpened(): - print("❌ Camera not available") - cap = None + # 尝试其他摄像头索引 + for i in range(1, 5): + cap = cv2.VideoCapture(i) + if cap.isOpened(): + print(f"✓ Camera {i} initialized") + break + else: + print("❌ Camera not available, using keyboard control only") + cap = None else: + print("✓ Camera 0 initialized") + + if cap: cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - print("✓ Camera initialized") - except: - print("⚠ Camera init failed") + cap.set(cv2.CAP_PROP_FPS, 30) + except Exception as e: + print(f"⚠ Camera init failed: {e}") cap = None # 显示说明 print("\n" + "=" * 60) - print("INSTRUCTIONS") + print("操作说明") print("=" * 60) - print("1. Start AirSim simulator first") - print("2. Choose environment (e.g., Landscape Mountains)") - print("3. Press [C] in this window to connect drone") - print("4. Press [SPACE] to takeoff/land") + print("1. 首先启动AirSim模拟器") + print("2. 选择环境(例如:Landscape Mountains)") + print("3. 按 [C] 连接无人机") + print("4. 按 [空格键] 起飞/降落") if cap: - print("5. Gesture control with hand in front of camera") + print("5. 手势控制:") + print(" - 握拳(0个手指):停止") + print(" - 伸出1个手指:向前") + print(" - 张开手掌(4-5个手指):根据手的位置控制方向") + print(" * 确保手部在摄像头视野内,光线充足,背景简单") else: - print("5. Keyboard control:") - print(" [W]Up [S]Down [A]Left [D]Right [F]Forward [X]Stop") - print("6. Press [ESC] to exit safely") + print("5. 键盘控制:") + print(" [W]向上 [S]向下 [A]向左 [D]向右 [F]向前 [X]停止") + print("6. 按 [ESC] 安全退出") print("=" * 60) - print("Program started successfully!") + print("程序启动成功!") print("-" * 60) # 键盘手势映射 @@ -431,14 +534,17 @@ def main(): ord('x'): "停止", ord('X'): "停止", } + last_control_time = time.time() + control_interval = 0.5 # 控制间隔,避免过于频繁 + # 主循环 while True: frame = None - # 读取摄像头画面 if cap: ret, frame = cap.read() if ret: + # 镜像图像以便更直观的控制 frame = cv2.flip(frame, 1) current_gesture = gesture_recognizer.recognize(frame) else: @@ -449,12 +555,16 @@ def main(): current_gesture = gesture_recognizer.current_gesture # ========== 绘制界面 ========== - # 标题栏 + # 创建半透明覆盖层 overlay = frame.copy() - cv2.rectangle(overlay, (0, 0), (640, 100), (0, 0, 0), -1) - frame = cv2.addWeighted(overlay, 0.7, frame, 0.3, 0) + cv2.rectangle(overlay, (0, 0), (640, 120), (0, 0, 0), -1) + frame = cv2.addWeighted(overlay, 0.6, frame, 0.4, 0) - # 主标题 + # 绘制底部操作提示区域 + cv2.rectangle(frame, (0, 400), (640, 480), (0, 0, 0), -1) + frame = cv2.addWeighted(frame, 1.0, frame, 0.0, 0) + + # 标题 frame = chinese_renderer.put_text(frame, "手势控制无人机系统", (10, 10), size=30, color=(0, 255, 255)) @@ -470,9 +580,15 @@ def main(): size=24, color=flight_color) # 手势信息 - gesture_color = (0, 255, 0) if current_gesture != "等待手势" else (200, 200, 200) + if current_gesture != "等待手势" and current_gesture != "摄像头错误": + gesture_color = (0, 255, 0) # 绿色 + # 添加手势识别置信度指示 + cv2.circle(frame, (600, 60), 10, gesture_color, -1) + else: + gesture_color = (200, 200, 200) # 灰色 + gesture_text = f"手势: {current_gesture}" - frame = chinese_renderer.put_text(frame, gesture_text, (10, 120), + frame = chinese_renderer.put_text(frame, gesture_text, (10, 110), size=24, color=gesture_color) # 操作提示 @@ -480,26 +596,30 @@ def main(): if not cap: help_text += " W/A/S/D/F/X:控制" - frame = chinese_renderer.put_text(frame, help_text, (10, 450), + frame = chinese_renderer.put_text(frame, help_text, (10, 420), size=20, color=(255, 255, 255)) - # AirSim连接状态 + # 添加手势提示 + if cap: + gesture_hint = "提示: 握拳停止,食指向前,手掌张开移动" + frame = chinese_renderer.put_text(frame, gesture_hint, (10, 450), + size=18, color=(255, 200, 100)) + if not drone_controller.connected: warning_text = "⚠ 请先启动AirSim模拟器,然后按C连接" - frame = chinese_renderer.put_text(frame, warning_text, (10, 400), + frame = chinese_renderer.put_text(frame, warning_text, (10, 360), size=18, color=(0, 165, 255)) - # 显示窗口 cv2.imshow('Gesture Controlled Drone - Press ESC to exit', frame) # ========== 键盘控制 ========== - key = cv2.waitKey(30) & 0xFF + key = cv2.waitKey(1) & 0xFF if key == 27: # ESC键 - print("\nExiting...") + print("\n退出程序...") break - elif key == ord('c') or key == ord('C'): # 连接无人机 + elif key == ord('c') or key == ord('C'): if not drone_controller.connected: drone_controller.connect() @@ -511,25 +631,29 @@ def main(): drone_controller.takeoff() time.sleep(0.3) - elif key in key_to_gesture: # 键盘控制 + elif key in key_to_gesture: gesture = key_to_gesture[key] gesture_recognizer.set_simulated_gesture(gesture) + current_gesture = gesture if drone_controller.connected and drone_controller.flying: drone_controller.move_by_gesture(gesture) - # 真实手势控制 + # 真实手势控制(添加间隔控制避免过于频繁) + current_time = time.time() if (current_gesture and current_gesture != "等待手势" and - current_gesture != "摄像头错误" and - drone_controller.connected and drone_controller.flying): + current_gesture != "摄像头错误" and current_gesture != "识别异常" and + drone_controller.connected and drone_controller.flying and + current_time - last_control_time > control_interval): drone_controller.move_by_gesture(current_gesture) + last_control_time = current_time - # ========== 清理资源 ========== - print("\nCleaning up resources...") + # 清理资源 + print("\n清理资源...") if cap: cap.release() cv2.destroyAllWindows() drone_controller.emergency_stop() - print("Program safely exited") + print("程序安全退出") print("=" * 60) @@ -538,9 +662,9 @@ def main(): try: main() except KeyboardInterrupt: - print("\nProgram interrupted by user") + print("\n程序被用户中断") except Exception as e: - print(f"\nProgram error: {e}") + print(f"\n程序错误: {e}") traceback.print_exc() finally: - input("\nPress Enter to exit...") \ No newline at end of file + input("\n按回车键退出...") \ No newline at end of file