Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 117 additions & 109 deletions src/gesture_controll_drone/gesture_detection.py
Original file line number Diff line number Diff line change
@@ -1,137 +1,145 @@
import cv2 # 导入OpenCV库,用于图像处理和视频捕获
import mediapipe as mp # 导入MediaPipe库,用于手部关键点检测
import numpy as np # 导入numpy库,用于数值计算
import cv2
import mediapipe as mp
import numpy as np

class GestureDetector:
"""手势检测类
基于MediaPipe实现单只手的关键点检测和基础手势识别
支持识别:张开手掌、握拳、食指指向、胜利手势、其他手势
"""

def __init__(self):
"""初始化函数:创建MediaPipe手部检测实例和绘图工具"""
# 初始化MediaPipe手部检测模块
"""基于MediaPipe的手势检测类(优化版)"""
# 手部关键点索引常量
WRIST = 0
THUMB_TIP = 4
THUMB_IP = 3
INDEX_TIP = 8
INDEX_PIP = 6
MIDDLE_TIP = 12
MIDDLE_PIP = 10
RING_TIP = 16
RING_PIP = 14
PINKY_TIP = 20
PINKY_PIP = 18

def __init__(self,
max_hands: int = 1,
detection_confidence: float = 0.7,
tracking_confidence: float = 0.5):
# 初始化MediaPipe手部检测
self.mp_hands = mp.solutions.hands
# 创建Hands对象,配置检测参数
self.hands = self.mp_hands.Hands(
static_image_mode=False, # 动态视频模式(False),非静态图片模式
max_num_hands=1, # 最多检测1只手
min_detection_confidence=0.7, # 检测置信度阈值,低于0.7则认为未检测到
min_tracking_confidence=0.5 # 跟踪置信度阈值,低于0.5则重新检测
static_image_mode=False,
max_num_hands=max_hands,
min_detection_confidence=detection_confidence,
min_tracking_confidence=tracking_confidence
)
# 初始化MediaPipe绘图工具,用于绘制手部关键点和连接线
# 初始化绘图工具
self.mp_drawing = mp.solutions.drawing_utils

self.draw_spec = self.mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2, circle_radius=2)

def detect_gestures(self, frame):
"""检测帧中的手势
Args:
frame: OpenCV读取的BGR格式视频帧
Returns:
frame: 绘制了手部关键点的帧
gesture: 识别出的手势名称(字符串)
landmarks: 手部关键点像素坐标列表,格式[(x1,y1), (x2,y2), ...]
"""
# 将BGR格式(OpenCV默认)转换为RGB格式(MediaPipe要求)
"""检测帧中的手势,返回处理后的帧、手势名称、关键点坐标"""
if frame is None or frame.size == 0:
return frame, "无效帧", None

# 颜色空间转换+性能优化
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# 处理帧,获取手部检测结果
rgb_frame.flags.writeable = False
results = self.hands.process(rgb_frame)
rgb_frame.flags.writeable = True

gesture = "未检测到手势"
landmarks = None

# 初始化返回值
gesture = "未检测到手势" # 默认手势状态
landmarks = None # 关键点坐标初始化为空

# 如果检测到至少一只手的关键点
if results.multi_hand_landmarks:
# 遍历检测到的每只手(此处最多1只)
for hand_landmarks in results.multi_hand_landmarks:
# 在帧上绘制手部关键点和连接线
self.mp_drawing.draw_landmarks(
frame, hand_landmarks, self.mp_hands.HAND_CONNECTIONS)

# 提取关键点像素坐标
landmarks = []
h, w, c = frame.shape # 获取帧的高度、宽度、通道数
for lm in hand_landmarks.landmark:
# 将MediaPipe的归一化坐标(0-1)转换为像素坐标
px = int(lm.x * w)
py = int(lm.y * h)
landmarks.append((px, py))

# 根据关键点坐标分类手势
gesture = self._classify_gesture(landmarks)
hand_landmarks = results.multi_hand_landmarks[0]
# 绘制关键点
self.mp_drawing.draw_landmarks(
frame, hand_landmarks, self.mp_hands.HAND_CONNECTIONS,
self.draw_spec, self.draw_spec
)
# 转换坐标并识别手势
landmarks = self._convert_landmarks_to_pixels(hand_landmarks, frame.shape)
gesture = self._classify_gesture(landmarks)

# 返回处理后的帧、手势名称、关键点坐标
return frame, gesture, landmarks

def _convert_landmarks_to_pixels(self, hand_landmarks, frame_shape):
"""将归一化坐标转换为像素坐标,限制坐标范围"""
h, w, _ = frame_shape
landmarks = []
for lm in hand_landmarks.landmark:
x = int(np.clip(lm.x * w, 0, w-1))
y = int(np.clip(lm.y * h, 0, h-1))
landmarks.append((x, y))
return landmarks

def _is_finger_open(self, landmarks, tip_idx, pip_idx):
"""判断非拇指手指是否张开"""
return landmarks[tip_idx][1] < landmarks[pip_idx][1]

def _is_thumb_open(self, landmarks):
"""判断拇指是否张开(适配左右手)"""
wrist_x = landmarks[self.WRIST][0]
thumb_tip_x = landmarks[self.THUMB_TIP][0]
thumb_ip_x = landmarks[self.THUMB_IP][0]

if thumb_tip_x > wrist_x: # 右手
return thumb_tip_x > thumb_ip_x + 10
else: # 左手
return thumb_tip_x < thumb_ip_x - 10

def _classify_gesture(self, landmarks):
"""根据关键点坐标分类手势(内部方法)
Args:
landmarks: 手部关键点像素坐标列表
Returns:
str: 识别出的手势名称
"""
# 校验关键点数量(正常手部关键点应为21个)
"""根据关键点坐标分类手势"""
if not landmarks or len(landmarks) < 21:
return "未检测到手势"

# 提取关键部位的坐标
thumb_tip = landmarks[4] # 拇指指尖
index_tip = landmarks[8] # 食指指尖
middle_tip = landmarks[12] # 中指指尖
ring_tip = landmarks[16] # 无名指指尖
pinky_tip = landmarks[20] # 小指指尖
wrist = landmarks[0] # 手腕

# 判断每根手指是否张开,存入列表
fingers = []
# 拇指判断:指尖x坐标 > 拇指近节指关节x坐标 视为张开(仅适配右手)
fingers.append(thumb_tip[0] > landmarks[3][0])
# 判断各手指状态
thumb_open = self._is_thumb_open(landmarks)
index_open = self._is_finger_open(landmarks, self.INDEX_TIP, self.INDEX_PIP)
middle_open = self._is_finger_open(landmarks, self.MIDDLE_TIP, self.MIDDLE_PIP)
ring_open = self._is_finger_open(landmarks, self.RING_TIP, self.RING_PIP)
pinky_open = self._is_finger_open(landmarks, self.PINKY_TIP, self.PINKY_PIP)

# 其他手指判断:指尖y坐标 < 手腕y坐标 视为张开(逻辑较简单)
for tip in [index_tip, middle_tip, ring_tip, pinky_tip]:
fingers.append(tip[1] < wrist[1])
finger_states = [thumb_open, index_open, middle_open, ring_open, pinky_open]

# 根据手指状态分类手势
if all(fingers): # 所有手指都张开
return "张开手掌"
elif not any(fingers): # 所有手指都闭合
# 手势分类逻辑
if not any(finger_states):
return "握拳"
elif fingers[1] and not any(fingers[2:]): # 仅食指张开
elif all(finger_states):
return "张开手掌"
elif index_open and not middle_open and not ring_open and not pinky_open:
return "食指指向"
elif fingers[1] and fingers[2] and not fingers[3] and not fingers[4]: # 食指+中指张开
elif index_open and middle_open and not ring_open and not pinky_open:
return "胜利手势"
elif all(fingers[1:5]): # 除拇指外其余四指张开
return "张开五指"
else: # 其他未匹配的手势
else:
return "其他手势"

# ------------------- 测试代码(可直接运行) -------------------
# 测试代码
if __name__ == "__main__":
# 创建手势检测器实例
detector = GestureDetector()
# 打开摄像头(0为默认摄像头)
detector = GestureDetector(max_hands=1, detection_confidence=0.7, tracking_confidence=0.5)
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)

# 循环读取视频帧
while cap.isOpened():
ret, frame = cap.read()
if not ret: # 读取帧失败则退出
break

# 检测手势
frame, gesture, landmarks = detector.detect_gestures(frame)

# 在帧上显示识别结果
cv2.putText(frame, f"Gesture: {gesture}", (10, 30),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)

# 显示处理后的帧
cv2.imshow("Gesture Detection", frame)

# 按下q键退出循环
if cv2.waitKey(1) & 0xFF == ord('q'):
break

# 释放资源
cap.release()
cv2.destroyAllWindows()
try:
while cap.isOpened():
ret, frame = cap.read()
if not ret:
print("无法读取摄像头画面,退出...")
break

frame = cv2.flip(frame, 1) # 镜像翻转
frame, gesture, _ = detector.detect_gestures(frame)

# 显示手势结果
cv2.putText(
frame, f"当前手势: {gesture}",
(20, 50), cv2.FONT_HERSHEY_SIMPLEX,
1.2, (0, 255, 0), 3
)

cv2.imshow("手势检测(按Q退出)", frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
except Exception as e:
print(f"运行出错: {e}")
finally:
cap.release()
cv2.destroyAllWindows()
Loading