Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
58 commits
Select commit Hold shift + click to select a range
11c5268
提交手势识别类,供后续主函数调用,同时添加了requirements.txt
Arrebol01 Nov 20, 2025
4e214f6
Merge branch 'OpenHUTB:main' into main
Arrebol01 Nov 21, 2025
b92bf27
清理移除无关的IDE和网站文件,并提交手势识别类以及require.txt文件
Arrebol01 Nov 21, 2025
e78bacf
Merge branch 'OpenHUTB:main' into main
Arrebol01 Nov 21, 2025
b0550a8
添加无人机控制模块
Arrebol01 Nov 22, 2025
c03d23d
Merge branch 'OpenHUTB:main' into main
Arrebol01 Nov 24, 2025
9ae0608
Merge branch 'OpenHUTB:main' into main
Arrebol01 Nov 25, 2025
dc537c0
Merge branch 'OpenHUTB:main' into main
Arrebol01 Nov 26, 2025
edaf42b
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 1, 2025
2f02eeb
添加了main.py主程序初步实现手势识别的显示窗口创建
Arrebol01 Dec 1, 2025
4ab80f7
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 2, 2025
73f398f
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 2, 2025
66b97f0
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 2, 2025
9e9a2db
更新.gitignore,剔除多余跟踪,添加主程序
Arrebol01 Dec 2, 2025
81af7c9
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 2, 2025
42a04e0
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 2, 2025
b77daa3
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 4, 2025
28887ad
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 4, 2025
931dc36
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 12, 2025
719b663
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 13, 2025
fa2d1bb
清理Python缓存文件
Arrebol01 Dec 13, 2025
7536bcf
完善README.md文件内容,并清除python缓存文件
Arrebol01 Dec 13, 2025
3dba771
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 13, 2025
df4bef4
将显示菜单更改为中文显示
Arrebol01 Dec 13, 2025
1b3075e
更改菜单栏显示为中文显示
Arrebol01 Dec 13, 2025
bca5ab8
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 14, 2025
55451ce
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 14, 2025
e151783
加入打开摄像头功能,实现手势识别
Arrebol01 Dec 14, 2025
de6e99a
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 14, 2025
90cbf47
创建无人机仿真模型
Arrebol01 Dec 14, 2025
02fc7cf
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 15, 2025
c490bff
添加无人机物理引擎
Arrebol01 Dec 15, 2025
6132063
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 15, 2025
e76b785
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 15, 2025
46d4d83
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 15, 2025
da2c0b6
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 16, 2025
3df8629
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 16, 2025
53e9146
整合整个项目内容
Arrebol01 Dec 16, 2025
8fff882
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 16, 2025
9c677e7
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 16, 2025
e8c3adb
添加训练数据的收集程序
Arrebol01 Dec 16, 2025
58e6601
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 16, 2025
dbe34cb
更改README.md当中文件结构,新增依赖
Arrebol01 Dec 16, 2025
7335468
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 16, 2025
3c40474
将requirements.txt转为UTF-8字符编码
Arrebol01 Dec 16, 2025
131176c
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 16, 2025
cbc4463
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 16, 2025
1e65692
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 17, 2025
2001e04
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 17, 2025
9993082
新添了学习分类模块,以及模型训练程序
Arrebol01 Dec 17, 2025
55d8696
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 17, 2025
0f214a0
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 17, 2025
574c38e
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 17, 2025
2ce9f61
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 17, 2025
849d0de
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 17, 2025
2300fa1
新添了学习增强的手势识别,修改主程序
Arrebol01 Dec 17, 2025
899913e
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 17, 2025
be97fde
Merge branch 'OpenHUTB:main' into main
Arrebol01 Dec 17, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
240 changes: 240 additions & 0 deletions src/Drone_hand_gesture_project/gesture_detector_enhanced.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,240 @@
import cv2
import os
import numpy as np
import mediapipe as mp
from gesture_classifier import GestureClassifier, EnsembleGestureClassifier
import PIL

try:
from PIL import Image, ImageDraw, ImageFont
import numpy as np
HAS_PIL = True
except ImportError:
HAS_PIL = False
print("⚠️ PIL未安装,中文显示不可用")

class EnhancedGestureDetector:
"""手势检测器(集成机器学习)"""

def __init__(self, ml_model_path=None, use_ml=True):
# 基础检测器
self.mp_hands = mp.solutions.hands
self.mp_drawing = mp.solutions.drawing_utils
self.mp_drawing_styles = mp.solutions.drawing_styles

self.hands = self.mp_hands.Hands(
static_image_mode=False,
max_num_hands=1,
min_detection_confidence=0.5,
min_tracking_confidence=0.5
)

# 机器学习模型
self.use_ml = use_ml
self.ml_classifier = None

if use_ml and ml_model_path:
# 检查是否是 ensemble 信息文件(很小的文件)
if os.path.exists(ml_model_path):
file_size = os.path.getsize(ml_model_path)
if file_size < 1024: # 小于1KB,可能是信息文件
print(f"⚠️ 检测到小文件 ({file_size}字节),可能是集成模型信息文件")
print(f" 尝试加载单个模型文件...")

# 尝试加载对应的单个模型
base_name = os.path.basename(ml_model_path)
if "ensemble" in base_name:
# 尝试加载 svm 模型
svm_path = ml_model_path.replace("ensemble", "svm")
if os.path.exists(svm_path):
print(f" 尝试加载: {svm_path}")
success = self.load_ml_model(svm_path)
else:
print(f" 未找到替代模型,使用规则检测")
self.use_ml = False
else:
# 正常加载
success = self.load_ml_model(ml_model_path)
else:
self.use_ml = False

# 手势到控制指令的映射(8个手势)
self.gesture_commands = {
"open_palm": "takeoff",
"closed_fist": "land",
"pointing_up": "up",
"pointing_down": "down",
"victory": "forward",
"thumb_up": "backward",
"thumb_down": "stop",
"ok_sign": "hover",
}

# 中文手势名称映射
self.chinese_gesture_names = {
"open_palm": "张开手掌",
"closed_fist": "握拳",
"victory": "胜利手势",
"thumb_up": "大拇指",
"thumb_down": "大拇指向下",
"pointing_up": "食指上指",
"pointing_down": "食指向下",
"ok_sign": "OK手势",
"none": "未检测到手"
}

# 中文指令名称映射
self.chinese_command_names = {
"takeoff": "起飞",
"land": "降落",
"up": "上升",
"down": "下降",
"forward": "前进",
"backward": "后退",
"stop": "停止",
"hover": "悬停",
"none": "无指令"
}

# 中文字体
self.chinese_font = None
if HAS_PIL:
try:
# Windows字体路径
font_paths = [
"C:/Windows/Fonts/msyh.ttc", # 微软雅黑
"C:/Windows/Fonts/simhei.ttf", # 黑体
"C:/Windows/Fonts/simsun.ttc", # 宋体
]

for font_path in font_paths:
try:
if os.path.exists(font_path):
self.chinese_font = ImageFont.truetype(font_path, 30)
print(f"✅ 加载中文字体: {os.path.basename(font_path)}")
break
except:
continue

if self.chinese_font is None:
print("⚠️ 无法加载系统字体,将使用英文显示")
except Exception as e:
print(f"⚠️ 字体初始化失败: {e}")


# 历史记录(用于平滑预测)
self.prediction_history = []
self.max_history = 5

def load_ml_model(self, model_path):
"""加载机器学习模型"""
try:
self.ml_classifier = GestureClassifier(model_path=model_path)
print(f"机器学习模型已加载: {model_path}")
return True
except Exception as e:
print(f"加载机器学习模型失败: {e}")
print("将使用规则检测器")
self.use_ml = False
return False

def extract_landmarks_for_ml(self, hand_landmarks):
"""提取关键点用于机器学习"""
landmarks = []
for landmark in hand_landmarks.landmark:
landmarks.extend([landmark.x, landmark.y, landmark.z])

# 确保是63维
if len(landmarks) < 63:
landmarks.extend([0.0] * (63 - len(landmarks)))
elif len(landmarks) > 63:
landmarks = landmarks[:63]

return landmarks

def detect_gestures(self, image, simulation_mode=False):
"""检测手势"""
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
results = self.hands.process(image_rgb)

gesture = "none" # 默认none
confidence = 0.0
landmarks_data = None

if results.multi_hand_landmarks:
for hand_landmarks in results.multi_hand_landmarks:
# 绘制手部关键点
self.mp_drawing.draw_landmarks(
image,
hand_landmarks,
self.mp_hands.HAND_CONNECTIONS,
self.mp_drawing_styles.get_default_hand_landmarks_style(),
self.mp_drawing_styles.get_default_hand_connections_style()
)

# 提取关键点
landmarks = self.extract_landmarks_for_ml(hand_landmarks)

# 使用机器学习或规则检测
if self.use_ml and self.ml_classifier and len(landmarks) == 63:
# 机器学习预测
gesture, confidence = self.ml_classifier.predict(landmarks)

# 应用平滑滤波
self.prediction_history.append((gesture, confidence))
if len(self.prediction_history) > self.max_history:
self.prediction_history.pop(0)

# 使用历史中的多数投票
if len(self.prediction_history) >= 3:
from collections import Counter
recent_gestures = [g for g, _ in self.prediction_history[-3:]]
most_common = Counter(recent_gestures).most_common(1)[0]
if most_common[1] >= 2: # 至少2/3一致
gesture = most_common[0]
# 计算平均置信度
matching_confs = [c for g, c in self.prediction_history if g == gesture]
if matching_confs:
confidence = np.mean(matching_confs)
else:
# 如果未使用ML,使用简单的规则检测
gesture, confidence = self._classify_by_rules(hand_landmarks)

# 显示信息
ml_info = " (ML)" if self.use_ml else " (规则)"
cv2.putText(image, f"Gesture: {gesture}{ml_info}", (10, 30),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
cv2.putText(image, f"Confidence: {confidence:.2f}", (10, 70),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2)

command = self.gesture_commands.get(gesture, "none")
cv2.putText(image, f"Command: {command}", (10, 110),
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2)

else:
# 无手部检测
cv2.putText(image, "未检测到手部", (10, 30),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
gesture = "none"
confidence = 0.0

return image, gesture, confidence, landmarks_data

def _classify_by_rules(self, hand_landmarks):
"""简单的规则分类器(备用)"""
# 获取关键点
landmarks = hand_landmarks.landmark

# 这里实现简单的规则检测逻辑
# 例如:计算手指是否伸直等

# 暂时返回一个默认值
return "open_palm", 0.5

def get_command(self, gesture):
"""获取控制指令"""
return self.gesture_commands.get(gesture, "none")

def release(self):
"""释放资源"""
self.hands.close()
Loading