diff --git a/src/Drone_hand_gesture_project/gesture_classifier.py b/src/Drone_hand_gesture_project/gesture_classifier.py new file mode 100644 index 0000000000..ccab2284dd --- /dev/null +++ b/src/Drone_hand_gesture_project/gesture_classifier.py @@ -0,0 +1,249 @@ +import numpy as np +import pickle +import os +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import StandardScaler +from sklearn.svm import SVC +from sklearn.ensemble import RandomForestClassifier +from sklearn.neural_network import MLPClassifier +from sklearn.metrics import classification_report, accuracy_score +import joblib + + +class GestureClassifier: + """机器学习手势分类器""" + + def __init__(self, model_type='svm', model_path=None): + self.model_type = model_type + self.model_path = model_path + self.scaler = StandardScaler() + self.model = None + self.gesture_classes = None + + # 如果提供了模型路径,加载模型 + if model_path and os.path.exists(model_path): + self.load_model(model_path) + + def create_model(self): + """创建分类模型""" + if self.model_type == 'svm': + self.model = SVC( + kernel='rbf', + C=10.0, + gamma='scale', + probability=True, + random_state=42 + ) + elif self.model_type == 'random_forest': + self.model = RandomForestClassifier( + n_estimators=100, + max_depth=10, + random_state=42, + n_jobs=-1 + ) + elif self.model_type == 'mlp': + self.model = MLPClassifier( + hidden_layer_sizes=(128, 64), + activation='relu', + solver='adam', + max_iter=1000, + random_state=42 + ) + else: + raise ValueError(f"未知的模型类型: {self.model_type}") + + def load_dataset(self, dataset_path): + """加载数据集""" + with open(dataset_path, 'rb') as f: + dataset = pickle.load(f) + + X = dataset['features'] + y = dataset['labels'] + self.gesture_classes = dataset['gesture_classes'] + + return X, y + + def preprocess_data(self, X_train, X_test): + """数据预处理""" + # 标准化特征 + X_train_scaled = self.scaler.fit_transform(X_train) + X_test_scaled = self.scaler.transform(X_test) + + return X_train_scaled, X_test_scaled + + def train(self, dataset_path, test_size=0.2, save_path=None): + """训练模型""" + # 加载数据 + X, y = self.load_dataset(dataset_path) + + # 检查数据 + print(f"数据集信息:") + print(f" 总样本数: {len(X)}") + print(f" 特征维度: {X.shape[1]}") + + unique_labels, counts = np.unique(y, return_counts=True) + print(" 各类样本数:") + for label, count in zip(unique_labels, counts): + class_name = self.gesture_classes[label] + print(f" {class_name}: {count}") + + if len(unique_labels) < 2: + raise ValueError(f"需要至少2个类别才能训练,当前只有 {len(unique_labels)} 个类别") + + # 划分训练集和测试集 + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=test_size, random_state=42, stratify=y + ) + + # 数据预处理 + X_train_scaled, X_test_scaled = self.preprocess_data(X_train, X_test) + + # 创建并训练模型 + self.create_model() + print(f"训练 {self.model_type} 模型...") + self.model.fit(X_train_scaled, y_train) + + # 评估模型 + train_acc = self.model.score(X_train_scaled, y_train) + test_acc = self.model.score(X_test_scaled, y_test) + + print(f"训练准确率: {train_acc:.4f}") + print(f"测试准确率: {test_acc:.4f}") + + # 详细分类报告 + y_pred = self.model.predict(X_test_scaled) + print("\n分类报告:") + print(classification_report(y_test, y_pred, + target_names=self.gesture_classes)) + + # 保存模型 + if save_path: + self.save_model(save_path) + + return test_acc + + def predict(self, landmarks): + """预测手势类别""" + if self.model is None: + raise ValueError("模型未训练或未加载") + + # 检查landmarks是否有效 + if len(landmarks) != 63: # 21个关键点 * 3坐标 + return "none", 0.0 + + # 预处理 + landmarks_scaled = self.scaler.transform([landmarks]) + + # 预测 + probabilities = self.model.predict_proba(landmarks_scaled)[0] + predicted_class_idx = np.argmax(probabilities) + confidence = probabilities[predicted_class_idx] + + predicted_class = self.gesture_classes[predicted_class_idx] + + return predicted_class, confidence + + def predict_batch(self, landmarks_list): + """批量预测""" + if not landmarks_list: + return [], [] + + # 预处理 + landmarks_array = np.array(landmarks_list) + landmarks_scaled = self.scaler.transform(landmarks_array) + + # 预测 + probabilities = self.model.predict_proba(landmarks_scaled) + predicted_indices = np.argmax(probabilities, axis=1) + confidences = np.max(probabilities, axis=1) + + predicted_classes = [self.gesture_classes[idx] for idx in predicted_indices] + + return predicted_classes, confidences + + def save_model(self, save_path): + """保存模型""" + model_data = { + 'model_type': self.model_type, + 'model': self.model, + 'scaler': self.scaler, + 'gesture_classes': self.gesture_classes + } + + joblib.dump(model_data, save_path) + print(f"模型已保存到: {save_path}") + + def load_model(self, model_path): + """加载模型""" + model_data = joblib.load(model_path) + + self.model_type = model_data['model_type'] + self.model = model_data['model'] + self.scaler = model_data['scaler'] + self.gesture_classes = model_data['gesture_classes'] + + print(f"模型已从 {model_path} 加载") + print(f"模型类型: {self.model_type}") + print(f"手势类别: {self.gesture_classes}") + + +class EnsembleGestureClassifier: + """集成学习手势分类器""" + + def __init__(self, model_paths=None): + self.models = [] + self.gesture_classes = None + + if model_paths: + self.load_models(model_paths) + + def load_models(self, model_paths): + """加载多个模型""" + for path in model_paths: + classifier = GestureClassifier(model_path=path) + self.models.append(classifier) + + # 使用第一个模型的类别 + if self.models: + self.gesture_classes = self.models[0].gesture_classes + + def predict(self, landmarks, voting='soft'): + """集成预测""" + if not self.models: + raise ValueError("没有加载任何模型") + + if len(landmarks) != 63: + return "none", 0.0 + + predictions = [] + confidences = [] + + for model in self.models: + pred_class, confidence = model.predict(landmarks) + predictions.append(pred_class) + confidences.append(confidence) + + if voting == 'hard': + # 硬投票 + from collections import Counter + most_common = Counter(predictions).most_common(1)[0] + final_prediction = most_common[0] + final_confidence = most_common[1] / len(predictions) + else: + # 软投票(平均置信度) + class_confidences = {} + for model, pred, conf in zip(self.models, predictions, confidences): + if pred not in class_confidences: + class_confidences[pred] = [] + class_confidences[pred].append(conf) + + # 平均每个类别的置信度 + avg_confidences = { + cls: np.mean(confs) for cls, confs in class_confidences.items() + } + + # 选择平均置信度最高的类别 + final_prediction = max(avg_confidences.items(), key=lambda x: x[1])[0] + final_confidence = avg_confidences[final_prediction] + + return final_prediction, final_confidence \ No newline at end of file diff --git a/src/Drone_hand_gesture_project/gesture_data_collector.py b/src/Drone_hand_gesture_project/gesture_data_collector.py index 8db9e2863f..4d1b5630ae 100644 --- a/src/Drone_hand_gesture_project/gesture_data_collector.py +++ b/src/Drone_hand_gesture_project/gesture_data_collector.py @@ -7,8 +7,6 @@ import mediapipe as mp import pickle from PIL import Image, ImageDraw, ImageFont -import sys - class GestureDataCollector: """手势数据收集器""" @@ -27,30 +25,16 @@ def __init__(self, data_dir="dataset"): min_tracking_confidence=0.5 ) - # 定义手势类别 + # 定义手势类别(8个手势,去掉no_hand) self.gesture_classes = [ - "张开手掌", # open_palm - "握拳", # closed_fist - "胜利手势", # victory - "大拇指向上", # thumb_up - "大拇指向下", # thumb_down - "食指上指", # pointing_up - "食指向下", # pointing_down - "OK手势", # ok_sign - "无手" # no_hand - ] - - # 手势类别的英文标识(用于保存文件) - self.gesture_english_names = [ - "open_palm", - "closed_fist", - "victory", - "thumb_up", - "thumb_down", - "pointing_up", - "pointing_down", - "ok_sign", - "no_hand" + "open_palm", # 张开手掌 + "closed_fist", # 握拳 + "victory", # 胜利手势 + "thumb_up", # 大拇指 + "thumb_down", # 大拇指向下 + "pointing_up", # 食指上指 + "pointing_down", # 食指向下 + "ok_sign", # OK手势 ] # 数据存储 @@ -58,31 +42,6 @@ def __init__(self, data_dir="dataset"): self.current_gesture = None self.collection_active = False - # 尝试加载中文字体 - self.font = None - try: - # Windows系统字体路径 - if sys.platform == 'win32': - font_path = 'C:/Windows/Fonts/simhei.ttf' # 黑体 - # Linux系统字体路径 - elif sys.platform == 'linux': - font_path = '/usr/share/fonts/truetype/wqy/wqy-microhei.ttc' # 文泉驿微米黑 - # macOS系统字体路径 - elif sys.platform == 'darwin': - font_path = '/System/Library/Fonts/PingFang.ttc' # 苹方字体 - else: - font_path = None - - if font_path and os.path.exists(font_path): - self.font = ImageFont.truetype(font_path, 30) - print(f"已加载中文字体: {font_path}") - else: - print("未找到中文字体,将使用默认字体") - self.font = ImageFont.load_default() - except Exception as e: - print(f"加载字体失败: {e}") - self.font = ImageFont.load_default() - def create_directories(self): """创建必要的目录""" directories = [ @@ -115,26 +74,31 @@ def collect_sample(self, gesture_class, image, landmarks): """收集一个样本""" timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") - # 获取对应的英文手势名称用于保存文件 - if gesture_class in self.gesture_classes: - english_name = self.gesture_english_names[self.gesture_classes.index(gesture_class)] - else: - english_name = gesture_class - # 保存图像 - img_filename = f"{english_name}_{timestamp}.jpg" + img_filename = f"{gesture_class}_{timestamp}.jpg" img_path = os.path.join(self.data_dir, "raw", "images", img_filename) cv2.imwrite(img_path, image) - # 保存关键点数据 + # 只保存有手部关键点的数据 + if landmarks and len(landmarks) > 0: + landmarks_data = landmarks[0] # 取第一个手 + # 确保是63维 + if len(landmarks_data) < 63: + landmarks_data.extend([0.0] * (63 - len(landmarks_data))) + elif len(landmarks_data) > 63: + landmarks_data = landmarks_data[:63] + else: + # 如果没有检测到手,跳过这个样本(不收集无手数据) + print(f"警告: {gesture_class} 未检测到手部,跳过此样本") + return None + data_entry = { 'timestamp': timestamp, 'gesture_class': gesture_class, - 'gesture_english': english_name, 'image_path': img_path, - 'landmarks': landmarks[0] if landmarks else [], # 取第一个手 - 'num_landmarks': 21 if landmarks else 0, - 'has_hand': len(landmarks) > 0 + 'landmarks': landmarks_data, + 'num_landmarks': 21, + 'has_hand': True } self.collected_data.append(data_entry) @@ -156,15 +120,19 @@ def save_dataset(self, filename="gesture_dataset.pkl"): y = [] for data in self.collected_data: - if data['has_hand'] and len(data['landmarks']) == 63: # 21个点 * 3坐标 + if data['has_hand'] and len(data['landmarks']) == 63: X.append(data['landmarks']) y.append(self.gesture_classes.index(data['gesture_class'])) + if len(X) == 0: + print("错误: 没有有效的训练数据!") + print("请确保每个手势都收集了足够的样本(每个手势至少10个)") + return None + dataset = { 'features': np.array(X), 'labels': np.array(y), 'gesture_classes': self.gesture_classes, - 'gesture_english_names': self.gesture_english_names, 'num_samples': len(X) } @@ -180,18 +148,6 @@ def save_dataset(self, filename="gesture_dataset.pkl"): return dataset - def put_chinese_text(self, img, text, position, font_color=(255, 255, 255)): - """在图像上绘制中文文本""" - # 将OpenCV图像转换为PIL图像 - img_pil = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) - draw = ImageDraw.Draw(img_pil) - - # 绘制文本 - draw.text(position, text, font=self.font, fill=font_color) - - # 将PIL图像转换回OpenCV图像 - return cv2.cvtColor(np.array(img_pil), cv2.COLOR_RGB2BGR) - def run_collection_app(self): """运行数据收集应用程序""" cap = cv2.VideoCapture(1) @@ -201,7 +157,7 @@ def run_collection_app(self): print("手势数据收集工具") print("=" * 50) print("命令:") - print(" 1-9: 选择手势类别") + print(" 1-8: 选择手势类别(共8个手势)") print(" c: 收集当前样本") print(" s: 停止收集当前手势") print(" w: 保存数据集") @@ -211,18 +167,17 @@ def run_collection_app(self): for i, gesture in enumerate(self.gesture_classes): print(f" {i + 1}: {gesture}") print("=" * 50) + print("提示: 每个手势建议收集20-30个样本") + print("确保手在摄像头中清晰可见") + print("=" * 50) while True: ret, frame = cap.read() if not ret: - print("无法读取摄像头画面") break frame = cv2.flip(frame, 1) - # 创建显示帧的副本用于绘制 - display_frame = frame.copy() - # 提取关键点 landmarks, hand_landmarks = self.extract_landmarks(frame) @@ -230,70 +185,72 @@ def run_collection_app(self): if hand_landmarks: for hand_landmarks in hand_landmarks: self.mp_drawing.draw_landmarks( - display_frame, hand_landmarks, self.mp_hands.HAND_CONNECTIONS) + frame, hand_landmarks, self.mp_hands.HAND_CONNECTIONS) - # 在图像上添加中文文本 - try: - # 第一行:当前手势 - if collecting_for: - status_text = f"当前手势: {collecting_for}" - else: - status_text = "当前手势: 无" - display_frame = self.put_chinese_text(display_frame, status_text, (10, 30), (0, 255, 0)) - - # 第二行:当前手势收集样本数 - display_frame = self.put_chinese_text(display_frame, f"当前收集样本数: {sample_count}", - (10, 70), (0, 255, 0)) + # 将OpenCV图像转换为PIL图像以支持中文 + pil_img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + draw = ImageDraw.Draw(pil_img) - # 第三行:总样本数 - display_frame = self.put_chinese_text(display_frame, f"总样本数: {len(self.collected_data)}", - (10, 110), (0, 255, 0)) + try: + # 尝试加载中文字体(Windows系统路径) + font_path = "C:/Windows/Fonts/msyh.ttc" # 微软雅黑 + font = ImageFont.truetype(font_path, 30) + except: + # 如果找不到字体,使用默认字体 + font = ImageFont.load_default() + print("使用默认字体,中文可能显示为方框") + + # 显示中文 + if collecting_for: + status_text = f"当前手势: {collecting_for}" + color = (0, 255, 0) + else: + status_text = "请选择手势 (1-8)" + color = (0, 255, 255) - # 第四行:操作提示 - display_frame = self.put_chinese_text(display_frame, "按1-9选择手势,c收集样本", (10, 150), - (255, 255, 255)) + # 绘制文字 + draw.text((10, 10), status_text, font=font, fill=color) + draw.text((10, 50), f"已收集样本: {sample_count}", font=font, fill=(0, 255, 0)) + draw.text((10, 90), "按1-8选择手势,c收集样本", font=font, fill=(255, 255, 255)) - # 第五行:操作提示 - display_frame = self.put_chinese_text(display_frame, "按w保存,q退出", (10, 190), - (255, 255, 255)) + # 显示手势类别提示 + y_pos = 130 + for i, gesture in enumerate(self.gesture_classes): + text = f"{i + 1}: {gesture}" + draw.text((10, y_pos), text, font=font, fill=(200, 200, 0)) + y_pos += 35 - except Exception as e: - print(f"绘制文字时出错: {e}") - # 如果无法绘制中文,使用英文 - status_text = f"Gesture: {collecting_for if collecting_for else 'None'}" - cv2.putText(display_frame, status_text, (10, 30), - cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) + # 将PIL图像转回OpenCV格式 + frame = cv2.cvtColor(np.array(pil_img), cv2.COLOR_RGB2BGR) - cv2.imshow('手势数据收集', display_frame) + cv2.imshow('手势数据收集', frame) key = cv2.waitKey(1) & 0xFF # 处理按键 if key == ord('q'): break - elif ord('1') <= key <= ord('9'): + elif ord('1') <= key <= ord('8'): # 只有1-8 gesture_idx = key - ord('1') if gesture_idx < len(self.gesture_classes): collecting_for = self.gesture_classes[gesture_idx] - sample_count = 0 print(f"开始收集手势: {collecting_for}") elif key == ord('c') and collecting_for: if landmarks: - self.collect_sample(collecting_for, frame, landmarks) - sample_count += 1 - print(f"已收集 {collecting_for} 样本 #{sample_count}") + result = self.collect_sample(collecting_for, frame, landmarks) + if result: # 只有成功收集才计数 + sample_count += 1 + print(f"已收集 {collecting_for} 样本 #{sample_count}") else: - print("未检测到手部!") + print("未检测到手部!请确保手在摄像头中清晰可见") elif key == ord('s'): collecting_for = None - sample_count = 0 print("停止收集") elif key == ord('w'): - if self.collected_data: - dataset = self.save_dataset() - print(f"数据集已保存,总样本数: {dataset['num_samples']}") - else: - print("没有数据可保存") + dataset = self.save_dataset() + if dataset: + sample_count = dataset['num_samples'] + print(f"数据集已保存,总样本数: {sample_count}") cap.release() cv2.destroyAllWindows() diff --git a/src/Drone_hand_gesture_project/train_gesture_model.py b/src/Drone_hand_gesture_project/train_gesture_model.py new file mode 100644 index 0000000000..c0f4be82c2 --- /dev/null +++ b/src/Drone_hand_gesture_project/train_gesture_model.py @@ -0,0 +1,78 @@ +import argparse +import os +from gesture_classifier import GestureClassifier, EnsembleGestureClassifier +import numpy as np + + +def main(): + parser = argparse.ArgumentParser(description='训练手势识别模型') + parser.add_argument('--dataset', type=str, default='dataset/processed/gesture_dataset.pkl', + help='数据集路径') + parser.add_argument('--model_type', type=str, default='ensemble', + choices=['svm', 'random_forest', 'mlp', 'ensemble'], + help='模型类型') + parser.add_argument('--output_dir', type=str, default='dataset/models', + help='输出目录') + parser.add_argument('--test_size', type=float, default=0.2, + help='测试集比例') + parser.add_argument('--collect_data', action='store_true', + help='是否先收集数据') + + args = parser.parse_args() + + # 创建输出目录 + os.makedirs(args.output_dir, exist_ok=True) + + # 如果需要收集数据 + if args.collect_data: + from gesture_data_collector import GestureDataCollector + print("启动数据收集...") + collector = GestureDataCollector() + collector.run_collection_app() + + if args.model_type == 'ensemble': + # 训练多个模型并集成 + models = [] + model_types = ['svm', 'random_forest', 'mlp'] + + for model_type in model_types: + print(f"\n训练 {model_type} 模型...") + classifier = GestureClassifier(model_type=model_type) + + model_path = os.path.join(args.output_dir, f'gesture_{model_type}.pkl') + accuracy = classifier.train(args.dataset, args.test_size, model_path) + + models.append(model_path) + print(f"{model_type} 准确率: {accuracy:.4f}") + + # 创建集成模型 + print("\n创建集成模型...") + ensemble = EnsembleGestureClassifier(models) + + # 保存集成模型信息 + ensemble_info = { + 'model_paths': models, + 'model_types': model_types + } + + import joblib + ensemble_path = os.path.join(args.output_dir, 'gesture_ensemble.pkl') + joblib.dump(ensemble_info, ensemble_path) + print(f"集成模型信息已保存到: {ensemble_path}") + + else: + # 训练单个模型 + print(f"训练 {args.model_type} 模型...") + classifier = GestureClassifier(model_type=args.model_type) + + model_path = os.path.join(args.output_dir, f'gesture_{args.model_type}.pkl') + accuracy = classifier.train(args.dataset, args.test_size, model_path) + + print(f"\n训练完成!") + print(f"模型类型: {args.model_type}") + print(f"准确率: {accuracy:.4f}") + print(f"模型已保存到: {model_path}") + + +if __name__ == "__main__": + main() \ No newline at end of file