diff --git a/.gitignore b/.gitignore index 0ee017c2b4..63d50ff1bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,17 @@ -# 虚拟环境 -venv/ -# 视频文件 -*.mp4 -*.avi -*.mov -# 结果目录 -lane_detection_results/ -# Python缓存 +# Python»º´棨ºËÐ쬽â¾ö»º´æÎļþÎÊÌ⣩ __pycache__/ *.pyc -# 日志文件 -lane_detection.log -# 构建目录 -src/openpilot_model/build/ -src/openpilot_model/log/ -# VS Code配置 +*.pyo +*.pyd +.pytest_cache/ +__pycache__/ + +# ϵͳ/±༭Æ÷»º´æ +.DS_Store +Thumbs.db .vscode/ +.idea/ + +# ÆäËû»º´棨°´Ð貹³䣩 +*.log +*.tmp diff --git a/src/AutonomousDriving_RL/eval_agent.py b/src/AutonomousDriving_RL/eval_agent.py index ff589eac57..712343cbde 100644 --- a/src/AutonomousDriving_RL/eval_agent.py +++ b/src/AutonomousDriving_RL/eval_agent.py @@ -1,46 +1,46 @@ -# eval_agent.py -""" -加载训练好的模型,在 CARLA 中演示智能体驾驶行为 -""" - -import argparse -import numpy as np -from stable_baselines3 import PPO -from carla_env.carla_env_multi_obs import CarlaEnvMultiObs - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--model_path", type=str, default="./checkpoints/best_model.zip", help="模型路径") - parser.add_argument("--steps", type=int, default=200, help="演示步数") - args = parser.parse_args() - - print("🔄 加载环境与模型...") - env = CarlaEnvMultiObs(keep_alive_after_exit=True) # 保留车辆便于观察 - model = PPO.load(args.model_path, env=env) - - print("▶️ 开始驾驶演示(运行 {} 步)...".format(args.steps)) - obs, _ = env.reset() - total_reward = 0.0 - - for step in range(args.steps): - action, _ = model.predict(obs, deterministic=True) - obs, reward, terminated, truncated, info = env.step(action) - total_reward += reward - - # 每 50 步打印状态 - if step % 50 == 0 or step == args.steps - 1: - x, y, vx, vy = obs - speed = np.linalg.norm([vx, vy]) - print(f" Step {step}: 位置=({x:.1f}, {y:.1f}), 速度={speed:.2f} m/s") - - if terminated or truncated: - break - - print(f"✅ 演示完成!总奖励: {total_reward:.2f}") - input("🛑 准备好后,请回到本窗口按 Enter 键退出...") - env.close() - - -if __name__ == "__main__": +# eval_agent.py +""" +加载训练好的模型,在 CARLA 中演示智能体驾驶行为 +""" + +import argparse +import numpy as np +from stable_baselines3 import PPO +from carla_env.carla_env_multi_obs import CarlaEnvMultiObs + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--model_path", type=str, default="./checkpoints/best_model.zip", help="模型路径") + parser.add_argument("--steps", type=int, default=200, help="演示步数") + args = parser.parse_args() + + print("🔄 加载环境与模型...") + env = CarlaEnvMultiObs(keep_alive_after_exit=True) # 保留车辆便于观察 + model = PPO.load(args.model_path, env=env) + + print("▶️ 开始驾驶演示(运行 {} 步)...".format(args.steps)) + obs, _ = env.reset() + total_reward = 0.0 + + for step in range(args.steps): + action, _ = model.predict(obs, deterministic=True) + obs, reward, terminated, truncated, info = env.step(action) + total_reward += reward + + # 每 50 步打印状态 + if step % 50 == 0 or step == args.steps - 1: + x, y, vx, vy = obs + speed = np.linalg.norm([vx, vy]) + print(f" Step {step}: 位置=({x:.1f}, {y:.1f}), 速度={speed:.2f} m/s") + + if terminated or truncated: + break + + print(f"✅ 演示完成!总奖励: {total_reward:.2f}") + input("🛑 准备好后,请回到本窗口按 Enter 键退出...") + env.close() + + +if __name__ == "__main__": main() \ No newline at end of file diff --git a/src/button_control_vehicle/main.py b/src/button_control_vehicle/main.py index 7d01b8d6f7..7fb5a41a45 100644 --- a/src/button_control_vehicle/main.py +++ b/src/button_control_vehicle/main.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +/#!usr/bin/env python3 """ CARLA全局路径规划节点 - 增强版 提供从起始点到随机目标点的路径规划服务,并将规划结果通过ROS消息发布和可视化 diff --git a/src/carla_tracking/README.md b/src/carla_tracking/README.md new file mode 100644 index 0000000000..17375217b5 --- /dev/null +++ b/src/carla_tracking/README.md @@ -0,0 +1,149 @@ +# CARLA_YOLO_ObjectTrackingTracking + +## 项目概述 +本项目实现了基于YOLOv5算法和CARLA仿真器的自动驾驶目标检测与跟踪功能。通过CARLA仿真环境生成动态交通场景,利用YOLOv5模型实时检测车辆、摩托车、公交车、卡车等交通参与者,结合内置优化版SORT跟踪算法实现目标的连续追踪,并通过OpenCV实时可视化检测与跟踪结果。 + +### 核心功能: +- 基于YOLOv5模型(支持yolov5s、yolov5m、yolov5x)进行实时目标检测,默认置信度阈值0.25 +- 集成优化版SORT跟踪算法(内置实现,无需额外依赖),通过卡尔曼滤波和历史位置平滑处理,实现对检测目标的稳定跟踪,支持多目标ID持续标注 +- 与CARLA仿真器深度集成,自动生成主车辆和NPC交通流,挂载RGB相机采集实时图像(分辨率640×480) +- 支持同步模式运行,保证检测帧率与仿真环境一致(默认0.05s/帧,20FPS) +- 实时可视化检测/跟踪结果,显示目标类别、置信度及跟踪ID,不同类别目标使用差异化颜色标注 + +## 安装步骤 + +### 前置条件: +- Python 3.7+(推荐3.7,与CARLA兼容性更佳) +- CARLA仿真器(支持0.9.x系列版本,版本需与Python API匹配) +- NVIDIA显卡(支持CUDA 11.3+,可选,用于加速模型推理) + +### 操作步骤: + +1. 下载并安装CARLA仿真器: +```plaintext +https://github.com/carla-simulator/carla/releases +``` +选择适合操作系统的版本,解压至任意路径(如D:\CARLA) + +2. 克隆或下载本项目代码: +```bash +git clone <项目仓库地址> +cd CARLA_YOLO_ObjectTracking +``` + +3. 安装Python依赖库: +```bash +# 安装PyTorch(支持CUDA加速) +pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu113 + +# 安装YOLOv5依赖 +pip install ultralytics + +# 安装其他依赖 +pip install numpy opencv-python scipy carla + +# 若使用CPU版本PyTorch +pip install torch torchvision +``` + +4. 安装CARLA Python API: +```bash +# 进入CARLA安装目录下的PythonAPI路径 +cd D:\CARLA\PythonAPI\carla\dist +# 安装对应版本的egg文件(需与Python版本匹配) +easy_install carla-<版本号>-py3.7-<平台>.egg +``` + +5. 下载YOLOv5预训练模型: +将模型文件(yolov5s.pt/yolov5m.pt/yolov5x.pt)放置在指定路径(默认路径为`D:\yolo\`,可在代码`load_detection_model`函数中修改) + +## 使用方法 + +1. 启动CARLA仿真器: +```bash +cd D:\CARLA +# Windows系统 +CarlaUE4.exe +# Linux系统 +./CarlaUE4.sh +``` +等待地图加载完成(显示3D城市场景) + +2. 运行主程序: +```bash +# 基础运行(默认yolov5m模型+SORT跟踪器) +python main.py + +# 可选参数配置 +python main.py --model yolov5x --tracker sort --host localhost --port 2000 --conf-thres 0.25 --iou-thres 0.45 +``` + +3. 操作指令: +- 按`q`键:退出程序 +- 程序运行时会自动生成主车辆、NPC车辆及相机,实时显示检测跟踪结果 + +## 功能说明 + +1. 目标检测: +- 支持检测类别:汽车(car)、摩托车(motorcycle)、公交车(bus)、卡车(truck)(对应COCO数据集ID:2、3、5、7) +- 模型选择:yolov5s(轻量快速)、yolov5m(平衡精度与速度)、yolov5x(高精度,适合对精度要求高的场景) +- 检测优化: + - 采用多帧投票机制(保留至少1帧出现的目标),提高小目标检测稳定性 + - 过滤过小检测框(宽度和高度均大于8像素),减少误检 + - 支持置信度阈值(--conf-thres)和NMS IOU阈值(--iou-thres)配置 + +2. 目标跟踪: +- 采用优化版SORT跟踪算法,核心改进包括: + - 卡尔曼滤波器参数优化,适配车辆运动特性 + - 引入历史位置平滑处理,减少跟踪抖动 + - 提高IOU匹配阈值,增强跟踪稳定性 + - 过滤过小检测框,减少误跟踪 +- 跟踪核心参数:max_age(最大消失帧数,默认5)、min_hits(最小命中数,默认3)、iou_threshold(IOU匹配阈值,默认0.4) +- 跟踪结果包含持续的目标ID,支持目标短暂消失后重新出现的ID匹配 + +3. CARLA交互: +- 自动清理仿真环境中的残留车辆和静态车辆,保证场景纯净 +- 主车辆开启自动驾驶模式,相机优化挂载于主车辆前方(x=2.0m,z=1.5m),更适合前方车辆检测 +- 自动生成NPC车辆并开启自动驾驶,构建动态交通场景(默认生成10辆) +- spectator视角自动跟随主车辆(后上方5m,俯角30°),方便观察全局场景 +- 同步模式运行,固定帧率0.05s/帧(20FPS),保证检测稳定性 + +## 项目结构 +- main.py:主程序脚本,包含以下核心组件: + - KalmanFilter:卡尔曼滤波器实现,用于目标运动状态预测与更新 + - Track:跟踪目标实体类,维护单目标的边界框、ID、生命周期及历史位置等信息 + - Sort:SORT跟踪器核心实现,处理多目标匹配与跟踪状态管理 + - 工具函数:包含边界框绘制(支持类别差异化颜色)、相机投影矩阵构建、CARLA环境清理等功能 + - CARLA交互模块:负责服务器连接、车辆生成、相机配置及同步控制 + - 检测与跟踪主逻辑:集成YOLOv5检测与SORT跟踪,实现端到端流程 +- YOLOv5预训练模型(yolov5s.pt/yolov5m.pt/yolov5x.pt):需放置在指定路径 + +## 常见问题 + +### CARLA连接失败: +- 确保CARLA仿真器已启动,且端口与脚本参数一致(默认2000) +- 检查CARLA版本与Python API版本是否匹配 +- 关闭防火墙或添加端口例外规则 + +### 模型加载错误: +- 确认模型文件已下载并放在正确路径(默认`D:\yolo\`) +- 检查模型文件名与代码中`model_paths`字典配置一致 +- 网络问题导致模型自动下载失败时,可手动下载并放置到指定路径 + +### 性能问题: +- 检测速度慢:可切换为yolov5s模型,或降低相机分辨率(修改`camera_bp`的`image_size_x`和`image_size_y`参数) +- 仿真卡顿:减少NPC车辆数量(修改`spawn_npcs`函数的`count`参数) +- 内存占用高:关闭不必要的后台程序,或降低模型复杂度 + +### 可视化窗口无响应: +- 确保OpenCV版本兼容(推荐4.5.x系列) +- 检查是否存在未处理的异常导致主循环中断 +- 尝试重新启动CARLA仿真器和程序 + +## 参考文档 +- [CARLA官方文档](https://carla.readthedocs.io/) +- [YOLOv5官方文档](https://docs.ultralytics.com/yolov5/) +- [SORT跟踪算法论文](https://arxiv.org/abs/1602.00763) + +## 许可协议 +本项目基于MIT许可协议开源,详情参见LICENSE文件。 \ No newline at end of file diff --git a/src/carla_tracking/main.py b/src/carla_tracking/main.py new file mode 100644 index 0000000000..60d0eab215 --- /dev/null +++ b/src/carla_tracking/main.py @@ -0,0 +1,516 @@ +import argparse +import carla +import os +import queue +import random +import cv2 +import numpy as np +import torch +from collections import deque + + +# -------------------------- 内置 SORT 跟踪器(优化版) -------------------------- +class KalmanFilter: + def __init__(self): + self.dt = 1.0 + self.x = np.zeros((4, 1)) # [x, y, vx, vy] + self.F = np.array([[1, 0, self.dt, 0], + [0, 1, 0, self.dt], + [0, 0, 1, 0], + [0, 0, 0, 1]]) + self.H = np.array([[1, 0, 0, 0], + [0, 1, 0, 0]]) + self.P = np.eye(4) * 1000 + # 优化过程噪声协方差,适合车辆运动特性 + self.Q = np.array([[1, 0, 0, 0], + [0, 1, 0, 0], + [0, 0, 5, 0], + [0, 0, 0, 5]]) + # 降低测量噪声,提高精度 + self.R = np.eye(2) * 5 + + def predict(self): + self.x = np.dot(self.F, self.x) + self.P = np.dot(np.dot(self.F, self.P), self.F.T) + self.Q + return self.x[:2] + + def update(self, z): + y = z - np.dot(self.H, self.x) + S = np.dot(np.dot(self.H, self.P), self.H.T) + self.R + K = np.dot(np.dot(self.P, self.H.T), np.linalg.inv(S)) + self.x = self.x + np.dot(K, y) + self.P = self.P - np.dot(np.dot(K, self.H), self.P) + return self.x[:2] + + +class Track: + def __init__(self, box, track_id): + self.id = track_id + self.kf = KalmanFilter() + self.x1, self.y1, self.x2, self.y2 = box + self.center = np.array([[(self.x1 + self.x2) / 2], [(self.y1 + self.y2) / 2]]) + self.kf.x[:2] = self.center + self.width = self.x2 - self.x1 + self.height = self.y2 - self.y1 + self.hits = 1 + self.age = 0 + # 新增:记录历史位置,用于平滑处理 + self.history = deque(maxlen=5) + self.history.append(self.center) + + def predict(self): + self.age += 1 + center = self.kf.predict() + # 新增:使用历史位置平滑预测结果 + self.history.append(center) + smoothed_center = np.mean(self.history, axis=0) + self.x1 = smoothed_center[0, 0] - self.width / 2 + self.y1 = smoothed_center[1, 0] - self.height / 2 + self.x2 = self.x1 + self.width + self.y2 = self.y1 + self.height + return [self.x1, self.y1, self.x2, self.y2] + + def update(self, box): + self.x1, self.y1, self.x2, self.y2 = box + self.center = np.array([[(self.x1 + self.x2) / 2], [(self.y1 + self.y2) / 2]]) + self.kf.update(self.center) + self.width = self.x2 - self.x1 + self.height = self.y2 - self.y1 + self.hits += 1 + self.age = 0 + self.history.append(self.center) + + def get_box(self): + return [self.x1, self.y1, self.x2, self.y2] + + +class Sort: + def __init__(self, max_age=5, min_hits=3, iou_threshold=0.4): # 提高IOU阈值 + self.max_age = max_age + self.min_hits = min_hits + self.iou_threshold = iou_threshold + self.tracks = [] + self.next_id = 1 + + def update(self, detections): + if len(detections) == 0: + for track in self.tracks: + track.age += 1 + self.tracks = [t for t in self.tracks if t.age <= self.max_age] + return np.array([[t.x1, t.y1, t.x2, t.y2, t.id] for t in self.tracks if t.hits >= self.min_hits]) + + if len(self.tracks) == 0: + for det in detections: + self.tracks.append(Track(det[:4], self.next_id)) + self.next_id += 1 + return np.array([[t.x1, t.y1, t.x2, t.y2, t.id] for t in self.tracks]) + + track_boxes = np.array([t.get_box() for t in self.tracks]) + iou_matrix = self._iou_batch(track_boxes, detections[:, :4]) + matches, unmatched_tracks, unmatched_dets = self._hungarian_algorithm(iou_matrix) + + # 优化匹配逻辑:只匹配高IOU的结果 + for track_idx, det_idx in matches: + if iou_matrix[track_idx, det_idx] >= self.iou_threshold: + self.tracks[track_idx].update(detections[det_idx][:4]) + + for track_idx in unmatched_tracks: + self.tracks[track_idx].age += 1 + + # 新增:过滤过小的检测框,减少误检(调整最小尺寸为8) + for det_idx in unmatched_dets: + box = detections[det_idx][:4] + width = box[2] - box[0] + height = box[3] - box[1] + if width > 8 and height > 8: # 减小最小尺寸过滤阈值,适配小目标 + self.tracks.append(Track(box, self.next_id)) + self.next_id += 1 + + self.tracks = [t for t in self.tracks if t.age <= self.max_age] + return np.array([[t.x1, t.y1, t.x2, t.y2, t.id] for t in self.tracks if t.hits >= self.min_hits]) + + def _iou_batch(self, b1, b2): + b1_x1, b1_y1, b1_x2, b1_y2 = b1[:, 0], b1[:, 1], b1[:, 2], b1[:, 3] + b2_x1, b2_y1, b2_x2, b2_y2 = b2[:, 0], b2[:, 1], b2[:, 2], b2[:, 3] + inter_x1 = np.maximum(b1_x1[:, None], b2_x1[None, :]) + inter_y1 = np.maximum(b1_y1[:, None], b2_y1[None, :]) + inter_x2 = np.minimum(b1_x2[:, None], b2_x2[None, :]) + inter_y2 = np.minimum(b1_y2[:, None], b2_y2[None, :]) + inter_area = np.maximum(0, inter_x2 - inter_x1) * np.maximum(0, inter_y2 - inter_y1) + b1_area = (b1_x2 - b1_x1) * (b1_y2 - b1_y1) + b2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + return inter_area / (b1_area[:, None] + b2_area[None, :] - inter_area + 1e-6) + + def _hungarian_algorithm(self, cost_matrix): + from scipy.optimize import linear_sum_assignment + row_ind, col_ind = linear_sum_assignment(-cost_matrix) + matches = np.array(list(zip(row_ind, col_ind))) + unmatched_tracks = [i for i in range(cost_matrix.shape[0]) if i not in matches[:, 0]] + unmatched_dets = [i for i in range(cost_matrix.shape[1]) if i not in matches[:, 1]] + return matches, unmatched_tracks, unmatched_dets + + +# -------------------------- YOLOv5 检测模型(优化版) -------------------------- +from ultralytics import YOLO + + +# -------------------------- 核心工具函数(优化版) -------------------------- +def draw_bounding_boxes(image, boxes, labels, class_names, track_ids=None, probs=None): + """绘制检测/跟踪框(含类别、置信度、跟踪ID)""" + for i, (box, label) in enumerate(zip(boxes, labels)): + x1, y1, x2, y2 = map(int, box) + # 过滤超出图像范围的框 + x1 = max(0, min(x1, image.shape[1] - 1)) + y1 = max(0, min(y1, image.shape[0] - 1)) + x2 = max(0, min(x2, image.shape[1] - 1)) + y2 = max(0, min(y2, image.shape[0] - 1)) + + # 根据类别设置不同颜色 + colors = {2: (0, 255, 0), 3: (0, 0, 255), 5: (255, 0, 0), 7: (255, 255, 0)} + color = colors.get(label, (0, 255, 0)) + cv2.rectangle(image, (x1, y1), (x2, y2), color, 2) + + # 拼接文本信息 + conf_text = f"{probs[i]:.2f}" if (probs is not None and i < len(probs)) else "" + label_text = f"{class_names[label]} {conf_text}".strip() + if track_ids and i < len(track_ids): + label_text += f"_ID:{track_ids[i]}" + + # 绘制文本背景(避免遮挡) + text_size = cv2.getTextSize(label_text, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 2)[0] + cv2.rectangle(image, (x1, y1 - text_size[1] - 5), (x1 + text_size[0], y1), color, -1) + cv2.putText(image, label_text, (x1, y1 - 3), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2) + return image + + +def build_projection_matrix(w, h, fov): + """构建相机投影矩阵""" + focal = w / (2.0 * np.tan(fov * np.pi / 360.0)) + return np.array([[focal, 0, w / 2], [0, focal, h / 2], [0, 0, 1]]) + + +def clear_npc(world): + """清理所有NPC车辆""" + if not world: + return + for actor in world.get_actors().filter('vehicle.*'): + try: + actor.destroy() + except: + pass + + +def clear_static_vehicle(world): + """清理静态车辆""" + if not world: + return + for actor in world.get_actors().filter('static.vehicle.*'): + try: + actor.destroy() + except: + pass + + +def clear(world, camera): + """清理所有资源""" + if camera: + try: + camera.destroy() + except: + pass + clear_npc(world) + clear_static_vehicle(world) + + +# -------------------------- CARLA 相关核心函数 -------------------------- +def camera_callback(image, rgb_image_queue): + """相机回调函数:将图像存入队列""" + rgb_image_queue.put(np.reshape(np.copy(image.raw_data), (image.height, image.width, 4))) + + +def setup_carla_client(host='localhost', port=2000): + """连接CARLA服务器并设置同步模式""" + client = carla.Client(host, port) + client.set_timeout(10.0) + world = client.get_world() + # 设置同步模式(保证帧率稳定) + settings = world.get_settings() + settings.synchronous_mode = True + settings.fixed_delta_seconds = 0.05 + world.apply_settings(settings) + return world, client + + +def spawn_ego_vehicle(world): + """生成主车辆(自车)- 兼容老版本CARLA""" + if not world: + return None + bp_lib = world.get_blueprint_library() + try: + vehicle_bp = bp_lib.find('vehicle.lincoln.mkz_2020') + except: + vehicle_bp = random.choice( + [bp for bp in bp_lib.filter('vehicle') if int(bp.get_attribute('number_of_wheels')) == 4]) + spawn_points = world.get_map().get_spawn_points() + if not spawn_points: + print("警告:没有可用的车辆生成点!") + return None + # 尝试生成车辆(多次尝试避免生成失败) + vehicle = None + for _ in range(5): + vehicle = world.try_spawn_actor(vehicle_bp, random.choice(spawn_points)) + if vehicle: + break + if vehicle: + vehicle.set_autopilot(True) + print("主车辆生成成功!") + else: + print("警告:无法生成主车辆!") + return vehicle + + +def spawn_camera(world, vehicle): + """生成相机传感器(挂载在自车)""" + if not world or not vehicle: + return None, None, None + bp_lib = world.get_blueprint_library() + camera_bp = bp_lib.find('sensor.camera.rgb') + # 提高相机分辨率以获取更多细节(略微降低性能) + camera_bp.set_attribute('image_size_x', '640') + camera_bp.set_attribute('image_size_y', '480') + camera_bp.set_attribute('fov', '90') + camera_bp.set_attribute('sensor_tick', '0.05') # 与世界帧率一致 + # 相机安装位置优化(更适合检测前方车辆) + camera_init_trans = carla.Transform(carla.Location(x=2.0, z=1.5)) + camera = world.spawn_actor(camera_bp, camera_init_trans, attach_to=vehicle) + # 图像队列(异步获取图像) + image_queue = queue.Queue() + camera.listen(lambda image: camera_callback(image, image_queue)) + return camera, image_queue, camera_bp + + +def spawn_npcs(world, count=10): + """生成NPC车辆(数量适中,避免CPU卡顿)""" + if not world: + return + bp_lib = world.get_blueprint_library() + vehicle_bp = bp_lib.filter('vehicle') + car_bp = [bp for bp in vehicle_bp if int(bp.get_attribute('number_of_wheels')) == 4] + spawn_points = world.get_map().get_spawn_points() + if not car_bp or not spawn_points: + print("警告:无法生成NPC车辆!") + return + spawned_count = 0 + for _ in range(count): + npc = world.try_spawn_actor(random.choice(car_bp), random.choice(spawn_points)) + if npc: + npc.set_autopilot(True) + spawned_count += 1 + print(f"成功生成 {spawned_count} 辆NPC车辆") + + +def load_detection_model(model_type): + """加载YOLOv5检测模型(使用自定义路径)""" + device = 'cuda' if torch.cuda.is_available() else 'cpu' + # 允许使用不同精度的模型,添加yolov5mu.pt路径 + model_paths = { + 'yolov5s': r"D:\yolo\yolov5s.pt", + 'yolov5m': r"D:\yolo\yolov5m.pt", + 'yolov5x': r"D:\yolo\yolov5x.pt", + 'yolov5mu': r"D:\yolo\yolov5mu.pt" # 新增yolov5mu模型路径 + } + + if model_type not in model_paths: + raise ValueError(f"不支持的模型类型:{model_type}") + + # 加载模型并启用自动混合精度 + model = YOLO(model_paths[model_type]) + model.to(device) + print(f"检测模型加载成功(设备:{device},路径:{model_paths[model_type]})") + return model, model.names + + +def setup_tracker(tracker_type): + """初始化跟踪器(默认SORT,稳定高效)""" + if tracker_type == 'sort': + return Sort(max_age=5, min_hits=3, iou_threshold=0.4), None + else: + raise ValueError(f"不支持的跟踪器类型:{tracker_type}") + + +# -------------------------- 主函数(优化版) -------------------------- +def main(): + parser = argparse.ArgumentParser(description='CARLA 目标检测与跟踪(优化版)') + # 新增yolov5mu作为可选模型 + parser.add_argument('--model', type=str, default='yolov5mu', choices=['yolov5s', 'yolov5m', 'yolov5x', 'yolov5mu'], + help='检测模型(yolov5x精度最高,yolov5s最轻便)') + parser.add_argument('--tracker', type=str, default='sort', choices=['sort'], help='跟踪器(仅保留稳定的SORT)') + parser.add_argument('--host', type=str, default='localhost', help='CARLA服务器地址') + parser.add_argument('--port', type=int, default=2000, help='CARLA服务器端口') + parser.add_argument('--conf-thres', type=float, default=0.25, help='检测置信度阈值(降低至0.25适配小目标)') + parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IOU阈值') + args = parser.parse_args() + + # 提前初始化所有变量,避免未定义错误 + world = None + camera = None + vehicle = None + image_queue = None + # 新增:记录历史检测结果用于后处理 + detection_history = deque(maxlen=3) + + try: + # 1. 初始化设备 + device = 'cuda' if torch.cuda.is_available() else 'cpu' + print(f"当前使用设备:{device}") + + # 2. 连接CARLA服务器(必须先启动CARLA) + print(f"正在连接CARLA服务器 {args.host}:{args.port}...") + world, client = setup_carla_client(args.host, args.port) + spectator = world.get_spectator() + print("CARLA服务器连接成功!") + + # 3. 清理环境(避免残留车辆) + clear_npc(world) + clear_static_vehicle(world) + print("环境清理完成!") + + # 4. 生成主车辆 + vehicle = spawn_ego_vehicle(world) + if not vehicle: + print("无法生成主车辆,程序退出!") + return + + # 5. 生成相机传感器 + camera, image_queue, camera_bp = spawn_camera(world, vehicle) + if not camera: + print("无法生成相机,程序退出!") + return + print("相机传感器生成成功!") + + # 6. 生成NPC车辆 + spawn_npcs(world, count=10) + + # 7. 加载检测模型和跟踪器 + model, class_names = load_detection_model(args.model) + tracker, _ = setup_tracker(args.tracker) + + # 8. 主循环(检测+跟踪) + print("开始目标检测与跟踪(按 'q' 键退出程序)") + while True: + # 同步CARLA世界时钟 + world.tick() + + # 移动视角到自车上方(方便观察) + ego_transform = vehicle.get_transform() + spectator_transform = carla.Transform( + ego_transform.transform(carla.Location(x=-5, z=8)), + carla.Rotation(yaw=ego_transform.rotation.yaw - 180, pitch=-30) + ) + spectator.set_transform(spectator_transform) + + # 获取相机图像(跳过空队列) + if image_queue.empty(): + continue + origin_image = image_queue.get() + # CARLA图像默认BGRA格式,转换为RGB + image = cv2.cvtColor(origin_image, cv2.COLOR_BGRA2RGB) + height, width, _ = image.shape + + # -------------------------- 目标检测(YOLOv5 优化版) -------------------------- + # 优化检测参数,使用NMS抑制重复检测 + results = model(image, conf=args.conf_thres, iou=args.iou_thres) + + boxes, labels, probs = [], [], [] + for r in results: + for box in r.boxes: + # 解析边界框(x1,y1,x2,y2) + x1, y1, x2, y2 = box.xyxy[0].cpu().numpy() + # 解析置信度和类别 + conf = box.conf[0].cpu().numpy() + cls = int(box.cls[0].cpu().numpy()) + # 只保留车辆相关类别(COCO数据集:car=2, motorcycle=3, bus=5, truck=7) + if cls in [2, 3, 5, 7]: + # 过滤过小目标(减小最小尺寸至8,适配小目标/远处车辆) + box_width = x2 - x1 + box_height = y2 - y1 + if box_width > 8 and box_height > 8: # 关键修改:从15→8 + boxes.append([x1, y1, x2, y2]) + labels.append(cls) + probs.append(conf) + + # 转换为numpy数组(避免空列表报错) + boxes = np.array(boxes) if boxes else np.array([]) + labels = np.array(labels) if labels else np.array([]) + probs = np.array(probs) if probs else np.array([]) + + # 新增:多帧投票机制(放宽至1帧即可保留,避免过滤小目标) + detection_history.append((boxes, labels, probs)) + if len(detection_history) == 3: # 累积3帧结果 + # 简单投票机制:保留至少在1帧中出现的目标(关键修改:从2→1) + combined_boxes = [] + combined_labels = [] + combined_probs = [] + + for i in range(len(boxes)): + current_box = boxes[i] + current_label = labels[i] + count = 1 # 当前帧计数 + + # 检查前两帧是否有相似目标 + for prev_boxes, prev_labels, _ in list(detection_history)[:-1]: + if len(prev_boxes) == 0: + continue + # 计算与历史框的IOU + ious = Sort()._iou_batch(np.array([current_box]), prev_boxes)[0] + if np.max(ious) > 0.5 and prev_labels[np.argmax(ious)] == current_label: + count += 1 + + # 关键修改:至少在1帧中出现就保留(适配小目标) + if count >= 1: + combined_boxes.append(current_box) + combined_labels.append(current_label) + combined_probs.append(probs[i]) + + boxes = np.array(combined_boxes) if combined_boxes else np.array([]) + labels = np.array(combined_labels) if combined_labels else np.array([]) + probs = np.array(combined_probs) if combined_probs else np.array([]) + + # -------------------------- 目标跟踪(SORT 优化版) -------------------------- + if args.tracker == 'sort' and len(boxes) > 0: + # 转换为SORT需要的格式:[x1,y1,x2,y2,conf] + dets = np.hstack([boxes, probs.reshape(-1, 1)]) if len(probs) > 0 else boxes + # 更新跟踪器 + tracks = tracker.update(dets) + if len(tracks) > 0: + track_boxes = tracks[:, :4] + track_ids = tracks[:, 4].astype(int) + # 绘制跟踪结果 + image = draw_bounding_boxes(image, track_boxes, labels[:len(track_boxes)], class_names, track_ids, probs[:len(track_boxes)]) + else: + # 无跟踪时绘制检测结果 + image = draw_bounding_boxes(image, boxes, labels, class_names, probs=probs) + + # 显示处理后的图像 + cv2.imshow('CARLA Detection & Tracking', cv2.cvtColor(image, cv2.COLOR_RGB2BGR)) + # 按q退出 + if cv2.waitKey(1) & 0xFF == ord('q'): + break + + except Exception as e: + print(f"程序出错:{str(e)}") + finally: + # 清理资源 + print("正在清理资源...") + clear(world, camera) + # 关闭同步模式 + if world: + settings = world.get_settings() + settings.synchronous_mode = False + world.apply_settings(settings) + cv2.destroyAllWindows() + print("程序已退出") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/carla_tracking/requirements.txt b/src/carla_tracking/requirements.txt new file mode 100644 index 0000000000..1ee0ed2e64 --- /dev/null +++ b/src/carla_tracking/requirements.txt @@ -0,0 +1,9 @@ +python >= 3.7 +numpy +opencv-python +torch +scipy +argparse +collections +carla +ultralytics \ No newline at end of file