diff --git a/src/Unmanned_vehicle_AD_DQN/Environment.py b/src/Unmanned_vehicle_AD_DQN/Environment.py index 5929383d5f..60d8eb6868 100644 --- a/src/Unmanned_vehicle_AD_DQN/Environment.py +++ b/src/Unmanned_vehicle_AD_DQN/Environment.py @@ -8,236 +8,222 @@ import cv2 import math from Hyperparameters import * +from collections import deque import carla from carla import ColorConverter -class CarEnv: - SHOW_CAM = SHOW_PREVIEW # 是否显示摄像头预览 - im_width = IM_WIDTH # 图像宽度 - im_height = IM_HEIGHT # 图像高度 +class EnhancedCarEnv: + SHOW_CAM = SHOW_PREVIEW + im_width = IM_WIDTH + im_height = IM_HEIGHT - def __init__(self): - self.actor_list = [] # 存储所有actor的列表 - self.sem_cam = None # 语义分割摄像头 - self.client = carla.Client("localhost", 2000) # CARLA客户端 - self.client.set_timeout(20.0) # 连接超时设置 - self.front_camera = None # 前置摄像头图像 - - # 新增变量 - self.last_action = 1 # 上一个动作,默认保持 - self.same_steer_counter = 0 # 连续同向转向计数器 - self.suggested_action = None # 建议的避让动作 - self.episode_start_time = None # 每轮开始时间 - self.last_ped_distance = float('inf') # 上次最近行人距离 - self.current_episode = 1 # 当前episode编号 - - # 加载世界和蓝图 + def __init__(self, obstacle_detection_mode='advanced'): + self.actor_list = [] + self.sem_cam = None + self.client = carla.Client("localhost", 2000) + self.client.set_timeout(20.0) self.world = self.client.load_world('Town03') + self.front_camera = None - # 设置观察者视角,让CARLA窗口显示 - self.setup_observer_view() + # 增强的状态跟踪 + self.last_action = 1 + self.same_steer_counter = 0 + self.suggested_action = None + self.episode_start_time = None + self.last_ped_distance = float('inf') + self.current_episode = 1 + self.obstacle_detection_mode = obstacle_detection_mode + + # 障碍物检测增强 + self.obstacle_history = deque(maxlen=10) # 障碍物历史 + self.obstacle_warning_level = 0 # 0-无警告, 1-轻微, 2-中等, 3-严重 + self.last_obstacle_type = None # 'pedestrian', 'vehicle', 'building' + + # 碰撞和避障统计 + self.collision_history = [] + self.near_miss_history = [] + self.successful_avoidance = 0 + self.obstacle_encounters = 0 + # 初始化设置 self.blueprint_library = self.world.get_blueprint_library() - self.model_3 = self.blueprint_library.filter("model3")[0] # Tesla Model3车辆 - - # 行人列表和碰撞历史 + self.model_3 = self.blueprint_library.filter("model3")[0] self.walker_list = [] - self.collision_history = [] - self.slow_counter = 0 # 慢速计数器 - self.steer_counter = 0 # 转向计数器,用于限制过度转向 + self.slow_counter = 0 + self.steer_counter = 0 + + # 设置观察者视角 + self.setup_observer_view() + + # 建筑物检测参数 + self.building_detection_enabled = True + self.building_proximity_threshold = 15.0 # 建筑物接近阈值(米) + + # 语义分割颜色映射(用于障碍物检测) + self.semantic_colors = { + 4: 'building', # 建筑物 - 蓝色 + 10: 'pedestrian', # 行人 - 红色 + 12: 'vehicle', # 车辆 - 绿色 + } def setup_observer_view(self): - """设置观察者视角,让用户可以在CARLA窗口中看到场景""" + """设置观察者视角""" try: - # 获取当前地图的生成点 - spawn_points = self.world.get_map().get_spawn_points() - if spawn_points: - # 选择一个合适的观察者位置 - spectator = self.world.get_spectator() - - # 设置观察者位置在车辆起始位置附近 - transform = carla.Transform() - transform.location.x = -81.0 - transform.location.y = -195.0 - transform.location.z = 15.0 # 提高视角高度 - transform.rotation.pitch = -45.0 # 向下倾斜视角 - transform.rotation.yaw = 0.0 - transform.rotation.roll = 0.0 - - spectator.set_transform(transform) - print("观察者视角已设置") + spectator = self.world.get_spectator() + transform = carla.Transform() + transform.location.x = -81.0 + transform.location.y = -195.0 + transform.location.z = 15.0 + transform.rotation.pitch = -45.0 + transform.rotation.yaw = 0.0 + transform.rotation.roll = 0.0 + spectator.set_transform(transform) + print("观察者视角已设置") except Exception as e: print(f"设置观察者视角时出错: {e}") - def spawn_pedestrians_general(self, number, isCross): - """生成指定数量的行人""" - # 记录要生成的数量 - target_number = number + def spawn_pedestrians_with_config(self, config): + """根据配置生成行人""" + cross_count = config.get('pedestrian_cross', 4) + normal_count = config.get('pedestrian_normal', 2) + + print(f"生成行人: 十字路口={cross_count}, 普通路段={normal_count}") + + # 清理现有行人 + self.cleanup_pedestrians() + + # 生成十字路口行人 + for _ in range(cross_count): + self.spawn_pedestrian_at_crossing() - # 记录成功生成的数量 - success_count = 0 + # 生成普通路段行人 + for _ in range(normal_count): + self.spawn_pedestrian_normal() + + def spawn_pedestrian_at_crossing(self): + """在十字路口生成行人""" + blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") + + # 两个主要十字路口位置 + crossings = [ + {'x_range': (-14, -10.5), 'y_range': (-188, -183)}, # 第一个十字路口右侧 + {'x_range': (17, 20.5), 'y_range': (-188, -183)}, # 第二个十字路口右侧 + {'x_range': (-14, -10.5), 'y_range': (-216, -210)}, # 第一个十字路口左侧 + {'x_range': (17, 20.5), 'y_range': (-216, -210)} # 第二个十字路口左侧 + ] + + crossing = random.choice(crossings) + x = random.uniform(crossing['x_range'][0], crossing['x_range'][1]) + y = random.uniform(crossing['y_range'][0], crossing['y_range'][1]) - # 尝试生成指定数量的行人 - attempts = 0 - max_attempts = number * 3 # 最多尝试3倍次数 + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - while success_count < target_number and attempts < max_attempts: - attempts += 1 - isLeft = random.choice([True, False]) # 随机选择左右侧 + try: + walker_bp = random.choice(blueprints_walkers) + + # 设置行人属性 + if walker_bp.has_attribute('is_invincible'): + walker_bp.set_attribute('is_invincible', 'false') + + npc = self.world.try_spawn_actor(walker_bp, spawn_point) - try: - if isLeft: - if self.spawn_pedestrians_left(isCross): - success_count += 1 + if npc is not None: + # 设置行人移动 + ped_control = carla.WalkerControl() + + # 十字路口行人更活跃 + ped_control.speed = random.uniform(0.8, 1.5) + + # 随机选择移动方向 + if random.random() > 0.5: + ped_control.direction.y = -1 # 南北向 + ped_control.direction.x = random.uniform(-0.2, 0.2) else: - if self.spawn_pedestrians_right(isCross): - success_count += 1 - except Exception as e: - # 如果生成失败,继续尝试 - print(f"生成行人失败: {e}") - continue + ped_control.direction.x = 1 # 东西向 + ped_control.direction.y = random.uniform(-0.2, 0.2) + + npc.apply_control(ped_control) + npc.set_simulate_physics(True) + self.walker_list.append(npc) + + # 添加随机移动模式 + self.setup_pedestrian_ai(npc) + + return True + except Exception as e: + print(f"生成十字路口行人失败: {e}") - print(f"成功生成 {success_count}/{target_number} 个行人 (isCross={isCross})") - return success_count + return False - def spawn_pedestrians_right(self, isCross): - """在右侧生成行人""" + def spawn_pedestrian_normal(self): + """在普通路段生成行人""" blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") - # 设置生成区域 - min_x = -50 - max_x = 140 - min_y = -188 - max_y = -183 - - # 如果是十字路口,调整生成位置 - if isCross: - isFirstCross = random.choice([True, False]) - if isFirstCross: - min_x = -14 - max_x = -10.5 - else: - min_x = 17 - max_x = 20.5 - - # 尝试多次生成直到成功 - for attempt in range(3): # 尝试3次 - # 随机生成位置 - x = random.uniform(min_x, max_x) - y = random.uniform(min_y, max_y) - - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - - # 避免在特定区域生成 - while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): - x = random.uniform(min_x, max_x) - y = random.uniform(min_y, max_y) - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - - # 尝试生成行人 - try: - walker_bp = random.choice(blueprints_walkers) - npc = self.world.try_spawn_actor(walker_bp, spawn_point) - - if npc is not None: - # 设置行人控制参数 - ped_control = carla.WalkerControl() - ped_control.speed = random.uniform(0.5, 1.0) # 随机速度 - ped_control.direction.y = -1 # 主要移动方向 - ped_control.direction.x = 0.15 # 轻微横向移动 - npc.apply_control(ped_control) - npc.set_simulate_physics(True) # 启用物理模拟 - self.walker_list.append(npc) # 添加到行人列表 - return True # 生成成功 - except Exception as e: - print(f"生成右侧行人失败 (尝试 {attempt+1}): {e}") - continue + # 普通路段位置 + if random.random() > 0.5: # 右侧 + x_range = (-50, 140) + y_range = (-188, -183) + direction_y = -1 + else: # 左侧 + x_range = (-50, 140) + y_range = (-216, -210) + direction_y = 1 - return False # 生成失败 - - def spawn_pedestrians_left(self, isCross): - """在左侧生成行人""" - blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") + x = random.uniform(x_range[0], x_range[1]) + y = random.uniform(y_range[0], y_range[1]) - # 设置生成区域 - min_x = -50 - max_x = 140 - min_y = -216 - max_y = -210 - - # 如果是十字路口,调整生成位置 - if isCross: - isFirstCross = random.choice([True, False]) - if isFirstCross: - min_x = -14 - max_x = -10.5 - else: - min_x = 17 - max_x = 20.5 - - # 尝试多次生成直到成功 - for attempt in range(3): # 尝试3次 - # 随机生成位置 - x = random.uniform(min_x, max_x) - y = random.uniform(min_y, max_y) - - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - - # 避免在特定区域生成 - while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): - x = random.uniform(min_x, max_x) - y = random.uniform(min_y, max_y) - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + # 避开十字路口区域 + while (-14 < x < -10.5) or (17 < x < 20.5) or (70 < x < 100): + x = random.uniform(x_range[0], x_range[1]) + + spawn_point = carla.Transform(carla.Location(x, y, 2.0)) + + try: + walker_bp = random.choice(blueprints_walkers) + npc = self.world.try_spawn_actor(walker_bp, spawn_point) + + if npc is not None: + ped_control = carla.WalkerControl() + ped_control.speed = random.uniform(0.5, 1.2) + ped_control.direction.y = direction_y + ped_control.direction.x = random.uniform(-0.1, 0.1) + + npc.apply_control(ped_control) + npc.set_simulate_physics(True) + self.walker_list.append(npc) + + return True + except Exception as e: + print(f"生成普通行人失败: {e}") + + return False - # 尝试生成行人 - try: - walker_bp = random.choice(blueprints_walkers) - npc = self.world.try_spawn_actor(walker_bp, spawn_point) + def setup_pedestrian_ai(self, pedestrian): + """设置行人AI行为""" + # 这里可以添加更复杂的行人AI逻辑 + # 目前使用简单的随机移动 + pass - if npc is not None: - # 设置行人控制参数 - ped_control = carla.WalkerControl() - ped_control.speed = random.uniform(0.7, 1.3) # 随机速度 - ped_control.direction.y = 1 # 主要移动方向 - ped_control.direction.x = -0.05 # 轻微横向移动 - npc.apply_control(ped_control) - npc.set_simulate_physics(True) # 启用物理模拟 - self.walker_list.append(npc) # 添加到行人列表 - return True # 生成成功 - except Exception as e: - print(f"生成左侧行人失败 (尝试 {attempt+1}): {e}") - continue - - return False # 生成失败 + def cleanup_pedestrians(self): + """清理所有行人""" + for walker in self.walker_list: + if walker.is_alive: + walker.destroy() + self.walker_list = [] - def reset(self, episode=1): - """重置环境,根据episode参数生成不同数量的行人""" + def reset(self, episode=1, curriculum_config=None): + """重置环境""" self.current_episode = episode - # 清理现有的行人和车辆 + # 清理现有的actors self.cleanup_actors() + self.cleanup_pedestrians() - # 重置行人列表 - self.walker_list = [] - - # 根据训练阶段生成不同数量的行人 - if episode < 100: # 第一阶段:少量行人 - print(f"Episode {episode}: 生成少量行人 (4十字路口 + 2非十字路口)") - self.spawn_pedestrians_general(4, True) # 十字路口行人 - self.spawn_pedestrians_general(2, False) # 非十字路口行人 - elif episode < 400: # 第二阶段:中等数量行人 - print(f"Episode {episode}: 生成中等数量行人 (6十字路口 + 3非十字路口)") - self.spawn_pedestrians_general(6, True) - self.spawn_pedestrians_general(3, False) - else: # 第三阶段:正常难度 - print(f"Episode {episode}: 生成正常数量行人 (8十字路口 + 4非十字路口)") - self.spawn_pedestrians_general(8, True) - self.spawn_pedestrians_general(4, False) - # 重置状态变量 self.collision_history = [] + self.near_miss_history = [] self.actor_list = [] self.slow_counter = 0 self.steer_counter = 0 @@ -246,7 +232,27 @@ def reset(self, episode=1): self.last_action = 1 self.episode_start_time = time.time() self.last_ped_distance = float('inf') - + self.obstacle_warning_level = 0 + self.last_obstacle_type = None + self.successful_avoidance = 0 + self.obstacle_encounters = 0 + + # 根据课程学习配置或episode阶段生成行人 + if curriculum_config: + self.spawn_pedestrians_with_config(curriculum_config) + else: + # 默认阶段划分 + if episode < 100: + config = {'pedestrian_cross': 2, 'pedestrian_normal': 1} + elif episode < 300: + config = {'pedestrian_cross': 4, 'pedestrian_normal': 2} + elif episode < 500: + config = {'pedestrian_cross': 6, 'pedestrian_normal': 3} + else: + config = {'pedestrian_cross': 8, 'pedestrian_normal': 4} + + self.spawn_pedestrians_with_config(config) + # 设置车辆生成点 spawn_point = carla.Transform() spawn_point.location.x = -81.0 @@ -259,103 +265,293 @@ def reset(self, episode=1): # 生成主车辆 self.vehicle = self.world.spawn_actor(self.model_3, spawn_point) self.actor_list.append(self.vehicle) - + # 设置语义分割摄像头 self.sem_cam = self.blueprint_library.find('sensor.camera.semantic_segmentation') self.sem_cam.set_attribute("image_size_x", f"{self.im_width}") self.sem_cam.set_attribute("image_size_y", f"{self.im_height}") - self.sem_cam.set_attribute("fov", f"110") # 视野角度 - + self.sem_cam.set_attribute("fov", f"110") + # 安装摄像头传感器 transform = carla.Transform(carla.Location(x=2.5, z=0.7)) self.sensor = self.world.spawn_actor(self.sem_cam, transform, attach_to=self.vehicle) self.actor_list.append(self.sensor) - self.sensor.listen(lambda data: self.process_img(data)) # 设置图像处理回调 - + self.sensor.listen(lambda data: self.process_img_enhanced(data)) + # 初始化车辆控制 self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) - time.sleep(2) # 等待环境稳定 - + time.sleep(1.5) # 等待环境稳定 + # 设置碰撞传感器 colsensor = self.blueprint_library.find("sensor.other.collision") self.colsensor = self.world.spawn_actor(colsensor, transform, attach_to=self.vehicle) self.actor_list.append(self.colsensor) - self.colsensor.listen(lambda event: self.collision_data(event)) # 设置碰撞检测回调 - + self.colsensor.listen(lambda event: self.collision_data(event)) + + # 设置车道入侵传感器(可选) + try: + lane_sensor = self.blueprint_library.find("sensor.other.lane_invasion") + self.lane_sensor = self.world.spawn_actor(lane_sensor, transform, attach_to=self.vehicle) + self.actor_list.append(self.lane_sensor) + self.lane_sensor.listen(lambda event: self.lane_invasion_data(event)) + except: + pass + # 等待摄像头初始化完成 while self.front_camera is None: time.sleep(0.01) - - # 设置跟随相机(用于观察) + + # 设置跟随相机 self.setup_follow_camera() - + # 记录episode开始时间并重置控制 self.episode_start = time.time() self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) - + + print(f"Episode {episode} 环境重置完成,有 {len(self.walker_list)} 个行人") + return self.front_camera def cleanup_actors(self): """清理所有actors""" - # 清理车辆 - vehicles = self.world.get_actors().filter('vehicle.*') - for vehicle in vehicles: - if vehicle.is_alive: - vehicle.destroy() - - # 清理行人 - walkers = self.world.get_actors().filter('walker.*') - for walker in walkers: - if walker.is_alive: - walker.destroy() - - # 清理传感器 for actor in self.actor_list: if actor.is_alive: actor.destroy() - self.actor_list = [] - self.walker_list = [] # 清空行人列表 def setup_follow_camera(self): - """设置跟随车辆的相机,用于在CARLA窗口中观察""" + """设置跟随车辆的相机""" try: - # 创建RGB相机 camera_bp = self.blueprint_library.find('sensor.camera.rgb') camera_bp.set_attribute('image_size_x', '800') camera_bp.set_attribute('image_size_y', '600') camera_bp.set_attribute('fov', '110') - # 相机位置相对于车辆(后方上方) camera_transform = carla.Transform(carla.Location(x=-8, z=6), carla.Rotation(pitch=-20)) follow_camera = self.world.spawn_actor(camera_bp, camera_transform, attach_to=self.vehicle) self.actor_list.append(follow_camera) - print("跟随相机已设置") except Exception as e: print(f"设置跟随相机时出错: {e}") def collision_data(self, event): """处理碰撞事件""" - self.collision_history.append(event) + self.collision_history.append({ + 'timestamp': time.time(), + 'actor': event.other_actor, + 'impulse': event.normal_impulse, + 'type': event.other_actor.type_id if event.other_actor else 'unknown' + }) + + # 记录碰撞类型 + if event.other_actor: + if 'walker' in event.other_actor.type_id: + self.last_obstacle_type = 'pedestrian' + print(f"⚠️ 与行人发生碰撞!") + elif 'vehicle' in event.other_actor.type_id: + self.last_obstacle_type = 'vehicle' + print(f"⚠️ 与车辆发生碰撞!") + else: + self.last_obstacle_type = 'building' + print(f"⚠️ 与建筑物发生碰撞!") - def process_img(self, image): - """处理摄像头图像""" - image.convert(carla.ColorConverter.CityScapesPalette) # 转换为CityScapes调色板 + def lane_invasion_data(self, event): + """处理车道入侵事件""" + # 可以用于检测偏离道路 + pass + def process_img_enhanced(self, image): + """增强的图像处理,包含障碍物检测""" + image.convert(carla.ColorConverter.CityScapesPalette) + # 处理原始图像数据 processed_image = np.array(image.raw_data) processed_image = processed_image.reshape((self.im_height, self.im_width, 4)) - processed_image = processed_image[:, :, :3] # 移除alpha通道 - + processed_image = processed_image[:, :, :3] + + # 缩小图像尺寸以减少内存使用 + target_height, target_width = 240, 320 + processed_image = cv2.resize(processed_image, (target_width, target_height), interpolation=cv2.INTER_LINEAR) + + # 障碍物检测和分析 + if self.obstacle_detection_mode != 'none': + obstacle_info = self.detect_obstacles_from_image(processed_image) + + # 更新障碍物历史 + self.obstacle_history.append(obstacle_info) + + # 计算警告级别 + self.update_obstacle_warning(obstacle_info) + # 显示预览(如果启用) if self.SHOW_CAM: - cv2.imshow("", processed_image) + # 在图像上绘制障碍物信息 + if hasattr(self, 'obstacle_warning_level') and self.obstacle_warning_level > 0: + warning_text = f"障碍物警告: 等级{self.obstacle_warning_level}" + cv2.putText(processed_image, warning_text, (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) + + cv2.imshow("语义分割视图", processed_image) cv2.waitKey(1) + + self.front_camera = processed_image + + def detect_obstacles_from_image(self, image): + """从语义分割图像中检测障碍物""" + obstacle_info = { + 'has_obstacle': False, + 'obstacle_type': None, + 'obstacle_distance': float('inf'), + 'obstacle_position': None, # (x, y) 在图像中的位置 + 'obstacle_size': 0, # 障碍物像素数量 + 'warning_level': 0 + } + + # 简化版的障碍物检测(基于颜色) + # 在实际应用中,应该使用更复杂的语义分割结果 + + # 检测红色区域(行人) + red_mask = cv2.inRange(image, (200, 0, 0), (255, 50, 50)) + red_pixels = cv2.countNonZero(red_mask) + + # 检测蓝色区域(建筑物) + blue_mask = cv2.inRange(image, (0, 0, 200), (50, 50, 255)) + blue_pixels = cv2.countNonZero(blue_mask) + + # 检测绿色区域(车辆) + green_mask = cv2.inRange(image, (0, 200, 0), (50, 255, 50)) + green_pixels = cv2.countNonZero(green_mask) + + # 确定主要障碍物类型 + max_pixels = max(red_pixels, blue_pixels, green_pixels) + + if max_pixels > 100: # 阈值 + obstacle_info['has_obstacle'] = True + obstacle_info['obstacle_size'] = max_pixels + + if max_pixels == red_pixels: + obstacle_info['obstacle_type'] = 'pedestrian' + # 计算障碍物在图像中的位置(中心) + moments = cv2.moments(red_mask) + if moments['m00'] != 0: + cx = int(moments['m10'] / moments['m00']) + cy = int(moments['m01'] / moments['m00']) + obstacle_info['obstacle_position'] = (cx, cy) + + # 根据位置估计距离(简化) + # 中心越靠近图像底部,距离越近 + distance_estimate = (self.im_height - cy) / self.im_height * 50 + 5 + obstacle_info['obstacle_distance'] = distance_estimate + + elif max_pixels == blue_pixels: + obstacle_info['obstacle_type'] = 'building' + elif max_pixels == green_pixels: + obstacle_info['obstacle_type'] = 'vehicle' + + return obstacle_info + + def update_obstacle_warning(self, obstacle_info): + """更新障碍物警告级别""" + if not obstacle_info['has_obstacle']: + self.obstacle_warning_level = 0 + return + + distance = obstacle_info['obstacle_distance'] + size = obstacle_info['obstacle_size'] + + # 根据距离和大小计算警告级别 + if distance < 5.0 or size > 1000: + self.obstacle_warning_level = 3 # 严重 + elif distance < 10.0 or size > 500: + self.obstacle_warning_level = 2 # 中等 + elif distance < 20.0 or size > 100: + self.obstacle_warning_level = 1 # 轻微 + else: + self.obstacle_warning_level = 0 # 无 + + self.last_obstacle_type = obstacle_info['obstacle_type'] - self.front_camera = processed_image # 更新前置摄像头图像 + def detect_obstacles_raycast(self): + """使用射线投射检测障碍物(更精确)""" + vehicle_location = self.vehicle.get_location() + vehicle_transform = self.vehicle.get_transform() + + obstacles = [] + + # 在前方多个方向发射射线 + for angle in range(-30, 31, 10): # -30度到30度,每10度一条射线 + # 计算射线方向 + ray_direction = vehicle_transform.rotation + ray_direction.yaw += angle + + # 创建射线终点 + ray_end = vehicle_location + carla.Location( + x=50 * math.cos(math.radians(ray_direction.yaw)), + y=50 * math.sin(math.radians(ray_direction.yaw)), + z=0 + ) + + # 执行射线投射 + raycast_result = self.world.cast_ray(vehicle_location, ray_end) + + if raycast_result: + hit_location, hit_actor = raycast_result + distance = vehicle_location.distance(hit_location) + + if distance < 30: # 只关心30米内的障碍物 + obstacle_type = 'unknown' + if hit_actor: + if 'walker' in hit_actor.type_id: + obstacle_type = 'pedestrian' + elif 'vehicle' in hit_actor.type_id: + obstacle_type = 'vehicle' + elif 'building' in hit_actor.type_id or 'static' in hit_actor.type_id: + obstacle_type = 'building' + + obstacles.append({ + 'type': obstacle_type, + 'distance': distance, + 'direction': angle, + 'location': hit_location + }) + + return obstacles + + def calculate_avoidance_direction(self, obstacles): + """计算最佳避让方向""" + if not obstacles: + return None + + # 分析障碍物分布 + left_obstacles = [o for o in obstacles if o['direction'] < 0] + right_obstacles = [o for o in obstacles if o['direction'] > 0] + center_obstacles = [o for o in obstacles if abs(o['direction']) <= 10] + + # 找到最近的障碍物 + min_distance = min(o['distance'] for o in obstacles) + nearest_obstacles = [o for o in obstacles if o['distance'] == min_distance] + + if not nearest_obstacles: + return None + + nearest = nearest_obstacles[0] + + # 根据最近的障碍物决定避让方向 + if nearest['distance'] < 5.0: # 紧急情况 + if nearest['direction'] < 0: # 障碍物在左侧,向右避让 + return 4 # 右转 + else: # 障碍物在右侧,向左避让 + return 3 # 左转 + elif nearest['distance'] < 10.0: # 警告情况 + # 选择障碍物较少的一侧 + if len(left_obstacles) < len(right_obstacles): + return 4 # 向右避让 + else: + return 3 # 向左避让 + + return None - def reward(self, speed_kmh, current_steer): - """增强的奖励函数 - 特别强调行人避障""" + def reward_enhanced(self, speed_kmh, current_steer): + """增强的奖励函数,特别注重避障""" reward = 0 done = False @@ -363,173 +559,272 @@ def reward(self, speed_kmh, current_steer): vehicle_location = self.vehicle.get_location() vehicle_rotation = self.vehicle.get_transform().rotation.yaw - # 1. 道路保持奖励(重要的基础) + # 1. 基础安全奖励 + reward += 0.1 # 每步存活奖励 + + # 2. 障碍物检测与避让(最高优先级) + obstacle_reward, obstacle_done, avoidance_success = self.calculate_obstacle_reward() + reward += obstacle_reward + if obstacle_done: + done = True + + # 记录避障成功 + if avoidance_success: + self.successful_avoidance += 1 + + # 3. 道路保持奖励 heading_error = abs(vehicle_rotation) + if heading_error < 5: + reward += 0.5 + elif heading_error < 15: + reward += 0.2 + elif heading_error < 30: + reward += 0.1 + else: + reward -= 0.2 - if heading_error < 5: # 完美保持方向 - reward += 0.8 # 略微降低权重,给行人避障更多空间 - elif heading_error < 15: # 良好保持 - reward += 0.4 - elif heading_error < 30: # 可接受 + # 4. 速度奖励(平衡避障和前进) + if 15 <= speed_kmh <= 35: + reward += 0.3 + elif 5 <= speed_kmh < 15: reward += 0.1 - else: # 方向偏差过大 - reward -= 0.3 * (heading_error / 30.0) # 降低惩罚强度 + elif 35 < speed_kmh <= 45: + reward += 0.05 + elif speed_kmh > 45: + reward -= 0.3 + else: + reward -= 0.05 + + # 5. 转向平滑性奖励 + steer_penalty = abs(current_steer) * 0.2 + reward -= steer_penalty + + # 6. 碰撞检测(已经在obstacle_reward中处理,但这里双重检查) + if len(self.collision_history) > 0: + # 检查最近2秒内是否有碰撞 + recent_collisions = [c for c in self.collision_history + if time.time() - c['timestamp'] < 2.0] + if recent_collisions: + reward = -20 + done = True + print(f"Episode {self.current_episode}: 发生碰撞!") + + # 7. 进度奖励 + progress = (vehicle_location.x + 81) / 236.0 + reward += progress * 0.3 + + # 8. 边界检查 + if vehicle_location.x > 155: + reward += 25 # 成功到达终点 + done = True + print(f"Episode {self.current_episode}: 成功到达终点!") + + # 根据避障表现给予额外奖励 + if self.successful_avoidance > 0: + reward += self.successful_avoidance * 2 + print(f"成功避让 {self.successful_avoidance} 次障碍物,获得额外奖励!") + + elif vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30: + reward -= 5 + done = True + print(f"Episode {self.current_episode}: 偏离道路!") - # 2. 行人避障(最高优先级) - 增强奖励机制 - min_ped_distance = float('inf') - closest_pedestrian = None + # 9. 时间惩罚(防止停滞) + elapsed_time = time.time() - self.episode_start_time + if elapsed_time > 60 and progress < 0.3: # 1分钟内进度不足30% + reward -= 0.01 * elapsed_time - # 统计有效行人数量 - active_pedestrians = 0 + # 限制极端奖励值 + reward = np.clip(reward, -25, 30) + return reward, done, { + 'obstacle_avoidance_success': avoidance_success, + 'obstacle_encounter': self.obstacle_encounters > 0, + 'progress': progress + } + + def calculate_obstacle_reward(self): + """计算障碍物相关奖励""" + reward = 0 + done = False + avoidance_success = False + + vehicle_location = self.vehicle.get_location() + + # 检测最近的障碍物 + min_distance = float('inf') + closest_obstacle = None + + # 检查行人 for walker in self.walker_list: if not walker.is_alive: continue - active_pedestrians += 1 ped_location = walker.get_location() dx = vehicle_location.x - ped_location.x dy = vehicle_location.y - ped_location.y distance = math.sqrt(dx**2 + dy**2) - if distance < min_ped_distance: - min_ped_distance = distance - closest_pedestrian = walker - - # 如果当前没有有效行人,重置距离 - if active_pedestrians == 0: - min_ped_distance = float('inf') - - # 行人距离分级奖励 - 增强避障激励 - if min_ped_distance < 100: # 只考虑100米内的行人 - if min_ped_distance < 3.0: # 紧急避让距离 - reward -= 8.0 # 增加惩罚 + if distance < min_distance: + min_distance = distance + closest_obstacle = walker + + self.last_ped_distance = min_distance + + # 障碍物距离处理 + if min_distance < 100: # 只考虑100米内的障碍物 + self.obstacle_encounters += 1 + + if min_distance < 3.0: # 碰撞距离 + reward -= 15 done = True - print(f"Episode {self.current_episode}: 与行人距离过近 ({min_ped_distance:.1f}m)!") - elif min_ped_distance < 5.0: # 危险距离 - reward -= 3.0 # 增加惩罚 + print(f"Episode {self.current_episode}: 与障碍物距离过近 ({min_distance:.1f}m)!") + + elif min_distance < 5.0: # 危险距离 + reward -= 5 # 计算避让方向 - if closest_pedestrian: - ped_y = closest_pedestrian.get_location().y + if closest_obstacle: + ped_y = closest_obstacle.get_location().y veh_y = vehicle_location.y - # 如果行人在车辆左侧,鼓励右转;反之鼓励左转 if ped_y < veh_y: # 行人在左侧 self.suggested_action = 4 # 右转 else: # 行人在右侧 self.suggested_action = 3 # 左转 - elif min_ped_distance < 8.0: # 预警距离 - reward -= 0.8 - elif min_ped_distance < 12.0: # 安全距离 - reward += 0.5 # 增加安全距离奖励 + + elif min_distance < 8.0: # 警告距离 + reward -= 1 + + elif min_distance < 12.0: # 安全距离,成功避让 + reward += 2 + avoidance_success = True + else: # 非常安全 - reward += 0.2 - - # 3. 成功避障奖励 - 当成功避开行人后给予额外奖励 - if self.last_ped_distance < 8.0 and min_ped_distance > self.last_ped_distance: - # 如果上次距离危险,这次距离更远了,说明成功避让 - reward += 0.3 - - self.last_ped_distance = min_ped_distance # 保存当前距离 - - # 4. 速度奖励(平衡避障和前进) - if 15 <= speed_kmh <= 35: # 理想速度区间 - reward += 0.4 - elif 5 <= speed_kmh < 15: # 较慢但安全(避障时可能需要减速) - reward += 0.2 - elif 35 < speed_kmh <= 45: # 稍快 - reward += 0.1 - elif speed_kmh > 45: # 过快,在行人环境中危险 - reward -= 0.5 # 增加惩罚 - else: # 停车或极慢 - reward -= 0.05 # 轻微惩罚 - - # 5. 转向平滑性奖励(避障时可能需要适当转向) - steer_penalty = abs(current_steer) * 0.3 # 降低惩罚,给避障转向更多空间 - reward -= steer_penalty + reward += 0.5 - # 6. 碰撞检测 - if len(self.collision_history) != 0: - reward = -15 # 增加碰撞惩罚 - done = True - print(f"Episode {self.current_episode}: 发生碰撞!") + # 检查历史避障成功(如果上次危险,这次安全了) + if hasattr(self, 'last_min_distance') and self.last_min_distance < 8.0 and min_distance > self.last_min_distance: + reward += 1.5 # 成功避让奖励 + avoidance_success = True - # 7. 进度奖励(次要于避障) - progress = (vehicle_location.x + 81) / 236.0 # 从-81到155 - reward += progress * 0.2 # 降低进度权重 + # 更新历史距离 + if hasattr(self, 'last_min_distance'): + self.last_min_distance = min_distance - # 8. 边界检查 - if vehicle_location.x > 155: # 成功到达终点 - reward += 20 # 增加到达终点的奖励 - done = True - print(f"Episode {self.current_episode}: 成功到达终点!") - elif vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30: # 偏离道路 - reward -= 3 # 降低偏离惩罚 - done = True - print(f"Episode {self.current_episode}: 偏离道路!") - - return reward, done + return reward, done, avoidance_success def step(self, action): - """执行动作并返回新状态 - 增强平滑性和安全性""" + """执行动作并返回新状态""" # 获取当前速度 velocity = self.vehicle.get_velocity() speed_kmh = 3.6 * math.sqrt(velocity.x**2 + velocity.y**2 + velocity.z**2) - # 5个动作: 0-减速, 1-保持, 2-加速, 3-左转, 4-右转 + # 根据障碍物警告级别调整动作 + adjusted_action = self.adjust_action_for_obstacles(action, speed_kmh) - # 根据速度调整转向幅度 - 速度越快,转向越平滑 - speed_factor = max(0.5, min(1.0, 30.0 / max(1.0, speed_kmh))) + # 速度相关的转向幅度 + speed_factor = max(0.4, min(1.0, 25.0 / max(1.0, speed_kmh))) # 基础控制参数 throttle = 0.0 brake = 0.0 steer = 0.0 - # 速度控制 - if action == 0: # 减速 + # 动作执行 + if adjusted_action == 0: # 减速 throttle = 0.0 - brake = 0.5 - elif action == 1: # 保持/轻微加速 + brake = 0.6 + elif adjusted_action == 1: # 保持 throttle = 0.3 brake = 0.0 - elif action == 2: # 加速 + elif adjusted_action == 2: # 加速 throttle = 0.7 brake = 0.0 - elif action == 3: # 左转 + elif adjusted_action == 3: # 左转 throttle = 0.4 brake = 0.0 - steer = -0.2 * speed_factor # 速度相关的转向幅度 - elif action == 4: # 右转 + steer = -0.25 * speed_factor + elif adjusted_action == 4: # 右转 throttle = 0.4 brake = 0.0 - steer = 0.2 * speed_factor # 速度相关的转向幅度 + steer = 0.25 * speed_factor - # 限制连续同向转向 - 防止过度转向 - if (action == 3 and self.last_action == 3) or (action == 4 and self.last_action == 4): + # 防止过度转向 + if (adjusted_action == 3 and self.last_action == 3) or (adjusted_action == 4 and self.last_action == 4): self.same_steer_counter += 1 - if self.same_steer_counter > 3: # 连续3次同向转向后强制回正 - steer *= 0.5 # 减小转向幅度 - throttle *= 0.8 # 减速 + if self.same_steer_counter > 3: + steer *= 0.5 + throttle *= 0.7 else: self.same_steer_counter = 0 - # 记录上一个动作 - self.last_action = action + self.last_action = adjusted_action # 应用控制 self.vehicle.apply_control(carla.VehicleControl( - throttle=throttle, - brake=brake, - steer=steer + throttle=throttle, + brake=brake, + steer=steer, + hand_brake=False, + reverse=False )) # 等待物理更新 time.sleep(0.05) - # 计算奖励和完成状态 - reward, done = self.reward(speed_kmh, steer) - - # 限制极端奖励值 - reward = np.clip(reward, -15, 20) + # 计算奖励 + reward, done, extra_info = self.reward_enhanced(speed_kmh, steer) + + # 为多目标优化准备额外信息 + obstacle_info = { + 'min_distance': self.last_ped_distance, + 'warning_level': self.obstacle_warning_level, + 'obstacle_type': self.last_obstacle_type, + 'avoidance_success': extra_info['obstacle_avoidance_success'] + } + + return self.front_camera, reward, done, { + 'obstacle_info': obstacle_info, + 'speed_kmh': speed_kmh, + 'progress': extra_info['progress'] + } + + def adjust_action_for_obstacles(self, action, speed_kmh): + """根据障碍物情况调整动作""" + # 如果有建议的避让动作,优先执行 + if self.suggested_action is not None: + adjusted = self.suggested_action + self.suggested_action = None + return adjusted + + # 根据障碍物警告级别调整 + if self.obstacle_warning_level >= 2: # 中等或严重警告 + if action == 2: # 如果原动作是加速,改为保持或减速 + return 1 if speed_kmh < 20 else 0 + elif action in [3, 4] and speed_kmh > 30: # 高速时避免急转 + return 1 # 改为保持 + + # 如果距离障碍物很近,强制减速 + if self.last_ped_distance < 6.0 and action == 2: + return 0 # 强制减速 + + return action + + def get_environment_info(self): + """获取环境信息""" + vehicle_location = self.vehicle.get_location() + velocity = self.vehicle.get_velocity() + speed_kmh = 3.6 * math.sqrt(velocity.x**2 + velocity.y**2) - return self.front_camera, reward, done, None \ No newline at end of file + return { + 'vehicle_location': (vehicle_location.x, vehicle_location.y), + 'speed_kmh': speed_kmh, + 'pedestrian_count': len(self.walker_list), + 'collision_count': len(self.collision_history), + 'successful_avoidance': self.successful_avoidance, + 'obstacle_warning_level': self.obstacle_warning_level, + 'last_obstacle_distance': self.last_ped_distance + } + + +# 为兼容性保留原类名 +CarEnv = EnhancedCarEnv \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py index bf5be3a954..113bf44652 100644 --- a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py +++ b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py @@ -1,110 +1,136 @@ # Hyperparameters.py -# 深度强化学习超参数配置 +# 深度强化学习超参数配置 - 优化避障性能 -DISCOUNT = 0.97 -# 未来奖励的折扣因子 - 提高未来奖励的重要性 +DISCOUNT = 0.98 +# 提高未来奖励的重要性,让智能体更关注长期安全 FPS = 60 -# 模拟环境的帧率 -MEMORY_FRACTION = 0.35 -# GPU内存分配比例 +MEMORY_FRACTION = 0.6 +# 增加GPU内存分配 REWARD_OFFSET = -100 -# 停止模拟的奖励阈值 -MIN_REPLAY_MEMORY_SIZE = 3_000 -# 开始训练前经验回放缓冲区的最小大小 - 增加以获得更稳定训练 +MIN_REPLAY_MEMORY_SIZE = 4_000 +# 增加最小回放缓冲区大小 -REPLAY_MEMORY_SIZE = 10_000 -# 经验回放缓冲区的最大容量 +REPLAY_MEMORY_SIZE = 15_000 +# 增加回放缓冲区容量 -MINIBATCH_SIZE = 64 -# 每次训练从经验回放中采样的经验数量 - 增加批次大小 +MINIBATCH_SIZE = 32 +# 减少批次大小以避免内存不足 PREDICTION_BATCH_SIZE = 1 -# 预测阶段使用的批次大小 TRAINING_BATCH_SIZE = MINIBATCH_SIZE // 4 -# 训练阶段使用的批次大小 -EPISODES = 800 # 从1000减少到800,因为减少了无行人阶段 -# 智能体训练的总轮次数 +EPISODES = 100 # 减少训练轮次以加快训练速度 +# 增加训练轮次以获得更好的避障性能 -SECONDS_PER_EPISODE = 60 -# 每轮训练的秒数 +SECONDS_PER_EPISODE = 70 +# 稍微增加每轮时间 -MIN_EPSILON = 0.01 -# 最小探索率 +MIN_EPSILON = 0.02 +# 稍微提高最小探索率 EPSILON = 1.0 -# 初始探索率 -EPSILON_DECAY = 0.998 -# 探索率的衰减率 - 减缓衰减速度 +EPSILON_DECAY = 0.995 +# 减缓探索率衰减 -MODEL_NAME = "YY_Optimized" -# 训练模型的名称标识 +MODEL_NAME = "YY_Enhanced_ObstacleAvoidance" +# 更新模型名称 -MIN_REWARD = 5 -# 被认为是"良好"或"积极"经验的最小奖励值 +MIN_REWARD = 8 +# 提高良好经验的最低奖励阈值 -UPDATE_TARGET_EVERY = 20 -# 目标网络更新的频率 - 增加以获得更稳定训练 +UPDATE_TARGET_EVERY = 25 +# 减缓目标网络更新频率 AGGREGATE_STATS_EVERY = 10 -# 计算和聚合统计信息(如平均得分、奖励)的频率 SHOW_PREVIEW = False -# 是否显示预览窗口 - 测试时设为False以显示CARLA主窗口 -IM_WIDTH = 640 -# 预览或模拟中捕获图像的宽度 +IM_WIDTH = 320 -IM_HEIGHT = 480 -# 预览或模拟中捕获图像的高度 +IM_HEIGHT = 240 SLOW_COUNTER = 330 -# 慢速计数器阈值 LOW_REWARD_THRESHOLD = -2 -# 低奖励阈值 -SUCCESSFUL_THRESHOLD = 3 -# 成功阈值 +SUCCESSFUL_THRESHOLD = 5 +# 提高成功阈值 -LEARNING_RATE = 0.00005 -# 优化器的学习率 - 降低以获得更稳定训练 +LEARNING_RATE = 0.00003 +# 降低学习率以获得更稳定训练 # PER (优先经验回放) 参数 -PER_ALPHA = 0.6 -# 优先级程度 (0 = 均匀采样, 1 = 完全优先级) +PER_ALPHA = 0.7 +# 提高优先级程度 -PER_BETA_START = 0.4 -# 重要性采样权重起始值 +PER_BETA_START = 0.5 +# 提高重要性采样起始值 -PER_BETA_FRAMES = 100000 -# beta线性增长的帧数 +PER_BETA_FRAMES = 120000 +# 增加beta线性增长的帧数 # 训练策略参数 USE_CURRICULUM_LEARNING = True -# 是否使用课程学习 USE_MULTI_OBJECTIVE = True -# 是否使用多目标优化 -USE_IMITATION_LEARNING = False -# 是否使用模仿学习(预训练) +USE_IMITATION_LEARNING = True +# 启用模仿学习 -# 多目标优化权重(这些权重会自动调整) -SAFETY_WEIGHT = 0.40 -EFFICIENCY_WEIGHT = 0.25 -COMFORT_WEIGHT = 0.20 -RULE_FOLLOWING_WEIGHT = 0.15 +USE_ATTENTION_MECHANISM = True +# 启用注意力机制 -# 课程学习参数 -CURRICULUM_STAGES = 5 -# 课程学习阶段数量 +USE_ENHANCED_MODEL = True +# 启用增强版模型 + +# 避障优化参数 +OBSTACLE_AVOIDANCE_WEIGHT = 0.15 +NEAR_MISS_REWARD = 2.0 +SAFE_DISTANCE_REWARD = 1.0 +COLLISION_PENALTY = -15.0 +DANGER_ZONE_PENALTY = -5.0 -CURRICULUM_SUCCESS_THRESHOLDS = [0.3, 0.5, 0.7, 0.85, 0.9] -# 每个阶段进入下一阶段所需的成功率阈值 \ No newline at end of file +# 课程学习参数 +CURRICULUM_STAGES = 6 +# 增加课程学习阶段 + +CURRICULUM_SUCCESS_THRESHOLDS = [0.4, 0.5, 0.6, 0.7, 0.75, 0.8] +# 调整成功率阈值 + +# 新增:障碍物检测参数 +OBSTACLE_DETECTION_MODE = 'advanced' # 'basic', 'advanced', 'none' +OBSTACLE_WARNING_THRESHOLDS = [20.0, 12.0, 8.0, 5.0] # 警告级别距离阈值 +BUILDING_PROXIMITY_THRESHOLD = 15.0 # 建筑物接近阈值(米) + +# 新增:奖励函数参数 +REWARD_CONFIG = { + 'base_survival': 0.1, + 'perfect_lane_keeping': 0.8, + 'good_lane_keeping': 0.4, + 'acceptable_lane_keeping': 0.1, + 'lane_deviation_penalty': 0.3, + 'ideal_speed_range': (18, 38), + 'ideal_speed_reward': 0.4, + 'low_speed_reward': 0.2, + 'high_speed_penalty': 0.3, + 'steer_smoothness_factor': 0.2, + 'progress_weight': 0.3, + 'destination_bonus': 25, + 'off_road_penalty': 5, + 'time_penalty_threshold': 60 # 秒 +} + +# 新增:动作调整参数 +ACTION_ADJUSTMENT = { + 'speed_factor_min': 0.4, + 'speed_factor_max': 1.0, + 'emergency_brake_distance': 6.0, + 'max_consecutive_steer': 3, + 'high_speed_steer_threshold': 30 +} \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Model.py b/src/Unmanned_vehicle_AD_DQN/Model.py index c419ae6c35..7c860fdcda 100644 --- a/src/Unmanned_vehicle_AD_DQN/Model.py +++ b/src/Unmanned_vehicle_AD_DQN/Model.py @@ -10,27 +10,31 @@ import matplotlib.pyplot as plt from collections import deque from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Concatenate, Conv2D, AveragePooling2D, Activation, \ - Flatten, Dropout, BatchNormalization, MaxPooling2D, Multiply, Add, Lambda, Subtract -from tensorflow.keras.optimizers import Adam + Flatten, Dropout, BatchNormalization, MaxPooling2D, Multiply, Add, Lambda, Subtract, UpSampling2D, Conv2DTranspose, \ + Reshape, Layer, LayerNormalization +from tensorflow.keras.optimizers import Adam, RMSprop from tensorflow.keras.models import Sequential, Model -from tensorflow.keras.callbacks import TensorBoard +from tensorflow.keras.callbacks import TensorBoard, EarlyStopping, ReduceLROnPlateau +from tensorflow.keras.regularizers import l2 import tensorflow as tf import tensorflow.keras.backend as backend -from threading import Thread +from threading import Thread, Lock from Environment import * from Hyperparameters import * import pickle import json from datetime import datetime +from scipy import ndimage -# 自定义TensorBoard类 -class ModifiedTensorBoard(TensorBoard): +# 增强的TensorBoard类 +class EnhancedTensorBoard(TensorBoard): def __init__(self, **kwargs): super().__init__(**kwargs) self._log_write_dir = self.log_dir self.step = 1 self.writer = tf.summary.create_file_writer(self.log_dir) + self.lock = Lock() def set_model(self, model): self.model = model @@ -50,302 +54,557 @@ def on_train_end(self, logs=None): pass def update_stats(self, **stats): - with self.writer.as_default(): - for key, value in stats.items(): - tf.summary.scalar(key, value, step=self.step) + with self.lock: + with self.writer.as_default(): + for key, value in stats.items(): + tf.summary.scalar(key, value, step=self.step) self.writer.flush() -# 优先经验回放缓冲区 -class PrioritizedReplayBuffer: +# 增强的优先经验回放缓冲区 +class EnhancedPrioritizedReplayBuffer: def __init__(self, max_size=REPLAY_MEMORY_SIZE, alpha=0.6, beta_start=0.4, beta_frames=100000): self.max_size = max_size - self.alpha = alpha # 优先级程度 (0 = 均匀采样, 1 = 完全优先级) - self.beta_start = beta_start # 重要性采样权重起始值 - self.beta_frames = beta_frames # beta线性增长的帧数 + self.alpha = alpha + self.beta_start = beta_start + self.beta_frames = beta_frames self.frame = 1 - # 使用循环缓冲区 + # 使用分段存储以提高效率 self.buffer = deque(maxlen=max_size) self.priorities = deque(maxlen=max_size) + self.obstacle_experiences = [] # 存储避障相关经验 + self.success_experiences = [] # 存储成功经验 def __len__(self): return len(self.buffer) def beta(self): - """线性递增的beta值,用于重要性采样权重""" return min(1.0, self.beta_start + self.frame * (1.0 - self.beta_start) / self.beta_frames) - def add(self, experience, error=None): + def add(self, experience, error=None, is_obstacle=False, is_success=False): """添加经验到缓冲区""" if error is None: - priority = max(self.priorities) if self.priorities else 1.0 + if self.priorities: + priority = max(self.priorities) * 0.8 + else: + priority = 1.0 else: priority = (abs(error) + 1e-5) ** self.alpha self.buffer.append(experience) self.priorities.append(priority) - def sample(self, batch_size): - """从缓冲区中采样一批经验""" + # 分类存储特殊经验 + if is_obstacle: + self.obstacle_experiences.append((experience, error)) + if len(self.obstacle_experiences) > self.max_size // 10: + self.obstacle_experiences.pop(0) + + if is_success: + self.success_experiences.append((experience, error)) + if len(self.success_experiences) > self.max_size // 10: + self.success_experiences.pop(0) + + self.frame += 1 + + def sample(self, batch_size, obstacle_ratio=0.3, success_ratio=0.2): + """改进的采样策略,确保包含避障和成功经验""" if len(self.buffer) == 0: return [], [], [], [] - # 计算采样概率 - priorities = np.array(self.priorities, dtype=np.float32) - probs = priorities ** self.alpha - probs /= probs.sum() - - # 采样索引 - indices = np.random.choice(len(self.buffer), batch_size, p=probs) - + # 基础采样(来自普通缓冲区) + base_size = int(batch_size * (1 - obstacle_ratio - success_ratio)) + probs = None + if base_size > 0: + priorities = np.array(self.priorities, dtype=np.float32) + if len(priorities) != len(self.buffer): + # 如果长度不匹配,重新初始化 priorities + self.priorities = [1.0] * len(self.buffer) + priorities = np.array(self.priorities, dtype=np.float32) + probs = priorities ** self.alpha + probs /= probs.sum() + base_indices = np.random.choice(len(self.buffer), base_size, p=probs) + else: + base_indices = [] + + # 避障经验采样 + obstacle_size = int(batch_size * obstacle_ratio) + obstacle_indices = [] + if self.obstacle_experiences and obstacle_size > 0: + obstacle_size = min(obstacle_size, len(self.obstacle_experiences)) + obstacle_samples = random.sample(self.obstacle_experiences, obstacle_size) + # 找到这些样本在缓冲区中的索引(近似) + for exp, _ in obstacle_samples: + try: + idx = list(self.buffer).index(exp) + obstacle_indices.append(idx) + except: + pass + + # 成功经验采样 + success_size = int(batch_size * success_ratio) + success_indices = [] + if self.success_experiences and success_size > 0: + success_size = min(success_size, len(self.success_experiences)) + success_samples = random.sample(self.success_experiences, success_size) + for exp, _ in success_samples: + try: + idx = list(self.buffer).index(exp) + success_indices.append(idx) + except: + pass + + # 合并所有索引 + all_indices = list(base_indices) + obstacle_indices + success_indices + + # 如果数量不够,用基础采样补足 + if len(all_indices) < batch_size: + additional = batch_size - len(all_indices) + if probs is not None: + additional_indices = np.random.choice(len(self.buffer), additional, p=probs) + else: + additional_indices = np.random.choice(len(self.buffer), additional) + all_indices.extend(additional_indices) + # 获取样本 - samples = [self.buffer[i] for i in indices] + samples = [self.buffer[i] for i in all_indices] # 计算重要性采样权重 total = len(self.buffer) - weights = (total * probs[indices]) ** (-self.beta()) - weights /= weights.max() # 归一化 + priorities_array = np.array(self.priorities, dtype=np.float32) + if len(priorities_array) != total: + # 如果长度不匹配,重新初始化 + self.priorities = [1.0] * total + priorities_array = np.array(self.priorities, dtype=np.float32) - # 更新帧计数器 - self.frame += 1 + weights = (total * (priorities_array[all_indices] ** self.alpha) / (priorities_array ** self.alpha).sum()) ** (-self.beta()) + + # 归一化权重 + max_weight = weights.max() + if max_weight > 0: + weights /= max_weight - return indices, samples, weights + return all_indices, samples, weights def update_priorities(self, indices, errors): """更新采样经验的优先级""" for idx, error in zip(indices, errors): if 0 <= idx < len(self.priorities): + # 对于避障相关的经验,给予更高的优先级权重 + is_obstacle_exp = any(idx == list(self.buffer).index(exp[0]) for exp in self.obstacle_experiences if exp[0] in self.buffer) + if is_obstacle_exp: + error *= 1.5 # 提高避障经验的优先级 self.priorities[idx] = (abs(error) + 1e-5) ** self.alpha -# 课程学习管理器 -class CurriculumManager: +# 障碍物注意力模块 +class ObstacleAttentionLayer(Layer): + def __init__(self, filters, kernel_size=3, **kwargs): + super(ObstacleAttentionLayer, self).__init__(**kwargs) + self.filters = filters + self.kernel_size = kernel_size + + def build(self, input_shape): + # 空间注意力 + self.spatial_conv = Conv2D(self.filters, (self.kernel_size, self.kernel_size), + padding='same', activation='relu') + self.spatial_attention = Conv2D(1, (1, 1), padding='same', activation='sigmoid') + + # 通道注意力 + self.channel_gap = GlobalAveragePooling2D() + self.channel_fc1 = Dense(self.filters // 8, activation='relu') + self.channel_fc2 = Dense(self.filters, activation='sigmoid') + + super(ObstacleAttentionLayer, self).build(input_shape) + + def call(self, inputs): + # 空间注意力 + spatial_features = self.spatial_conv(inputs) + spatial_attention = self.spatial_attention(spatial_features) + + # 通道注意力 + channel_weights = self.channel_gap(inputs) + channel_weights = self.channel_fc1(channel_weights) + channel_weights = self.channel_fc2(channel_weights) + channel_weights = Reshape((1, 1, self.filters))(channel_weights) + + # 合并注意力 + attended = Multiply()([inputs, spatial_attention]) + attended = Multiply()([attended, channel_weights]) + + # 残差连接 + output = Add()([inputs, attended]) + return output + + +# 课程学习管理器 - 增强版 +class EnhancedCurriculumManager: def __init__(self, env): self.env = env self.current_stage = 0 - self.stage_thresholds = [0.3, 0.5, 0.7, 0.85] # 成功率阈值 + self.stage_progress = 0.0 # 阶段内进度(0-1) + + # 增强的阶段配置,更专注于避障训练 self.stage_configs = [ - # 阶段0: 入门 + # 阶段0: 基础避障训练 + { + 'name': '基础避障', + 'pedestrian_cross': 2, + 'pedestrian_normal': 1, + 'pedestrian_speed_min': 0.3, + 'pedestrian_speed_max': 0.8, + 'max_episode_steps': 900, + 'success_threshold': 0.4, + 'obstacle_focus': 0.8, # 避障训练权重 + 'speed_limit': 25 + }, + # 阶段1: 简单场景 { - 'pedestrian_cross': 4, # 十字路口行人数量 - 'pedestrian_normal': 2, # 普通路段行人数量 - 'pedestrian_speed_min': 0.5, # 行人最低速度 - 'pedestrian_speed_max': 1.0, # 行人最高速度 - 'max_episode_steps': 1200, # 最大步数 (20秒 * 60FPS) - 'success_threshold': 0.3 # 进入下一阶段成功率 + 'name': '简单场景', + 'pedestrian_cross': 4, + 'pedestrian_normal': 2, + 'pedestrian_speed_min': 0.5, + 'pedestrian_speed_max': 1.2, + 'max_episode_steps': 1200, + 'success_threshold': 0.5, + 'obstacle_focus': 0.7, + 'speed_limit': 30 }, - # 阶段1: 初级 + # 阶段2: 中等难度 { + 'name': '中等难度', 'pedestrian_cross': 6, 'pedestrian_normal': 3, 'pedestrian_speed_min': 0.7, - 'pedestrian_speed_max': 1.3, - 'max_episode_steps': 1800, # 30秒 - 'success_threshold': 0.5 + 'pedestrian_speed_max': 1.5, + 'max_episode_steps': 1800, + 'success_threshold': 0.6, + 'obstacle_focus': 0.6, + 'speed_limit': 35 }, - # 阶段2: 中级 + # 阶段3: 复杂场景 { + 'name': '复杂场景', 'pedestrian_cross': 8, 'pedestrian_normal': 4, 'pedestrian_speed_min': 0.8, - 'pedestrian_speed_max': 1.5, - 'max_episode_steps': 2400, # 40秒 - 'success_threshold': 0.7 + 'pedestrian_speed_max': 1.8, + 'max_episode_steps': 2400, + 'success_threshold': 0.7, + 'obstacle_focus': 0.5, + 'speed_limit': 40 }, - # 阶段3: 高级 (正常难度) + # 阶段4: 挑战模式 { + 'name': '挑战模式', 'pedestrian_cross': 10, 'pedestrian_normal': 5, 'pedestrian_speed_min': 1.0, - 'pedestrian_speed_max': 2.0, - 'max_episode_steps': 3600, # 60秒 - 'success_threshold': 0.85 + 'pedestrian_speed_max': 2.2, + 'max_episode_steps': 3000, + 'success_threshold': 0.75, + 'obstacle_focus': 0.4, + 'speed_limit': 45 }, - # 阶段4: 专家 (挑战) + # 阶段5: 专家模式 { + 'name': '专家模式', 'pedestrian_cross': 12, 'pedestrian_normal': 6, 'pedestrian_speed_min': 1.2, 'pedestrian_speed_max': 2.5, 'max_episode_steps': 3600, - 'success_threshold': 0.9 + 'success_threshold': 0.8, + 'obstacle_focus': 0.3, + 'speed_limit': 50 } ] # 训练历史 - self.success_history = deque(maxlen=20) # 记录最近20轮的成功情况 - self.reward_history = deque(maxlen=50) # 记录最近50轮的奖励 + self.success_history = deque(maxlen=20) + self.reward_history = deque(maxlen=50) + self.obstacle_avoidance_history = deque(maxlen=30) # 避障成功率历史 - def update_stage(self, success, reward): + def update_stage(self, success, reward, obstacle_avoidance_score=0): """更新训练阶段""" # 记录历史 self.success_history.append(1 if success else 0) self.reward_history.append(reward) + if obstacle_avoidance_score > 0: + self.obstacle_avoidance_history.append(obstacle_avoidance_score) - # 计算最近成功率 + # 计算统计数据 if len(self.success_history) >= 10: success_rate = sum(self.success_history) / len(self.success_history) avg_reward = np.mean(self.reward_history) if self.reward_history else 0 + obstacle_rate = np.mean(self.obstacle_avoidance_history) if self.obstacle_avoidance_history else 0 + + # 动态更新阶段进度 + current_config = self.get_current_config() + target_threshold = current_config['success_threshold'] - print(f"课程学习 - 当前阶段: {self.current_stage}, 成功率: {success_rate:.2f}, 平均奖励: {avg_reward:.2f}") + # 计算阶段内进度 + if target_threshold > 0: + self.stage_progress = min(1.0, success_rate / target_threshold) # 检查是否可以进入下一阶段 if self.current_stage < len(self.stage_configs) - 1: - next_stage_threshold = self.stage_configs[self.current_stage]['success_threshold'] - if success_rate >= next_stage_threshold and avg_reward > 5: - self.current_stage += 1 - print(f"🎉 课程学习: 进阶到阶段 {self.current_stage}!") - return True + next_stage = self.current_stage + 1 + next_threshold = self.stage_configs[next_stage]['success_threshold'] + + # 进阶条件:达到成功率阈值且有一定的避障表现 + if (success_rate >= current_config['success_threshold'] and + avg_reward > 3 and + obstacle_rate > 0.6): + + self.current_stage = next_stage + self.stage_progress = 0.0 + print(f"🎉 课程学习: 进阶到阶段 {self.current_stage} - {self.stage_configs[self.current_stage]['name']}!") + return True, 'advance' - # 如果表现太差,退回上一阶段 - if self.current_stage > 0 and success_rate < 0.2: + # 如果表现持续不佳,退回上一阶段 + if (self.current_stage > 0 and + success_rate < 0.2 and + len(self.success_history) >= 15): + self.current_stage -= 1 - print(f"⚠️ 课程学习: 退回阶段 {self.current_stage}") - return True + self.stage_progress = 0.5 # 退回后给予中等进度 + print(f"⚠️ 课程学习: 退回阶段 {self.current_stage} - {self.stage_configs[self.current_stage]['name']}") + return True, 'regress' - return False + return False, 'maintain' def get_current_config(self): """获取当前阶段的配置""" return self.stage_configs[min(self.current_stage, len(self.stage_configs) - 1)] - def apply_to_environment(self): - """将当前阶段配置应用到环境""" + def get_stage_info(self): + """获取阶段信息""" config = self.get_current_config() - # 注意:这里需要修改Environment.py中的行人生成逻辑来支持这些参数 - # 暂时返回配置,由外部调用者处理 - return config + return { + 'stage': self.current_stage, + 'name': config['name'], + 'progress': self.stage_progress, + 'pedestrian_total': config['pedestrian_cross'] + config['pedestrian_normal'], + 'difficulty': self.current_stage / len(self.stage_configs) + } -# 多目标优化器 -class MultiObjectiveOptimizer: +# 多目标优化器 - 增强版,专注于避障 +class EnhancedMultiObjectiveOptimizer: def __init__(self): - # 定义优化目标及其权重(可动态调整) + # 动态权重调整,初期更注重安全 self.objectives = { 'safety': { - 'weight': 0.4, + 'weight': 0.50, # 提高安全权重 'description': '安全避障和避免碰撞', - 'metrics': ['collision_avoidance', 'pedestrian_distance'] + 'target_value': 0.9, + 'current_performance': 0.0, + 'improvement_rate': 0.0 }, 'efficiency': { - 'weight': 0.25, + 'weight': 0.20, 'description': '快速到达目的地', - 'metrics': ['progress_speed', 'total_time'] + 'target_value': 0.7, + 'current_performance': 0.0, + 'improvement_rate': 0.0 }, 'comfort': { - 'weight': 0.2, + 'weight': 0.15, 'description': '平稳驾驶体验', - 'metrics': ['smoothness', 'steering_changes'] + 'target_value': 0.6, + 'current_performance': 0.0, + 'improvement_rate': 0.0 }, - 'rule_following': { + 'obstacle_avoidance': { # 新增:专门针对障碍物避让 'weight': 0.15, - 'description': '遵守交通规则', - 'metrics': ['lane_keeping', 'speed_limit'] + 'description': '有效避让行人和建筑物', + 'target_value': 0.8, + 'current_performance': 0.0, + 'improvement_rate': 0.0 } } # 指标跟踪 self.metrics_history = { - 'safety': [], - 'efficiency': [], - 'comfort': [], - 'rule_following': [] + 'safety': deque(maxlen=100), + 'efficiency': deque(maxlen=100), + 'comfort': deque(maxlen=100), + 'obstacle_avoidance': deque(maxlen=100) } - def compute_composite_reward(self, metrics): - """计算综合奖励值""" + # 避障特别奖励参数 + self.obstacle_avoidance_bonus = { + 'near_miss': 2.0, # 成功避让奖励 + 'safe_distance': 1.0, # 保持安全距离奖励 + 'collision_penalty': -10.0, # 碰撞惩罚 + 'danger_zone_penalty': -3.0 # 进入危险区域惩罚 + } + + def compute_composite_reward(self, metrics, obstacle_info=None): + """计算综合奖励值,特别关注避障""" composite = 0 + # 基础目标计算 for obj_name, obj_info in self.objectives.items(): if obj_name in metrics: - # 归一化处理每个目标的贡献 normalized_value = self._normalize_metric(metrics[obj_name], obj_name) composite += normalized_value * obj_info['weight'] - # 记录指标历史 + # 更新性能记录 self.metrics_history[obj_name].append(normalized_value) + self.objectives[obj_name]['current_performance'] = np.mean( + self.metrics_history[obj_name]) if self.metrics_history[obj_name] else 0 - # 特殊惩罚项 + # 特别避障奖励 + if obstacle_info: + composite += self._compute_obstacle_avoidance_reward(obstacle_info) + + # 特殊惩罚项(增强) if metrics.get('collision', False): - composite -= 10 + composite += self.obstacle_avoidance_bonus['collision_penalty'] if metrics.get('off_road', False): composite -= 5 if metrics.get('dangerous_action', False): - composite -= 3 + composite -= 4 + if metrics.get('near_miss', False): # 成功避让奖励 + composite += self.obstacle_avoidance_bonus['near_miss'] return composite + def _compute_obstacle_avoidance_reward(self, obstacle_info): + """计算避障特别奖励""" + reward = 0 + + # 根据障碍物距离给予奖励 + min_distance = obstacle_info.get('min_distance', float('inf')) + if min_distance < 100: # 只考虑100米内的障碍物 + if min_distance > 15: # 非常安全 + reward += self.obstacle_avoidance_bonus['safe_distance'] * 0.5 + elif min_distance > 10: # 安全 + reward += self.obstacle_avoidance_bonus['safe_distance'] * 0.3 + elif min_distance > 5: # 警告距离 + reward -= self.obstacle_avoidance_bonus['danger_zone_penalty'] * 0.5 + else: # 危险距离 + reward -= self.obstacle_avoidance_bonus['danger_zone_penalty'] + + # 成功避让奖励 + if obstacle_info.get('avoidance_success', False): + reward += self.obstacle_avoidance_bonus['near_miss'] + + return reward + def _normalize_metric(self, value, metric_name): - """归一化指标值到[0, 1]范围""" - # 不同指标的归一化方式不同 + """归一化指标值""" normalization_rules = { - 'safety': lambda x: min(max(x / 10, 0), 1), # 假设安全分满分10 - 'efficiency': lambda x: min(max(x / 100, 0), 1), # 效率分满分100 - 'comfort': lambda x: min(max((x + 5) / 10, 0), 1), # 舒适度[-5, 5] -> [0, 1] - 'rule_following': lambda x: min(max(x, 0), 1) # 规则遵循度[0, 1] + 'safety': lambda x: min(max(x / 10, 0), 1), + 'efficiency': lambda x: min(max(x / 100, 0), 1), + 'comfort': lambda x: min(max((x + 5) / 10, 0), 1), + 'obstacle_avoidance': lambda x: min(max(x, 0), 1) } if metric_name in normalization_rules: return normalization_rules[metric_name](value) - return min(max(value, 0), 1) # 默认截断到[0, 1] + return min(max(value, 0), 1) - def adjust_weights(self, performance_feedback): - """根据性能反馈动态调整权重""" - # 如果某个目标表现持续较差,增加其权重 + def adjust_weights(self, performance_feedback, training_stage=0): + """动态调整权重,考虑训练阶段""" + # 首先更新指标历史 + if performance_feedback: + for obj_name, value in performance_feedback.items(): + if obj_name in self.metrics_history: + self.metrics_history[obj_name].append(value) + + # 根据训练阶段调整基础权重 + if training_stage < 2: # 初期阶段更注重安全 + self.objectives['safety']['weight'] = 0.6 + self.objectives['efficiency']['weight'] = 0.15 + self.objectives['obstacle_avoidance']['weight'] = 0.15 + self.objectives['comfort']['weight'] = 0.10 + elif training_stage < 4: # 中期平衡 + self.objectives['safety']['weight'] = 0.5 + self.objectives['efficiency']['weight'] = 0.2 + self.objectives['obstacle_avoidance']['weight'] = 0.15 + self.objectives['comfort']['weight'] = 0.15 + else: # 后期注重效率 + self.objectives['safety']['weight'] = 0.4 + self.objectives['efficiency']['weight'] = 0.25 + self.objectives['obstacle_avoidance']['weight'] = 0.2 + self.objectives['comfort']['weight'] = 0.15 + + # 基于性能反馈微调 recent_performance = {} for obj in self.objectives: if len(self.metrics_history[obj]) >= 10: - recent_avg = np.mean(self.metrics_history[obj][-10:]) + recent_avg = np.mean(list(self.metrics_history[obj])[-10:]) recent_performance[obj] = recent_avg if recent_performance: - # 找到表现最差的目标 - worst_obj = min(recent_performance, key=recent_performance.get) - best_obj = max(recent_performance, key=recent_performance.get) + # 如果某个目标表现持续低于阈值,增加其权重 + for obj_name, obj_info in self.objectives.items(): + if obj_name in recent_performance: + performance = recent_performance[obj_name] + target = obj_info['target_value'] + + if performance < target * 0.7: # 表现严重不足 + adjustment = 0.03 + obj_info['weight'] += adjustment + # 从表现最好的目标中扣除 + best_obj = max(recent_performance, key=recent_performance.get) + if best_obj != obj_name: + self.objectives[best_obj]['weight'] -= adjustment - # 如果最差目标表现低于阈值,增加其权重 - if recent_performance[worst_obj] < 0.3: - adjustment = 0.05 - self.objectives[worst_obj]['weight'] += adjustment - self.objectives[best_obj]['weight'] -= adjustment - - # 确保权重总和为1 - total = sum(obj['weight'] for obj in self.objectives.values()) - for obj in self.objectives: - self.objectives[obj]['weight'] /= total - - print(f"动态权重调整: {worst_obj}权重↑ {adjustment:.3f}, {best_obj}权重↓ {adjustment:.3f}") + # 确保权重总和为1 + total = sum(obj['weight'] for obj in self.objectives.values()) + for obj in self.objectives: + self.objectives[obj]['weight'] /= total def get_performance_report(self): """生成性能报告""" - report = "多目标优化性能报告:\n" - report += "=" * 50 + "\n" + report = "多目标优化性能报告 (增强避障版):\n" + report += "=" * 60 + "\n" for obj_name, obj_info in self.objectives.items(): history = self.metrics_history[obj_name] if history: - avg = np.mean(history[-20:]) if len(history) >= 20 else np.mean(history) - report += f"{obj_name}(权重:{obj_info['weight']:.2f}): 平均得分={avg:.3f}\n" - report += f" 描述: {obj_info['description']}\n" + history_list = list(history) + avg = np.mean(history_list[-20:]) if len(history_list) >= 20 else np.mean(history_list) + trend = "↑" if len(history_list) >= 2 and history_list[-1] > history_list[-2] else "↓" + report += f"{obj_name:20s} 权重:{obj_info['weight']:.2f} 得分:{avg:.3f}{trend}\n" + report += f" 目标值:{obj_info['target_value']:.2f} - {obj_info['description']}\n" + + # 计算总体避障成功率 + if self.metrics_history['obstacle_avoidance']: + avoidance_rate = np.mean(self.metrics_history['obstacle_avoidance']) + report += f"\n总体避障成功率: {avoidance_rate:.2%}\n" return report -# 模仿学习管理器 -class ImitationLearningManager: +# 模仿学习管理器 - 增强版 +class EnhancedImitationLearningManager: def __init__(self, expert_data_path=None): self.expert_data_path = expert_data_path self.expert_data = [] self.is_pretrained = False + self.avoidance_demos = [] # 专门存储避障演示 def load_expert_data(self, path): """加载专家示范数据""" try: if os.path.exists(path): with open(path, 'rb') as f: - self.expert_data = pickle.load(f) + data = pickle.load(f) + if isinstance(data, dict) and 'demonstrations' in data: + self.expert_data = data['demonstrations'] + if 'avoidance_demos' in data: + self.avoidance_demos = data['avoidance_demos'] + else: + self.expert_data = data print(f"已加载 {len(self.expert_data)} 条专家示范数据") + if self.avoidance_demos: + print(f"已加载 {len(self.avoidance_demos)} 条避障专用演示") return True else: print(f"专家数据文件不存在: {path}") @@ -354,100 +613,212 @@ def load_expert_data(self, path): print(f"加载专家数据失败: {e}") return False - def collect_expert_demonstration(self, env, num_episodes=10): - """收集专家示范数据(可以手动控制或使用规则控制器)""" + def collect_expert_demonstration(self, env, num_episodes=3, focus_avoidance=True): + """收集专家示范数据,特别关注避障场景""" print(f"开始收集专家示范数据 ({num_episodes}个episodes)...") demonstrations = [] + avoidance_demos = [] for episode in range(num_episodes): print(f"收集专家示范 Episode {episode + 1}/{num_episodes}") + # 调整环境难度,专注于避障 + if focus_avoidance: + config = {'pedestrian_cross': 8, 'pedestrian_normal': 4} + env.spawn_pedestrians_with_config(config) + state = env.reset(episode) done = False episode_data = [] + obstacle_encountered = False while not done: - # 这里可以使用规则控制器或手动控制 - # 示例:简单的规则控制器 - action = self._rule_based_controller(env) + # 使用增强的规则控制器,特别注重避障 + action = self._enhanced_rule_based_controller(env) new_state, reward, done, _ = env.step(action) - # 保存示范数据 - episode_data.append({ - 'state': state.copy(), - 'action': action, - 'reward': reward, - 'next_state': new_state.copy(), - 'done': done - }) + # 检查是否遇到障碍物 + min_distance = getattr(env, 'last_ped_distance', float('inf')) + if min_distance < 8.0: + obstacle_encountered = True + + # 只保存避障相关的示范数据 + if min_distance < 8.0: # 只在有障碍物时保存数据 + demo_entry = { + 'state': state.copy(), + 'action': action, + 'reward': reward, + 'next_state': new_state.copy(), + 'done': done, + 'obstacle_nearby': min_distance < 8.0, + 'obstacle_distance': min_distance + } + + episode_data.append(demo_entry) + avoidance_demos.append(demo_entry) state = new_state + # 如果整个episode中有避障场景,保存为避障演示 + if obstacle_encountered: + avoidance_demos.extend(episode_data) + demonstrations.extend(episode_data) env.cleanup_actors() # 保存专家数据 - self.expert_data = demonstrations + self.expert_data = avoidance_demos # 只保存避障演示 + self.avoidance_demos = avoidance_demos + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - save_path = f"expert_data_{timestamp}.pkl" + save_data = { + 'demonstrations': avoidance_demos, # 只保存避障演示 + 'avoidance_demos': avoidance_demos, + 'collection_date': timestamp, + 'num_episodes': num_episodes, + 'focus_avoidance': focus_avoidance + } + + save_path = f"expert_data_enhanced_{timestamp}.pkl" with open(save_path, 'wb') as f: - pickle.dump(demonstrations, f) + pickle.dump(save_data, f) - print(f"专家示范数据已保存到: {save_path}, 共 {len(demonstrations)} 条记录") + print(f"专家示范数据已保存到: {save_path}") + print(f"共 {len(demonstrations)} 条记录,其中 {len(avoidance_demos)} 条避障专用演示") return True - def _rule_based_controller(self, env): - """基于规则的控制器(作为专家示范)""" + def _enhanced_rule_based_controller(self, env): + """增强的基于规则的控制器,特别注重避障""" # 获取车辆状态 vehicle_location = env.vehicle.get_location() velocity = env.vehicle.get_velocity() speed_kmh = 3.6 * math.sqrt(velocity.x**2 + velocity.y**2) - # 简单规则:保持速度在20-40 km/h,避免障碍物 - if speed_kmh < 20: + # 检查前方障碍物 + obstacle_info = self._check_obstacles_ahead(env) + has_obstacle = obstacle_info['has_obstacle'] + obstacle_distance = obstacle_info['distance'] + obstacle_direction = obstacle_info['direction'] # 'left', 'right', 'center' + + # 避障优先级最高 + if has_obstacle: + if obstacle_distance < 5.0: # 紧急避让 + if obstacle_direction == 'left': + return 4 # 右转避让 + elif obstacle_direction == 'right': + return 3 # 左转避让 + else: # 正前方 + return 0 # 紧急制动 + elif obstacle_distance < 10.0: # 预警距离 + if speed_kmh > 20: + return 0 # 减速 + elif obstacle_direction == 'center': + # 轻微转向避让 + return 3 if random.random() > 0.5 else 4 + else: + return 1 # 保持警惕 + + # 速度控制 + if speed_kmh < 15: return 2 # 加速 - elif speed_kmh > 40: + elif speed_kmh > 35: return 0 # 减速 else: - # 检查前方障碍物 - has_obstacle_ahead = self._check_obstacle_ahead(env) - if has_obstacle_ahead: - return 0 # 减速 - else: - return 1 # 保持 - - return 1 # 默认保持 + return 1 # 保持 - def _check_obstacle_ahead(self, env): - """检查前方是否有障碍物(简化版本)""" - # 这里可以添加更复杂的障碍物检测逻辑 - # 暂时返回False - return False + def _check_obstacles_ahead(self, env): + """检查前方障碍物,返回详细信息""" + vehicle_location = env.vehicle.get_location() + + has_obstacle = False + min_distance = float('inf') + obstacle_direction = 'center' + + # 检查行人 + for walker in env.walker_list: + if not walker.is_alive: + continue + + ped_location = walker.get_location() + dx = ped_location.x - vehicle_location.x + dy = ped_location.y - vehicle_location.y + + # 只考虑前方的行人(车辆朝向为0度) + if dx > 0 and abs(dy) < 20: # 前方20米内 + distance = math.sqrt(dx**2 + dy**2) + if distance < min_distance: + min_distance = distance + has_obstacle = True + + # 判断障碍物方向 + if dy > 2: # 在车辆右侧 + obstacle_direction = 'right' + elif dy < -2: # 在车辆左侧 + obstacle_direction = 'left' + else: + obstacle_direction = 'center' + + return { + 'has_obstacle': has_obstacle, + 'distance': min_distance, + 'direction': obstacle_direction + } - def pretrain_with_behavioral_cloning(self, model, epochs=20): - """使用行为克隆进行预训练""" + def pretrain_with_behavioral_cloning(self, model, epochs=20, focus_avoidance=True): + """使用行为克隆进行预训练,可选专注避障""" if not self.expert_data: print("没有专家数据可用,跳过预训练") return model print(f"开始行为克隆预训练 ({epochs}个epochs)...") + # 选择训练数据 + if focus_avoidance and self.avoidance_demos: + print(f"使用 {len(self.avoidance_demos)} 条避障专用演示进行训练") + training_data = self.avoidance_demos + else: + print(f"使用 {len(self.expert_data)} 条常规演示进行训练") + training_data = self.expert_data + # 准备训练数据 states = [] actions = [] - for demo in self.expert_data: + for demo in training_data: states.append(demo['state']) actions.append(demo['action']) + # 数据增强:对图像进行轻微变换以增强泛化 + augmented_states = [] + augmented_actions = [] + + for state, action in zip(states, actions): + # 原始数据 + augmented_states.append(state) + augmented_actions.append(action) + + # 添加轻微亮度变化 + if random.random() > 0.7: + bright_state = np.clip(state * random.uniform(0.8, 1.2), 0, 255).astype(np.uint8) + augmented_states.append(bright_state) + augmented_actions.append(action) + + # 添加轻微对比度变化 + if random.random() > 0.7: + contrast = random.uniform(0.8, 1.2) + mean = np.mean(state) + contrast_state = np.clip((state - mean) * contrast + mean, 0, 255).astype(np.uint8) + augmented_states.append(contrast_state) + augmented_actions.append(action) + # 将状态归一化 - states = np.array(states) / 255.0 + states_array = np.array(augmented_states) / 255.0 # 将动作转换为one-hot编码 - actions_onehot = tf.keras.utils.to_categorical(actions, num_classes=5) + actions_onehot = tf.keras.utils.to_categorical(augmented_actions, num_classes=5) # 备份原始编译设置 original_loss = model.loss @@ -461,13 +832,20 @@ def pretrain_with_behavioral_cloning(self, model, epochs=20): metrics=['accuracy'] ) + # 添加回调函数 + callbacks = [ + EarlyStopping(monitor='val_accuracy', patience=5, restore_best_weights=True), + ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=1e-6) + ] + # 训练模型模仿专家行为 history = model.fit( - states, actions_onehot, - batch_size=32, + states_array, actions_onehot, + batch_size=16, epochs=epochs, validation_split=0.2, - verbose=1 + verbose=1, + callbacks=callbacks ) print(f"预训练完成 - 最终准确率: {history.history['accuracy'][-1]:.3f}") @@ -481,217 +859,169 @@ def pretrain_with_behavioral_cloning(self, model, epochs=20): self.is_pretrained = True return model - - def train_with_dagger(self, model, env, iterations=5, episodes_per_iter=5): - """使用DAgger算法进行训练""" - print(f"开始DAgger训练 ({iterations}次迭代,每次{episodes_per_iter}个episodes)...") - - aggregated_data = self.expert_data.copy() - - for iteration in range(iterations): - print(f"\nDAgger 迭代 {iteration + 1}/{iterations}") - - # 使用当前策略收集数据 - new_demos = [] - - for episode in range(episodes_per_iter): - print(f" 收集数据 Episode {episode + 1}/{episodes_per_iter}") - - state = env.reset(episode) - done = False - - while not done: - # 使用当前策略选择动作 - qs = model.predict(np.array(state).reshape(-1, *state.shape) / 255)[0] - action = np.argmax(qs) - - # 执行动作 - new_state, reward, done, _ = env.step(action) - - # 专家纠正(这里可以添加专家纠正逻辑) - # 如果策略动作与专家建议不同,使用专家动作 - expert_action = self._rule_based_controller(env) - - # 保存数据(使用专家纠正后的动作) - new_demos.append({ - 'state': state.copy(), - 'action': expert_action, # 使用专家动作 - 'reward': reward, - 'next_state': new_state.copy(), - 'done': done - }) - - state = new_state - - env.cleanup_actors() - - # 合并数据 - aggregated_data.extend(new_demos) - - # 在合并数据上重新训练 - states = [d['state'] for d in aggregated_data] - actions = [d['action'] for d in aggregated_data] - - states = np.array(states) / 255.0 - actions_onehot = tf.keras.utils.to_categorical(actions, num_classes=5) - - # 训练模型 - history = model.fit( - states, actions_onehot, - batch_size=32, - epochs=10, - validation_split=0.1, - verbose=0 - ) - - print(f" 训练完成 - 准确率: {history.history['accuracy'][-1]:.3f}") - - print("DAgger训练完成!") - return model -# DQN智能体类 - 升级版(整合训练策略) -class DQNAgent: - def __init__(self, use_dueling=True, use_per=True, use_curriculum=True, use_multi_objective=True): - # 创建主网络和目标网络 +# DQN智能体类 - 完全重写,专注于避障 +class EnhancedDQNAgent: + def __init__(self, use_dueling=True, use_per=True, use_curriculum=True, + use_multi_objective=True, use_attention=True, use_enhanced_model=True): + + # 配置参数 self.use_dueling = use_dueling self.use_per = use_per self.use_curriculum = use_curriculum self.use_multi_objective = use_multi_objective + self.use_attention = use_attention + self.use_enhanced_model = use_enhanced_model - if use_dueling: + # 创建主网络和目标网络 + if use_enhanced_model: + self.model = self.create_enhanced_model() + self.target_model = self.create_enhanced_model() + elif use_attention: + self.model = self.create_attention_model() + self.target_model = self.create_attention_model() + elif use_dueling: self.model = self.create_dueling_model() self.target_model = self.create_dueling_model() else: - self.model = self.create_model() - self.target_model = self.create_model() + self.model = self.create_basic_model() + self.target_model = self.create_basic_model() self.target_model.set_weights(self.model.get_weights()) - # 经验回放缓冲区 - 使用PER或标准缓冲区 + # 经验回放缓冲区 if use_per: - self.replay_buffer = PrioritizedReplayBuffer(max_size=REPLAY_MEMORY_SIZE) + self.replay_buffer = EnhancedPrioritizedReplayBuffer(max_size=REPLAY_MEMORY_SIZE) else: self.replay_memory = deque(maxlen=REPLAY_MEMORY_SIZE) # 自定义TensorBoard - self.tensorboard = ModifiedTensorBoard(log_dir=f"logs/{MODEL_NAME}-{int(time.time())}") - self.target_update_counter = 0 # 目标网络更新计数器 + self.tensorboard = EnhancedTensorBoard( + log_dir=f"logs/{MODEL_NAME}-enhanced-{int(time.time())}", + histogram_freq=0, + write_graph=True, + write_images=False + ) + + self.target_update_counter = 0 + self.train_step_counter = 0 # 训练控制标志 self.terminate = False self.last_logged_episode = 0 self.training_initialized = False + self.training_paused = False # 训练策略组件 self.curriculum_manager = None self.multi_objective_optimizer = None self.imitation_manager = None + # 避障性能跟踪 + self.obstacle_avoidance_stats = { + 'success_count': 0, + 'total_encounters': 0, + 'near_misses': 0, + 'collisions': 0 + } + + # 锁用于线程安全 + self.training_lock = Lock() + def setup_training_strategies(self, env=None): """设置训练策略组件""" if self.use_curriculum and env: - self.curriculum_manager = CurriculumManager(env) - print("课程学习管理器已启用") + self.curriculum_manager = EnhancedCurriculumManager(env) + print("增强版课程学习管理器已启用") if self.use_multi_objective: - self.multi_objective_optimizer = MultiObjectiveOptimizer() - print("多目标优化器已启用") + self.multi_objective_optimizer = EnhancedMultiObjectiveOptimizer() + print("增强版多目标优化器已启用") - # 模仿学习管理器(需要时手动启用) - self.imitation_manager = ImitationLearningManager() - - def create_model(self): - """创建标准深度Q网络模型""" - # 使用函数式API + # 模仿学习管理器 + self.imitation_manager = EnhancedImitationLearningManager() + print("增强版模仿学习管理器已启用") + + def create_basic_model(self): + """创建基础深度Q网络模型""" inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) - # 第一卷积块 + # 特征提取层 x = Conv2D(32, (5, 5), strides=(2, 2), padding='same')(inputs) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 第二卷积块 x = Conv2D(64, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 第三卷积块 x = Conv2D(128, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 空间注意力机制 - attention = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(x) - x = Multiply()([x, attention]) - # 展平层 x = Flatten()(x) # 全连接层 - x = Dense(512, activation='relu')(x) - x = Dropout(0.3)(x) - x = Dense(256, activation='relu')(x) + x = Dense(512, activation='relu', kernel_regularizer=l2(1e-4))(x) x = Dropout(0.3)(x) - x = Dense(128, activation='relu')(x) + x = Dense(256, activation='relu', kernel_regularizer=l2(1e-4))(x) x = Dropout(0.2)(x) - x = Dense(64, activation='relu')(x) + x = Dense(128, activation='relu')(x) x = Dropout(0.1)(x) - # 输出层 - 5个动作 + # 输出层 outputs = Dense(5, activation='linear')(x) # 创建模型 model = Model(inputs=inputs, outputs=outputs) # 编译模型 - model.compile(loss="huber", optimizer=Adam(learning_rate=LEARNING_RATE), metrics=["mae"]) + model.compile( + loss="huber", + optimizer=Adam(learning_rate=LEARNING_RATE, clipnorm=1.0), + metrics=["mae"] + ) return model def create_dueling_model(self): - """创建Dueling DQN模型架构""" + """创建Dueling DQN模型""" inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) # 共享的特征提取层 - # 第一卷积块 x = Conv2D(32, (5, 5), strides=(2, 2), padding='same')(inputs) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 第二卷积块 x = Conv2D(64, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 第三卷积块 x = Conv2D(128, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 空间注意力机制 - attention = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(x) - x = Multiply()([x, attention]) - # 展平层 x = Flatten()(x) # 共享的全连接层 - shared = Dense(512, activation='relu')(x) + shared = Dense(512, activation='relu', kernel_regularizer=l2(1e-4))(x) shared = Dropout(0.3)(shared) shared = Dense(256, activation='relu')(shared) - # 价值流 (V(s)) + # 价值流 value_stream = Dense(128, activation='relu')(shared) value_stream = Dropout(0.2)(value_stream) value = Dense(1, activation='linear', name='value')(value_stream) - # 优势流 (A(s,a)) + # 优势流 advantage_stream = Dense(128, activation='relu')(shared) advantage_stream = Dropout(0.2)(advantage_stream) advantage = Dense(5, activation='linear', name='advantage')(advantage_stream) @@ -705,163 +1035,382 @@ def create_dueling_model(self): model = Model(inputs=inputs, outputs=q_values) # 编译模型 - model.compile(loss="huber", optimizer=Adam(learning_rate=LEARNING_RATE), metrics=["mae"]) + model.compile( + loss="huber", + optimizer=Adam(learning_rate=LEARNING_RATE, clipnorm=1.0), + metrics=["mae"] + ) return model - - def update_replay_memory(self, transition): + + def create_attention_model(self): + """创建带注意力机制的模型""" + inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) + + # 第一卷积块 + x = Conv2D(32, (5, 5), strides=(2, 2), padding='same')(inputs) + x = Activation('relu')(x) + x = BatchNormalization()(x) + x = MaxPooling2D(pool_size=(2, 2))(x) + + # 第二卷积块 + 注意力 + x = Conv2D(64, (3, 3), padding='same')(x) + x = Activation('relu')(x) + x = BatchNormalization()(x) + x = ObstacleAttentionLayer(64)(x) # 添加注意力层 + x = MaxPooling2D(pool_size=(2, 2))(x) + + # 第三卷积块 + 注意力 + x = Conv2D(128, (3, 3), padding='same')(x) + x = Activation('relu')(x) + x = BatchNormalization()(x) + x = ObstacleAttentionLayer(128)(x) # 第二层注意力 + x = MaxPooling2D(pool_size=(2, 2))(x) + + # 展平层 + x = Flatten()(x) + + # 全连接层 + x = Dense(512, activation='relu', kernel_regularizer=l2(1e-4))(x) + x = Dropout(0.3)(x) + x = Dense(256, activation='relu')(x) + x = Dropout(0.2)(x) + + # Dueling架构 + # 价值流 + value_stream = Dense(128, activation='relu')(x) + value_stream = Dropout(0.2)(value_stream) + value = Dense(1, activation='linear', name='value')(value_stream) + + # 优势流 + advantage_stream = Dense(128, activation='relu')(x) + advantage_stream = Dropout(0.2)(advantage_stream) + advantage = Dense(5, activation='linear', name='advantage')(advantage_stream) + + # 合并 + mean_advantage = Lambda(lambda a: tf.reduce_mean(a, axis=1, keepdims=True))(advantage) + advantage_centered = Subtract()([advantage, mean_advantage]) + q_values = Add()([value, advantage_centered]) + + # 创建模型 + model = Model(inputs=inputs, outputs=q_values) + + # 编译模型 + model.compile( + loss="huber", + optimizer=Adam(learning_rate=LEARNING_RATE, clipnorm=1.0), + metrics=["mae"] + ) + + return model + + def create_enhanced_model(self): + """创建增强版模型,专门用于避障""" + inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) + + # 多尺度特征提取 + # 分支1: 大感受野,检测远处障碍物 + branch1 = Conv2D(32, (7, 7), strides=(2, 2), padding='same')(inputs) + branch1 = Activation('relu')(branch1) + branch1 = BatchNormalization()(branch1) + branch1 = MaxPooling2D(pool_size=(2, 2))(branch1) + + # 分支2: 中等感受野 + branch2 = Conv2D(32, (5, 5), strides=(2, 2), padding='same')(inputs) + branch2 = Activation('relu')(branch2) + branch2 = BatchNormalization()(branch2) + branch2 = MaxPooling2D(pool_size=(2, 2))(branch2) + + # 分支3: 小感受野,检测近处细节 + branch3 = Conv2D(32, (3, 3), strides=(2, 2), padding='same')(inputs) + branch3 = Activation('relu')(branch3) + branch3 = BatchNormalization()(branch3) + branch3 = MaxPooling2D(pool_size=(2, 2))(branch3) + + # 合并分支 + merged = Concatenate()([branch1, branch2, branch3]) + + # 深度特征提取 + x = Conv2D(128, (3, 3), padding='same')(merged) + x = Activation('relu')(x) + x = BatchNormalization()(x) + x = ObstacleAttentionLayer(128)(x) # 注意力机制 + x = MaxPooling2D(pool_size=(2, 2))(x) + + x = Conv2D(256, (3, 3), padding='same')(x) + x = Activation('relu')(x) + x = BatchNormalization()(x) + x = ObstacleAttentionLayer(256)(x) # 第二层注意力 + x = MaxPooling2D(pool_size=(2, 2))(x) + + # 展平层 + x = Flatten()(x) + + # 密集连接层 + x = Dense(1024, activation='relu', kernel_regularizer=l2(1e-4))(x) + x = Dropout(0.4)(x) + x = Dense(512, activation='relu', kernel_regularizer=l2(1e-4))(x) + x = Dropout(0.3)(x) + x = Dense(256, activation='relu')(x) + x = Dropout(0.2)(x) + + # 双流输出(Dueling DQN) + # 价值流 + value_stream = Dense(128, activation='relu')(x) + value_stream = Dropout(0.2)(value_stream) + value = Dense(1, activation='linear', name='value')(value_stream) + + # 优势流(特别关注避障动作) + advantage_stream = Dense(128, activation='relu')(x) + advantage_stream = Dropout(0.2)(advantage_stream) + advantage = Dense(5, activation='linear', name='advantage')(advantage_stream) + + # 合并: Q(s,a) = V(s) + (A(s,a) - mean(A(s,a))) + mean_advantage = Lambda(lambda a: tf.reduce_mean(a, axis=1, keepdims=True))(advantage) + advantage_centered = Subtract()([advantage, mean_advantage]) + q_values = Add()([value, advantage_centered]) + + # 创建模型 + model = Model(inputs=inputs, outputs=q_values) + + # 编译模型 + optimizer = Adam( + learning_rate=LEARNING_RATE, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-7, + clipnorm=1.0 + ) + + model.compile( + loss="huber", + optimizer=optimizer, + metrics=["mae"] + ) + + print("增强版模型创建完成(专门用于避障)") + return model + + def update_replay_memory(self, transition, is_obstacle=False, is_success=False): """更新经验回放缓冲区""" - # transition = (当前状态, 动作, 奖励, 新状态, 完成标志) if self.use_per: - # PER: 初始添加时使用最大优先级 - self.replay_buffer.add(transition, error=1.0) # 初始误差设为1.0 + self.replay_buffer.add(transition, error=1.0, + is_obstacle=is_obstacle, + is_success=is_success) else: self.replay_memory.append(transition) - - def minibatch_chooser(self): - """改进的经验采样策略""" - if self.use_per: - # PER采样 - if len(self.replay_buffer) < MIN_REPLAY_MEMORY_SIZE: - return [], [], [], [] - - indices, samples, weights = self.replay_buffer.sample(MINIBATCH_SIZE) - return indices, samples, weights - else: - # 标准采样 - if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: - return random.sample(self.replay_memory, min(len(self.replay_memory), MINIBATCH_SIZE)) - - # 分类经验样本 - positive_samples = [] # 高奖励经验 - negative_samples = [] # 负奖励/碰撞经验 - neutral_samples = [] # 中性奖励经验 - - for sample in self.replay_memory: - _, _, reward, _, done = sample - - if done and reward < -5: # 碰撞或严重错误 - negative_samples.append(sample) - elif reward > 1: # 积极经验 - positive_samples.append(sample) - else: # 中性经验 - neutral_samples.append(sample) - - # 平衡采样 - batch = [] - - # 采样负经验 (20%) - num_negative = min(len(negative_samples), MINIBATCH_SIZE // 5) - batch.extend(random.sample(negative_samples, num_negative)) - # 采样正经验 (30%) - num_positive = min(len(positive_samples), MINIBATCH_SIZE // 3) - batch.extend(random.sample(positive_samples, num_positive)) - - # 用中性经验补全批次 - remaining = MINIBATCH_SIZE - len(batch) - if remaining > 0: - batch.extend(random.sample(neutral_samples, min(remaining, len(neutral_samples)))) - - # 如果还不够,从整个记忆库随机采样 - if len(batch) < MINIBATCH_SIZE: - additional = MINIBATCH_SIZE - len(batch) - batch.extend(random.sample(self.replay_memory, additional)) - - random.shuffle(batch) # 打乱批次 - return batch - + # 如果是重要经验,额外存储 + if is_obstacle or is_success: + self.replay_memory.append(transition) # 重要经验重复存储 + def train(self): """训练DQN网络""" - if self.use_per: - if len(self.replay_buffer) < MIN_REPLAY_MEMORY_SIZE: + with self.training_lock: + if self.training_paused: return - # PER: 采样并获取权重 - indices, minibatch, weights = self.replay_buffer.sample(MINIBATCH_SIZE) - if len(minibatch) == 0: - return - else: - if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: - return + if self.use_per: + if len(self.replay_buffer) < MIN_REPLAY_MEMORY_SIZE: + return + + # PER采样,特别关注避障经验 + indices, minibatch, weights = self.replay_buffer.sample( + MINIBATCH_SIZE, + obstacle_ratio=0.3, # 30%避障经验 + success_ratio=0.2 # 20%成功经验 + ) + if len(minibatch) == 0: + return + else: + if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: + return + + # 标准采样 + minibatch = random.sample(self.replay_memory, + min(MINIBATCH_SIZE, len(self.replay_memory))) + weights = np.ones(len(minibatch)) + + # 准备训练数据 + current_states = np.array([transition[0] for transition in minibatch]) / 255 + current_qs_list = self.model.predict(current_states, + batch_size=PREDICTION_BATCH_SIZE, + verbose=0) + + new_current_states = np.array([transition[3] for transition in minibatch]) / 255 + future_qs_list = self.target_model.predict(new_current_states, + batch_size=PREDICTION_BATCH_SIZE, + verbose=0) + + x = [] # 输入状态 + y = [] # 目标Q值 + errors = [] # TD误差 + + # 计算目标Q值(Double DQN风格) + for index, (current_state, action, reward, new_state, done) in enumerate(minibatch): + if not done: + # Double DQN: 使用主网络选择动作,目标网络评估 + next_qs = self.model.predict(np.array([new_state]) / 255, verbose=0)[0] + best_action = np.argmax(next_qs) + max_future_q = future_qs_list[index][best_action] + new_q = reward + DISCOUNT * max_future_q + else: + new_q = reward + + current_qs = current_qs_list[index].copy() + old_q = current_qs[action] + current_qs[action] = new_q - # 标准采样 - minibatch = self.minibatch_chooser() - weights = np.ones(len(minibatch)) # 标准训练权重为1 + # 计算TD误差 + td_error = abs(new_q - old_q) + errors.append(td_error) - # 准备训练数据 - current_states = np.array([transition[0] for transition in minibatch]) / 255 - current_qs_list = self.model.predict(current_states, batch_size=PREDICTION_BATCH_SIZE) - - new_current_states = np.array([transition[3] for transition in minibatch]) / 255 - future_qs_list = self.target_model.predict(new_current_states, batch_size=PREDICTION_BATCH_SIZE) - - x = [] # 输入状态 - y = [] # 目标Q值 - errors = [] # TD误差(用于PER) - - # 计算目标Q值 - for index, (current_state, action, reward, new_state, done) in enumerate(minibatch): - if not done: - # 使用贝尔曼方程计算目标Q值 - max_future_q = np.max(future_qs_list[index]) - new_q = reward + DISCOUNT * max_future_q - else: - new_q = reward # 终止状态 + x.append(current_state) + y.append(current_qs) - current_qs = current_qs_list[index].copy() - old_q = current_qs[action] # 用于计算TD误差 - current_qs[action] = new_q # 更新对应动作的Q值 - - # 计算TD误差 - td_error = abs(new_q - old_q) - errors.append(td_error) - - x.append(current_state) - y.append(current_qs) - - # PER: 更新优先级 - if self.use_per and len(errors) > 0: - self.replay_buffer.update_priorities(indices, errors) - - # 记录日志判断 - log_this_step = False - if self.tensorboard.step > self.last_logged_episode: - log_this_step = True - self.last_logged_episode = self.tensorboard.step - - # 训练模型(带样本权重) - self.model.fit(np.array(x) / 255, np.array(y), - batch_size=TRAINING_BATCH_SIZE, - sample_weight=weights if self.use_per else None, - verbose=0, shuffle=False, - callbacks=[self.tensorboard] if log_this_step else None) - - # 更新目标网络 - if log_this_step: - self.target_update_counter += 1 - - if self.target_update_counter > UPDATE_TARGET_EVERY: - print("目标网络已更新") - self.target_model.set_weights(self.model.get_weights()) - self.target_update_counter = 0 + # PER: 更新优先级 + if self.use_per and len(errors) > 0: + self.replay_buffer.update_priorities(indices, errors) + + # 记录日志 + log_this_step = False + if self.tensorboard.step > self.last_logged_episode: + log_this_step = True + self.last_logged_episode = self.tensorboard.step + + # 训练模型 + self.model.fit( + np.array(x) / 255, + np.array(y), + batch_size=TRAINING_BATCH_SIZE, + sample_weight=weights if self.use_per else None, + verbose=0, + shuffle=False, + callbacks=[self.tensorboard] if log_this_step else None + ) + + self.train_step_counter += 1 + + # 更新目标网络 + if log_this_step: + self.target_update_counter += 1 + if self.target_update_counter > UPDATE_TARGET_EVERY: + print("目标网络已更新") + self.target_model.set_weights(self.model.get_weights()) + self.target_update_counter = 0 + def train_in_loop(self): """在单独线程中持续训练""" # 预热训练 x = np.random.uniform(size=(1, IM_HEIGHT, IM_WIDTH, 3)).astype(np.float32) - y = np.random.uniform(size=(1, 5)).astype(np.float32) # 改为5个输出 + y = np.random.uniform(size=(1, 5)).astype(np.float32) self.model.fit(x, y, verbose=False, batch_size=1) self.training_initialized = True + print("训练线程已启动") + # 持续训练循环 while True: if self.terminate: + print("训练线程终止") return - self.train() - time.sleep(0.01) # 控制训练频率 + + try: + self.train() + time.sleep(0.005) # 更高的训练频率 + except Exception as e: + print(f"训练过程中出错: {e}") + time.sleep(0.1) def get_qs(self, state): """获取状态的Q值""" - return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255)[0] \ No newline at end of file + return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255, + verbose=0)[0] + + def update_obstacle_stats(self, success, is_collision=False, is_near_miss=False): + """更新避障统计""" + self.obstacle_avoidance_stats['total_encounters'] += 1 + if success: + self.obstacle_avoidance_stats['success_count'] += 1 + if is_collision: + self.obstacle_avoidance_stats['collisions'] += 1 + if is_near_miss: + self.obstacle_avoidance_stats['near_misses'] += 1 + + def get_obstacle_avoidance_rate(self): + """获取避障成功率""" + total = self.obstacle_avoidance_stats['total_encounters'] + if total == 0: + return 0.0 + return self.obstacle_avoidance_stats['success_count'] / total + + def pause_training(self): + """暂停训练""" + self.training_paused = True + + def resume_training(self): + """恢复训练""" + self.training_paused = False + + def save_model(self, path, include_stats=True): + """保存模型""" + try: + # 尝试使用Keras的标准保存方法 + self.model.save(path) + print(f"模型已保存到: {path}") + except Exception as e: + print(f"标准保存失败,尝试替代方法: {e}") + try: + # 尝试保存权重 + weights_path = path.replace('.model', '_weights.h5') + self.model.save_weights(weights_path) + print(f"模型权重已保存到: {weights_path}") + + # 保存模型架构 + config_path = path.replace('.model', '_config.json') + with open(config_path, 'w') as f: + f.write(self.model.to_json()) + print(f"模型架构已保存到: {config_path}") + except Exception as e2: + print(f"权重保存也失败: {e2}") + print("跳过模型保存,继续训练...") + + if include_stats: + # 保存训练统计 + stats = { + 'obstacle_avoidance_stats': self.obstacle_avoidance_stats, + 'train_step_counter': self.train_step_counter, + 'model_config': { + 'use_dueling': self.use_dueling, + 'use_per': self.use_per, + 'use_attention': self.use_attention, + 'use_enhanced_model': self.use_enhanced_model + } + } + + stats_path = path.replace('.model', '_stats.pkl') + with open(stats_path, 'wb') as f: + pickle.dump(stats, f) + print(f"训练统计已保存到: {stats_path}") + + def load_model(self, path): + """加载模型""" + try: + # 定义自定义层 + custom_objects = { + 'ObstacleAttentionLayer': ObstacleAttentionLayer, + 'Add': Add, + 'Subtract': Subtract, + 'Lambda': Lambda, + 'Multiply': Multiply + } + + self.model = tf.keras.models.load_model(path, custom_objects=custom_objects) + self.target_model.set_weights(self.model.get_weights()) + print(f"模型已从 {path} 加载") + return True + except Exception as e: + print(f"加载模型失败: {e}") + return False \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/main.py b/src/Unmanned_vehicle_AD_DQN/main.py index de06a3dddc..4ed20e1f71 100644 --- a/src/Unmanned_vehicle_AD_DQN/main.py +++ b/src/Unmanned_vehicle_AD_DQN/main.py @@ -21,7 +21,65 @@ from Model import * from Hyperparameters import * -def extended_reward_calculation(env, action, reward, done, step_info): + +def manage_top_models(agent, score, top_models, max_models=10): + """ + 管理最佳模型,只保留前10个最佳模型 + 当有更好的模型时,替换最差的 + """ + import os + + # 创建模型文件名 + model_suffix = f"enhanced_top_{score:.2f}" + model_filename = f'{MODEL_NAME}_{model_suffix}.model' + model_path = f'models/{model_filename}' + + # 检查是否应该保存这个模型 + should_save = False + model_to_remove = None + + if len(top_models) < max_models: + # 还没有10个模型,直接保存 + should_save = True + elif score > min(score for score, _ in top_models): + # 比当前最差的模型更好,替换它 + should_save = True + # 找到最差的模型 + min_score_idx = min(range(len(top_models)), key=lambda i: top_models[i][0]) + _, model_to_remove = top_models[min_score_idx] + # 从列表中移除 + top_models.pop(min_score_idx) + + if should_save: + # 保存新模型 + try: + agent.save_model(model_path) + top_models.append((score, model_filename)) + print(f"保存最佳模型: 得分={score:.2f}, 排名={len(top_models)}") + except Exception as e: + print(f"保存模型失败: {e}") + print("继续训练,不保存此模型...") + + # 删除被替换的模型 + if model_to_remove: + remove_path = f'models/{model_to_remove}' + if os.path.exists(remove_path): + try: + # 删除模型目录 + import shutil + if os.path.isdir(remove_path): + shutil.rmtree(remove_path) + else: + os.remove(remove_path) + print(f"删除较差模型: {model_to_remove}") + except Exception as e: + print(f"删除模型失败 {model_to_remove}: {e}") + + # 按得分排序(降序) + top_models.sort(key=lambda x: x[0], reverse=True) + + +def extended_reward_calculation(env, action, reward, done, step_info, obstacle_info=None): """ 扩展的奖励计算函数,用于多目标优化 """ @@ -33,19 +91,19 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 计算多目标指标 metrics = {} - # 1. 安全性指标 - # 最近行人距离 + # 1. 安全性指标 - 增强 min_ped_distance = env.last_ped_distance if hasattr(env, 'last_ped_distance') else float('inf') safety_score = 0 + if min_ped_distance < 100: - if min_ped_distance > 12: + if min_ped_distance > 15: safety_score = 10 # 非常安全 - elif min_ped_distance > 8: - safety_score = 7 # 安全 - elif min_ped_distance > 5: - safety_score = 3 # 警告 + elif min_ped_distance > 10: + safety_score = 8 # 安全 + elif min_ped_distance > 6: + safety_score = 5 # 注意 elif min_ped_distance > 3: - safety_score = 1 # 危险 + safety_score = 2 # 危险 else: safety_score = 0 # 极危险 @@ -56,73 +114,98 @@ def extended_reward_calculation(env, action, reward, done, step_info): efficiency_score = progress * 100 # 进度百分比 metrics['efficiency'] = efficiency_score - # 3. 舒适度指标 + # 3. 舒适度指标 - 增强 + comfort_score = 5 # 基础分数 + # 转向平滑性 if hasattr(env, 'last_action') and env.last_action in [3, 4]: if env.same_steer_counter > 2: # 连续同向转向 - comfort_score = -2 # 不舒适 + comfort_score -= 3 # 不舒适 else: - comfort_score = 2 # 舒适 - else: - comfort_score = 5 # 直行,最舒适 + comfort_score += 1 # 舒适转向 + + # 加减速平滑性 + if action in [0, 2]: # 减速或加速 + if hasattr(env, 'last_action') and env.last_action in [0, 2]: + comfort_score -= 1 # 连续加减速不舒适 metrics['comfort'] = comfort_score - # 4. 规则遵循指标 - # 速度是否在合理范围内 - if 20 <= speed_kmh <= 40: + # 4. 障碍物避让指标(新增) + if obstacle_info: + avoidance_score = 0 + min_distance = obstacle_info.get('min_distance', float('inf')) + warning_level = obstacle_info.get('warning_level', 0) + avoidance_success = obstacle_info.get('avoidance_success', False) + + if avoidance_success: + avoidance_score = 8 # 成功避让 + elif min_distance > 12: + avoidance_score = 6 # 保持安全距离 + elif min_distance > 8: + avoidance_score = 4 # 中等距离 + elif min_distance > 5: + avoidance_score = 2 # 近距离但安全 + else: + avoidance_score = 0 # 危险 + + metrics['obstacle_avoidance'] = avoidance_score + + # 5. 规则遵循指标 + if 18 <= speed_kmh <= 38: rule_score = 1.0 - elif 15 <= speed_kmh < 20 or 40 < speed_kmh <= 45: + elif 15 <= speed_kmh < 18 or 38 < speed_kmh <= 42: rule_score = 0.7 else: rule_score = 0.3 metrics['rule_following'] = rule_score - # 5. 特殊事件 + # 6. 特殊事件 metrics['collision'] = len(env.collision_history) > 0 metrics['off_road'] = vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30 metrics['dangerous_action'] = speed_kmh > 45 and action in [3, 4] # 高速急转 + metrics['near_miss'] = min_ped_distance < 3.0 and min_ped_distance > 1.0 # 惊险避让 return metrics -if __name__ == '__main__': - FPS = 60 # 帧率 - ep_rewards = [-200] # 存储每轮奖励 - # 为了结果可重复性(注释掉) - # random.seed(1) - # np.random.seed(1) - # tf.compat.v1.set_random_seed(1) - - # GPU内存配置,主要用于多智能体训练 +def train_enhanced_agent(): + """增强版训练函数""" + FPS = 60 + ep_rewards = [-200] + + # GPU内存配置 gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) tf.compat.v1.keras.backend.set_session( tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) - - # 创建模型保存目录 - if not os.path.isdir('models'): - os.makedirs('models') - # 创建专家数据目录 - if not os.path.isdir('expert_data'): - os.makedirs('expert_data') - - # 创建智能体和环境 - 启用所有高级功能 - agent = DQNAgent( + # 创建目录 + for dir_name in ['models', 'expert_data', 'logs', 'training_stats']: + if not os.path.isdir(dir_name): + os.makedirs(dir_name) + + # 创建智能体和环境 - 启用所有增强功能 + print("=" * 60) + print("初始化增强版智能体和环境") + print("=" * 60) + + agent = EnhancedDQNAgent( use_dueling=True, use_per=True, use_curriculum=True, - use_multi_objective=True + use_multi_objective=True, + use_attention=True, + use_enhanced_model=True # 使用增强版模型 ) - env = CarEnv() + env = CarEnv(obstacle_detection_mode='advanced') # 设置训练策略 agent.setup_training_strategies(env) - - # 可选:使用模仿学习进行预训练 - use_imitation_pretraining = False # 设置为True启用模仿学习预训练 + + # 模仿学习预训练(可选) + use_imitation_pretraining = True if use_imitation_pretraining: print("=" * 60) @@ -130,40 +213,52 @@ def extended_reward_calculation(env, action, reward, done, step_info): print("=" * 60) # 检查是否有现有的专家数据 - expert_files = glob.glob("expert_data/*.pkl") + expert_files = glob.glob("expert_data/*enhanced*.pkl") if expert_files: - # 使用最新的专家数据 latest_expert = max(expert_files, key=os.path.getctime) agent.imitation_manager.load_expert_data(latest_expert) else: - # 收集新的专家数据 + # 收集新的专家数据,特别关注避障 print("未找到专家数据,开始收集...") - agent.imitation_manager.collect_expert_demonstration(env, num_episodes=5) + agent.imitation_manager.collect_expert_demonstration( + env, + num_episodes=3, + focus_avoidance=True + ) - # 使用行为克隆进行预训练 - agent.model = agent.imitation_manager.pretrain_with_behavioral_cloning(agent.model, epochs=15) + # 使用行为克隆进行预训练,专注避障 + agent.model = agent.imitation_manager.pretrain_with_behavioral_cloning( + agent.model, + epochs=20, + focus_avoidance=True + ) agent.target_model.set_weights(agent.model.get_weights()) print("模仿学习预训练完成!") - - # 启动训练线程并等待训练初始化完成 + + # 启动训练线程 trainer_thread = Thread(target=agent.train_in_loop, daemon=True) trainer_thread.start() while not agent.training_initialized: time.sleep(0.01) - + # 预热Q网络 agent.get_qs(np.ones((env.im_height, env.im_width, 3))) - + # 训练统计变量 - best_score = -float('inf') # 最佳得分 - success_count = 0 # 成功次数计数 - scores = [] # 存储每轮得分 - avg_scores = [] # 存储平均得分 - - # 记录PER相关统计 - per_stats = { - 'avg_td_error': [], - 'buffer_size': [] + best_score = -float('inf') + success_count = 0 + scores = [] + avg_scores = [] + + # 最佳模型管理器 - 只保留前10个最佳模型 + top_models = [] # 存储 (score, filename) 元组的列表 + + # 避障统计 + avoidance_stats = { + 'success_rate': [], + 'near_misses': [], + 'collisions': [], + 'encounters': [] } # 多目标统计 @@ -171,26 +266,37 @@ def extended_reward_calculation(env, action, reward, done, step_info): 'safety': [], 'efficiency': [], 'comfort': [], - 'rule_following': [] + 'obstacle_avoidance': [] } # 课程学习阶段记录 curriculum_stages = [] + curriculum_progress = [] # 迭代训练轮次 - epds = [] + print("=" * 60) + print("开始主要训练阶段") + print("=" * 60) + for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'): - env.collision_hist = [] # 重置碰撞历史 - agent.tensorboard.step = episode # 设置TensorBoard步数 - + env.collision_hist = [] + agent.tensorboard.step = episode + # 应用课程学习配置 if agent.curriculum_manager: config = agent.curriculum_manager.get_current_config() - # 这里可以根据配置调整环境难度 - # 例如:调整行人数量、速度等 - print(f"课程学习 - 阶段 {agent.curriculum_manager.current_stage}: " - f"行人(十字路口={config['pedestrian_cross']}, 普通={config['pedestrian_normal']})") - curriculum_stages.append(agent.curriculum_manager.current_stage) + stage_info = agent.curriculum_manager.get_stage_info() + + # 重置环境时应用课程配置 + current_state = env.reset(episode, curriculum_config=config) + + print(f"课程学习 - 阶段 {stage_info['stage']}: {stage_info['name']}") + print(f" 进度: {stage_info['progress']:.2%}, 行人数量: {stage_info['pedestrian_total']}") + + curriculum_stages.append(stage_info['stage']) + curriculum_progress.append(stage_info['progress']) + else: + current_state = env.reset(episode) # 重置每轮统计 score = 0 @@ -201,46 +307,46 @@ def extended_reward_calculation(env, action, reward, done, step_info): 'safety': [], 'efficiency': [], 'comfort': [], - 'rule_following': [] + 'obstacle_avoidance': [] } - - # 重置环境并获取初始状态 - current_state = env.reset(episode) - + # 重置完成标志 done = False episode_start = time.time() - + # 应用课程学习的最大步数限制 if agent.curriculum_manager: config = agent.curriculum_manager.get_current_config() max_steps_per_episode = config['max_episode_steps'] else: max_steps_per_episode = SECONDS_PER_EPISODE * FPS - - # 仅在给定秒数内运行 + + # 步进循环 while not done and step < max_steps_per_episode: - # 选择动作策略 if np.random.random() > Hyperparameters.EPSILON: - # 从Q网络获取动作(利用) + # 从Q网络获取动作 qs = agent.get_qs(current_state) action = np.argmax(qs) - if episode % 20 == 0: # 减少打印频率 - print(f'动作: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}, {qs[3]:>5.2f}, {qs[4]:>5.2f}] {action}') + + # 减少打印频率,仅在调试时启用 + if episode % 50 == 0 and step % 30 == 0: + print(f'Step {step}: Q值 [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}, {qs[3]:>5.2f}, {qs[4]:>5.2f}] 选择动作 {action}') else: # 随机选择动作(探索) action = np.random.randint(0, 5) - # 添加延迟以匹配60FPS - time.sleep(1 / FPS) - + time.sleep(1 / FPS) # 匹配帧率 + # 执行动作并获取结果 - new_state, reward, done, _ = env.step(action) + new_state, reward, done, extra_info = env.step(action) + + # 提取障碍物信息 + obstacle_info = extra_info.get('obstacle_info', {}) if extra_info else {} # 计算多目标指标 if agent.multi_objective_optimizer: step_info = {'step': step, 'action': action} - metrics = extended_reward_calculation(env, action, reward, done, step_info) + metrics = extended_reward_calculation(env, action, reward, done, step_info, obstacle_info) # 记录指标 for key in episode_metrics: @@ -248,42 +354,77 @@ def extended_reward_calculation(env, action, reward, done, step_info): episode_metrics[key].append(metrics[key]) # 使用多目标优化器计算综合奖励 - composite_reward = agent.multi_objective_optimizer.compute_composite_reward(metrics) + composite_reward = agent.multi_objective_optimizer.compute_composite_reward( + metrics, + obstacle_info + ) reward = composite_reward # 使用综合奖励 - score += reward # 累加奖励 + score += reward - # 更新经验回放 - agent.update_replay_memory((current_state, action, reward, new_state, done)) - current_state = new_state # 更新当前状态 - + # 判断是否为避障相关经验 + is_obstacle_experience = ( + obstacle_info.get('min_distance', float('inf')) < 10.0 or + obstacle_info.get('warning_level', 0) > 0 + ) + + # 判断是否为成功经验 + is_success_experience = (reward > 2.0 and not done) + + # 更新经验回放(特别标记重要经验) + agent.update_replay_memory( + (current_state, action, reward, new_state, done), + is_obstacle=is_obstacle_experience, + is_success=is_success_experience + ) + + current_state = new_state step += 1 - + if done: break - - # 本轮结束 - 销毁所有actor - env.cleanup_actors() - + + # 本轮结束 + + # 更新避障统计 - 在清理之前获取信息 + env_info = env.get_environment_info() + if env_info['obstacle_warning_level'] > 0: + avoidance_stats['encounters'].append(1) + else: + avoidance_stats['encounters'].append(0) + + avoidance_stats['success_rate'].append(env_info['successful_avoidance']) + avoidance_stats['collisions'].append(1 if env_info['collision_count'] > 0 else 0) + + # 更新课程学习 + success = score > 8 # 提高成功阈值 + obstacle_avoidance_score = env_info['successful_avoidance'] / max(1, env_info['pedestrian_count']) + + if agent.curriculum_manager: + stage_changed, change_type = agent.curriculum_manager.update_stage( + success, + score, + obstacle_avoidance_score + ) + if stage_changed: + stage_info = agent.curriculum_manager.get_stage_info() + print(f"课程学习阶段已更新: {stage_info['name']} (变更类型: {change_type})") + # 计算本轮平均指标 avg_metrics = {} for key, values in episode_metrics.items(): if values: avg_metrics[key] = np.mean(values) - # 记录到统计中 if key in multi_obj_stats: multi_obj_stats[key].append(avg_metrics[key]) - # 更新课程学习 - success = score > 5 # 成功完成的阈值 - if agent.curriculum_manager: - stage_changed = agent.curriculum_manager.update_stage(success, score) - if stage_changed: - print(f"课程学习阶段已更新: {agent.curriculum_manager.current_stage}") + env.cleanup_actors() # 更新多目标优化器权重 if agent.multi_objective_optimizer and episode % 10 == 0: - agent.multi_objective_optimizer.adjust_weights(avg_metrics) + training_stage = agent.curriculum_manager.current_stage if agent.curriculum_manager else 0 + agent.multi_objective_optimizer.adjust_weights(avg_metrics, training_stage) + if episode % 50 == 0: print(agent.multi_objective_optimizer.get_performance_report()) @@ -291,67 +432,60 @@ def extended_reward_calculation(env, action, reward, done, step_info): if success: success_count += 1 - # 动态保存最佳模型 - if score > best_score: - best_score = score - model_suffix = f"advanced_best_{score:.2f}" - agent.model.save(f'models/{MODEL_NAME}_{model_suffix}.model') - print(f"新的最佳模型已保存: 得分={score:.2f}") - + # 管理最佳模型(只保留前10个) + manage_top_models(agent, score, top_models) + + # 定期保存检查点 + if episode % 50 == 0: + model_suffix = f"enhanced_checkpoint_ep{episode}" + agent.save_model(f'models/{MODEL_NAME}_{model_suffix}.model') + # 记录得分统计 scores.append(score) avg_scores.append(np.mean(scores[-10:]) if len(scores) >= 10 else np.mean(scores)) - - # 记录PER缓冲区信息 - if hasattr(agent, 'replay_buffer'): - per_stats['buffer_size'].append(len(agent.replay_buffer)) - + # 定期聚合统计信息 if not episode % AGGREGATE_STATS_EVERY or episode == 1: average_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:]) if len(scores) >= AGGREGATE_STATS_EVERY else np.mean(scores) min_reward = min(scores[-AGGREGATE_STATS_EVERY:]) if len(scores) >= AGGREGATE_STATS_EVERY else min(scores) max_reward = max(scores[-AGGREGATE_STATS_EVERY:]) if len(scores) >= AGGREGATE_STATS_EVERY else max(scores) - # 添加PER统计到TensorBoard + # 准备TensorBoard统计 stats_dict = { 'reward_avg': average_reward, 'reward_min': min_reward, 'reward_max': max_reward, - 'epsilon': Hyperparameters.EPSILON + 'epsilon': Hyperparameters.EPSILON, + 'success_rate': success_count / episode if episode > 0 else 0 } - if hasattr(agent, 'replay_buffer'): - avg_buffer = np.mean(per_stats['buffer_size'][-AGGREGATE_STATS_EVERY:]) if per_stats['buffer_size'] else 0 - stats_dict['buffer_size'] = avg_buffer + # 添加避障统计 + if avoidance_stats['encounters']: + encounter_rate = np.mean(avoidance_stats['encounters'][-AGGREGATE_STATS_EVERY:]) if len(avoidance_stats['encounters']) >= AGGREGATE_STATS_EVERY else np.mean(avoidance_stats['encounters']) + stats_dict['obstacle_encounter_rate'] = encounter_rate # 添加多目标指标 if agent.multi_objective_optimizer: - for obj in ['safety', 'efficiency', 'comfort', 'rule_following']: + for obj in ['safety', 'efficiency', 'comfort', 'obstacle_avoidance']: if multi_obj_stats[obj]: recent_avg = np.mean(multi_obj_stats[obj][-AGGREGATE_STATS_EVERY:]) if len(multi_obj_stats[obj]) >= AGGREGATE_STATS_EVERY else np.mean(multi_obj_stats[obj]) stats_dict[f'{obj}_score'] = recent_avg agent.tensorboard.update_stats(**stats_dict) - - # 保存模型,仅当最小奖励达到设定值时 - if min_reward >= MIN_REWARD and (episode not in epds): - model_suffix = f"advanced_{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min" - agent.model.save(f'models/{MODEL_NAME}_{model_suffix}__{int(time.time())}.model') - - epds.append(episode) # 打印训练信息 - info_str = f'轮次: {episode:3d}, 得分: {score:6.2f}, 成功: {success_count:3d}' + info_str = f'轮次: {episode:3d}, 得分: {score:6.2f}, 成功: {success_count:3d}, 避障成功: {env_info["successful_avoidance"]}' if agent.curriculum_manager: - info_str += f', 阶段: {agent.curriculum_manager.current_stage}' + stage_info = agent.curriculum_manager.get_stage_info() + info_str += f', 阶段: {stage_info["stage"]}({stage_info["name"]})' print(info_str) # 衰减探索率 if Hyperparameters.EPSILON > Hyperparameters.MIN_EPSILON: Hyperparameters.EPSILON *= Hyperparameters.EPSILON_DECAY Hyperparameters.EPSILON = max(Hyperparameters.MIN_EPSILON, Hyperparameters.EPSILON) - - # 设置训练线程终止标志并等待其结束 + + # 训练结束 agent.terminate = True trainer_thread.join() @@ -361,110 +495,168 @@ def extended_reward_calculation(env, action, reward, done, step_info): final_avg_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:] if len(scores) >= AGGREGATE_STATS_EVERY else scores) final_min_reward = min(scores[-AGGREGATE_STATS_EVERY:] if len(scores) >= AGGREGATE_STATS_EVERY else scores) - model_suffix = "advanced_final" - agent.model.save( - f'models/{MODEL_NAME}_{model_suffix}__{final_max_reward:_>7.2f}max_{final_avg_reward:_>7.2f}avg_{final_min_reward:_>7.2f}min__{int(time.time())}.model') + model_suffix = "enhanced_final" + agent.save_model( + f'models/{MODEL_NAME}_{model_suffix}__{final_max_reward:_>7.2f}max_{final_avg_reward:_>7.2f}avg_{final_min_reward:_>7.2f}min__{int(time.time())}.model' + ) # 保存训练统计数据 training_stats = { 'scores': scores, 'avg_scores': avg_scores, 'multi_obj_stats': multi_obj_stats, + 'avoidance_stats': avoidance_stats, 'curriculum_stages': curriculum_stages, + 'curriculum_progress': curriculum_progress, + 'top_models': top_models, # 添加最佳模型信息 'final_scores': { 'max': final_max_reward, 'avg': final_avg_reward, 'min': final_min_reward + }, + 'training_config': { + 'episodes': EPISODES, + 'epsilon_decay': EPSILON_DECAY, + 'model_name': MODEL_NAME } } - stats_file = f'training_stats_{int(time.time())}.pkl' + timestamp = int(time.time()) + stats_file = f'training_stats/training_stats_enhanced_{timestamp}.pkl' + with open(stats_file, 'wb') as f: pickle.dump(training_stats, f) + print(f"训练统计数据已保存到: {stats_file}") - - # 绘制训练曲线 - fig, axes = plt.subplots(3, 2, figsize=(15, 15)) - - # 1. 得分曲线 - axes[0, 0].plot(scores, label='每轮得分', alpha=0.6, linewidth=1) - axes[0, 0].plot(avg_scores, label='平均得分(最近10轮)', linewidth=2, color='red') - axes[0, 0].set_ylabel('得分') - axes[0, 0].set_xlabel('训练轮次') - axes[0, 0].set_title('训练进度 - 得分曲线') - axes[0, 0].legend() - axes[0, 0].grid(True, alpha=0.3) - - # 2. 探索率衰减曲线 - eps_values = [max(MIN_EPSILON, 1.0 * (EPSILON_DECAY ** i)) for i in range(len(scores))] - axes[0, 1].plot(eps_values, color='red', linewidth=2) - axes[0, 1].set_ylabel('探索率 (ε)') - axes[0, 1].set_xlabel('训练轮次') - axes[0, 1].set_title('探索率衰减曲线') - axes[0, 1].grid(True, alpha=0.3) - - # 3. PER缓冲区大小 - if per_stats['buffer_size']: - axes[1, 0].plot(per_stats['buffer_size'], color='green', linewidth=2) - axes[1, 0].axhline(y=REPLAY_MEMORY_SIZE, color='r', linestyle='--', alpha=0.5, label='最大容量') - axes[1, 0].set_ylabel('缓冲区大小') - axes[1, 0].set_xlabel('训练轮次') - axes[1, 0].set_title('PER缓冲区使用情况') - axes[1, 0].legend() - axes[1, 0].grid(True, alpha=0.3) - - # 4. 课程学习阶段变化 - if curriculum_stages: - axes[1, 1].plot(curriculum_stages, color='purple', linewidth=2, drawstyle='steps-post') - axes[1, 1].set_ylabel('课程学习阶段') - axes[1, 1].set_xlabel('训练轮次') - axes[1, 1].set_title('课程学习阶段变化') - axes[1, 1].grid(True, alpha=0.3) - - # 5. 多目标指标 - if multi_obj_stats['safety']: - colors = ['blue', 'green', 'orange', 'purple'] - for i, (key, values) in enumerate(multi_obj_stats.items()): - if values: - # 计算滑动平均 - window = 10 - if len(values) >= window: - smoothed = np.convolve(values, np.ones(window)/window, mode='valid') - axes[2, 0].plot(range(len(smoothed)), smoothed, label=key, color=colors[i], alpha=0.7) - else: - axes[2, 0].plot(values, label=key, color=colors[i], alpha=0.7) - - axes[2, 0].set_ylabel('分数') - axes[2, 0].set_xlabel('训练轮次') - axes[2, 0].set_title('多目标优化指标') - axes[2, 0].legend() - axes[2, 0].grid(True, alpha=0.3) - - # 6. 成功率统计 - success_rates = [] - for i in range(len(scores)): - window = scores[max(0, i-9):i+1] - success_rate = sum(1 for s in window if s > 5) / len(window) * 100 - success_rates.append(success_rate) - - axes[2, 1].plot(success_rates, color='darkred', linewidth=2) - axes[2, 1].axhline(y=80, color='g', linestyle='--', alpha=0.5, label='目标成功率80%') - axes[2, 1].set_ylabel('成功率 (%)') - axes[2, 1].set_xlabel('训练轮次') - axes[2, 1].set_title('最近10轮成功率') - axes[2, 1].legend() - axes[2, 1].grid(True, alpha=0.3) - - plt.suptitle('高级训练策略 - 综合训练报告', fontsize=16, fontweight='bold') - plt.tight_layout() - plt.show() + + # 生成训练报告 + generate_training_report(training_stats) print("\n" + "="*60) - print("训练完成!") + print("增强版训练完成!") print("="*60) print(f"最终统计:") print(f" 总轮次: {EPISODES}") print(f" 最佳得分: {best_score:.2f}") print(f" 平均得分: {np.mean(scores):.2f}") print(f" 成功率: {(success_count/EPISODES)*100:.1f}%") - print(f" 最终探索率: {Hyperparameters.EPSILON:.4f}") \ No newline at end of file + print(f" 最终探索率: {Hyperparameters.EPSILON:.4f}") + print(f" 总避障成功次数: {sum(avoidance_stats['success_rate'])}") + + return agent + + +def generate_training_report(stats): + """生成训练报告图表""" + try: + fig, axes = plt.subplots(3, 2, figsize=(15, 15)) + + # 1. 得分曲线 + axes[0, 0].plot(stats['scores'], label='每轮得分', alpha=0.6, linewidth=1) + axes[0, 0].plot(stats['avg_scores'], label='平均得分(最近10轮)', linewidth=2, color='red') + axes[0, 0].set_ylabel('得分') + axes[0, 0].set_xlabel('训练轮次') + axes[0, 0].set_title('训练进度 - 得分曲线') + axes[0, 0].legend() + axes[0, 0].grid(True, alpha=0.3) + + # 2. 探索率衰减曲线 + eps_values = [max(MIN_EPSILON, 1.0 * (EPSILON_DECAY ** i)) for i in range(len(stats['scores']))] + axes[0, 1].plot(eps_values, color='red', linewidth=2) + axes[0, 1].set_ylabel('探索率 (ε)') + axes[0, 1].set_xlabel('训练轮次') + axes[0, 1].set_title('探索率衰减曲线') + axes[0, 1].grid(True, alpha=0.3) + + # 3. 多目标指标 + colors = ['blue', 'green', 'orange', 'purple'] + obj_keys = ['safety', 'efficiency', 'comfort', 'obstacle_avoidance'] + + for i, key in enumerate(obj_keys): + if key in stats['multi_obj_stats'] and stats['multi_obj_stats'][key]: + values = stats['multi_obj_stats'][key] + window = 10 + if len(values) >= window: + smoothed = np.convolve(values, np.ones(window)/window, mode='valid') + axes[1, 0].plot(range(len(smoothed)), smoothed, label=key, color=colors[i], alpha=0.7) + else: + axes[1, 0].plot(values, label=key, color=colors[i], alpha=0.7) + + axes[1, 0].set_ylabel('分数') + axes[1, 0].set_xlabel('训练轮次') + axes[1, 0].set_title('多目标优化指标') + axes[1, 0].legend() + axes[1, 0].grid(True, alpha=0.3) + + # 4. 课程学习阶段变化 + if 'curriculum_stages' in stats and stats['curriculum_stages']: + axes[1, 1].plot(stats['curriculum_stages'], color='purple', linewidth=2, drawstyle='steps-post') + axes[1, 1].set_ylabel('课程学习阶段') + axes[1, 1].set_xlabel('训练轮次') + axes[1, 1].set_title('课程学习阶段变化') + axes[1, 1].grid(True, alpha=0.3) + + # 5. 避障成功率 + if 'avoidance_stats' in stats and 'success_rate' in stats['avoidance_stats']: + success_rates = [] + success_values = stats['avoidance_stats']['success_rate'] + + for i in range(len(success_values)): + window = success_values[max(0, i-9):i+1] + if window: + success_rates.append(np.mean(window)) + + axes[2, 0].plot(success_rates, color='darkgreen', linewidth=2) + axes[2, 0].axhline(y=0.7, color='g', linestyle='--', alpha=0.5, label='目标成功率70%') + axes[2, 0].set_ylabel('平均避障成功次数') + axes[2, 0].set_xlabel('训练轮次') + axes[2, 0].set_title('避障性能') + axes[2, 0].legend() + axes[2, 0].grid(True, alpha=0.3) + + # 6. 成功率统计 + success_rates = [] + scores = stats['scores'] + + for i in range(len(scores)): + window = scores[max(0, i-9):i+1] + success_rate = sum(1 for s in window if s > 8) / len(window) * 100 + success_rates.append(success_rate) + + axes[2, 1].plot(success_rates, color='darkred', linewidth=2) + axes[2, 1].axhline(y=70, color='g', linestyle='--', alpha=0.5, label='目标成功率70%') + axes[2, 1].set_ylabel('成功率 (%)') + axes[2, 1].set_xlabel('训练轮次') + axes[2, 1].set_title('最近10轮成功率') + axes[2, 1].legend() + axes[2, 1].grid(True, alpha=0.3) + + plt.suptitle('增强版训练策略 - 综合训练报告', fontsize=16, fontweight='bold') + plt.tight_layout() + + # 保存图表 + timestamp = int(time.time()) + chart_file = f'training_stats/training_chart_enhanced_{timestamp}.png' + plt.savefig(chart_file, dpi=150, bbox_inches='tight') + plt.show() + + print(f"训练图表已保存到: {chart_file}") + + # 显示最佳模型总结 + print("\n" + "="*60) + print("最佳模型总结 (前10个)") + print("="*60) + top_models = stats.get('top_models', []) + if top_models: + for i, (score, filename) in enumerate(top_models, 1): + print(f"{i:2d}. 得分: {score:8.2f} - {filename}") + else: + print("没有保存任何模型") + print("="*60) + + except Exception as e: + print(f"生成训练报告时出错: {e}") + + +if __name__ == '__main__': + train_enhanced_agent() \ No newline at end of file