diff --git a/src/Unmanned_vehicle_AD_DQN/Environment.py b/src/Unmanned_vehicle_AD_DQN/Environment.py index 0a8394be95..ef1615321e 100644 --- a/src/Unmanned_vehicle_AD_DQN/Environment.py +++ b/src/Unmanned_vehicle_AD_DQN/Environment.py @@ -16,96 +16,97 @@ try: from Hyperparameters import * except ImportError: - # 如果在导入Hyperparameters时出错,使用默认值 SHOW_PREVIEW = False - IM_WIDTH = 160 # 修改默认值 - IM_HEIGHT = 120 # 修改默认值 + IM_WIDTH = 160 + IM_HEIGHT = 120 class CarEnv: - SHOW_CAM = SHOW_PREVIEW # 是否显示摄像头预览 - im_width = IM_WIDTH # 图像宽度 - im_height = IM_HEIGHT # 图像高度 + SHOW_CAM = SHOW_PREVIEW + im_width = IM_WIDTH + im_height = IM_HEIGHT def __init__(self): - self.actor_list = [] # 存储所有actor的列表 - self.sem_cam = None # 语义分割摄像头 - self.client = carla.Client("localhost", 2000) # CARLA客户端 - self.client.set_timeout(20.0) # 连接超时设置 - self.front_camera = None # 前置摄像头图像 - - # 新增变量 - self.last_action = 1 # 上一个动作,默认保持 - self.same_steer_counter = 0 # 连续同向转向计数器 - self.suggested_action = None # 建议的避让动作 - self.episode_start_time = None # 每轮开始时间 - self.last_ped_distance = float('inf') # 上次最近行人距离 - self.current_episode = 1 # 当前episode编号 - self.obstacle_detected_time = None # 障碍物检测到的时间 - self.reaction_start_time = None # 反应开始时间 - self.proactive_action_count = 0 # 主动避障动作计数 - - # 新增:多帧缓存用于运动检测 + self.actor_list = [] + self.sem_cam = None + self.client = carla.Client("localhost", 2000) + self.client.set_timeout(20.0) + self.front_camera = None + + # 道路参数 + self.road_center_y = -195 + self.road_left = -216 # 道路左侧边界 + self.road_right = -183 # 道路右侧边界 + + # 原有变量 + self.last_action = 1 + self.same_steer_counter = 0 + self.suggested_action = None + self.episode_start_time = None + self.last_ped_distance = float('inf') + self.current_episode = 1 + self.obstacle_detected_time = None + self.reaction_start_time = None + self.proactive_action_count = 0 self.frame_buffer = deque(maxlen=3) self.motion_detected = False - # 加载世界和蓝图 + # 加载世界 try: self.world = self.client.load_world('Town03') except: - # 如果加载失败,使用当前世界 self.world = self.client.get_world() - # 设置观察者视角,让CARLA窗口显示 + # 设置观察者视角 self.setup_observer_view() self.blueprint_library = self.world.get_blueprint_library() - self.model_3 = self.blueprint_library.filter("model3")[0] # Tesla Model3车辆 - + self.model_3 = self.blueprint_library.filter("model3")[0] + # 行人列表和碰撞历史 self.walker_list = [] self.collision_history = [] - self.slow_counter = 0 # 慢速计数器 - self.steer_counter = 0 # 转向计数器,用于限制过度转向 + self.slow_counter = 0 + self.steer_counter = 0 def setup_observer_view(self): - """设置观察者视角,让用户可以在CARLA窗口中看到场景""" + """设置观察者视角""" try: - # 获取当前地图的生成点 - spawn_points = self.world.get_map().get_spawn_points() - if spawn_points: - # 选择一个合适的观察者位置 - spectator = self.world.get_spectator() - - # 设置观察者位置在车辆起始位置附近 - transform = carla.Transform() - transform.location.x = -81.0 - transform.location.y = -195.0 - transform.location.z = 15.0 # 提高视角高度 - transform.rotation.pitch = -45.0 # 向下倾斜视角 - transform.rotation.yaw = 0.0 - transform.rotation.roll = 0.0 - - spectator.set_transform(transform) - print("观察者视角已设置") + spectator = self.world.get_spectator() + transform = carla.Transform() + transform.location.x = -81.0 + transform.location.y = -195.0 + transform.location.z = 15.0 + transform.rotation.pitch = -45.0 + transform.rotation.yaw = 0.0 + transform.rotation.roll = 0.0 + + spectator.set_transform(transform) + print("观察者视角已设置") except Exception as e: print(f"设置观察者视角时出错: {e}") + def check_road_boundary(self, location): + """检查是否在道路边界内 - 简化版""" + # 检查是否在道路范围内 + if self.road_left <= location.y <= self.road_right: + # 在道路内,计算到道路中心的距离 + distance_to_center = abs(location.y - self.road_center_y) + return distance_to_center, False # 在道路内 + else: + # 超出道路边界 + return float('inf'), True + def spawn_pedestrians_general(self, number, isCross): """生成指定数量的行人""" - # 记录要生成的数量 target_number = number - - # 记录成功生成的数量 success_count = 0 - - # 尝试生成指定数量的行人 attempts = 0 - max_attempts = number * 3 # 最多尝试3倍次数 + max_attempts = number * 3 while success_count < target_number and attempts < max_attempts: attempts += 1 - isLeft = random.choice([True, False]) # 随机选择左右侧 + isLeft = random.choice([True, False]) try: if isLeft: @@ -115,7 +116,6 @@ def spawn_pedestrians_general(self, number, isCross): if self.spawn_pedestrians_right(isCross): success_count += 1 except Exception as e: - # 如果生成失败,继续尝试 continue print(f"成功生成 {success_count}/{target_number} 个行人 (isCross={isCross})") @@ -125,13 +125,12 @@ def spawn_pedestrians_right(self, isCross): """在右侧生成行人""" blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") - # 设置生成区域 + # 设置生成区域 - 在道路右侧的人行道上 min_x = -50 max_x = 140 - min_y = -188 - max_y = -183 + min_y = self.road_right + 2 # 道路右侧外2米 + max_y = self.road_right + 5 - # 如果是十字路口,调整生成位置 if isCross: isFirstCross = random.choice([True, False]) if isFirstCross: @@ -141,51 +140,39 @@ def spawn_pedestrians_right(self, isCross): min_x = 17 max_x = 20.5 - # 尝试多次生成直到成功 - for attempt in range(3): # 尝试3次 - # 随机生成位置 + for attempt in range(3): x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - # 避免在特定区域生成 - while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): - x = random.uniform(min_x, max_x) - y = random.uniform(min_y, max_y) - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - - # 尝试生成行人 try: walker_bp = random.choice(blueprints_walkers) npc = self.world.try_spawn_actor(walker_bp, spawn_point) if npc is not None: - # 设置行人控制参数 ped_control = carla.WalkerControl() - ped_control.speed = random.uniform(0.5, 1.0) # 随机速度 - ped_control.direction.y = -1 # 主要移动方向 - ped_control.direction.x = 0.15 # 轻微横向移动 + ped_control.speed = random.uniform(0.5, 1.0) + ped_control.direction.y = -1 # 向道路方向移动 + ped_control.direction.x = 0.15 npc.apply_control(ped_control) - npc.set_simulate_physics(True) # 启用物理模拟 - self.walker_list.append(npc) # 添加到行人列表 - return True # 生成成功 + npc.set_simulate_physics(True) + self.walker_list.append(npc) + return True except Exception as e: continue - return False # 生成失败 + return False def spawn_pedestrians_left(self, isCross): """在左侧生成行人""" blueprints_walkers = self.world.get_blueprint_library().filter("walker.pedestrian.*") - # 设置生成区域 min_x = -50 max_x = 140 - min_y = -216 - max_y = -210 + min_y = self.road_left - 5 # 道路左侧外5米 + max_y = self.road_left - 2 - # 如果是十字路口,调整生成位置 if isCross: isFirstCross = random.choice([True, False]) if isFirstCross: @@ -195,42 +182,32 @@ def spawn_pedestrians_left(self, isCross): min_x = 17 max_x = 20.5 - # 尝试多次生成直到成功 - for attempt in range(3): # 尝试3次 - # 随机生成位置 + for attempt in range(3): x = random.uniform(min_x, max_x) y = random.uniform(min_y, max_y) spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - # 避免在特定区域生成 - while (-10 < spawn_point.location.x < 17) or (70 < spawn_point.location.x < 100): - x = random.uniform(min_x, max_x) - y = random.uniform(min_y, max_y) - spawn_point = carla.Transform(carla.Location(x, y, 2.0)) - - # 尝试生成行人 try: walker_bp = random.choice(blueprints_walkers) npc = self.world.try_spawn_actor(walker_bp, spawn_point) if npc is not None: - # 设置行人控制参数 ped_control = carla.WalkerControl() - ped_control.speed = random.uniform(0.7, 1.3) # 随机速度 - ped_control.direction.y = 1 # 主要移动方向 - ped_control.direction.x = -0.05 # 轻微横向移动 + ped_control.speed = random.uniform(0.7, 1.3) + ped_control.direction.y = 1 # 向道路方向移动 + ped_control.direction.x = -0.05 npc.apply_control(ped_control) - npc.set_simulate_physics(True) # 启用物理模拟 - self.walker_list.append(npc) # 添加到行人列表 - return True # 生成成功 + npc.set_simulate_physics(True) + self.walker_list.append(npc) + return True except Exception as e: continue - return False # 生成失败 + return False def reset(self, episode=1): - """重置环境,根据episode参数生成不同数量的行人""" + """重置环境""" self.current_episode = episode # 清理现有的行人和车辆 @@ -239,16 +216,16 @@ def reset(self, episode=1): # 重置行人列表 self.walker_list = [] - # 根据训练阶段生成不同数量的行人 - if episode < 100: # 第一阶段:少量行人 + # 根据训练阶段生成行人 + if episode < 100: print(f"Episode {episode}: 生成少量行人 (4十字路口 + 2非十字路口)") - self.spawn_pedestrians_general(4, True) # 十字路口行人 - self.spawn_pedestrians_general(2, False) # 非十字路口行人 - elif episode < 400: # 第二阶段:中等数量行人 + self.spawn_pedestrians_general(4, True) + self.spawn_pedestrians_general(2, False) + elif episode < 400: print(f"Episode {episode}: 生成中等数量行人 (6十字路口 + 3非十字路口)") self.spawn_pedestrians_general(6, True) self.spawn_pedestrians_general(3, False) - else: # 第三阶段:正常难度 + else: print(f"Episode {episode}: 生成正常数量行人 (8十字路口 + 4非十字路口)") self.spawn_pedestrians_general(8, True) self.spawn_pedestrians_general(4, False) @@ -269,10 +246,10 @@ def reset(self, episode=1): self.frame_buffer.clear() self.motion_detected = False - # 设置车辆生成点 + # 设置车辆生成点 - 在道路中心 spawn_point = carla.Transform() spawn_point.location.x = -81.0 - spawn_point.location.y = -195.0 + spawn_point.location.y = self.road_center_y # 道路中心 spawn_point.location.z = 2.0 spawn_point.rotation.roll = 0.0 spawn_point.rotation.pitch = 0.0 @@ -286,41 +263,83 @@ def reset(self, episode=1): self.sem_cam = self.blueprint_library.find('sensor.camera.semantic_segmentation') self.sem_cam.set_attribute("image_size_x", f"{self.im_width}") self.sem_cam.set_attribute("image_size_y", f"{self.im_height}") - self.sem_cam.set_attribute("fov", f"110") # 视野角度 + self.sem_cam.set_attribute("fov", f"110") # 安装摄像头传感器 transform = carla.Transform(carla.Location(x=2.5, z=0.7)) self.sensor = self.world.spawn_actor(self.sem_cam, transform, attach_to=self.vehicle) self.actor_list.append(self.sensor) - self.sensor.listen(lambda data: self.process_img(data)) # 设置图像处理回调 + self.sensor.listen(lambda data: self.process_img(data)) # 初始化车辆控制 self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) - time.sleep(2) # 等待环境稳定 + time.sleep(2) # 设置碰撞传感器 colsensor = self.blueprint_library.find("sensor.other.collision") self.colsensor = self.world.spawn_actor(colsensor, transform, attach_to=self.vehicle) self.actor_list.append(self.colsensor) - self.colsensor.listen(lambda event: self.collision_data(event)) # 设置碰撞检测回调 + self.colsensor.listen(lambda event: self.collision_data(event)) # 等待摄像头初始化完成 start_time = time.time() while self.front_camera is None and time.time() - start_time < 5: time.sleep(0.01) - # 设置跟随相机(用于观察) + # 设置跟随相机 self.setup_follow_camera() # 记录episode开始时间并重置控制 self.episode_start = time.time() self.vehicle.apply_control(carla.VehicleControl(throttle=0.0, brake=0.0, steer=0.0)) - return self.front_camera + # 返回增强的状态信息 + return self.get_enhanced_state() + + def get_enhanced_state(self): + """获取增强的状态信息(图像+位置+速度+方向)""" + # 基础图像状态 + base_state = self.front_camera + + # 获取车辆状态信息 + if hasattr(self, 'vehicle') and self.vehicle is not None: + vehicle_location = self.vehicle.get_location() + vehicle_transform = self.vehicle.get_transform() + velocity = self.vehicle.get_velocity() + + # 计算速度 + speed_kmh = 3.6 * math.sqrt(velocity.x**2 + velocity.y**2 + velocity.z**2) + + # 计算方向 + heading = vehicle_transform.rotation.yaw + + # 计算到道路中心的距离 + distance_to_center = abs(vehicle_location.y - self.road_center_y) + + # 创建状态字典 + state_info = { + 'image': base_state, + 'location': np.array([vehicle_location.x, vehicle_location.y]), # 2维 + 'speed': np.array([speed_kmh]), # 1维 + 'heading': np.array([heading]), # 1维 + 'distance_to_center': np.array([distance_to_center]), # 1维 - 新增 + 'last_action': np.array([self.last_action]) if hasattr(self, 'last_action') else np.array([1]) # 1维 + } + + return state_info + + # 默认返回 + return { + 'image': base_state, + 'location': np.zeros(2), + 'speed': np.array([0]), + 'heading': np.array([0]), + 'distance_to_center': np.array([0]), + 'last_action': np.array([1]) + } def cleanup_actors(self): """清理所有actors""" - # 清理车辆 try: vehicles = self.world.get_actors().filter('vehicle.*') for vehicle in vehicles: @@ -329,7 +348,6 @@ def cleanup_actors(self): except: pass - # 清理行人 try: walkers = self.world.get_actors().filter('walker.*') for walker in walkers: @@ -338,7 +356,6 @@ def cleanup_actors(self): except: pass - # 清理传感器 for actor in self.actor_list: try: if actor.is_alive: @@ -347,18 +364,16 @@ def cleanup_actors(self): pass self.actor_list = [] - self.walker_list = [] # 清空行人列表 + self.walker_list = [] def setup_follow_camera(self): - """设置跟随车辆的相机,用于在CARLA窗口中观察""" + """设置跟随车辆的相机""" try: - # 创建RGB相机 camera_bp = self.blueprint_library.find('sensor.camera.rgb') camera_bp.set_attribute('image_size_x', '800') camera_bp.set_attribute('image_size_y', '600') camera_bp.set_attribute('fov', '110') - # 相机位置相对于车辆(后方上方) camera_transform = carla.Transform(carla.Location(x=-8, z=6), carla.Rotation(pitch=-20)) follow_camera = self.world.spawn_actor(camera_bp, camera_transform, attach_to=self.vehicle) self.actor_list.append(follow_camera) @@ -373,49 +388,43 @@ def collision_data(self, event): def process_img(self, image): """处理摄像头图像""" try: - image.convert(carla.ColorConverter.CityScapesPalette) # 转换为CityScapes调色板 + image.convert(carla.ColorConverter.CityScapesPalette) - # 处理原始图像数据 processed_image = np.array(image.raw_data) processed_image = processed_image.reshape((self.im_height, self.im_width, 4)) - processed_image = processed_image[:, :, :3] # 移除alpha通道 + processed_image = processed_image[:, :, :3] - # 添加简单的图像增强 - # 增加对比度以突出障碍物 + # 图像增强 processed_image = cv2.convertScaleAbs(processed_image, alpha=1.2, beta=10) - # 存储到帧缓冲区用于运动检测 + # 运动检测 if len(self.frame_buffer) == 0: self.frame_buffer.append(processed_image.copy()) else: - # 简单的运动检测 if len(self.frame_buffer) >= 2: prev_frame = self.frame_buffer[-1] gray_prev = cv2.cvtColor(prev_frame, cv2.COLOR_RGB2GRAY) gray_current = cv2.cvtColor(processed_image, cv2.COLOR_RGB2GRAY) - # 计算帧间差异 diff = cv2.absdiff(gray_prev, gray_current) _, diff_thresh = cv2.threshold(diff, 25, 255, cv2.THRESH_BINARY) - # 如果差异足够大,检测到运动 motion_pixels = np.sum(diff_thresh > 0) motion_ratio = motion_pixels / (self.im_width * self.im_height) - self.motion_detected = motion_ratio > 0.01 # 1%的像素变化 + self.motion_detected = motion_ratio > 0.01 self.frame_buffer.append(processed_image.copy()) - # 显示预览(如果启用) if self.SHOW_CAM: cv2.imshow("", processed_image) cv2.waitKey(1) - self.front_camera = processed_image # 更新前置摄像头图像 + self.front_camera = processed_image except Exception as e: print(f"处理图像时出错: {e}") def reward(self, speed_kmh, current_steer): - """增强的奖励函数 - 特别强调行人避障和快速反应""" + """增强的奖励函数 - 简化版""" reward = 0 done = False @@ -423,23 +432,41 @@ def reward(self, speed_kmh, current_steer): vehicle_location = self.vehicle.get_location() vehicle_rotation = self.vehicle.get_transform().rotation.yaw - # 1. 道路保持奖励 + # 1. 道路保持奖励 - 基于到道路中心的距离 + distance_to_center = abs(vehicle_location.y - self.road_center_y) + + if distance_to_center < 5: # 完美保持在中心 + reward += 1.0 + elif distance_to_center < 10: # 良好保持 + reward += 0.5 + elif distance_to_center < 15: # 可接受 + reward += 0.1 + else: # 偏离较大 + reward -= 0.5 + + # 2. 道路边界检查 + boundary_distance, out_of_boundary = self.check_road_boundary(vehicle_location) + + if out_of_boundary: + reward -= 20.0 # 大幅惩罚 + done = True + print(f"Episode {self.current_episode}: 驶出道路边界! y={vehicle_location.y:.1f}, 范围[{self.road_left}, {self.road_right}]") + elif distance_to_center > 20: # 严重偏离但还在道路内 + reward -= 10.0 + done = True + print(f"Episode {self.current_episode}: 严重偏离道路中心! 距离: {distance_to_center:.1f}m") + + # 3. 方向保持奖励 heading_error = abs(vehicle_rotation) - if heading_error < 5: # 完美保持方向 + if heading_error < 5: reward += 0.5 - elif heading_error < 15: # 良好保持 + elif heading_error < 15: reward += 0.2 - elif heading_error < 30: # 可接受 - reward += 0.05 - else: # 方向偏差过大 - reward -= 0.2 * (heading_error / 30.0) - # 2. 行人避障(最高优先级) - 增强反应机制 + # 4. 行人避障 min_ped_distance = float('inf') closest_pedestrian = None - - # 统计有效行人数量 active_pedestrians = 0 for walker in self.walker_list: @@ -456,187 +483,159 @@ def reward(self, speed_kmh, current_steer): min_ped_distance = distance closest_pedestrian = walker - # 如果当前没有有效行人,重置距离 - if active_pedestrians == 0: - min_ped_distance = float('inf') - - # 行人检测和反应时间记录 current_time = time.time() - # 检测到行人进入危险区域 - if min_ped_distance < 100: # 只考虑100米内的行人 + if min_ped_distance < 100: if self.obstacle_detected_time is None: self.obstacle_detected_time = current_time self.reaction_start_time = current_time - # 行人距离分级奖励/惩罚 - 强调提前反应 - if min_ped_distance < 3.0: # 紧急避让距离 - reward -= 15.0 # 大幅增加惩罚 + # 行人距离分级奖励/惩罚 + if min_ped_distance < 3.0: + reward -= 25.0 done = True print(f"Episode {self.current_episode}: 与行人碰撞! 距离: {min_ped_distance:.1f}m") - elif min_ped_distance < 5.0: # 危险距离 - reward -= 5.0 # 增加惩罚 + elif min_ped_distance < 5.0: + reward -= 8.0 - # 计算避让方向 if closest_pedestrian: ped_y = closest_pedestrian.get_location().y veh_y = vehicle_location.y - # 如果行人在车辆左侧,鼓励右转;反之鼓励左转 - if ped_y < veh_y: # 行人在左侧 - self.suggested_action = 4 # 右转 - else: # 行人在右侧 - self.suggested_action = 3 # 左转 - - # 反应时间惩罚 - if self.reaction_start_time: - reaction_time = current_time - self.reaction_start_time - if reaction_time > 0.5: # 反应超过0.5秒 - reward -= 2.0 * (reaction_time - 0.5) + if ped_y < veh_y: + self.suggested_action = 4 # 行人在左侧,右转 + else: + self.suggested_action = 3 # 行人在右侧,左转 - elif min_ped_distance < 8.0: # 预警距离 - reward -= 1.5 + elif min_ped_distance < 8.0: + reward -= 3.0 - # 计算避让方向(提前建议) if closest_pedestrian: ped_y = closest_pedestrian.get_location().y veh_y = vehicle_location.y - if ped_y < veh_y: # 行人在左侧 - self.suggested_action = 4 # 右转 - else: # 行人在右侧 - self.suggested_action = 3 # 左转 + if ped_y < veh_y: + self.suggested_action = 4 + else: + self.suggested_action = 3 - # 提前反应的奖励 - if self.last_ped_distance < 12.0 and min_ped_distance > self.last_ped_distance: - # 如果上次距离更近,这次距离更远了,说明正在成功避让 - reward += 1.0 - self.proactive_action_count += 1 - - elif min_ped_distance < 12.0: # 安全距离 - # 检测到行人但距离安全,给予轻微奖励 - reward += 0.3 - - # 如果检测到运动,给予额外奖励 - if self.motion_detected: - reward += 0.2 - - # 提前规划奖励 - if self.last_ped_distance < 8.0: - # 从危险距离进入安全距离,成功避障 - reward += 2.0 + elif min_ped_distance < 12.0: + # 预警距离,轻微惩罚 + reward -= 0.5 else: - # 没有检测到行人,重置反应计时 self.obstacle_detected_time = None self.reaction_start_time = None - self.last_ped_distance = min_ped_distance # 保存当前距离 + self.last_ped_distance = min_ped_distance - # 3. 速度奖励(平衡避障和前进) - if 20 <= speed_kmh <= 35: # 理想速度区间 + # 5. 速度奖励 + if 20 <= speed_kmh <= 35: reward += 0.5 - elif 10 <= speed_kmh < 20: # 较慢但安全 + elif 10 <= speed_kmh < 20: reward += 0.3 - elif 35 < speed_kmh <= 40: # 稍快 - reward += 0.1 - elif speed_kmh > 40: # 过快,在行人环境中危险 - reward -= 1.0 # 增加惩罚 - else: # 停车或极慢 - reward -= 0.1 # 轻微惩罚 + elif speed_kmh > 40: + reward -= 1.0 - # 4. 转向平滑性奖励 - steer_penalty = abs(current_steer) * 0.2 # 降低惩罚,给避障转向更多空间 + # 6. 转向平滑性奖励 + steer_penalty = abs(current_steer) * 0.1 reward -= steer_penalty - # 5. 碰撞检测 + # 7. 碰撞检测 if len(self.collision_history) != 0: - reward = -20 # 增加碰撞惩罚 + reward = -30 done = True print(f"Episode {self.current_episode}: 发生碰撞!") - # 6. 进度奖励 - progress = (vehicle_location.x + 81) / 236.0 # 从-81到155 - reward += progress * 0.3 + # 8. 进度奖励 + progress = (vehicle_location.x + 81) / 236.0 + reward += progress * 0.5 - # 7. 边界检查 - if vehicle_location.x > 155: # 成功到达终点 - reward += 20 # 增加到达终点的奖励 + # 9. 边界检查 + if vehicle_location.x > 155: + reward += 30 done = True print(f"Episode {self.current_episode}: 成功到达终点!") - elif vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30: # 偏离道路 - reward -= 5 # 增加偏离惩罚 + elif vehicle_location.x < -90: + reward -= 10 done = True - print(f"Episode {self.current_episode}: 偏离道路!") + print(f"Episode {self.current_episode}: 反向行驶过远!") - # 8. 主动避障奖励 + # 10. 主动避障奖励 if self.proactive_action_count > 0: reward += self.proactive_action_count * 0.5 # 限制奖励范围 - reward = max(min(reward, 30), -25) + reward = max(min(reward, 40), -35) return reward, done def step(self, action): - """执行动作并返回新状态 - 增强平滑性和安全性""" - # 获取当前速度 + """执行动作并返回新状态""" velocity = self.vehicle.get_velocity() speed_kmh = 3.6 * math.sqrt(velocity.x**2 + velocity.y**2 + velocity.z**2) - # 5个动作: 0-减速, 1-保持, 2-加速, 3-左转, 4-右转 + # 获取当前位置和到道路中心的距离 + vehicle_location = self.vehicle.get_location() + distance_to_center = abs(vehicle_location.y - self.road_center_y) + + # 道路保持因子 - 偏离越大,转向越谨慎 + road_keeping_factor = max(0.3, 1.0 - distance_to_center / 30.0) - # 根据速度调整转向幅度 - 速度越快,转向越平滑 speed_factor = max(0.5, min(1.0, 30.0 / max(1.0, speed_kmh))) - # 基础控制参数 throttle = 0.0 brake = 0.0 steer = 0.0 - # 速度控制 - 优化动作响应 - if action == 0: # 减速 + if action == 0: throttle = 0.0 - brake = 0.7 # 增加制动强度 - elif action == 1: # 保持/轻微加速 - throttle = 0.4 # 增加油门 + brake = 0.7 + elif action == 1: + throttle = 0.4 brake = 0.0 - elif action == 2: # 加速 + elif action == 2: throttle = 0.8 brake = 0.0 - elif action == 3: # 左转 - throttle = 0.5 # 增加转向时的油门 + elif action == 3: + throttle = 0.5 brake = 0.0 - steer = -0.25 * speed_factor # 增加转向幅度 - elif action == 4: # 右转 + steer = -0.25 * speed_factor * road_keeping_factor + elif action == 4: throttle = 0.5 brake = 0.0 - steer = 0.25 * speed_factor # 增加转向幅度 + steer = 0.25 * speed_factor * road_keeping_factor + + # 自动道路保持:如果偏离较大,自动轻微回正 + if distance_to_center > 10: + auto_correction = 0.05 * (distance_to_center - 10) / 10.0 + if vehicle_location.y < self.road_center_y: # 在道路左侧 + steer += auto_correction # 轻微右转 + else: # 在道路右侧 + steer -= auto_correction # 轻微左转 + throttle = min(throttle, 0.6) # 减速 # 如果有建议的避让动作,调整当前动作 if self.suggested_action is not None: - # 如果当前动作与建议动作不同,调整转向幅度 if self.suggested_action in [3, 4] and action in [3, 4]: - # 如果都是转向动作,但方向不同,增加建议方向的转向 if self.suggested_action != action: - if self.suggested_action == 3: # 建议左转 + if self.suggested_action == 3: steer = -0.3 * speed_factor - else: # 建议右转 + else: steer = 0.3 * speed_factor - elif self.suggested_action == 0: # 建议减速 - brake = max(brake, 0.8) # 强制制动 + elif self.suggested_action == 0: + brake = max(brake, 0.8) throttle = 0.0 - # 限制连续同向转向 - 防止过度转向 + # 限制连续同向转向 if (action == 3 and self.last_action == 3) or (action == 4 and self.last_action == 4): self.same_steer_counter += 1 - if self.same_steer_counter > 2: # 连续2次同向转向后强制回正 - steer *= 0.3 # 大幅减小转向幅度 - throttle *= 0.6 # 减速 - brake = 0.2 # 轻微制动 + if self.same_steer_counter > 2: + steer *= 0.3 + throttle *= 0.6 + brake = 0.2 else: self.same_steer_counter = 0 - # 记录上一个动作 self.last_action = action # 应用控制 @@ -646,13 +645,13 @@ def step(self, action): steer=steer )) - # 减少等待时间,加快反应 - time.sleep(0.03) # 从0.05减少到0.03 + time.sleep(0.03) - # 计算奖励和完成状态 reward, done = self.reward(speed_kmh, steer) - # 限制极端奖励值 - reward = np.clip(reward, -25, 30) + reward = np.clip(reward, -35, 40) + + # 获取增强的状态 + new_enhanced_state = self.get_enhanced_state() - return self.front_camera, reward, done, None \ No newline at end of file + return new_enhanced_state, reward, done, None \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py index e701e4507e..0ed367cd96 100644 --- a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py +++ b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py @@ -1,114 +1,79 @@ -# Hyperparameters.py +# Hyperparameters.py - 修复版 # 深度强化学习超参数配置 DISCOUNT = 0.97 -# 未来奖励的折扣因子 - 提高未来奖励的重要性 FPS = 60 -# 模拟环境的帧率 MEMORY_FRACTION = 0.35 -# GPU内存分配比例 REWARD_OFFSET = -100 -# 停止模拟的奖励阈值 -MIN_REPLAY_MEMORY_SIZE = 1_000 # 减少预热样本数,让训练更快开始 -# 开始训练前经验回放缓冲区的最小大小 +MIN_REPLAY_MEMORY_SIZE = 1_000 -REPLAY_MEMORY_SIZE = 15_000 # 增加经验回放容量 -# 经验回放缓冲区的最大容量 +REPLAY_MEMORY_SIZE = 20_000 -MINIBATCH_SIZE = 128 # 增加批次大小 -# 每次训练从经验回放中采样的经验数量 +MINIBATCH_SIZE = 64 # 减小批次大小以提高稳定性 PREDICTION_BATCH_SIZE = 1 -# 预测阶段使用的批次大小 TRAINING_BATCH_SIZE = MINIBATCH_SIZE // 4 -# 训练阶段使用的批次大小 -EPISODES = 500 # 增加训练轮次 -# 智能体训练的总轮次数 +EPISODES = 300 # 减少训练轮次以快速测试 SECONDS_PER_EPISODE = 60 -# 每轮训练的秒数 -MIN_EPSILON = 0.05 # 增加最小探索率,保持一定的探索 -# 最小探索率 +MIN_EPSILON = 0.05 EPSILON = 1.0 -# 初始探索率 -EPSILON_DECAY = 0.995 # 减缓衰减速度 -# 探索率的衰减率 +EPSILON_DECAY = 0.995 -MODEL_NAME = "YY_Optimized_v2" -# 训练模型的名称标识 +MODEL_NAME = "YY_Optimized_v3" -MIN_REWARD = -100 # 降低阈值,更容易保存模型 -# 被认为是"良好"或"积极"经验的最小奖励值 +MIN_REWARD = -100 -UPDATE_TARGET_EVERY = 15 # 更频繁地更新目标网络 -# 目标网络更新的频率 +UPDATE_TARGET_EVERY = 15 -AGGREGATE_STATS_EVERY = 5 # 更频繁地记录统计信息 -# 计算和聚合统计信息(如平均得分、奖励)的频率 +AGGREGATE_STATS_EVERY = 5 SHOW_PREVIEW = False -# 是否显示预览窗口 - 测试时设为False以显示CARLA主窗口 -IM_WIDTH = 160 # 减少图像尺寸,加快处理速度 -# 预览或模拟中捕获图像的宽度 +IM_WIDTH = 160 -IM_HEIGHT = 120 # 减少图像尺寸,加快处理速度 -# 预览或模拟中捕获图像的高度 +IM_HEIGHT = 120 SLOW_COUNTER = 330 -# 慢速计数器阈值 LOW_REWARD_THRESHOLD = -2 -# 低奖励阈值 SUCCESSFUL_THRESHOLD = 3 -# 成功阈值 -LEARNING_RATE = 0.0001 # 增加学习率,加速收敛 -# 优化器的学习率 - -# PER (优先经验回放) 参数 -PER_ALPHA = 0.7 # 增加优先级影响 -# 优先级程度 (0 = 均匀采样, 1 = 完全优先级) +LEARNING_RATE = 0.0001 +# PER参数 +PER_ALPHA = 0.7 PER_BETA_START = 0.5 -# 重要性采样权重起始值 - -PER_BETA_FRAMES = 50000 # 减少beta增长时间 -# beta线性增长的帧数 +PER_BETA_FRAMES = 50000 # 训练策略参数 USE_CURRICULUM_LEARNING = True -# 是否使用课程学习 - USE_MULTI_OBJECTIVE = True -# 是否使用多目标优化 - USE_IMITATION_LEARNING = False -# 是否使用模仿学习(预训练) - -# 多目标优化权重(这些权重会自动调整) -SAFETY_WEIGHT = 0.50 # 提高安全权重 -EFFICIENCY_WEIGHT = 0.20 -COMFORT_WEIGHT = 0.15 -RULE_FOLLOWING_WEIGHT = 0.15 # 课程学习参数 -CURRICULUM_STAGES = 5 -# 课程学习阶段数量 +CURRICULUM_STAGES = 6 +CURRICULUM_SUCCESS_THRESHOLDS = [0.3, 0.4, 0.5, 0.6, 0.7, 0.8] + +# 静态障碍物参数 +STATIC_OBSTACLE_PENALTY_MULTIPLIER = 1.5 +STATIC_SAFE_DISTANCE = 5.0 +ROAD_BOUNDARY_PENALTY = 10.0 -CURRICULUM_SUCCESS_THRESHOLDS = [0.3, 0.5, 0.7, 0.85, 0.9] -# 每个阶段进入下一阶段所需的成功率阈值 +# 交叉路口参数 +INTERSECTION_SAFE_SPEED = 25.0 +INTERSECTION_STEER_REDUCTION = 0.6 -# 新增:反应时间参数 -REACTION_TIME_PENALTY = 0.1 # 对反应迟钝的惩罚系数 -PROACTIVE_REWARD = 0.5 # 对提前行动的奖励 \ No newline at end of file +# 反应时间参数 +REACTION_TIME_PENALTY = 0.1 +PROACTIVE_REWARD = 0.5 \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Model.py b/src/Unmanned_vehicle_AD_DQN/Model.py index b1c7d7f4ed..9b9ea822aa 100644 --- a/src/Unmanned_vehicle_AD_DQN/Model.py +++ b/src/Unmanned_vehicle_AD_DQN/Model.py @@ -26,7 +26,6 @@ try: from Hyperparameters import * except ImportError: - # 如果在导入Hyperparameters时出错,使用默认值 DISCOUNT = 0.97 MEMORY_FRACTION = 0.35 REPLAY_MEMORY_SIZE = 10000 @@ -36,10 +35,9 @@ TRAINING_BATCH_SIZE = 16 UPDATE_TARGET_EVERY = 20 LEARNING_RATE = 0.00005 - IM_HEIGHT = 480 - IM_WIDTH = 640 - MODEL_NAME = "YY_Optimized_v2" - + IM_HEIGHT = 120 + IM_WIDTH = 160 + MODEL_NAME = "YY_Optimized_v3" # 自定义TensorBoard类 class ModifiedTensorBoard(TensorBoard): @@ -72,7 +70,6 @@ def update_stats(self, **stats): tf.summary.scalar(key, value, step=self.step) self.writer.flush() - # 注意力机制模块 class AttentionModule: @staticmethod @@ -114,32 +111,89 @@ def spatial_attention(input_feature): @staticmethod def cbam_block(input_feature, ratio=8): """完整的CBAM注意力模块""" - # 通道注意力 x = AttentionModule.channel_attention(input_feature, ratio) - # 空间注意力 x = AttentionModule.spatial_attention(x) return x -# DQN智能体类 - 升级版(整合训练策略) +class EnhancedStateProcessor: + """增强状态处理器 - 处理多模态输入""" + + @staticmethod + def process_state(state_dict): + """处理增强的状态字典""" + # 提取图像 + image_state = state_dict['image'] + + # 提取其他状态信息 + location = state_dict.get('location', np.zeros(2)) # 2维 + speed = state_dict.get('speed', np.array([0])) # 1维 + heading = state_dict.get('heading', np.array([0])) # 1维 + last_action = state_dict.get('last_action', np.array([1])) # 1维 + + # 归一化处理 + location_norm = location / np.array([200, 200]) # 假设最大范围 + speed_norm = speed / 100.0 # 假设最大速度100km/h + heading_norm = heading / 180.0 # 归一化到[-1, 1] + + # 动作历史向量 + action_history = np.zeros(5) + if last_action.size > 0: + action_idx = int(last_action[0]) + if 0 <= action_idx < 5: + action_history[action_idx] = 1.0 + + # 合并状态向量 - 总共10维 + vector_state = np.concatenate([ + location_norm, # 2维 + speed_norm, # 1维 + heading_norm, # 1维 + action_history # 5维 + ]) + + # 填充到10维 + if len(vector_state) != 10: + vector_state = np.pad(vector_state, (0, 10 - len(vector_state)), 'constant') + + return image_state, vector_state + + @staticmethod + def batch_process(states): + """批量处理状态""" + batch_images = [] + batch_vectors = [] + + for state in states: + if isinstance(state, dict): + img, vec = EnhancedStateProcessor.process_state(state) + else: + img = state + vec = np.zeros(10) # 10维向量 + + batch_images.append(img) + batch_vectors.append(vec) + + return np.array(batch_images), np.array(batch_vectors) + + +# DQN智能体类 - 支持多模态输入 class DQNAgent: def __init__(self, use_dueling=True, use_per=True, use_curriculum=True, use_multi_objective=True): - # 创建主网络和目标网络 self.use_dueling = use_dueling self.use_per = use_per self.use_curriculum = use_curriculum self.use_multi_objective = use_multi_objective if use_dueling: - self.model = self.create_dueling_model() - self.target_model = self.create_dueling_model() + self.model = self.create_enhanced_dueling_model() + self.target_model = self.create_enhanced_dueling_model() else: - self.model = self.create_model() - self.target_model = self.create_model() + self.model = self.create_enhanced_model() + self.target_model = self.create_enhanced_model() self.target_model.set_weights(self.model.get_weights()) - # 经验回放缓冲区 - 使用PER或标准缓冲区 + # 经验回放缓冲区 if use_per: self.replay_buffer = PrioritizedReplayBuffer(max_size=REPLAY_MEMORY_SIZE) else: @@ -147,7 +201,7 @@ def __init__(self, use_dueling=True, use_per=True, use_curriculum=True, use_mult # 自定义TensorBoard self.tensorboard = ModifiedTensorBoard(log_dir=f"logs/{MODEL_NAME}-{int(time.time())}") - self.target_update_counter = 0 # 目标网络更新计数器 + self.target_update_counter = 0 # 训练控制标志 self.terminate = False @@ -159,7 +213,10 @@ def __init__(self, use_dueling=True, use_per=True, use_curriculum=True, use_mult self.multi_objective_optimizer = None self.imitation_manager = None - # 新增:反应时间追踪 + # 状态处理器 + self.state_processor = EnhancedStateProcessor() + + # 反应时间追踪 self.reaction_times = deque(maxlen=100) self.last_obstacle_distance = float('inf') @@ -175,17 +232,16 @@ def setup_training_strategies(self, env=None): self.multi_objective_optimizer = MultiObjectiveOptimizer() print("多目标优化器已启用") - # 模仿学习管理器(需要时手动启用) from TrainingStrategies import ImitationLearningManager self.imitation_manager = ImitationLearningManager() - def create_model(self): - """创建标准深度Q网络模型 - 优化版""" - # 使用函数式API - inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) + def create_enhanced_model(self): + """创建增强的深度Q网络模型 - 支持多模态输入""" + # 图像输入流 + image_input = Input(shape=(IM_HEIGHT, IM_WIDTH, 3), name='image_input') - # 第一卷积块 - 使用深度可分离卷积加快计算 - x = SeparableConv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) + # 第一卷积块 + x = SeparableConv2D(32, (3, 3), strides=(1, 1), padding='same')(image_input) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) @@ -202,20 +258,34 @@ def create_model(self): x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 注意力机制 - 增强对障碍物的关注 + # 注意力机制 x = AttentionModule.cbam_block(x) - # 第四卷积块 - 增加一层特征提取 + # 第四卷积块 - 静态障碍物特征提取 x = SeparableConv2D(256, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = SpatialDropout2D(0.1)(x) - # 展平层 - x = Flatten()(x) + # 展平图像特征 + image_features = Flatten()(x) + image_features = Dense(256, activation='relu')(image_features) + image_features = Dropout(0.3)(image_features) + + # 向量输入流(位置、速度、方向等)- 10维 + vector_input = Input(shape=(10,), name='vector_input') - # 全连接层 - 优化网络结构 - x = Dense(256, activation='relu')(x) + # 向量处理 + vector_features = Dense(64, activation='relu')(vector_input) + vector_features = BatchNormalization()(vector_features) + vector_features = Dense(32, activation='relu')(vector_features) + vector_features = Dropout(0.2)(vector_features) + + # 合并特征 + merged = Concatenate()([image_features, vector_features]) + + # 决策层 + x = Dense(256, activation='relu')(merged) x = Dropout(0.3)(x) x = Dense(128, activation='relu')(x) x = Dropout(0.3)(x) @@ -226,51 +296,61 @@ def create_model(self): outputs = Dense(5, activation='linear')(x) # 创建模型 - model = Model(inputs=inputs, outputs=outputs) + model = Model(inputs=[image_input, vector_input], outputs=outputs) - # 编译模型 - 使用自适应学习率优化器 + # 编译模型 model.compile(loss="huber", optimizer=Adam(learning_rate=LEARNING_RATE, clipnorm=1.0), metrics=["mae"]) return model - def create_dueling_model(self): - """创建Dueling DQN模型架构 - 优化版""" - inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) + def create_enhanced_dueling_model(self): + """创建增强的Dueling DQN模型""" + # 图像输入流 + image_input = Input(shape=(IM_HEIGHT, IM_WIDTH, 3), name='image_input') - # 共享的特征提取层 - # 第一卷积块 - x = SeparableConv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) + # 共享的图像特征提取 + x = SeparableConv2D(32, (3, 3), strides=(1, 1), padding='same')(image_input) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 第二卷积块 x = SeparableConv2D(64, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 第三卷积块 x = SeparableConv2D(128, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 注意力机制 x = AttentionModule.cbam_block(x) - # 第四卷积块 x = SeparableConv2D(256, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = SpatialDropout2D(0.1)(x) - # 展平层 - x = Flatten()(x) + # 展平图像特征 + image_features = Flatten()(x) + image_features = Dense(256, activation='relu')(image_features) + image_features = Dropout(0.3)(image_features) + + # 向量输入流 - 10维 + vector_input = Input(shape=(10,), name='vector_input') + + # 向量处理 + vector_features = Dense(64, activation='relu')(vector_input) + vector_features = BatchNormalization()(vector_features) + vector_features = Dense(32, activation='relu')(vector_features) + vector_features = Dropout(0.2)(vector_features) - # 共享的全连接层 - shared = Dense(256, activation='relu')(x) + # 合并特征 + merged = Concatenate()([image_features, vector_features]) + + # 共享层 + shared = Dense(256, activation='relu')(merged) shared = Dropout(0.3)(shared) shared = Dense(128, activation='relu')(shared) @@ -290,7 +370,7 @@ def create_dueling_model(self): q_values = Add()([value, advantage_centered]) # 创建模型 - model = Model(inputs=inputs, outputs=q_values) + model = Model(inputs=[image_input, vector_input], outputs=q_values) # 编译模型 model.compile(loss="huber", @@ -301,67 +381,68 @@ def create_dueling_model(self): def update_replay_memory(self, transition, reaction_time=None): """更新经验回放缓冲区""" - # transition = (当前状态, 动作, 奖励, 新状态, 完成标志) if reaction_time is not None: - # 记录反应时间 self.reaction_times.append(reaction_time) - # 如果反应时间过长,增加惩罚 - if reaction_time > 1.0: # 反应时间超过1秒 + if reaction_time > 1.0: transition = list(transition) - transition[2] -= 0.5 # 减少奖励 + transition[2] -= 0.5 transition = tuple(transition) if self.use_per: - # PER: 初始添加时使用最大优先级 - self.replay_buffer.add(transition, error=1.0) # 初始误差设为1.0 + self.replay_buffer.add(transition, error=1.0) else: self.replay_memory.append(transition) def minibatch_chooser(self): - """改进的经验采样策略 - 优先采样危险情况""" + """改进的经验采样策略""" if self.use_per: - # PER采样 if len(self.replay_buffer) < MIN_REPLAY_MEMORY_SIZE: return [] indices, samples, weights = self.replay_buffer.sample(MINIBATCH_SIZE) return samples else: - # 标准采样 - 增加危险经验的权重 if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: return random.sample(self.replay_memory, min(len(self.replay_memory), MINIBATCH_SIZE)) # 分类经验样本 - danger_samples = [] # 接近碰撞的经验 - collision_samples = [] # 碰撞经验 - positive_samples = [] # 成功避障经验 - neutral_samples = [] # 中性经验 + static_collision_samples = [] + danger_samples = [] + collision_samples = [] + positive_samples = [] + neutral_samples = [] for sample in self.replay_memory: state, action, reward, next_state, done = sample - if done and reward < -5: # 碰撞经验 + if done and reward < -20: + static_collision_samples.append(sample) + elif done and reward < -5: collision_samples.append(sample) - elif reward < -2 and not done: # 危险但未碰撞 + elif reward < -2 and not done: danger_samples.append(sample) - elif reward > 2: # 积极经验(成功避障) + elif reward > 2: positive_samples.append(sample) - else: # 中性经验 + else: neutral_samples.append(sample) - # 平衡采样 - 增加危险经验的权重 + # 平衡采样 batch = [] - # 采样碰撞经验 (25%) - num_collision = min(len(collision_samples), MINIBATCH_SIZE // 4) + # 采样静态障碍物碰撞经验 + num_static = min(len(static_collision_samples), MINIBATCH_SIZE // 5) + batch.extend(random.sample(static_collision_samples, num_static)) + + # 采样行人碰撞经验 + num_collision = min(len(collision_samples), MINIBATCH_SIZE * 3 // 20) batch.extend(random.sample(collision_samples, num_collision)) - # 采样危险经验 (30%) - num_danger = min(len(danger_samples), MINIBATCH_SIZE * 3 // 10) + # 采样危险经验 + num_danger = min(len(danger_samples), MINIBATCH_SIZE // 4) batch.extend(random.sample(danger_samples, num_danger)) - # 采样成功避障经验 (25%) + # 采样成功避障经验 num_positive = min(len(positive_samples), MINIBATCH_SIZE // 4) batch.extend(random.sample(positive_samples, num_positive)) @@ -370,12 +451,11 @@ def minibatch_chooser(self): if remaining > 0: batch.extend(random.sample(neutral_samples, min(remaining, len(neutral_samples)))) - # 如果还不够,从整个记忆库随机采样 if len(batch) < MINIBATCH_SIZE: additional = MINIBATCH_SIZE - len(batch) batch.extend(random.sample(self.replay_memory, additional)) - random.shuffle(batch) # 打乱批次 + random.shuffle(batch) return batch def train(self): @@ -384,7 +464,6 @@ def train(self): if len(self.replay_buffer) < MIN_REPLAY_MEMORY_SIZE: return - # PER: 采样并获取权重 indices, minibatch, weights = self.replay_buffer.sample(MINIBATCH_SIZE) if len(minibatch) == 0: return @@ -392,39 +471,81 @@ def train(self): if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: return - # 标准采样 minibatch = self.minibatch_chooser() - weights = np.ones(len(minibatch)) # 标准训练权重为1 - - # 准备训练数据 - current_states = np.array([transition[0] for transition in minibatch]) / 255 - current_qs_list = self.model.predict(current_states, batch_size=PREDICTION_BATCH_SIZE, verbose=0) + weights = np.ones(len(minibatch)) - new_current_states = np.array([transition[3] for transition in minibatch]) / 255 - future_qs_list = self.target_model.predict(new_current_states, batch_size=PREDICTION_BATCH_SIZE, verbose=0) - - x = [] # 输入状态 - y = [] # 目标Q值 - errors = [] # TD误差(用于PER) + # 处理批次数据 + batch_images = [] + batch_vectors = [] + next_batch_images = [] + next_batch_vectors = [] + + for transition in minibatch: + current_state, action, reward, new_state, done = transition + + # 处理当前状态 + if isinstance(current_state, dict): + img, vec = self.state_processor.process_state(current_state) + else: + img = current_state + vec = np.zeros(10) # 10维向量 + + batch_images.append(img) + batch_vectors.append(vec) + + # 处理下一个状态 + if isinstance(new_state, dict): + next_img, next_vec = self.state_processor.process_state(new_state) + else: + next_img = new_state + next_vec = np.zeros(10) + + next_batch_images.append(next_img) + next_batch_vectors.append(next_vec) + + # 归一化图像 + batch_images = np.array(batch_images) / 255 + next_batch_images = np.array(next_batch_images) / 255 + batch_vectors = np.array(batch_vectors) + next_batch_vectors = np.array(next_batch_vectors) + + # 预测Q值 + current_qs_list = self.model.predict([batch_images, batch_vectors], + batch_size=PREDICTION_BATCH_SIZE, + verbose=0) + future_qs_list = self.target_model.predict([next_batch_images, next_batch_vectors], + batch_size=PREDICTION_BATCH_SIZE, + verbose=0) + + x_images = [] + x_vectors = [] + y = [] + errors = [] # 计算目标Q值 for index, (current_state, action, reward, new_state, done) in enumerate(minibatch): if not done: - # 使用贝尔曼方程计算目标Q值 max_future_q = np.max(future_qs_list[index]) new_q = reward + DISCOUNT * max_future_q else: - new_q = reward # 终止状态 + new_q = reward current_qs = current_qs_list[index].copy() - old_q = current_qs[action] # 用于计算TD误差 - current_qs[action] = new_q # 更新对应动作的Q值 + old_q = current_qs[action] + current_qs[action] = new_q - # 计算TD误差 td_error = abs(new_q - old_q) errors.append(td_error) - x.append(current_state) + # 处理当前状态用于训练 + if isinstance(current_state, dict): + img, vec = self.state_processor.process_state(current_state) + else: + img = current_state + vec = np.zeros(10) + + x_images.append(img) + x_vectors.append(vec) y.append(current_qs) # PER: 更新优先级 @@ -437,8 +558,9 @@ def train(self): log_this_step = True self.last_logged_episode = self.tensorboard.step - # 训练模型(带样本权重) - self.model.fit(np.array(x) / 255, np.array(y), + # 训练模型 + self.model.fit([np.array(x_images)/255, np.array(x_vectors)], + np.array(y), batch_size=TRAINING_BATCH_SIZE, sample_weight=weights if self.use_per else None, verbose=0, shuffle=False, @@ -456,10 +578,11 @@ def train(self): def train_in_loop(self): """在单独线程中持续训练""" # 预热训练 - x = np.random.uniform(size=(1, IM_HEIGHT, IM_WIDTH, 3)).astype(np.float32) - y = np.random.uniform(size=(1, 5)).astype(np.float32) # 改为5个输出 + dummy_image = np.random.uniform(size=(1, IM_HEIGHT, IM_WIDTH, 3)).astype(np.float32) + dummy_vector = np.random.uniform(size=(1, 10)).astype(np.float32) + y = np.random.uniform(size=(1, 5)).astype(np.float32) - self.model.fit(x, y, verbose=False, batch_size=1) + self.model.fit([dummy_image, dummy_vector], y, verbose=False, batch_size=1) self.training_initialized = True # 持续训练循环 @@ -467,11 +590,20 @@ def train_in_loop(self): if self.terminate: return self.train() - time.sleep(0.01) # 控制训练频率 + time.sleep(0.01) def get_qs(self, state): """获取状态的Q值""" - return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255, verbose=0)[0] + if isinstance(state, dict): + image_state, vector_state = self.state_processor.process_state(state) + image_input = np.array(image_state).reshape(-1, *image_state.shape) / 255 + vector_input = np.array(vector_state).reshape(-1, *vector_state.shape) + return self.model.predict([image_input, vector_input], verbose=0)[0] + else: + # 向后兼容 + image_input = np.array(state).reshape(-1, *state.shape) / 255 + dummy_vector = np.zeros((1, 10)) + return self.model.predict([image_input, dummy_vector], verbose=0)[0] def get_average_reaction_time(self): """获取平均反应时间""" diff --git a/src/Unmanned_vehicle_AD_DQN/Test.py b/src/Unmanned_vehicle_AD_DQN/Test.py index b90d26188f..07b9766bdc 100644 --- a/src/Unmanned_vehicle_AD_DQN/Test.py +++ b/src/Unmanned_vehicle_AD_DQN/Test.py @@ -9,7 +9,6 @@ from tensorflow.keras.models import load_model from Environment import CarEnv, MEMORY_FRACTION from Hyperparameters import * -from TrainingStrategies import * import os import json import glob @@ -66,10 +65,10 @@ def select_best_model(model_files, preferred_keywords=None, excluded_keywords=No return None if preferred_keywords is None: - preferred_keywords = ["best", "advanced", "dueling_per", "v2"] + preferred_keywords = ["best", "advanced", "dueling_per", "v3", "final"] if excluded_keywords is None: - excluded_keywords = ["min", "avg", "final"] # 排除统计文件 + excluded_keywords = ["min", "avg", "stage"] # 排除统计文件和中间阶段文件 # 评分系统:根据关键词和文件属性给模型打分 scored_models = [] @@ -86,7 +85,7 @@ def select_best_model(model_files, preferred_keywords=None, excluded_keywords=No # 排除包含特定关键词的文件 exclude = False for keyword in excluded_keywords: - if keyword.lower() in filename.lower() and not filename.lower().endswith(".model"): + if keyword.lower() in filename.lower() and "stage" not in filename.lower(): exclude = True break @@ -96,7 +95,7 @@ def select_best_model(model_files, preferred_keywords=None, excluded_keywords=No # 基于文件大小和修改时间打分 try: file_size = os.path.getsize(file_path) / (1024 * 1024) # MB - if file_size > 100: # 大于100MB的模型可能更复杂 + if file_size > 50: # 大于50MB的模型可能更复杂 score += 5 # 文件修改时间(越新越好) @@ -130,10 +129,42 @@ def select_best_model(model_files, preferred_keywords=None, excluded_keywords=No def get_safe_action_advanced(model, state, env, previous_action, uncertainty_threshold=1.0): """ 高级安全动作选择,结合模型预测、安全规则、不确定性估计和多目标优化 + 支持多模态输入(图像+向量) """ - # 模型预测 - state_normalized = np.array(state).reshape(-1, *state.shape) / 255 - qs = model.predict(state_normalized, verbose=0)[0] + # 提取状态信息 + if isinstance(state, dict): + # 多模态输入 + image_state = state['image'] + + # 获取车辆状态信息 + vehicle_location = env.vehicle.get_location() + velocity = env.vehicle.get_velocity() + speed_kmh = 3.6 * np.linalg.norm([velocity.x, velocity.y, velocity.z]) + vehicle_transform = env.vehicle.get_transform() + heading = vehicle_transform.rotation.yaw + + # 构建向量状态(10维) + vector_state = np.array([ + vehicle_location.x / 200.0, # 归一化 + vehicle_location.y / 200.0, + speed_kmh / 100.0, # 归一化速度 + heading / 180.0, # 归一化方向 + 0, 0, 0, 0, 0, 0 # 动作历史占位符 + ]) + + # 设置上一个动作 + vector_state[4 + previous_action] = 1.0 if previous_action < 5 else 0 + + # 模型预测 + image_input = np.array(image_state).reshape(-1, *image_state.shape) / 255 + vector_input = vector_state.reshape(1, -1) + + qs = model.predict([image_input, vector_input], verbose=0)[0] + else: + # 单图像输入(向后兼容) + state_normalized = np.array(state).reshape(-1, *state.shape) / 255 + dummy_vector = np.zeros((1, 10)) + qs = model.predict([state_normalized, dummy_vector], verbose=0)[0] # 获取车辆速度 velocity = env.vehicle.get_velocity() @@ -151,7 +182,7 @@ def get_safe_action_advanced(model, state, env, previous_action, uncertainty_thr qs[1] *= 1.2 # 提高保持倾向 qs[2] *= 1.3 # 提高加速倾向 - # 2. 行人避障优先级 - 增强版 + # 2. 行人避障优先级 if hasattr(env, 'suggested_action') and env.suggested_action is not None: # 根据距离调整优先级 min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) @@ -187,7 +218,7 @@ def get_safe_action_advanced(model, state, env, previous_action, uncertainty_thr else: # 偏右,鼓励左转 qs[3] += 1.5 - # 6. 紧急情况处理 - 增强版 + # 6. 紧急情况处理 min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) if min_ped_distance < 5.0: # 紧急避让距离 # 大幅调整Q值以确保安全 @@ -221,7 +252,7 @@ def get_safe_action_advanced(model, state, env, previous_action, uncertainty_thr def run_test_episode(model, env, episode_num, use_advanced_safety=True): - """运行单个测试episode""" + """运行单个测试episode - 支持多模态输入""" print(f"\n{'='*50}") print(f"测试 Episode {episode_num}") print(f"{'='*50}") @@ -250,9 +281,35 @@ def run_test_episode(model, env, episode_num, use_advanced_safety=True): if use_advanced_safety: action, qs = get_safe_action_advanced(model, current_state, env, previous_action) else: - # 基础动作选择 - state_normalized = np.array(current_state).reshape(-1, *current_state.shape) / 255 - qs = model.predict(state_normalized, verbose=0)[0] + # 基础动作选择 - 需要处理多模态输入 + if isinstance(current_state, dict): + # 多模态输入 + image_input = np.array(current_state['image']).reshape(-1, *current_state['image'].shape) / 255 + + # 构建向量输入 + vehicle_location = env.vehicle.get_location() + velocity = env.vehicle.get_velocity() + speed_kmh = 3.6 * np.linalg.norm([velocity.x, velocity.y, velocity.z]) + vehicle_transform = env.vehicle.get_transform() + heading = vehicle_transform.rotation.yaw + + vector_state = np.array([ + vehicle_location.x / 200.0, + vehicle_location.y / 200.0, + speed_kmh / 100.0, + heading / 180.0, + 0, 0, 0, 0, 0, 0 + ]) + vector_state[4 + previous_action] = 1.0 if previous_action < 5 else 0 + + vector_input = vector_state.reshape(1, -1) + qs = model.predict([image_input, vector_input], verbose=0)[0] + else: + # 单图像输入 + state_normalized = np.array(current_state).reshape(-1, *current_state.shape) / 255 + dummy_vector = np.zeros((1, 10)) + qs = model.predict([state_normalized, dummy_vector], verbose=0)[0] + action = np.argmax(qs) previous_action = action @@ -311,7 +368,7 @@ def run_test_episode(model, env, episode_num, use_advanced_safety=True): def load_model_with_fallback(model_path): - """加载模型,支持多种格式和回退机制""" + """加载模型,支持多种格式和回退机制 - 支持多模态输入模型""" print(f"尝试加载模型: {model_path}") # 如果是相对路径,尝试转换为绝对路径(相对于脚本目录) @@ -342,20 +399,33 @@ def load_model_with_fallback(model_path): 'Add': tf.keras.layers.Add, 'Subtract': tf.keras.layers.Subtract, 'Lambda': tf.keras.layers.Lambda, - 'Multiply': tf.keras.layers.Multiply + 'Multiply': tf.keras.layers.Multiply, + 'Concatenate': tf.keras.layers.Concatenate, + 'SeparableConv2D': tf.keras.layers.SeparableConv2D, + 'SpatialDropout2D': tf.keras.layers.SpatialDropout2D } try: # 尝试加载完整模型 - model = load_model(model_path, custom_objects=custom_objects) + model = load_model(model_path, custom_objects=custom_objects, compile=False) print(f"✅ 模型加载成功 (使用自定义层)") + + # 编译模型 + model.compile(loss="huber", + optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001, clipnorm=1.0), + metrics=["mae"]) return model except Exception as e1: print(f"使用自定义层加载失败: {e1}") try: # 尝试不加载自定义层 - model = load_model(model_path) + model = load_model(model_path, compile=False) print(f"✅ 模型加载成功 (基础加载)") + + # 编译模型 + model.compile(loss="huber", + optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001, clipnorm=1.0), + metrics=["mae"]) return model except Exception as e2: print(f"基础加载失败: {e2}") @@ -368,14 +438,19 @@ def load_model_with_fallback(model_path): custom_objects=custom_objects ) print(f"✅ 模型加载成功 (不编译)") + + # 编译模型 + model.compile(loss="huber", + optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001, clipnorm=1.0), + metrics=["mae"]) return model except Exception as e3: print(f"所有加载尝试失败: {e3}") raise ValueError(f"无法加载模型: {model_path}") -def comprehensive_model_evaluation(model_path, num_episodes=5): - """综合模型评估""" +def comprehensive_model_evaluation(model_path, num_episodes=3): + """综合模型评估 - 支持多模态输入""" print(f"\n{'='*60}") print(f"开始综合模型评估") print(f"模型路径: {model_path}") @@ -390,13 +465,38 @@ def comprehensive_model_evaluation(model_path, num_episodes=5): # 加载模型 model = load_model_with_fallback(model_path) + # 检查模型输入 + print(f"模型输入数量: {len(model.inputs)}") + for i, inp in enumerate(model.inputs): + print(f" 输入{i}: {inp.shape}") + # 创建环境 env = CarEnv() env.SHOW_CAM = False # 预热模型 print("预热模型...") - model.predict(np.ones((1, env.im_height, env.im_width, 3)), verbose=0) + # 创建测试输入 + dummy_image = np.ones((1, env.im_height, env.im_width, 3)) / 255 + dummy_vector = np.zeros((1, 10)) + + try: + # 根据模型输入数量进行预测 + if len(model.inputs) == 2: + qs = model.predict([dummy_image, dummy_vector], verbose=0) + print(f"✅ 多模态模型预热成功,输出形状: {qs.shape}") + else: + qs = model.predict(dummy_image, verbose=0) + print(f"✅ 单输入模型预热成功,输出形状: {qs.shape}") + except Exception as e: + print(f"⚠️ 模型预热失败: {e}") + print("尝试使用单输入进行预热...") + try: + qs = model.predict(dummy_image, verbose=0) + print(f"✅ 单输入预热成功") + except Exception as e2: + print(f"❌ 预热失败: {e2}") + print("模型可能不兼容,尝试继续...") # 运行测试 results = { @@ -424,6 +524,8 @@ def comprehensive_model_evaluation(model_path, num_episodes=5): print("\n测试被用户中断") except Exception as e: print(f"测试过程中发生错误: {e}") + import traceback + traceback.print_exc() finally: # 清理环境 env.cleanup_actors() @@ -533,7 +635,7 @@ def interactive_model_selection(model_files): def main(): """主函数 - 自动查找和测试模型""" print(f"\n{'='*60}") - print("自动驾驶模型测试系统 - 优化版") + print("自动驾驶模型测试系统 - 多模态输入版") print(f"{'='*60}") # 显示当前脚本所在目录 @@ -570,7 +672,7 @@ def main(): return # 开始测试 - comprehensive_model_evaluation(selected_model, num_episodes=3) + comprehensive_model_evaluation(selected_model, num_episodes=2) def quick_test(): @@ -603,6 +705,42 @@ def quick_test(): comprehensive_model_evaluation(selected_model, num_episodes=1) +def simple_test_model(model_path): + """简单测试模型 - 用于调试""" + print(f"\n简单测试模型: {model_path}") + + # 加载模型 + try: + model = load_model_with_fallback(model_path) + print(f"✅ 模型加载成功") + print(f"模型输入: {len(model.inputs)} 个") + + # 测试模型预测 + env = CarEnv() + + # 创建测试输入 + dummy_image = np.ones((1, env.im_height, env.im_width, 3)) / 255 + dummy_vector = np.zeros((1, 10)) + + if len(model.inputs) == 2: + print("测试多模态输入...") + predictions = model.predict([dummy_image, dummy_vector], verbose=0) + print(f"预测成功! 输出形状: {predictions.shape}") + else: + print("测试单输入...") + predictions = model.predict(dummy_image, verbose=0) + print(f"预测成功! 输出形状: {predictions.shape}") + + env.cleanup_actors() + return True + + except Exception as e: + print(f"❌ 测试失败: {e}") + import traceback + traceback.print_exc() + return False + + if __name__ == '__main__': import argparse @@ -610,10 +748,14 @@ def quick_test(): parser.add_argument('--quick', action='store_true', help='快速测试模式') parser.add_argument('--model', type=str, help='指定模型文件路径') parser.add_argument('--episodes', type=int, default=3, help='测试轮次数量') + parser.add_argument('--test', type=str, help='简单测试模型文件') args = parser.parse_args() - if args.model: + if args.test: + # 简单测试模式 + simple_test_model(args.test) + elif args.model: # 使用指定的模型文件 script_dir = get_script_directory() diff --git a/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py b/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py index 6f35c41a38..a866437da5 100644 --- a/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py +++ b/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py @@ -9,125 +9,175 @@ from tensorflow.keras.optimizers import Adam +# 静态障碍物检测器 +class StaticObstacleDetector: + def __init__(self): + self.static_obstacle_history = deque(maxlen=50) + self.collision_patterns = [] + + def detect_pattern(self, location, heading, action, reward): + """检测静态障碍物碰撞模式""" + if reward < -20: + pattern = { + 'location': location, + 'heading': heading, + 'action': action, + 'timestamp': datetime.now() + } + self.static_obstacle_history.append(pattern) + + if len(self.static_obstacle_history) >= 10: + self.analyze_collision_patterns() + + def analyze_collision_patterns(self): + """分析碰撞模式""" + if len(self.static_obstacle_history) == 0: + return + + print(f"静态障碍物碰撞分析: 总次数={len(self.static_obstacle_history)}") + + def get_safe_action_suggestion(self, current_location, current_heading): + """获取安全动作建议""" + suggestions = [] + + if len(self.static_obstacle_history) > 0: + for pattern in list(self.static_obstacle_history)[-5:]: + loc = pattern['location'] + distance = math.sqrt( + (current_location[0] - loc[0])**2 + + (current_location[1] - loc[1])**2 + ) + + if distance < 10.0: + dangerous_action = pattern['action'] + suggestions.append({ + 'avoid_action': dangerous_action, + 'suggested_actions': [0, 1, 2] if dangerous_action in [3, 4] else [3, 4], + 'reason': '历史碰撞区域' + }) + + return suggestions + + # 课程学习管理器 class CurriculumManager: def __init__(self, env): self.env = env self.current_stage = 0 - self.stage_thresholds = [0.3, 0.5, 0.7, 0.85] # 成功率阈值 + self.stage_thresholds = [0.3, 0.5, 0.7, 0.85, 0.9] + self.stage_configs = [ - # 阶段0: 入门 - 更早引入行人,但数量少速度慢 + # 阶段0: 入门 { - 'pedestrian_cross': 3, # 增加一点难度 - 'pedestrian_normal': 2, - 'pedestrian_speed_min': 0.3, # 降低行人速度 - 'pedestrian_speed_max': 0.8, - 'max_episode_steps': 900, # 减少最大步数,加速训练 + 'pedestrian_cross': 2, + 'pedestrian_normal': 1, + 'static_obstacle_penalty': 0.5, + 'max_episode_steps': 800, 'success_threshold': 0.3, + 'difficulty_name': '入门' + }, + # 阶段1: 简单 + { + 'pedestrian_cross': 4, + 'pedestrian_normal': 2, + 'static_obstacle_penalty': 1.0, + 'max_episode_steps': 1000, + 'success_threshold': 0.4, 'difficulty_name': '简单' }, - # 阶段1: 初级 + # 阶段2: 中等 { - 'pedestrian_cross': 5, + 'pedestrian_cross': 6, 'pedestrian_normal': 3, - 'pedestrian_speed_min': 0.5, - 'pedestrian_speed_max': 1.0, - 'max_episode_steps': 1200, + 'static_obstacle_penalty': 2.0, + 'max_episode_steps': 1200, 'success_threshold': 0.5, - 'difficulty_name': '初级' + 'difficulty_name': '中等' }, - # 阶段2: 中级 - 增加反应时间训练 + # 阶段3: 困难 { - 'pedestrian_cross': 7, + 'pedestrian_cross': 8, 'pedestrian_normal': 4, - 'pedestrian_speed_min': 0.7, - 'pedestrian_speed_max': 1.3, - 'max_episode_steps': 1500, - 'success_threshold': 0.6, # 提高阈值 - 'difficulty_name': '中级' + 'static_obstacle_penalty': 3.0, + 'max_episode_steps': 1500, + 'success_threshold': 0.6, + 'difficulty_name': '困难' }, - # 阶段3: 高级 + # 阶段4: 专家 { - 'pedestrian_cross': 9, - 'pedestrian_normal': 5, - 'pedestrian_speed_min': 0.9, - 'pedestrian_speed_max': 1.6, - 'max_episode_steps': 1800, - 'success_threshold': 0.7, - 'difficulty_name': '高级' - }, - # 阶段4: 专家 (正常难度) - { - 'pedestrian_cross': 10, + 'pedestrian_cross': 10, 'pedestrian_normal': 6, - 'pedestrian_speed_min': 1.0, - 'pedestrian_speed_max': 2.0, - 'max_episode_steps': 2400, - 'success_threshold': 0.8, + 'static_obstacle_penalty': 4.0, + 'max_episode_steps': 1800, + 'success_threshold': 0.7, 'difficulty_name': '专家' }, - # 阶段5: 大师 (挑战) + # 阶段5: 大师 { - 'pedestrian_cross': 12, + 'pedestrian_cross': 12, 'pedestrian_normal': 8, - 'pedestrian_speed_min': 1.2, - 'pedestrian_speed_max': 2.5, - 'max_episode_steps': 3000, - 'success_threshold': 0.85, + 'static_obstacle_penalty': 5.0, + 'max_episode_steps': 2400, + 'success_threshold': 0.8, 'difficulty_name': '大师' } ] # 训练历史 - self.success_history = deque(maxlen=20) # 记录最近20轮的成功情况 - self.reward_history = deque(maxlen=50) # 记录最近50轮的奖励 - self.reaction_time_history = deque(maxlen=50) # 记录反应时间 + self.success_history = deque(maxlen=20) + self.reward_history = deque(maxlen=50) + self.reaction_time_history = deque(maxlen=50) + self.static_collision_history = deque(maxlen=20) + + # 静态障碍物检测器 + self.static_detector = StaticObstacleDetector() - def update_stage(self, success, reward, reaction_time=None): + def update_stage(self, success, reward, reaction_time=None, static_collision=False): """更新训练阶段""" - # 记录历史 self.success_history.append(1 if success else 0) self.reward_history.append(reward) if reaction_time is not None: self.reaction_time_history.append(reaction_time) + if static_collision: + self.static_collision_history.append(1) + else: + self.static_collision_history.append(0) - # 计算最近成功率 if len(self.success_history) >= 10: success_rate = sum(self.success_history) / len(self.success_history) avg_reward = np.mean(self.reward_history) if self.reward_history else 0 - # 每20轮打印一次 + static_collision_rate = sum(self.static_collision_history) / len(self.static_collision_history) if self.static_collision_history else 0 + if len(self.success_history) % 20 == 0: stage_info = self.get_current_config() - print(f"课程学习 - 阶段: {self.current_stage}({stage_info['difficulty_name']}), " - f"成功率: {success_rate:.2f}, 平均奖励: {avg_reward:.2f}") + print(f"课程学习 - 阶段: {self.current_stage}({stage_info['difficulty_name']})") + print(f" 成功率: {success_rate:.2f}, 平均奖励: {avg_reward:.2f}") + print(f" 静态碰撞率: {static_collision_rate:.2f}") if self.reaction_time_history: avg_rt = np.mean(self.reaction_time_history) print(f" 平均反应时间: {avg_rt:.2f}秒") - # 检查是否可以进入下一阶段 if self.current_stage < len(self.stage_configs) - 1: next_stage_threshold = self.stage_configs[self.current_stage]['success_threshold'] - # 不仅要看成功率,还要看反应时间(如果可用) - can_advance = success_rate >= next_stage_threshold and avg_reward > 5 - if can_advance and self.reaction_time_history: - avg_rt = np.mean(self.reaction_time_history) - # 要求反应时间小于1秒 - if avg_rt < 1.0: - self.current_stage += 1 - print(f"🎉 课程学习: 进阶到阶段 {self.current_stage}!") - print(f" 新配置: {self.stage_configs[self.current_stage]['difficulty_name']}") - return True - elif can_advance: + can_advance = ( + success_rate >= next_stage_threshold and + avg_reward > 3 and + static_collision_rate < 0.2 + ) + + if can_advance: self.current_stage += 1 print(f"🎉 课程学习: 进阶到阶段 {self.current_stage}!") print(f" 新配置: {self.stage_configs[self.current_stage]['difficulty_name']}") return True - # 如果表现太差或反应时间太长,退回上一阶段 - if self.current_stage > 0 and (success_rate < 0.2 or - (self.reaction_time_history and np.mean(self.reaction_time_history) > 2.0)): + if self.current_stage > 0 and ( + success_rate < 0.2 or + static_collision_rate > 0.4 or + (self.reaction_time_history and np.mean(self.reaction_time_history) > 2.0) + ): self.current_stage -= 1 print(f"⚠️ 课程学习: 退回阶段 {self.current_stage}") return True @@ -147,20 +197,19 @@ def apply_to_environment(self): # 多目标优化器 class MultiObjectiveOptimizer: def __init__(self): - # 定义优化目标及其权重(可动态调整) self.objectives = { 'reaction_time': { - 'weight': 0.25, # 新增:反应时间权重 + 'weight': 0.20, 'description': '快速反应避障', 'metrics': ['reaction_time', 'proactive_actions'] }, 'safety': { - 'weight': 0.30, + 'weight': 0.35, 'description': '安全避障和避免碰撞', - 'metrics': ['collision_avoidance', 'pedestrian_distance'] + 'metrics': ['collision_avoidance', 'pedestrian_distance', 'static_obstacle_distance'] }, 'efficiency': { - 'weight': 0.25, + 'weight': 0.20, 'description': '快速到达目的地', 'metrics': ['progress_speed', 'total_time'] }, @@ -169,20 +218,19 @@ def __init__(self): 'description': '平稳驾驶体验', 'metrics': ['smoothness', 'steering_changes'] }, - 'rule_following': { - 'weight': 0.05, - 'description': '遵守交通规则', - 'metrics': ['lane_keeping', 'speed_limit'] + 'static_avoidance': { + 'weight': 0.10, + 'description': '避免静态障碍物', + 'metrics': ['static_collision', 'static_distance'] } } - # 指标跟踪 self.metrics_history = { 'reaction_time': [], 'safety': [], 'efficiency': [], 'comfort': [], - 'rule_following': [] + 'static_avoidance': [] } def compute_composite_reward(self, metrics): @@ -191,53 +239,52 @@ def compute_composite_reward(self, metrics): for obj_name, obj_info in self.objectives.items(): if obj_name in metrics: - # 归一化处理每个目标的贡献 normalized_value = self._normalize_metric(metrics[obj_name], obj_name) composite += normalized_value * obj_info['weight'] - # 记录指标历史 self.metrics_history[obj_name].append(normalized_value) # 特殊奖励/惩罚项 if metrics.get('collision', False): - composite -= 10 # 增加碰撞惩罚 + composite -= 12 + if metrics.get('static_collision', False): + composite -= 15 if metrics.get('off_road', False): - composite -= 5 # 增加偏离道路惩罚 - if metrics.get('dangerous_action', False): - composite -= 3 # 增加危险动作惩罚 + composite -= 8 - # 新增:反应时间相关奖励/惩罚 if 'reaction_time' in metrics: rt = metrics['reaction_time'] - if rt < 0.5: # 快速反应 - composite += 2 - elif rt > 1.5: # 反应太慢 - composite -= 3 + if rt < 0.3: + composite += 3 + elif rt > 1.2: + composite -= 4 - # 新增:主动避障奖励 if metrics.get('proactive_action', False): - composite += 1.5 + composite += 2.0 + + if metrics.get('static_distance', 100) > 15: + composite += 1.0 + elif metrics.get('static_distance', 100) < 5: + composite -= 3.0 return composite def _normalize_metric(self, value, metric_name): """归一化指标值到[0, 1]范围""" - # 不同指标的归一化方式不同 normalization_rules = { - 'reaction_time': lambda x: max(0, 1 - x/3), # 反应时间越短越好 + 'reaction_time': lambda x: max(0, 1 - x/3), 'safety': lambda x: min(max(x / 10, 0), 1), 'efficiency': lambda x: min(max(x / 100, 0), 1), 'comfort': lambda x: min(max((x + 5) / 10, 0), 1), - 'rule_following': lambda x: min(max(x, 0), 1) + 'static_avoidance': lambda x: min(max(1 - x/5, 0), 1) } if metric_name in normalization_rules: return normalization_rules[metric_name](value) - return min(max(value, 0), 1) # 默认截断到[0, 1] + return min(max(value, 0), 1) def adjust_weights(self, performance_feedback): """根据性能反馈动态调整权重""" - # 如果某个目标表现持续较差,增加其权重 recent_performance = {} for obj in self.objectives: if len(self.metrics_history[obj]) >= 10: @@ -245,17 +292,14 @@ def adjust_weights(self, performance_feedback): recent_performance[obj] = recent_avg if recent_performance: - # 找到表现最差的目标 worst_obj = min(recent_performance, key=recent_performance.get) best_obj = max(recent_performance, key=recent_performance.get) - # 如果最差目标表现低于阈值,增加其权重 if recent_performance[worst_obj] < 0.3: - adjustment = 0.03 # 调整幅度 + adjustment = 0.04 self.objectives[worst_obj]['weight'] += adjustment self.objectives[best_obj]['weight'] -= adjustment - # 确保权重总和为1 total = sum(obj['weight'] for obj in self.objectives.values()) for obj in self.objectives: self.objectives[obj]['weight'] /= total @@ -278,7 +322,7 @@ def get_performance_report(self): return report -# 模仿学习管理器(保持不变,略作修改) +# 模仿学习管理器 class ImitationLearningManager: def __init__(self, expert_data_path=None): self.expert_data_path = expert_data_path @@ -301,16 +345,15 @@ def load_expert_data(self, path): return False -# 优先经验回放缓冲区(增加对危险经验的优先级) +# 优先经验回放缓冲区 class PrioritizedReplayBuffer: - def __init__(self, max_size=15000, alpha=0.7, beta_start=0.5, beta_frames=50000): + def __init__(self, max_size=20000, alpha=0.7, beta_start=0.5, beta_frames=50000): self.max_size = max_size - self.alpha = alpha # 优先级程度 (0 = 均匀采样, 1 = 完全优先级) - self.beta_start = beta_start # 重要性采样权重起始值 - self.beta_frames = beta_frames # beta线性增长的帧数 + self.alpha = alpha + self.beta_start = beta_start + self.beta_frames = beta_frames self.frame = 1 - # 使用循环缓冲区 self.buffer = deque(maxlen=max_size) self.priorities = deque(maxlen=max_size) @@ -328,9 +371,13 @@ def add(self, experience, error=None): else: priority = (abs(error) + 1e-5) ** self.alpha - # 如果是危险经验(负奖励较大),增加优先级 - reward = experience[2] - if reward < -2: # 危险经验 + state, action, reward, next_state, done = experience + + # 静态碰撞检测 + if reward < -20: + priority *= 2.0 + + elif reward < -5: priority *= 1.5 self.buffer.append(experience) @@ -341,23 +388,18 @@ def sample(self, batch_size): if len(self.buffer) == 0: return [], [], [] - # 计算采样概率 priorities = np.array(self.priorities, dtype=np.float32) probs = priorities ** self.alpha probs /= probs.sum() - # 采样索引 indices = np.random.choice(len(self.buffer), min(batch_size, len(self.buffer)), p=probs, replace=False) - # 获取样本 samples = [self.buffer[i] for i in indices] - # 计算重要性采样权重 total = len(self.buffer) weights = (total * probs[indices]) ** (-self.beta()) - weights /= weights.max() # 归一化 + weights /= weights.max() - # 更新帧计数器 self.frame += 1 return indices, samples, weights diff --git a/src/Unmanned_vehicle_AD_DQN/main.py b/src/Unmanned_vehicle_AD_DQN/main.py index f69fb91625..7b861df789 100644 --- a/src/Unmanned_vehicle_AD_DQN/main.py +++ b/src/Unmanned_vehicle_AD_DQN/main.py @@ -1,4 +1,4 @@ -# main_fixed.py - 修复模型保存问题的版本 +# main.py import glob import os import sys @@ -47,10 +47,18 @@ def save_model_with_retry(model, filepath, max_retries=3): print(f"❌ 无法保存模型: {filepath}") return False +def create_dummy_state(env): + """创建虚拟状态用于测试""" + return { + 'image': np.ones((env.im_height, env.im_width, 3)), + 'location': np.array([-81.0, -195.0]), # 2维 + 'speed': np.array([0.0]), + 'heading': np.array([0.0]), + 'last_action': np.array([1]) + } + def extended_reward_calculation(env, action, reward, done, step_info): - """ - 扩展的奖励计算函数,用于多目标优化 - """ + """扩展的奖励计算函数""" # 获取车辆状态 vehicle_location = env.vehicle.get_location() velocity = env.vehicle.get_velocity() @@ -75,63 +83,73 @@ def extended_reward_calculation(env, action, reward, done, step_info): metrics['proactive_action'] = proactive_action - # 3. 安全性指标 - 基于最近行人距离 + # 3. 安全性指标 min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) safety_score = 0 if min_ped_distance < 100: if min_ped_distance > 12: - safety_score = 10 # 非常安全 + safety_score = 10 elif min_ped_distance > 8: - safety_score = 7 # 安全 + safety_score = 7 elif min_ped_distance > 5: - safety_score = 3 # 警告 + safety_score = 3 elif min_ped_distance > 3: - safety_score = 1 # 危险 + safety_score = 1 else: - safety_score = 0 # 极危险 + safety_score = 0 metrics['safety'] = safety_score - # 4. 效率指标 - 基于进度 - progress = (vehicle_location.x + 81) / 236.0 # 从-81到155 - efficiency_score = progress * 100 # 进度百分比 + # 4. 静态障碍物指标 + if hasattr(env, 'check_static_obstacles'): + static_distance, _ = env.check_static_obstacles(vehicle_location) + metrics['static_distance'] = static_distance + + if static_distance == 0: + metrics['static_collision'] = True + else: + metrics['static_collision'] = False + + # 5. 道路边界指标 + if hasattr(env, 'check_road_boundary'): + boundary_distance, out_of_boundary = env.check_road_boundary(vehicle_location) + metrics['off_road'] = out_of_boundary + + # 6. 效率指标 + progress = (vehicle_location.x + 81) / 236.0 + efficiency_score = progress * 100 metrics['efficiency'] = efficiency_score - # 5. 舒适度指标 - 基于转向平滑性 - comfort_score = 5 # 默认舒适 + # 7. 舒适度指标 + comfort_score = 5 if hasattr(env, 'last_action') and env.last_action in [3, 4]: - if getattr(env, 'same_steer_counter', 0) > 2: # 连续同向转向 - comfort_score = 2 # 稍不舒适 + if getattr(env, 'same_steer_counter', 0) > 2: + comfort_score = 2 elif getattr(env, 'same_steer_counter', 0) > 1: - comfort_score = 3 # 一般 + comfort_score = 3 else: - comfort_score = 4 # 舒适 + comfort_score = 4 else: - comfort_score = 5 # 直行,最舒适 + comfort_score = 5 metrics['comfort'] = comfort_score - # 6. 规则遵循指标 - 基于速度 - rule_score = 0.3 # 默认较低分数 + # 8. 规则遵循指标 + rule_score = 0.3 - if 20 <= speed_kmh <= 35: # 理想速度范围 + if 20 <= speed_kmh <= 35: rule_score = 1.0 elif 15 <= speed_kmh < 20 or 35 < speed_kmh <= 40: rule_score = 0.7 - elif 10 <= speed_kmh < 15 or 40 < speed_kmh <= 45: - rule_score = 0.5 - elif 5 <= speed_kmh < 10: - rule_score = 0.4 metrics['rule_following'] = rule_score - # 7. 特殊事件 + # 9. 碰撞检测 metrics['collision'] = len(getattr(env, 'collision_history', [])) > 0 - metrics['off_road'] = vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30 - # 8. 危险动作检测 - if speed_kmh > 40 and action in [3, 4]: # 高速急转 + # 10. 危险动作检测 + if speed_kmh > 40 and action in [3, 4]: metrics['dangerous_action'] = True else: metrics['dangerous_action'] = False @@ -139,9 +157,12 @@ def extended_reward_calculation(env, action, reward, done, step_info): return metrics if __name__ == '__main__': - FPS = 60 # 帧率 - ep_rewards = [-200] # 存储每轮奖励 + FPS = 60 + ep_rewards = [-200] + print("自动驾驶模型训练开始...") + print("=" * 60) + # 确保models目录存在 models_dir = ensure_models_directory() @@ -151,6 +172,7 @@ def extended_reward_calculation(env, action, reward, done, step_info): tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) # 创建智能体和环境 + print("创建智能体和环境...") agent = DQNAgent( use_dueling=True, use_per=True, @@ -163,301 +185,223 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 设置训练策略 agent.setup_training_strategies(env) - # 启动训练线程并等待训练初始化完成 + # 预热模型 + print("预热模型...") + dummy_state = create_dummy_state(env) + + try: + qs = agent.get_qs(dummy_state) + print(f"✅ 模型预热成功,Q值形状: {qs.shape}") + except Exception as e: + print(f"⚠️ 模型预热失败: {e}") + # 尝试直接调用predict进行预热 + dummy_image = np.ones((1, env.im_height, env.im_width, 3)) / 255 + dummy_vector = np.zeros((1, 10)) + try: + qs = agent.model.predict([dummy_image, dummy_vector], verbose=0) + print(f"✅ 使用直接预测方法预热成功,Q值形状: {qs.shape}") + except Exception as e2: + print(f"❌ 直接预测也失败: {e2}") + print("检查模型输入维度...") + print(f"模型输入: {agent.model.input}") + print(f"模型输出: {agent.model.output}") + sys.exit(1) + + # 启动训练线程 + print("启动训练线程...") trainer_thread = Thread(target=agent.train_in_loop, daemon=True) trainer_thread.start() + + # 等待训练初始化完成 + print("等待训练初始化完成...") + start_time = time.time() while not agent.training_initialized: time.sleep(0.01) - - # 预热Q网络 - agent.get_qs(np.ones((env.im_height, env.im_width, 3))) + if time.time() - start_time > 30: + print("⚠️ 训练初始化超时,继续执行...") + break + + print("✅ 训练初始化完成") # 训练统计变量 - best_score = -float('inf') # 最佳得分 - success_count = 0 # 成功次数计数 - scores = [] # 存储每轮得分 - avg_scores = [] # 存储平均得分 - - # 记录PER相关统计 - per_stats = { - 'avg_td_error': [], - 'buffer_size': [] - } + best_score = -float('inf') + success_count = 0 + scores = [] + avg_scores = [] - # 多目标统计 + # 其他统计变量 + per_stats = {'buffer_size': []} multi_obj_stats = { - 'reaction_time': [], - 'safety': [], - 'efficiency': [], - 'comfort': [], - 'rule_following': [] + 'reaction_time': [], 'safety': [], 'efficiency': [], + 'comfort': [], 'static_avoidance': [] } - - # 课程学习阶段记录 curriculum_stages = [] - - # 反应时间统计 reaction_time_stats = [] - + static_collision_stats = [] + # 迭代训练轮次 - epds = [] + print(f"\n开始训练,共 {EPISODES} 轮...") + print("=" * 60) + for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'): - env.collision_hist = [] # 重置碰撞历史 - agent.tensorboard.step = episode # 设置TensorBoard步数 + env.collision_hist = [] + agent.tensorboard.step = episode # 应用课程学习配置 if agent.curriculum_manager: config = agent.curriculum_manager.get_current_config() - if episode % 50 == 0: # 每50轮打印一次 - print(f"课程学习 - 阶段 {agent.curriculum_manager.current_stage}({config['difficulty_name']}): " - f"行人(十字路口={config['pedestrian_cross']}, 普通={config['pedestrian_normal']})") + if episode % 50 == 0: + print(f"课程学习 - 阶段 {agent.curriculum_manager.current_stage}({config['difficulty_name']})") curriculum_stages.append(agent.curriculum_manager.current_stage) # 重置每轮统计 score = 0 step = 1 - - # 多目标指标记录 episode_metrics = { - 'reaction_time': [], - 'safety': [], - 'efficiency': [], - 'comfort': [], - 'rule_following': [] + 'reaction_time': [], 'safety': [], 'efficiency': [], + 'comfort': [], 'static_avoidance': [] } - # 重置环境并获取初始状态 - current_state = env.reset(episode) + # 重置环境 + try: + current_state = env.reset(episode) + except Exception as e: + print(f"❌ 重置环境失败: {e}") + continue - # 重置完成标志 done = False episode_start = time.time() + static_collision_occurred = False - # 应用课程学习的最大步数限制 + # 最大步数 + max_steps_per_episode = SECONDS_PER_EPISODE * FPS if agent.curriculum_manager: config = agent.curriculum_manager.get_current_config() max_steps_per_episode = config['max_episode_steps'] - else: - max_steps_per_episode = SECONDS_PER_EPISODE * FPS - # 仅在给定秒数内运行 + # 运行episode while not done and step < max_steps_per_episode: - # 选择动作策略 + # 选择动作 if np.random.random() > Hyperparameters.EPSILON: - # 从Q网络获取动作(利用) - qs = agent.get_qs(current_state) - action = np.argmax(qs) - - # 如果有建议的避让动作,考虑采纳 - if hasattr(env, 'suggested_action') and env.suggested_action is not None: - # 检查建议动作的Q值 - suggested_q = qs[env.suggested_action] - current_best_q = qs[action] + try: + qs = agent.get_qs(current_state) + action = np.argmax(qs) - # 如果建议动作的Q值接近最佳动作,采纳建议 - if suggested_q > current_best_q * 0.8: - action = env.suggested_action + # 安全检查:如果接近静态障碍物,调整动作 + if hasattr(env, 'check_static_obstacles'): + vehicle_location = env.vehicle.get_location() + static_distance, _ = env.check_static_obstacles(vehicle_location) + + if static_distance < 5.0: + if action in [3, 4] and qs[0] > qs[action] * 0.7: + action = 0 - if episode % 100 == 0 and step % 100 == 0: # 减少打印频率 - print(f'Ep {episode} Step {step}: Q值 [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}, {qs[3]:>5.2f}, {qs[4]:>5.2f}] 动作: {action}') + except Exception as e: + print(f"⚠️ 获取Q值失败: {e}") + action = np.random.randint(0, 5) else: - # 随机选择动作(探索) action = np.random.randint(0, 5) - # 添加延迟以匹配60FPS + + # 探索时的安全检查 + if hasattr(env, 'check_static_obstacles'): + vehicle_location = env.vehicle.get_location() + static_distance, _ = env.check_static_obstacles(vehicle_location) + + if static_distance < 3.0: + safe_actions = [0, 3, 4] + action = np.random.choice(safe_actions) + time.sleep(1 / FPS) - # 执行动作并获取结果 - new_state, reward, done, _ = env.step(action) - - # 计算反应时间 - reaction_time = 0 - if hasattr(env, 'obstacle_detected_time') and env.obstacle_detected_time is not None: - if hasattr(env, 'reaction_start_time') and env.reaction_start_time is not None: - reaction_time = time.time() - env.reaction_start_time - + # 执行动作 + try: + new_state, reward, done, _ = env.step(action) + except Exception as e: + print(f"❌ 执行动作失败: {e}") + break + + # 检测静态碰撞 + static_collision = False + if hasattr(env, 'check_static_obstacles'): + vehicle_location = env.vehicle.get_location() + static_distance, _ = env.check_static_obstacles(vehicle_location) + if static_distance == 0: + static_collision = True + static_collision_occurred = True + # 计算多目标指标 if agent.multi_objective_optimizer: - step_info = {'step': step, 'action': action} - metrics = extended_reward_calculation(env, action, reward, done, step_info) - - # 记录指标 - for key in episode_metrics: - if key in metrics: - episode_metrics[key].append(metrics[key]) - - # 使用多目标优化器计算综合奖励 - composite_reward = agent.multi_objective_optimizer.compute_composite_reward(metrics) - reward = composite_reward # 使用综合奖励 - - score += reward # 累加奖励 + try: + step_info = {'step': step, 'action': action} + metrics = extended_reward_calculation(env, action, reward, done, step_info) + + for key in episode_metrics: + if key in metrics: + episode_metrics[key].append(metrics[key]) + + composite_reward = agent.multi_objective_optimizer.compute_composite_reward(metrics) + reward = composite_reward + except Exception as e: + print(f"⚠️ 计算多目标奖励失败: {e}") + + score += reward - # 更新经验回放(带反应时间信息) - agent.update_replay_memory((current_state, action, reward, new_state, done), - reaction_time=reaction_time) - current_state = new_state # 更新当前状态 + # 更新经验回放 + try: + agent.update_replay_memory((current_state, action, reward, new_state, done)) + except Exception as e: + print(f"⚠️ 更新经验回放失败: {e}") + current_state = new_state step += 1 if done: break - # 本轮结束 - 销毁所有actor - env.cleanup_actors() - - # 计算平均反应时间 - if episode_metrics['reaction_time']: - avg_reaction_time = np.mean([rt for rt in episode_metrics['reaction_time'] if rt > 0]) - reaction_time_stats.append(avg_reaction_time) - - # 计算本轮平均指标 - avg_metrics = {} - for key, values in episode_metrics.items(): - if values: - # 过滤掉零值(无反应时) - if key == 'reaction_time': - filtered_values = [v for v in values if v > 0] - avg_metrics[key] = np.mean(filtered_values) if filtered_values else 0 - else: - avg_metrics[key] = np.mean(values) - # 记录到统计中 - if key in multi_obj_stats: - multi_obj_stats[key].append(avg_metrics[key]) - - # 更新课程学习(带反应时间) - success = score > 5 # 成功完成的阈值 - avg_rt = avg_metrics.get('reaction_time', 0) - if agent.curriculum_manager: - stage_changed = agent.curriculum_manager.update_stage(success, score, avg_rt) - if stage_changed: - print(f"课程学习阶段已更新: {agent.curriculum_manager.current_stage}") - - # 阶段变化时保存模型 - model_path = f'{models_dir}/{MODEL_NAME}_stage_{agent.curriculum_manager.current_stage}_ep_{episode}.model' - save_model_with_retry(agent.model, model_path) - - # 更新多目标优化器权重 - if agent.multi_objective_optimizer and episode % 20 == 0: - agent.multi_objective_optimizer.adjust_weights(avg_metrics) + # 清理环境 + try: + env.cleanup_actors() + except: + pass + + # 记录统计 + scores.append(score) + static_collision_stats.append(1 if static_collision_occurred else 0) # 更新成功计数 - if success: + if score > 5: success_count += 1 - - # ============================================ - # 修复:简化模型保存条件 - # ============================================ - - # 1. 定期保存模型(每10轮) + + # 保存模型 if episode % 10 == 0: model_path = f'{models_dir}/{MODEL_NAME}_ep{episode}_score{score:.1f}.model' save_model_with_retry(agent.model, model_path) - # 2. 保存最佳模型(如果比之前好) if score > best_score: best_score = score model_path = f'{models_dir}/{MODEL_NAME}_best_ep{episode}_score{score:.1f}.model' save_model_with_retry(agent.model, model_path) print(f"🏆 新的最佳模型: Episode {episode}, 得分: {score:.2f}") - - # 3. 保存里程碑模型(每50轮) - if episode % 50 == 0: - model_path = f'{models_dir}/{MODEL_NAME}_milestone_ep{episode}.model' - save_model_with_retry(agent.model, model_path) - print(f"🎯 里程碑模型: Episode {episode}") - - # 记录得分统计 - scores.append(score) - avg_scores.append(np.mean(scores[-10:]) if len(scores) >= 10 else np.mean(scores)) - - # 记录PER缓冲区信息 - if hasattr(agent, 'replay_buffer'): - per_stats['buffer_size'].append(len(agent.replay_buffer)) - - # 定期聚合统计信息 - if not episode % AGGREGATE_STATS_EVERY or episode == 1: - average_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:]) if len(scores) >= AGGREGATE_STATS_EVERY else np.mean(scores) - min_reward = min(scores[-AGGREGATE_STATS_EVERY:]) if len(scores) >= AGGREGATE_STATS_EVERY else min(scores) - max_reward = max(scores[-AGGREGATE_STATS_EVERY:]) if len(scores) >= AGGREGATE_STATS_EVERY else max(scores) - - # 添加PER统计到TensorBoard - stats_dict = { - 'reward_avg': average_reward, - 'reward_min': min_reward, - 'reward_max': max_reward, - 'epsilon': Hyperparameters.EPSILON - } - - if hasattr(agent, 'replay_buffer'): - avg_buffer = np.mean(per_stats['buffer_size'][-AGGREGATE_STATS_EVERY:]) if per_stats['buffer_size'] else 0 - stats_dict['buffer_size'] = avg_buffer - - # 添加多目标指标 - if agent.multi_objective_optimizer: - for obj in ['reaction_time', 'safety', 'efficiency', 'comfort', 'rule_following']: - if multi_obj_stats[obj]: - recent_avg = np.mean(multi_obj_stats[obj][-AGGREGATE_STATS_EVERY:]) if len(multi_obj_stats[obj]) >= AGGREGATE_STATS_EVERY else np.mean(multi_obj_stats[obj]) - stats_dict[f'{obj}_score'] = recent_avg - - # 添加反应时间统计 - if reaction_time_stats: - avg_rt = np.mean(reaction_time_stats[-AGGREGATE_STATS_EVERY:]) if len(reaction_time_stats) >= AGGREGATE_STATS_EVERY else np.mean(reaction_time_stats) - stats_dict['reaction_time'] = avg_rt - - agent.tensorboard.update_stats(**stats_dict) - epds.append(episode) - # 打印训练信息 - if episode % 10 == 0: # 每10轮打印一次 - avg_rt = np.mean(reaction_time_stats[-10:]) if len(reaction_time_stats) >= 10 else 0 - info_str = f'轮次: {episode:3d}, 得分: {score:6.2f}, 成功: {success_count:3d}, 反应时间: {avg_rt:.2f}s' - if agent.curriculum_manager: - info_str += f', 阶段: {agent.curriculum_manager.current_stage}' - print(info_str) - + if episode % 10 == 0: + avg_score = np.mean(scores[-10:]) if len(scores) >= 10 else np.mean(scores) + print(f'轮次: {episode:3d}, 得分: {score:6.2f}, 最近10轮平均: {avg_score:6.2f}, 成功: {success_count:3d}') + # 衰减探索率 if Hyperparameters.EPSILON > Hyperparameters.MIN_EPSILON: Hyperparameters.EPSILON *= Hyperparameters.EPSILON_DECAY Hyperparameters.EPSILON = max(Hyperparameters.MIN_EPSILON, Hyperparameters.EPSILON) - # 设置训练线程终止标志并等待其结束 + # 结束训练 agent.terminate = True trainer_thread.join() - # ============================================ - # 修复:始终保存最终模型 - # ============================================ - # 保存最终模型 final_model_path = f'{models_dir}/{MODEL_NAME}_final_ep{EPISODES}_avg{np.mean(scores):.1f}.model' - save_model_with_retry(agent.model, final_model_path) - print(f"✅ 最终模型已保存: {final_model_path}") - - # 同时保存目标网络 - final_target_path = f'{models_dir}/{MODEL_NAME}_target_final.model' - save_model_with_retry(agent.target_model, final_target_path) - print(f"✅ 目标网络已保存: {final_target_path}") - - # 保存训练统计数据 - training_stats = { - 'scores': scores, - 'avg_scores': avg_scores, - 'multi_obj_stats': multi_obj_stats, - 'reaction_time_stats': reaction_time_stats, - 'curriculum_stages': curriculum_stages, - 'final_scores': { - 'max': max(scores) if scores else 0, - 'avg': np.mean(scores) if scores else 0, - 'min': min(scores) if scores else 0, - } - } - - stats_file = f'training_stats_{int(time.time())}.pkl' - with open(stats_file, 'wb') as f: - pickle.dump(training_stats, f) - print(f"📊 训练统计数据已保存到: {stats_file}") + if save_model_with_retry(agent.model, final_model_path): + print(f"✅ 最终模型已保存: {final_model_path}") - # 打印最终统计 print("\n" + "="*60) print("训练完成!") print("="*60) @@ -466,15 +410,15 @@ def extended_reward_calculation(env, action, reward, done, step_info): print(f" 最佳得分: {max(scores) if scores else 0:.2f}") print(f" 平均得分: {np.mean(scores) if scores else 0:.2f}") print(f" 成功率: {(success_count/EPISODES)*100:.1f}%") - print(f" 平均反应时间: {np.mean(reaction_time_stats) if reaction_time_stats else 0:.2f}秒") + print(f" 静态碰撞率: {np.mean(static_collision_stats) if static_collision_stats else 0:.2%}") print(f" 最终探索率: {Hyperparameters.EPSILON:.4f}") # 显示保存的模型文件 print(f"\n已保存的模型文件:") model_files = glob.glob(f'{models_dir}/*.model') if model_files: - for model_file in sorted(model_files, key=os.path.getmtime): - file_size = os.path.getsize(model_file) / (1024 * 1024) # MB + for model_file in sorted(model_files, key=os.path.getmtime)[-10:]: + file_size = os.path.getsize(model_file) / (1024 * 1024) print(f" 📁 {os.path.basename(model_file)} ({file_size:.1f} MB)") else: - print(" ⚠️ 没有找到模型文件,请检查保存路径和权限") \ No newline at end of file + print(" ⚠️ 没有找到模型文件") \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/requirements.txt b/src/Unmanned_vehicle_AD_DQN/requirements.txt index 44823773cd..33f0018cf2 100644 --- a/src/Unmanned_vehicle_AD_DQN/requirements.txt +++ b/src/Unmanned_vehicle_AD_DQN/requirements.txt @@ -3,4 +3,4 @@ numpy==1.19.5 tensorflow==2.6.0 tqdm==4.67.1 matplotlib==3.5.3 -opencv-python==4.12.0.88 +opencv-python==4.12.0.88 \ No newline at end of file