diff --git a/src/Unmanned_vehicle_AD_DQN/Environment.py b/src/Unmanned_vehicle_AD_DQN/Environment.py index a9d245b04c..5929383d5f 100644 --- a/src/Unmanned_vehicle_AD_DQN/Environment.py +++ b/src/Unmanned_vehicle_AD_DQN/Environment.py @@ -7,19 +7,11 @@ import numpy as np import cv2 import math +from Hyperparameters import * import carla from carla import ColorConverter -# 导入超参数 -try: - from Hyperparameters import * -except ImportError: - # 如果在导入Hyperparameters时出错,使用默认值 - SHOW_PREVIEW = False - IM_WIDTH = 640 - IM_HEIGHT = 480 - class CarEnv: SHOW_CAM = SHOW_PREVIEW # 是否显示摄像头预览 @@ -42,11 +34,7 @@ def __init__(self): self.current_episode = 1 # 当前episode编号 # 加载世界和蓝图 - try: - self.world = self.client.load_world('Town03') - except: - # 如果加载失败,使用当前世界 - self.world = self.client.get_world() + self.world = self.client.load_world('Town03') # 设置观察者视角,让CARLA窗口显示 self.setup_observer_view() @@ -108,6 +96,7 @@ def spawn_pedestrians_general(self, number, isCross): success_count += 1 except Exception as e: # 如果生成失败,继续尝试 + print(f"生成行人失败: {e}") continue print(f"成功生成 {success_count}/{target_number} 个行人 (isCross={isCross})") @@ -163,6 +152,7 @@ def spawn_pedestrians_right(self, isCross): self.walker_list.append(npc) # 添加到行人列表 return True # 生成成功 except Exception as e: + print(f"生成右侧行人失败 (尝试 {attempt+1}): {e}") continue return False # 生成失败 @@ -217,6 +207,7 @@ def spawn_pedestrians_left(self, isCross): self.walker_list.append(npc) # 添加到行人列表 return True # 生成成功 except Exception as e: + print(f"生成左侧行人失败 (尝试 {attempt+1}): {e}") continue return False # 生成失败 @@ -292,8 +283,7 @@ def reset(self, episode=1): self.colsensor.listen(lambda event: self.collision_data(event)) # 设置碰撞检测回调 # 等待摄像头初始化完成 - start_time = time.time() - while self.front_camera is None and time.time() - start_time < 5: + while self.front_camera is None: time.sleep(0.01) # 设置跟随相机(用于观察) @@ -308,30 +298,21 @@ def reset(self, episode=1): def cleanup_actors(self): """清理所有actors""" # 清理车辆 - try: - vehicles = self.world.get_actors().filter('vehicle.*') - for vehicle in vehicles: - if vehicle.is_alive: - vehicle.destroy() - except: - pass + vehicles = self.world.get_actors().filter('vehicle.*') + for vehicle in vehicles: + if vehicle.is_alive: + vehicle.destroy() # 清理行人 - try: - walkers = self.world.get_actors().filter('walker.*') - for walker in walkers: - if walker.is_alive: - walker.destroy() - except: - pass + walkers = self.world.get_actors().filter('walker.*') + for walker in walkers: + if walker.is_alive: + walker.destroy() # 清理传感器 for actor in self.actor_list: - try: - if actor.is_alive: - actor.destroy() - except: - pass + if actor.is_alive: + actor.destroy() self.actor_list = [] self.walker_list = [] # 清空行人列表 @@ -359,22 +340,19 @@ def collision_data(self, event): def process_img(self, image): """处理摄像头图像""" - try: - image.convert(carla.ColorConverter.CityScapesPalette) # 转换为CityScapes调色板 + image.convert(carla.ColorConverter.CityScapesPalette) # 转换为CityScapes调色板 - # 处理原始图像数据 - processed_image = np.array(image.raw_data) - processed_image = processed_image.reshape((self.im_height, self.im_width, 4)) - processed_image = processed_image[:, :, :3] # 移除alpha通道 + # 处理原始图像数据 + processed_image = np.array(image.raw_data) + processed_image = processed_image.reshape((self.im_height, self.im_width, 4)) + processed_image = processed_image[:, :, :3] # 移除alpha通道 - # 显示预览(如果启用) - if self.SHOW_CAM: - cv2.imshow("", processed_image) - cv2.waitKey(1) + # 显示预览(如果启用) + if self.SHOW_CAM: + cv2.imshow("", processed_image) + cv2.waitKey(1) - self.front_camera = processed_image # 更新前置摄像头图像 - except Exception as e: - print(f"处理图像时出错: {e}") + self.front_camera = processed_image # 更新前置摄像头图像 def reward(self, speed_kmh, current_steer): """增强的奖励函数 - 特别强调行人避障""" @@ -454,13 +432,13 @@ def reward(self, speed_kmh, current_steer): self.last_ped_distance = min_ped_distance # 保存当前距离 # 4. 速度奖励(平衡避障和前进) - if 20 <= speed_kmh <= 35: # 理想速度区间 + if 15 <= speed_kmh <= 35: # 理想速度区间 reward += 0.4 - elif 10 <= speed_kmh < 20: # 较慢但安全(避障时可能需要减速) + elif 5 <= speed_kmh < 15: # 较慢但安全(避障时可能需要减速) reward += 0.2 - elif 35 < speed_kmh <= 40: # 稍快 + elif 35 < speed_kmh <= 45: # 稍快 reward += 0.1 - elif speed_kmh > 40: # 过快,在行人环境中危险 + elif speed_kmh > 45: # 过快,在行人环境中危险 reward -= 0.5 # 增加惩罚 else: # 停车或极慢 reward -= 0.05 # 轻微惩罚 @@ -471,7 +449,7 @@ def reward(self, speed_kmh, current_steer): # 6. 碰撞检测 if len(self.collision_history) != 0: - reward = -10 # 减少碰撞惩罚 + reward = -15 # 增加碰撞惩罚 done = True print(f"Episode {self.current_episode}: 发生碰撞!") @@ -481,7 +459,7 @@ def reward(self, speed_kmh, current_steer): # 8. 边界检查 if vehicle_location.x > 155: # 成功到达终点 - reward += 15 # 减少到达终点的奖励 + reward += 20 # 增加到达终点的奖励 done = True print(f"Episode {self.current_episode}: 成功到达终点!") elif vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30: # 偏离道路 @@ -489,9 +467,6 @@ def reward(self, speed_kmh, current_steer): done = True print(f"Episode {self.current_episode}: 偏离道路!") - # 限制奖励范围 - reward = max(min(reward, 20), -15) - return reward, done def step(self, action): @@ -535,7 +510,6 @@ def step(self, action): if self.same_steer_counter > 3: # 连续3次同向转向后强制回正 steer *= 0.5 # 减小转向幅度 throttle *= 0.8 # 减速 - brake = 0.1 # 轻微制动 else: self.same_steer_counter = 0 diff --git a/src/Unmanned_vehicle_AD_DQN/Model.py b/src/Unmanned_vehicle_AD_DQN/Model.py index 19daa0e80c..c419ae6c35 100644 --- a/src/Unmanned_vehicle_AD_DQN/Model.py +++ b/src/Unmanned_vehicle_AD_DQN/Model.py @@ -17,27 +17,11 @@ import tensorflow as tf import tensorflow.keras.backend as backend from threading import Thread - -# 导入本地模块 -from TrainingStrategies import PrioritizedReplayBuffer - -# 导入超参数 -try: - from Hyperparameters import * -except ImportError: - # 如果在导入Hyperparameters时出错,使用默认值 - DISCOUNT = 0.97 - MEMORY_FRACTION = 0.35 - REPLAY_MEMORY_SIZE = 10000 - MIN_REPLAY_MEMORY_SIZE = 3000 - MINIBATCH_SIZE = 64 - PREDICTION_BATCH_SIZE = 1 - TRAINING_BATCH_SIZE = 16 - UPDATE_TARGET_EVERY = 20 - LEARNING_RATE = 0.00005 - IM_HEIGHT = 480 - IM_WIDTH = 640 - MODEL_NAME = "YY_Optimized" +from Environment import * +from Hyperparameters import * +import pickle +import json +from datetime import datetime # 自定义TensorBoard类 @@ -72,6 +56,500 @@ def update_stats(self, **stats): self.writer.flush() +# 优先经验回放缓冲区 +class PrioritizedReplayBuffer: + def __init__(self, max_size=REPLAY_MEMORY_SIZE, alpha=0.6, beta_start=0.4, beta_frames=100000): + self.max_size = max_size + self.alpha = alpha # 优先级程度 (0 = 均匀采样, 1 = 完全优先级) + self.beta_start = beta_start # 重要性采样权重起始值 + self.beta_frames = beta_frames # beta线性增长的帧数 + self.frame = 1 + + # 使用循环缓冲区 + self.buffer = deque(maxlen=max_size) + self.priorities = deque(maxlen=max_size) + + def __len__(self): + return len(self.buffer) + + def beta(self): + """线性递增的beta值,用于重要性采样权重""" + return min(1.0, self.beta_start + self.frame * (1.0 - self.beta_start) / self.beta_frames) + + def add(self, experience, error=None): + """添加经验到缓冲区""" + if error is None: + priority = max(self.priorities) if self.priorities else 1.0 + else: + priority = (abs(error) + 1e-5) ** self.alpha + + self.buffer.append(experience) + self.priorities.append(priority) + + def sample(self, batch_size): + """从缓冲区中采样一批经验""" + if len(self.buffer) == 0: + return [], [], [], [] + + # 计算采样概率 + priorities = np.array(self.priorities, dtype=np.float32) + probs = priorities ** self.alpha + probs /= probs.sum() + + # 采样索引 + indices = np.random.choice(len(self.buffer), batch_size, p=probs) + + # 获取样本 + samples = [self.buffer[i] for i in indices] + + # 计算重要性采样权重 + total = len(self.buffer) + weights = (total * probs[indices]) ** (-self.beta()) + weights /= weights.max() # 归一化 + + # 更新帧计数器 + self.frame += 1 + + return indices, samples, weights + + def update_priorities(self, indices, errors): + """更新采样经验的优先级""" + for idx, error in zip(indices, errors): + if 0 <= idx < len(self.priorities): + self.priorities[idx] = (abs(error) + 1e-5) ** self.alpha + + +# 课程学习管理器 +class CurriculumManager: + def __init__(self, env): + self.env = env + self.current_stage = 0 + self.stage_thresholds = [0.3, 0.5, 0.7, 0.85] # 成功率阈值 + self.stage_configs = [ + # 阶段0: 入门 + { + 'pedestrian_cross': 4, # 十字路口行人数量 + 'pedestrian_normal': 2, # 普通路段行人数量 + 'pedestrian_speed_min': 0.5, # 行人最低速度 + 'pedestrian_speed_max': 1.0, # 行人最高速度 + 'max_episode_steps': 1200, # 最大步数 (20秒 * 60FPS) + 'success_threshold': 0.3 # 进入下一阶段成功率 + }, + # 阶段1: 初级 + { + 'pedestrian_cross': 6, + 'pedestrian_normal': 3, + 'pedestrian_speed_min': 0.7, + 'pedestrian_speed_max': 1.3, + 'max_episode_steps': 1800, # 30秒 + 'success_threshold': 0.5 + }, + # 阶段2: 中级 + { + 'pedestrian_cross': 8, + 'pedestrian_normal': 4, + 'pedestrian_speed_min': 0.8, + 'pedestrian_speed_max': 1.5, + 'max_episode_steps': 2400, # 40秒 + 'success_threshold': 0.7 + }, + # 阶段3: 高级 (正常难度) + { + 'pedestrian_cross': 10, + 'pedestrian_normal': 5, + 'pedestrian_speed_min': 1.0, + 'pedestrian_speed_max': 2.0, + 'max_episode_steps': 3600, # 60秒 + 'success_threshold': 0.85 + }, + # 阶段4: 专家 (挑战) + { + 'pedestrian_cross': 12, + 'pedestrian_normal': 6, + 'pedestrian_speed_min': 1.2, + 'pedestrian_speed_max': 2.5, + 'max_episode_steps': 3600, + 'success_threshold': 0.9 + } + ] + + # 训练历史 + self.success_history = deque(maxlen=20) # 记录最近20轮的成功情况 + self.reward_history = deque(maxlen=50) # 记录最近50轮的奖励 + + def update_stage(self, success, reward): + """更新训练阶段""" + # 记录历史 + self.success_history.append(1 if success else 0) + self.reward_history.append(reward) + + # 计算最近成功率 + if len(self.success_history) >= 10: + success_rate = sum(self.success_history) / len(self.success_history) + avg_reward = np.mean(self.reward_history) if self.reward_history else 0 + + print(f"课程学习 - 当前阶段: {self.current_stage}, 成功率: {success_rate:.2f}, 平均奖励: {avg_reward:.2f}") + + # 检查是否可以进入下一阶段 + if self.current_stage < len(self.stage_configs) - 1: + next_stage_threshold = self.stage_configs[self.current_stage]['success_threshold'] + if success_rate >= next_stage_threshold and avg_reward > 5: + self.current_stage += 1 + print(f"🎉 课程学习: 进阶到阶段 {self.current_stage}!") + return True + + # 如果表现太差,退回上一阶段 + if self.current_stage > 0 and success_rate < 0.2: + self.current_stage -= 1 + print(f"⚠️ 课程学习: 退回阶段 {self.current_stage}") + return True + + return False + + def get_current_config(self): + """获取当前阶段的配置""" + return self.stage_configs[min(self.current_stage, len(self.stage_configs) - 1)] + + def apply_to_environment(self): + """将当前阶段配置应用到环境""" + config = self.get_current_config() + # 注意:这里需要修改Environment.py中的行人生成逻辑来支持这些参数 + # 暂时返回配置,由外部调用者处理 + return config + + +# 多目标优化器 +class MultiObjectiveOptimizer: + def __init__(self): + # 定义优化目标及其权重(可动态调整) + self.objectives = { + 'safety': { + 'weight': 0.4, + 'description': '安全避障和避免碰撞', + 'metrics': ['collision_avoidance', 'pedestrian_distance'] + }, + 'efficiency': { + 'weight': 0.25, + 'description': '快速到达目的地', + 'metrics': ['progress_speed', 'total_time'] + }, + 'comfort': { + 'weight': 0.2, + 'description': '平稳驾驶体验', + 'metrics': ['smoothness', 'steering_changes'] + }, + 'rule_following': { + 'weight': 0.15, + 'description': '遵守交通规则', + 'metrics': ['lane_keeping', 'speed_limit'] + } + } + + # 指标跟踪 + self.metrics_history = { + 'safety': [], + 'efficiency': [], + 'comfort': [], + 'rule_following': [] + } + + def compute_composite_reward(self, metrics): + """计算综合奖励值""" + composite = 0 + + for obj_name, obj_info in self.objectives.items(): + if obj_name in metrics: + # 归一化处理每个目标的贡献 + normalized_value = self._normalize_metric(metrics[obj_name], obj_name) + composite += normalized_value * obj_info['weight'] + + # 记录指标历史 + self.metrics_history[obj_name].append(normalized_value) + + # 特殊惩罚项 + if metrics.get('collision', False): + composite -= 10 + if metrics.get('off_road', False): + composite -= 5 + if metrics.get('dangerous_action', False): + composite -= 3 + + return composite + + def _normalize_metric(self, value, metric_name): + """归一化指标值到[0, 1]范围""" + # 不同指标的归一化方式不同 + normalization_rules = { + 'safety': lambda x: min(max(x / 10, 0), 1), # 假设安全分满分10 + 'efficiency': lambda x: min(max(x / 100, 0), 1), # 效率分满分100 + 'comfort': lambda x: min(max((x + 5) / 10, 0), 1), # 舒适度[-5, 5] -> [0, 1] + 'rule_following': lambda x: min(max(x, 0), 1) # 规则遵循度[0, 1] + } + + if metric_name in normalization_rules: + return normalization_rules[metric_name](value) + return min(max(value, 0), 1) # 默认截断到[0, 1] + + def adjust_weights(self, performance_feedback): + """根据性能反馈动态调整权重""" + # 如果某个目标表现持续较差,增加其权重 + recent_performance = {} + for obj in self.objectives: + if len(self.metrics_history[obj]) >= 10: + recent_avg = np.mean(self.metrics_history[obj][-10:]) + recent_performance[obj] = recent_avg + + if recent_performance: + # 找到表现最差的目标 + worst_obj = min(recent_performance, key=recent_performance.get) + best_obj = max(recent_performance, key=recent_performance.get) + + # 如果最差目标表现低于阈值,增加其权重 + if recent_performance[worst_obj] < 0.3: + adjustment = 0.05 + self.objectives[worst_obj]['weight'] += adjustment + self.objectives[best_obj]['weight'] -= adjustment + + # 确保权重总和为1 + total = sum(obj['weight'] for obj in self.objectives.values()) + for obj in self.objectives: + self.objectives[obj]['weight'] /= total + + print(f"动态权重调整: {worst_obj}权重↑ {adjustment:.3f}, {best_obj}权重↓ {adjustment:.3f}") + + def get_performance_report(self): + """生成性能报告""" + report = "多目标优化性能报告:\n" + report += "=" * 50 + "\n" + + for obj_name, obj_info in self.objectives.items(): + history = self.metrics_history[obj_name] + if history: + avg = np.mean(history[-20:]) if len(history) >= 20 else np.mean(history) + report += f"{obj_name}(权重:{obj_info['weight']:.2f}): 平均得分={avg:.3f}\n" + report += f" 描述: {obj_info['description']}\n" + + return report + + +# 模仿学习管理器 +class ImitationLearningManager: + def __init__(self, expert_data_path=None): + self.expert_data_path = expert_data_path + self.expert_data = [] + self.is_pretrained = False + + def load_expert_data(self, path): + """加载专家示范数据""" + try: + if os.path.exists(path): + with open(path, 'rb') as f: + self.expert_data = pickle.load(f) + print(f"已加载 {len(self.expert_data)} 条专家示范数据") + return True + else: + print(f"专家数据文件不存在: {path}") + return False + except Exception as e: + print(f"加载专家数据失败: {e}") + return False + + def collect_expert_demonstration(self, env, num_episodes=10): + """收集专家示范数据(可以手动控制或使用规则控制器)""" + print(f"开始收集专家示范数据 ({num_episodes}个episodes)...") + + demonstrations = [] + + for episode in range(num_episodes): + print(f"收集专家示范 Episode {episode + 1}/{num_episodes}") + + state = env.reset(episode) + done = False + episode_data = [] + + while not done: + # 这里可以使用规则控制器或手动控制 + # 示例:简单的规则控制器 + action = self._rule_based_controller(env) + + new_state, reward, done, _ = env.step(action) + + # 保存示范数据 + episode_data.append({ + 'state': state.copy(), + 'action': action, + 'reward': reward, + 'next_state': new_state.copy(), + 'done': done + }) + + state = new_state + + demonstrations.extend(episode_data) + env.cleanup_actors() + + # 保存专家数据 + self.expert_data = demonstrations + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + save_path = f"expert_data_{timestamp}.pkl" + + with open(save_path, 'wb') as f: + pickle.dump(demonstrations, f) + + print(f"专家示范数据已保存到: {save_path}, 共 {len(demonstrations)} 条记录") + return True + + def _rule_based_controller(self, env): + """基于规则的控制器(作为专家示范)""" + # 获取车辆状态 + vehicle_location = env.vehicle.get_location() + velocity = env.vehicle.get_velocity() + speed_kmh = 3.6 * math.sqrt(velocity.x**2 + velocity.y**2) + + # 简单规则:保持速度在20-40 km/h,避免障碍物 + if speed_kmh < 20: + return 2 # 加速 + elif speed_kmh > 40: + return 0 # 减速 + else: + # 检查前方障碍物 + has_obstacle_ahead = self._check_obstacle_ahead(env) + if has_obstacle_ahead: + return 0 # 减速 + else: + return 1 # 保持 + + return 1 # 默认保持 + + def _check_obstacle_ahead(self, env): + """检查前方是否有障碍物(简化版本)""" + # 这里可以添加更复杂的障碍物检测逻辑 + # 暂时返回False + return False + + def pretrain_with_behavioral_cloning(self, model, epochs=20): + """使用行为克隆进行预训练""" + if not self.expert_data: + print("没有专家数据可用,跳过预训练") + return model + + print(f"开始行为克隆预训练 ({epochs}个epochs)...") + + # 准备训练数据 + states = [] + actions = [] + + for demo in self.expert_data: + states.append(demo['state']) + actions.append(demo['action']) + + # 将状态归一化 + states = np.array(states) / 255.0 + + # 将动作转换为one-hot编码 + actions_onehot = tf.keras.utils.to_categorical(actions, num_classes=5) + + # 备份原始编译设置 + original_loss = model.loss + original_optimizer = model.optimizer + original_metrics = model.metrics_names + + # 重新编译模型用于分类任务 + model.compile( + optimizer=Adam(learning_rate=0.0001), + loss='categorical_crossentropy', + metrics=['accuracy'] + ) + + # 训练模型模仿专家行为 + history = model.fit( + states, actions_onehot, + batch_size=32, + epochs=epochs, + validation_split=0.2, + verbose=1 + ) + + print(f"预训练完成 - 最终准确率: {history.history['accuracy'][-1]:.3f}") + + # 恢复原始编译设置 + model.compile( + optimizer=original_optimizer, + loss=original_loss, + metrics=original_metrics + ) + + self.is_pretrained = True + return model + + def train_with_dagger(self, model, env, iterations=5, episodes_per_iter=5): + """使用DAgger算法进行训练""" + print(f"开始DAgger训练 ({iterations}次迭代,每次{episodes_per_iter}个episodes)...") + + aggregated_data = self.expert_data.copy() + + for iteration in range(iterations): + print(f"\nDAgger 迭代 {iteration + 1}/{iterations}") + + # 使用当前策略收集数据 + new_demos = [] + + for episode in range(episodes_per_iter): + print(f" 收集数据 Episode {episode + 1}/{episodes_per_iter}") + + state = env.reset(episode) + done = False + + while not done: + # 使用当前策略选择动作 + qs = model.predict(np.array(state).reshape(-1, *state.shape) / 255)[0] + action = np.argmax(qs) + + # 执行动作 + new_state, reward, done, _ = env.step(action) + + # 专家纠正(这里可以添加专家纠正逻辑) + # 如果策略动作与专家建议不同,使用专家动作 + expert_action = self._rule_based_controller(env) + + # 保存数据(使用专家纠正后的动作) + new_demos.append({ + 'state': state.copy(), + 'action': expert_action, # 使用专家动作 + 'reward': reward, + 'next_state': new_state.copy(), + 'done': done + }) + + state = new_state + + env.cleanup_actors() + + # 合并数据 + aggregated_data.extend(new_demos) + + # 在合并数据上重新训练 + states = [d['state'] for d in aggregated_data] + actions = [d['action'] for d in aggregated_data] + + states = np.array(states) / 255.0 + actions_onehot = tf.keras.utils.to_categorical(actions, num_classes=5) + + # 训练模型 + history = model.fit( + states, actions_onehot, + batch_size=32, + epochs=10, + validation_split=0.1, + verbose=0 + ) + + print(f" 训练完成 - 准确率: {history.history['accuracy'][-1]:.3f}") + + print("DAgger训练完成!") + return model + + # DQN智能体类 - 升级版(整合训练策略) class DQNAgent: def __init__(self, use_dueling=True, use_per=True, use_curriculum=True, use_multi_objective=True): @@ -113,17 +591,14 @@ def __init__(self, use_dueling=True, use_per=True, use_curriculum=True, use_mult def setup_training_strategies(self, env=None): """设置训练策略组件""" if self.use_curriculum and env: - from TrainingStrategies import CurriculumManager self.curriculum_manager = CurriculumManager(env) print("课程学习管理器已启用") if self.use_multi_objective: - from TrainingStrategies import MultiObjectiveOptimizer self.multi_objective_optimizer = MultiObjectiveOptimizer() print("多目标优化器已启用") # 模仿学习管理器(需要时手动启用) - from TrainingStrategies import ImitationLearningManager self.imitation_manager = ImitationLearningManager() def create_model(self): @@ -131,8 +606,8 @@ def create_model(self): # 使用函数式API inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) - # 第一卷积块 - 使用较小的卷积核 - x = Conv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) + # 第一卷积块 + x = Conv2D(32, (5, 5), strides=(2, 2), padding='same')(inputs) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) @@ -156,14 +631,14 @@ def create_model(self): # 展平层 x = Flatten()(x) - # 全连接层 - 减少神经元数量 + # 全连接层 + x = Dense(512, activation='relu')(x) + x = Dropout(0.3)(x) x = Dense(256, activation='relu')(x) x = Dropout(0.3)(x) x = Dense(128, activation='relu')(x) - x = Dropout(0.3)(x) - x = Dense(64, activation='relu')(x) x = Dropout(0.2)(x) - x = Dense(32, activation='relu')(x) + x = Dense(64, activation='relu')(x) x = Dropout(0.1)(x) # 输出层 - 5个动作 @@ -182,7 +657,7 @@ def create_dueling_model(self): # 共享的特征提取层 # 第一卷积块 - x = Conv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) + x = Conv2D(32, (5, 5), strides=(2, 2), padding='same')(inputs) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) @@ -207,17 +682,17 @@ def create_dueling_model(self): x = Flatten()(x) # 共享的全连接层 - shared = Dense(256, activation='relu')(x) + shared = Dense(512, activation='relu')(x) shared = Dropout(0.3)(shared) - shared = Dense(128, activation='relu')(shared) + shared = Dense(256, activation='relu')(shared) # 价值流 (V(s)) - value_stream = Dense(64, activation='relu')(shared) + value_stream = Dense(128, activation='relu')(shared) value_stream = Dropout(0.2)(value_stream) value = Dense(1, activation='linear', name='value')(value_stream) # 优势流 (A(s,a)) - advantage_stream = Dense(64, activation='relu')(shared) + advantage_stream = Dense(128, activation='relu')(shared) advantage_stream = Dropout(0.2)(advantage_stream) advantage = Dense(5, activation='linear', name='advantage')(advantage_stream) @@ -248,10 +723,10 @@ def minibatch_chooser(self): if self.use_per: # PER采样 if len(self.replay_buffer) < MIN_REPLAY_MEMORY_SIZE: - return [] + return [], [], [], [] indices, samples, weights = self.replay_buffer.sample(MINIBATCH_SIZE) - return samples + return indices, samples, weights else: # 标准采样 if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: @@ -316,10 +791,10 @@ def train(self): # 准备训练数据 current_states = np.array([transition[0] for transition in minibatch]) / 255 - current_qs_list = self.model.predict(current_states, batch_size=PREDICTION_BATCH_SIZE, verbose=0) + current_qs_list = self.model.predict(current_states, batch_size=PREDICTION_BATCH_SIZE) new_current_states = np.array([transition[3] for transition in minibatch]) / 255 - future_qs_list = self.target_model.predict(new_current_states, batch_size=PREDICTION_BATCH_SIZE, verbose=0) + future_qs_list = self.target_model.predict(new_current_states, batch_size=PREDICTION_BATCH_SIZE) x = [] # 输入状态 y = [] # 目标Q值 @@ -389,4 +864,4 @@ def train_in_loop(self): def get_qs(self, state): """获取状态的Q值""" - return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255, verbose=0)[0] \ No newline at end of file + return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255)[0] \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Test.py b/src/Unmanned_vehicle_AD_DQN/Test.py index ccd53b468c..d284b8245e 100644 --- a/src/Unmanned_vehicle_AD_DQN/Test.py +++ b/src/Unmanned_vehicle_AD_DQN/Test.py @@ -9,7 +9,6 @@ from tensorflow.keras.models import load_model from Environment import CarEnv, MEMORY_FRACTION from Hyperparameters import * -from TrainingStrategies import * import os import json import glob diff --git a/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py b/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py deleted file mode 100644 index cb5e611506..0000000000 --- a/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py +++ /dev/null @@ -1,530 +0,0 @@ -# TrainingStrategies.py -import os -import math -import pickle -import numpy as np -from datetime import datetime -from collections import deque -import tensorflow as tf -from tensorflow.keras.optimizers import Adam - - -# 课程学习管理器 -class CurriculumManager: - def __init__(self, env): - self.env = env - self.current_stage = 0 - self.stage_thresholds = [0.3, 0.5, 0.7, 0.85] # 成功率阈值 - self.stage_configs = [ - # 阶段0: 入门 - { - 'pedestrian_cross': 2, # 十字路口行人数量(减少) - 'pedestrian_normal': 1, # 普通路段行人数量(减少) - 'pedestrian_speed_min': 0.5, # 行人最低速度 - 'pedestrian_speed_max': 1.0, # 行人最高速度 - 'max_episode_steps': 1200, # 最大步数 (20秒 * 60FPS) - 'success_threshold': 0.3 # 进入下一阶段成功率 - }, - # 阶段1: 初级 - { - 'pedestrian_cross': 4, # 逐步增加 - 'pedestrian_normal': 2, - 'pedestrian_speed_min': 0.7, - 'pedestrian_speed_max': 1.3, - 'max_episode_steps': 1800, # 30秒 - 'success_threshold': 0.5 - }, - # 阶段2: 中级 - { - 'pedestrian_cross': 6, - 'pedestrian_normal': 3, - 'pedestrian_speed_min': 0.8, - 'pedestrian_speed_max': 1.5, - 'max_episode_steps': 2400, # 40秒 - 'success_threshold': 0.7 - }, - # 阶段3: 高级 (正常难度) - { - 'pedestrian_cross': 8, - 'pedestrian_normal': 4, - 'pedestrian_speed_min': 1.0, - 'pedestrian_speed_max': 2.0, - 'max_episode_steps': 3600, # 60秒 - 'success_threshold': 0.85 - }, - # 阶段4: 专家 (挑战) - { - 'pedestrian_cross': 10, # 适当减少 - 'pedestrian_normal': 5, - 'pedestrian_speed_min': 1.2, - 'pedestrian_speed_max': 2.5, - 'max_episode_steps': 3600, - 'success_threshold': 0.9 - } - ] - - # 训练历史 - self.success_history = deque(maxlen=20) # 记录最近20轮的成功情况 - self.reward_history = deque(maxlen=50) # 记录最近50轮的奖励 - - def update_stage(self, success, reward): - """更新训练阶段""" - # 记录历史 - self.success_history.append(1 if success else 0) - self.reward_history.append(reward) - - # 计算最近成功率 - if len(self.success_history) >= 10: - success_rate = sum(self.success_history) / len(self.success_history) - avg_reward = np.mean(self.reward_history) if self.reward_history else 0 - - # 减少打印频率 - if len(self.success_history) % 10 == 0: - print(f"课程学习 - 当前阶段: {self.current_stage}, 成功率: {success_rate:.2f}, 平均奖励: {avg_reward:.2f}") - - # 检查是否可以进入下一阶段 - if self.current_stage < len(self.stage_configs) - 1: - next_stage_threshold = self.stage_configs[self.current_stage]['success_threshold'] - if success_rate >= next_stage_threshold and avg_reward > 5: - self.current_stage += 1 - print(f"🎉 课程学习: 进阶到阶段 {self.current_stage}!") - return True - - # 如果表现太差,退回上一阶段 - if self.current_stage > 0 and success_rate < 0.2: - self.current_stage -= 1 - print(f"⚠️ 课程学习: 退回阶段 {self.current_stage}") - return True - - return False - - def get_current_config(self): - """获取当前阶段的配置""" - return self.stage_configs[min(self.current_stage, len(self.stage_configs) - 1)] - - def apply_to_environment(self): - """将当前阶段配置应用到环境""" - config = self.get_current_config() - return config - - -# 多目标优化器 -class MultiObjectiveOptimizer: - def __init__(self): - # 定义优化目标及其权重(可动态调整) - self.objectives = { - 'safety': { - 'weight': 0.35, # 稍微降低安全权重 - 'description': '安全避障和避免碰撞', - 'metrics': ['collision_avoidance', 'pedestrian_distance'] - }, - 'efficiency': { - 'weight': 0.30, # 提高效率权重 - 'description': '快速到达目的地', - 'metrics': ['progress_speed', 'total_time'] - }, - 'comfort': { - 'weight': 0.20, - 'description': '平稳驾驶体验', - 'metrics': ['smoothness', 'steering_changes'] - }, - 'rule_following': { - 'weight': 0.15, - 'description': '遵守交通规则', - 'metrics': ['lane_keeping', 'speed_limit'] - } - } - - # 指标跟踪 - self.metrics_history = { - 'safety': [], - 'efficiency': [], - 'comfort': [], - 'rule_following': [] - } - - def compute_composite_reward(self, metrics): - """计算综合奖励值""" - composite = 0 - - for obj_name, obj_info in self.objectives.items(): - if obj_name in metrics: - # 归一化处理每个目标的贡献 - normalized_value = self._normalize_metric(metrics[obj_name], obj_name) - composite += normalized_value * obj_info['weight'] - - # 记录指标历史 - self.metrics_history[obj_name].append(normalized_value) - - # 特殊惩罚项 - if metrics.get('collision', False): - composite -= 8 # 减少碰撞惩罚 - if metrics.get('off_road', False): - composite -= 3 # 减少偏离道路惩罚 - if metrics.get('dangerous_action', False): - composite -= 2 # 减少危险动作惩罚 - - return composite - - def _normalize_metric(self, value, metric_name): - """归一化指标值到[0, 1]范围""" - # 不同指标的归一化方式不同 - normalization_rules = { - 'safety': lambda x: min(max(x / 10, 0), 1), # 假设安全分满分10 - 'efficiency': lambda x: min(max(x / 100, 0), 1), # 效率分满分100 - 'comfort': lambda x: min(max((x + 5) / 10, 0), 1), # 舒适度[-5, 5] -> [0, 1] - 'rule_following': lambda x: min(max(x, 0), 1) # 规则遵循度[0, 1] - } - - if metric_name in normalization_rules: - return normalization_rules[metric_name](value) - return min(max(value, 0), 1) # 默认截断到[0, 1] - - def adjust_weights(self, performance_feedback): - """根据性能反馈动态调整权重""" - # 如果某个目标表现持续较差,增加其权重 - recent_performance = {} - for obj in self.objectives: - if len(self.metrics_history[obj]) >= 10: - recent_avg = np.mean(self.metrics_history[obj][-10:]) - recent_performance[obj] = recent_avg - - if recent_performance: - # 找到表现最差的目标 - worst_obj = min(recent_performance, key=recent_performance.get) - best_obj = max(recent_performance, key=recent_performance.get) - - # 如果最差目标表现低于阈值,增加其权重 - if recent_performance[worst_obj] < 0.3: - adjustment = 0.02 # 减少调整幅度 - self.objectives[worst_obj]['weight'] += adjustment - self.objectives[best_obj]['weight'] -= adjustment - - # 确保权重总和为1 - total = sum(obj['weight'] for obj in self.objectives.values()) - for obj in self.objectives: - self.objectives[obj]['weight'] /= total - - if adjustment != 0: - print(f"动态权重调整: {worst_obj}权重↑ {adjustment:.3f}, {best_obj}权重↓ {adjustment:.3f}") - - def get_performance_report(self): - """生成性能报告""" - report = "多目标优化性能报告:\n" - report += "=" * 50 + "\n" - - for obj_name, obj_info in self.objectives.items(): - history = self.metrics_history[obj_name] - if history: - avg = np.mean(history[-20:]) if len(history) >= 20 else np.mean(history) - report += f"{obj_name}(权重:{obj_info['weight']:.2f}): 平均得分={avg:.3f}\n" - report += f" 描述: {obj_info['description']}\n" - - return report - - -# 模仿学习管理器 -class ImitationLearningManager: - def __init__(self, expert_data_path=None): - self.expert_data_path = expert_data_path - self.expert_data = [] - self.is_pretrained = False - - def load_expert_data(self, path): - """加载专家示范数据""" - try: - if os.path.exists(path): - with open(path, 'rb') as f: - self.expert_data = pickle.load(f) - print(f"已加载 {len(self.expert_data)} 条专家示范数据") - return True - else: - print(f"专家数据文件不存在: {path}") - return False - except Exception as e: - print(f"加载专家数据失败: {e}") - return False - - def collect_expert_demonstration(self, env, num_episodes=10): - """收集专家示范数据(可以手动控制或使用规则控制器)""" - print(f"开始收集专家示范数据 ({num_episodes}个episodes)...") - - demonstrations = [] - - for episode in range(num_episodes): - print(f"收集专家示范 Episode {episode + 1}/{num_episodes}") - - state = env.reset(episode) - done = False - episode_data = [] - - step_count = 0 - max_steps = 60 * 60 # 最大60秒 - - while not done and step_count < max_steps: - # 这里可以使用规则控制器或手动控制 - # 示例:简单的规则控制器 - action = self._rule_based_controller(env) - - new_state, reward, done, _ = env.step(action) - - # 保存示范数据 - episode_data.append({ - 'state': state.copy(), - 'action': action, - 'reward': reward, - 'next_state': new_state.copy(), - 'done': done - }) - - state = new_state - step_count += 1 - - demonstrations.extend(episode_data) - env.cleanup_actors() - - # 保存专家数据 - self.expert_data = demonstrations - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - save_path = f"expert_data_{timestamp}.pkl" - - with open(save_path, 'wb') as f: - pickle.dump(demonstrations, f) - - print(f"专家示范数据已保存到: {save_path}, 共 {len(demonstrations)} 条记录") - return True - - def _rule_based_controller(self, env): - """基于规则的控制器(作为专家示范)""" - # 获取车辆状态 - vehicle_location = env.vehicle.get_location() - velocity = env.vehicle.get_velocity() - speed_kmh = 3.6 * math.sqrt(velocity.x**2 + velocity.y**2) - - # 简单规则:保持速度在20-35 km/h,避免障碍物 - if speed_kmh < 20: - return 2 # 加速 - elif speed_kmh > 35: - return 0 # 减速 - else: - # 检查前方障碍物 - has_obstacle_ahead = self._check_obstacle_ahead(env) - if has_obstacle_ahead: - return 0 # 减速 - else: - return 1 # 保持 - - return 1 # 默认保持 - - def _check_obstacle_ahead(self, env): - """检查前方是否有障碍物(简化版本)""" - # 这里可以添加更复杂的障碍物检测逻辑 - # 暂时返回False - return False - - def pretrain_with_behavioral_cloning(self, model, epochs=20): - """使用行为克隆进行预训练""" - if not self.expert_data: - print("没有专家数据可用,跳过预训练") - return model - - print(f"开始行为克隆预训练 ({epochs}个epochs)...") - - # 准备训练数据 - states = [] - actions = [] - - for demo in self.expert_data: - states.append(demo['state']) - actions.append(demo['action']) - - # 将状态归一化 - states = np.array(states) / 255.0 - - # 将动作转换为one-hot编码 - actions_onehot = tf.keras.utils.to_categorical(actions, num_classes=5) - - # 备份原始编译设置 - original_loss = model.loss - original_optimizer = model.optimizer - original_metrics = model.metrics_names - - # 重新编译模型用于分类任务 - model.compile( - optimizer=Adam(learning_rate=0.0001), - loss='categorical_crossentropy', - metrics=['accuracy'] - ) - - # 训练模型模仿专家行为 - history = model.fit( - states, actions_onehot, - batch_size=32, - epochs=epochs, - validation_split=0.2, - verbose=1 - ) - - print(f"预训练完成 - 最终准确率: {history.history['accuracy'][-1]:.3f}") - - # 恢复原始编译设置 - model.compile( - optimizer=original_optimizer, - loss=original_loss, - metrics=original_metrics - ) - - self.is_pretrained = True - return model - - def train_with_dagger(self, model, env, iterations=5, episodes_per_iter=5): - """使用DAgger算法进行训练""" - print(f"开始DAgger训练 ({iterations}次迭代,每次{episodes_per_iter}个episodes)...") - - aggregated_data = self.expert_data.copy() - - for iteration in range(iterations): - print(f"\nDAgger 迭代 {iteration + 1}/{iterations}") - - # 使用当前策略收集数据 - new_demos = [] - - for episode in range(episodes_per_iter): - print(f" 收集数据 Episode {episode + 1}/{episodes_per_iter}") - - state = env.reset(episode) - done = False - step_count = 0 - max_steps = 60 * 60 - - while not done and step_count < max_steps: - # 使用当前策略选择动作 - qs = model.predict(np.array(state).reshape(-1, *state.shape) / 255, verbose=0)[0] - action = np.argmax(qs) - - # 执行动作 - new_state, reward, done, _ = env.step(action) - - # 专家纠正(这里可以添加专家纠正逻辑) - # 如果策略动作与专家建议不同,使用专家动作 - expert_action = self._rule_based_controller(env) - - # 保存数据(使用专家纠正后的动作) - new_demos.append({ - 'state': state.copy(), - 'action': expert_action, # 使用专家动作 - 'reward': reward, - 'next_state': new_state.copy(), - 'done': done - }) - - state = new_state - step_count += 1 - - env.cleanup_actors() - - # 合并数据 - aggregated_data.extend(new_demos) - - # 在合并数据上重新训练 - states = [d['state'] for d in aggregated_data] - actions = [d['action'] for d in aggregated_data] - - states = np.array(states) / 255.0 - actions_onehot = tf.keras.utils.to_categorical(actions, num_classes=5) - - # 备份原始编译设置 - original_loss = model.loss - original_optimizer = model.optimizer - original_metrics = model.metrics_names - - # 重新编译用于分类 - model.compile( - optimizer=Adam(learning_rate=0.0001), - loss='categorical_crossentropy', - metrics=['accuracy'] - ) - - # 训练模型 - history = model.fit( - states, actions_onehot, - batch_size=32, - epochs=10, - validation_split=0.1, - verbose=0 - ) - - # 恢复原始编译设置 - model.compile( - optimizer=original_optimizer, - loss=original_loss, - metrics=original_metrics - ) - - print(f" 训练完成 - 准确率: {history.history['accuracy'][-1]:.3f}") - - print("DAgger训练完成!") - return model - - -# 优先经验回放缓冲区 -class PrioritizedReplayBuffer: - def __init__(self, max_size=10000, alpha=0.6, beta_start=0.4, beta_frames=100000): - self.max_size = max_size - self.alpha = alpha # 优先级程度 (0 = 均匀采样, 1 = 完全优先级) - self.beta_start = beta_start # 重要性采样权重起始值 - self.beta_frames = beta_frames # beta线性增长的帧数 - self.frame = 1 - - # 使用循环缓冲区 - self.buffer = deque(maxlen=max_size) - self.priorities = deque(maxlen=max_size) - - def __len__(self): - return len(self.buffer) - - def beta(self): - """线性递增的beta值,用于重要性采样权重""" - return min(1.0, self.beta_start + self.frame * (1.0 - self.beta_start) / self.beta_frames) - - def add(self, experience, error=None): - """添加经验到缓冲区""" - if error is None: - priority = max(self.priorities) if self.priorities else 1.0 - else: - priority = (abs(error) + 1e-5) ** self.alpha - - self.buffer.append(experience) - self.priorities.append(priority) - - def sample(self, batch_size): - """从缓冲区中采样一批经验""" - if len(self.buffer) == 0: - return [], [], [] - - # 计算采样概率 - priorities = np.array(self.priorities, dtype=np.float32) - probs = priorities ** self.alpha - probs /= probs.sum() - - # 采样索引 - indices = np.random.choice(len(self.buffer), min(batch_size, len(self.buffer)), p=probs, replace=False) - - # 获取样本 - samples = [self.buffer[i] for i in indices] - - # 计算重要性采样权重 - total = len(self.buffer) - weights = (total * probs[indices]) ** (-self.beta()) - weights /= weights.max() # 归一化 - - # 更新帧计数器 - self.frame += 1 - - return indices, samples, weights - - def update_priorities(self, indices, errors): - """更新采样经验的优先级""" - for idx, error in zip(indices, errors): - if 0 <= idx < len(self.priorities): - self.priorities[idx] = (abs(error) + 1e-5) ** self.alpha \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/main.py b/src/Unmanned_vehicle_AD_DQN/main.py index e627c3006d..de06a3dddc 100644 --- a/src/Unmanned_vehicle_AD_DQN/main.py +++ b/src/Unmanned_vehicle_AD_DQN/main.py @@ -15,15 +15,10 @@ from threading import Thread from tqdm import tqdm -import pickle -# 导入本地模块 -from Environment import CarEnv -from Model import DQNAgent -from TrainingStrategies import CurriculumManager, MultiObjectiveOptimizer, ImitationLearningManager import Hyperparameters - -# 从Hyperparameters导入所有参数 +from Environment import * +from Model import * from Hyperparameters import * def extended_reward_calculation(env, action, reward, done, step_info): @@ -38,8 +33,9 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 计算多目标指标 metrics = {} - # 1. 安全性指标 - 基于最近行人距离 - min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) + # 1. 安全性指标 + # 最近行人距离 + min_ped_distance = env.last_ped_distance if hasattr(env, 'last_ped_distance') else float('inf') safety_score = 0 if min_ped_distance < 100: if min_ped_distance > 12: @@ -55,49 +51,38 @@ def extended_reward_calculation(env, action, reward, done, step_info): metrics['safety'] = safety_score - # 2. 效率指标 - 基于进度 + # 2. 效率指标 progress = (vehicle_location.x + 81) / 236.0 # 从-81到155 efficiency_score = progress * 100 # 进度百分比 metrics['efficiency'] = efficiency_score - # 3. 舒适度指标 - 基于转向平滑性 - comfort_score = 5 # 默认舒适 - + # 3. 舒适度指标 + # 转向平滑性 if hasattr(env, 'last_action') and env.last_action in [3, 4]: - if getattr(env, 'same_steer_counter', 0) > 2: # 连续同向转向 - comfort_score = 2 # 稍不舒适 - elif getattr(env, 'same_steer_counter', 0) > 1: - comfort_score = 3 # 一般 + if env.same_steer_counter > 2: # 连续同向转向 + comfort_score = -2 # 不舒适 else: - comfort_score = 4 # 舒适 + comfort_score = 2 # 舒适 else: comfort_score = 5 # 直行,最舒适 metrics['comfort'] = comfort_score - # 4. 规则遵循指标 - 基于速度 - rule_score = 0.3 # 默认较低分数 - - if 20 <= speed_kmh <= 35: # 理想速度范围 + # 4. 规则遵循指标 + # 速度是否在合理范围内 + if 20 <= speed_kmh <= 40: rule_score = 1.0 - elif 15 <= speed_kmh < 20 or 35 < speed_kmh <= 40: + elif 15 <= speed_kmh < 20 or 40 < speed_kmh <= 45: rule_score = 0.7 - elif 10 <= speed_kmh < 15 or 40 < speed_kmh <= 45: - rule_score = 0.5 - elif 5 <= speed_kmh < 10: - rule_score = 0.4 + else: + rule_score = 0.3 metrics['rule_following'] = rule_score # 5. 特殊事件 - metrics['collision'] = len(getattr(env, 'collision_history', [])) > 0 + metrics['collision'] = len(env.collision_history) > 0 metrics['off_road'] = vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30 - - # 6. 危险动作检测 - if speed_kmh > 40 and action in [3, 4]: # 高速急转 - metrics['dangerous_action'] = True - else: - metrics['dangerous_action'] = False + metrics['dangerous_action'] = speed_kmh > 45 and action in [3, 4] # 高速急转 return metrics @@ -105,7 +90,12 @@ def extended_reward_calculation(env, action, reward, done, step_info): FPS = 60 # 帧率 ep_rewards = [-200] # 存储每轮奖励 - # GPU内存配置 + # 为了结果可重复性(注释掉) + # random.seed(1) + # np.random.seed(1) + # tf.compat.v1.set_random_seed(1) + + # GPU内存配置,主要用于多智能体训练 gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) tf.compat.v1.keras.backend.set_session( tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) @@ -118,7 +108,7 @@ def extended_reward_calculation(env, action, reward, done, step_info): if not os.path.isdir('expert_data'): os.makedirs('expert_data') - # 创建智能体和环境 + # 创建智能体和环境 - 启用所有高级功能 agent = DQNAgent( use_dueling=True, use_per=True, @@ -196,9 +186,10 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 应用课程学习配置 if agent.curriculum_manager: config = agent.curriculum_manager.get_current_config() - if episode % 50 == 0: # 每50轮打印一次 - print(f"课程学习 - 阶段 {agent.curriculum_manager.current_stage}: " - f"行人(十字路口={config['pedestrian_cross']}, 普通={config['pedestrian_normal']})") + # 这里可以根据配置调整环境难度 + # 例如:调整行人数量、速度等 + print(f"课程学习 - 阶段 {agent.curriculum_manager.current_stage}: " + f"行人(十字路口={config['pedestrian_cross']}, 普通={config['pedestrian_normal']})") curriculum_stages.append(agent.curriculum_manager.current_stage) # 重置每轮统计 @@ -229,13 +220,14 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 仅在给定秒数内运行 while not done and step < max_steps_per_episode: + # 选择动作策略 if np.random.random() > Hyperparameters.EPSILON: # 从Q网络获取动作(利用) qs = agent.get_qs(current_state) action = np.argmax(qs) - if episode % 50 == 0 and step % 30 == 0: # 减少打印频率 - print(f'Ep {episode} Step {step}: Q值 [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}, {qs[3]:>5.2f}, {qs[4]:>5.2f}] 动作: {action}') + if episode % 20 == 0: # 减少打印频率 + print(f'动作: [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}, {qs[3]:>5.2f}, {qs[4]:>5.2f}] {action}') else: # 随机选择动作(探索) action = np.random.randint(0, 5) @@ -290,9 +282,9 @@ def extended_reward_calculation(env, action, reward, done, step_info): print(f"课程学习阶段已更新: {agent.curriculum_manager.current_stage}") # 更新多目标优化器权重 - if agent.multi_objective_optimizer and episode % 20 == 0: + if agent.multi_objective_optimizer and episode % 10 == 0: agent.multi_objective_optimizer.adjust_weights(avg_metrics) - if episode % 100 == 0: + if episode % 50 == 0: print(agent.multi_objective_optimizer.get_performance_report()) # 更新成功计数 @@ -349,11 +341,10 @@ def extended_reward_calculation(env, action, reward, done, step_info): epds.append(episode) # 打印训练信息 - if episode % 10 == 0: # 每10轮打印一次 - info_str = f'轮次: {episode:3d}, 得分: {score:6.2f}, 成功: {success_count:3d}' - if agent.curriculum_manager: - info_str += f', 阶段: {agent.curriculum_manager.current_stage}' - print(info_str) + info_str = f'轮次: {episode:3d}, 得分: {score:6.2f}, 成功: {success_count:3d}' + if agent.curriculum_manager: + info_str += f', 阶段: {agent.curriculum_manager.current_stage}' + print(info_str) # 衰减探索率 if Hyperparameters.EPSILON > Hyperparameters.MIN_EPSILON: