diff --git a/src/Unmanned_vehicle_AD_DQN/Environment.py b/src/Unmanned_vehicle_AD_DQN/Environment.py index 0a8394be95..a9d245b04c 100644 --- a/src/Unmanned_vehicle_AD_DQN/Environment.py +++ b/src/Unmanned_vehicle_AD_DQN/Environment.py @@ -7,7 +7,6 @@ import numpy as np import cv2 import math -from collections import deque import carla from carla import ColorConverter @@ -18,8 +17,8 @@ except ImportError: # 如果在导入Hyperparameters时出错,使用默认值 SHOW_PREVIEW = False - IM_WIDTH = 160 # 修改默认值 - IM_HEIGHT = 120 # 修改默认值 + IM_WIDTH = 640 + IM_HEIGHT = 480 class CarEnv: @@ -41,13 +40,6 @@ def __init__(self): self.episode_start_time = None # 每轮开始时间 self.last_ped_distance = float('inf') # 上次最近行人距离 self.current_episode = 1 # 当前episode编号 - self.obstacle_detected_time = None # 障碍物检测到的时间 - self.reaction_start_time = None # 反应开始时间 - self.proactive_action_count = 0 # 主动避障动作计数 - - # 新增:多帧缓存用于运动检测 - self.frame_buffer = deque(maxlen=3) - self.motion_detected = False # 加载世界和蓝图 try: @@ -263,11 +255,6 @@ def reset(self, episode=1): self.last_action = 1 self.episode_start_time = time.time() self.last_ped_distance = float('inf') - self.obstacle_detected_time = None - self.reaction_start_time = None - self.proactive_action_count = 0 - self.frame_buffer.clear() - self.motion_detected = False # 设置车辆生成点 spawn_point = carla.Transform() @@ -379,31 +366,6 @@ def process_img(self, image): processed_image = np.array(image.raw_data) processed_image = processed_image.reshape((self.im_height, self.im_width, 4)) processed_image = processed_image[:, :, :3] # 移除alpha通道 - - # 添加简单的图像增强 - # 增加对比度以突出障碍物 - processed_image = cv2.convertScaleAbs(processed_image, alpha=1.2, beta=10) - - # 存储到帧缓冲区用于运动检测 - if len(self.frame_buffer) == 0: - self.frame_buffer.append(processed_image.copy()) - else: - # 简单的运动检测 - if len(self.frame_buffer) >= 2: - prev_frame = self.frame_buffer[-1] - gray_prev = cv2.cvtColor(prev_frame, cv2.COLOR_RGB2GRAY) - gray_current = cv2.cvtColor(processed_image, cv2.COLOR_RGB2GRAY) - - # 计算帧间差异 - diff = cv2.absdiff(gray_prev, gray_current) - _, diff_thresh = cv2.threshold(diff, 25, 255, cv2.THRESH_BINARY) - - # 如果差异足够大,检测到运动 - motion_pixels = np.sum(diff_thresh > 0) - motion_ratio = motion_pixels / (self.im_width * self.im_height) - self.motion_detected = motion_ratio > 0.01 # 1%的像素变化 - - self.frame_buffer.append(processed_image.copy()) # 显示预览(如果启用) if self.SHOW_CAM: @@ -415,7 +377,7 @@ def process_img(self, image): print(f"处理图像时出错: {e}") def reward(self, speed_kmh, current_steer): - """增强的奖励函数 - 特别强调行人避障和快速反应""" + """增强的奖励函数 - 特别强调行人避障""" reward = 0 done = False @@ -423,19 +385,19 @@ def reward(self, speed_kmh, current_steer): vehicle_location = self.vehicle.get_location() vehicle_rotation = self.vehicle.get_transform().rotation.yaw - # 1. 道路保持奖励 + # 1. 道路保持奖励(重要的基础) heading_error = abs(vehicle_rotation) if heading_error < 5: # 完美保持方向 - reward += 0.5 + reward += 0.8 # 略微降低权重,给行人避障更多空间 elif heading_error < 15: # 良好保持 - reward += 0.2 + reward += 0.4 elif heading_error < 30: # 可接受 - reward += 0.05 + reward += 0.1 else: # 方向偏差过大 - reward -= 0.2 * (heading_error / 30.0) + reward -= 0.3 * (heading_error / 30.0) # 降低惩罚强度 - # 2. 行人避障(最高优先级) - 增强反应机制 + # 2. 行人避障(最高优先级) - 增强奖励机制 min_ped_distance = float('inf') closest_pedestrian = None @@ -460,24 +422,14 @@ def reward(self, speed_kmh, current_steer): if active_pedestrians == 0: min_ped_distance = float('inf') - # 行人检测和反应时间记录 - current_time = time.time() - - # 检测到行人进入危险区域 + # 行人距离分级奖励 - 增强避障激励 if min_ped_distance < 100: # 只考虑100米内的行人 - if self.obstacle_detected_time is None: - self.obstacle_detected_time = current_time - self.reaction_start_time = current_time - - # 行人距离分级奖励/惩罚 - 强调提前反应 if min_ped_distance < 3.0: # 紧急避让距离 - reward -= 15.0 # 大幅增加惩罚 + reward -= 8.0 # 增加惩罚 done = True - print(f"Episode {self.current_episode}: 与行人碰撞! 距离: {min_ped_distance:.1f}m") - + print(f"Episode {self.current_episode}: 与行人距离过近 ({min_ped_distance:.1f}m)!") elif min_ped_distance < 5.0: # 危险距离 - reward -= 5.0 # 增加惩罚 - + reward -= 3.0 # 增加惩罚 # 计算避让方向 if closest_pedestrian: ped_y = closest_pedestrian.get_location().y @@ -487,93 +439,58 @@ def reward(self, speed_kmh, current_steer): self.suggested_action = 4 # 右转 else: # 行人在右侧 self.suggested_action = 3 # 左转 - - # 反应时间惩罚 - if self.reaction_start_time: - reaction_time = current_time - self.reaction_start_time - if reaction_time > 0.5: # 反应超过0.5秒 - reward -= 2.0 * (reaction_time - 0.5) - elif min_ped_distance < 8.0: # 预警距离 - reward -= 1.5 - - # 计算避让方向(提前建议) - if closest_pedestrian: - ped_y = closest_pedestrian.get_location().y - veh_y = vehicle_location.y - if ped_y < veh_y: # 行人在左侧 - self.suggested_action = 4 # 右转 - else: # 行人在右侧 - self.suggested_action = 3 # 左转 - - # 提前反应的奖励 - if self.last_ped_distance < 12.0 and min_ped_distance > self.last_ped_distance: - # 如果上次距离更近,这次距离更远了,说明正在成功避让 - reward += 1.0 - self.proactive_action_count += 1 - + reward -= 0.8 elif min_ped_distance < 12.0: # 安全距离 - # 检测到行人但距离安全,给予轻微奖励 - reward += 0.3 - - # 如果检测到运动,给予额外奖励 - if self.motion_detected: - reward += 0.2 - - # 提前规划奖励 - if self.last_ped_distance < 8.0: - # 从危险距离进入安全距离,成功避障 - reward += 2.0 - - else: - # 没有检测到行人,重置反应计时 - self.obstacle_detected_time = None - self.reaction_start_time = None + reward += 0.5 # 增加安全距离奖励 + else: # 非常安全 + reward += 0.2 + + # 3. 成功避障奖励 - 当成功避开行人后给予额外奖励 + if self.last_ped_distance < 8.0 and min_ped_distance > self.last_ped_distance: + # 如果上次距离危险,这次距离更远了,说明成功避让 + reward += 0.3 self.last_ped_distance = min_ped_distance # 保存当前距离 - # 3. 速度奖励(平衡避障和前进) + # 4. 速度奖励(平衡避障和前进) if 20 <= speed_kmh <= 35: # 理想速度区间 - reward += 0.5 - elif 10 <= speed_kmh < 20: # 较慢但安全 - reward += 0.3 + reward += 0.4 + elif 10 <= speed_kmh < 20: # 较慢但安全(避障时可能需要减速) + reward += 0.2 elif 35 < speed_kmh <= 40: # 稍快 reward += 0.1 elif speed_kmh > 40: # 过快,在行人环境中危险 - reward -= 1.0 # 增加惩罚 + reward -= 0.5 # 增加惩罚 else: # 停车或极慢 - reward -= 0.1 # 轻微惩罚 + reward -= 0.05 # 轻微惩罚 - # 4. 转向平滑性奖励 - steer_penalty = abs(current_steer) * 0.2 # 降低惩罚,给避障转向更多空间 + # 5. 转向平滑性奖励(避障时可能需要适当转向) + steer_penalty = abs(current_steer) * 0.3 # 降低惩罚,给避障转向更多空间 reward -= steer_penalty - # 5. 碰撞检测 + # 6. 碰撞检测 if len(self.collision_history) != 0: - reward = -20 # 增加碰撞惩罚 + reward = -10 # 减少碰撞惩罚 done = True print(f"Episode {self.current_episode}: 发生碰撞!") - # 6. 进度奖励 + # 7. 进度奖励(次要于避障) progress = (vehicle_location.x + 81) / 236.0 # 从-81到155 - reward += progress * 0.3 + reward += progress * 0.2 # 降低进度权重 - # 7. 边界检查 + # 8. 边界检查 if vehicle_location.x > 155: # 成功到达终点 - reward += 20 # 增加到达终点的奖励 + reward += 15 # 减少到达终点的奖励 done = True print(f"Episode {self.current_episode}: 成功到达终点!") elif vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30: # 偏离道路 - reward -= 5 # 增加偏离惩罚 + reward -= 3 # 降低偏离惩罚 done = True print(f"Episode {self.current_episode}: 偏离道路!") - # 8. 主动避障奖励 - if self.proactive_action_count > 0: - reward += self.proactive_action_count * 0.5 - # 限制奖励范围 - reward = max(min(reward, 30), -25) + reward = max(min(reward, 20), -15) return reward, done @@ -593,46 +510,32 @@ def step(self, action): brake = 0.0 steer = 0.0 - # 速度控制 - 优化动作响应 + # 速度控制 if action == 0: # 减速 throttle = 0.0 - brake = 0.7 # 增加制动强度 + brake = 0.5 elif action == 1: # 保持/轻微加速 - throttle = 0.4 # 增加油门 + throttle = 0.3 brake = 0.0 elif action == 2: # 加速 - throttle = 0.8 + throttle = 0.7 brake = 0.0 elif action == 3: # 左转 - throttle = 0.5 # 增加转向时的油门 + throttle = 0.4 brake = 0.0 - steer = -0.25 * speed_factor # 增加转向幅度 + steer = -0.2 * speed_factor # 速度相关的转向幅度 elif action == 4: # 右转 - throttle = 0.5 + throttle = 0.4 brake = 0.0 - steer = 0.25 * speed_factor # 增加转向幅度 - - # 如果有建议的避让动作,调整当前动作 - if self.suggested_action is not None: - # 如果当前动作与建议动作不同,调整转向幅度 - if self.suggested_action in [3, 4] and action in [3, 4]: - # 如果都是转向动作,但方向不同,增加建议方向的转向 - if self.suggested_action != action: - if self.suggested_action == 3: # 建议左转 - steer = -0.3 * speed_factor - else: # 建议右转 - steer = 0.3 * speed_factor - elif self.suggested_action == 0: # 建议减速 - brake = max(brake, 0.8) # 强制制动 - throttle = 0.0 + steer = 0.2 * speed_factor # 速度相关的转向幅度 # 限制连续同向转向 - 防止过度转向 if (action == 3 and self.last_action == 3) or (action == 4 and self.last_action == 4): self.same_steer_counter += 1 - if self.same_steer_counter > 2: # 连续2次同向转向后强制回正 - steer *= 0.3 # 大幅减小转向幅度 - throttle *= 0.6 # 减速 - brake = 0.2 # 轻微制动 + if self.same_steer_counter > 3: # 连续3次同向转向后强制回正 + steer *= 0.5 # 减小转向幅度 + throttle *= 0.8 # 减速 + brake = 0.1 # 轻微制动 else: self.same_steer_counter = 0 @@ -646,13 +549,13 @@ def step(self, action): steer=steer )) - # 减少等待时间,加快反应 - time.sleep(0.03) # 从0.05减少到0.03 + # 等待物理更新 + time.sleep(0.05) # 计算奖励和完成状态 reward, done = self.reward(speed_kmh, steer) # 限制极端奖励值 - reward = np.clip(reward, -25, 30) + reward = np.clip(reward, -15, 20) return self.front_camera, reward, done, None \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py index e701e4507e..bf5be3a954 100644 --- a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py +++ b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py @@ -13,14 +13,14 @@ REWARD_OFFSET = -100 # 停止模拟的奖励阈值 -MIN_REPLAY_MEMORY_SIZE = 1_000 # 减少预热样本数,让训练更快开始 -# 开始训练前经验回放缓冲区的最小大小 +MIN_REPLAY_MEMORY_SIZE = 3_000 +# 开始训练前经验回放缓冲区的最小大小 - 增加以获得更稳定训练 -REPLAY_MEMORY_SIZE = 15_000 # 增加经验回放容量 +REPLAY_MEMORY_SIZE = 10_000 # 经验回放缓冲区的最大容量 -MINIBATCH_SIZE = 128 # 增加批次大小 -# 每次训练从经验回放中采样的经验数量 +MINIBATCH_SIZE = 64 +# 每次训练从经验回放中采样的经验数量 - 增加批次大小 PREDICTION_BATCH_SIZE = 1 # 预测阶段使用的批次大小 @@ -28,40 +28,40 @@ TRAINING_BATCH_SIZE = MINIBATCH_SIZE // 4 # 训练阶段使用的批次大小 -EPISODES = 500 # 增加训练轮次 +EPISODES = 800 # 从1000减少到800,因为减少了无行人阶段 # 智能体训练的总轮次数 SECONDS_PER_EPISODE = 60 # 每轮训练的秒数 -MIN_EPSILON = 0.05 # 增加最小探索率,保持一定的探索 +MIN_EPSILON = 0.01 # 最小探索率 EPSILON = 1.0 # 初始探索率 -EPSILON_DECAY = 0.995 # 减缓衰减速度 -# 探索率的衰减率 +EPSILON_DECAY = 0.998 +# 探索率的衰减率 - 减缓衰减速度 -MODEL_NAME = "YY_Optimized_v2" +MODEL_NAME = "YY_Optimized" # 训练模型的名称标识 -MIN_REWARD = -100 # 降低阈值,更容易保存模型 +MIN_REWARD = 5 # 被认为是"良好"或"积极"经验的最小奖励值 -UPDATE_TARGET_EVERY = 15 # 更频繁地更新目标网络 -# 目标网络更新的频率 +UPDATE_TARGET_EVERY = 20 +# 目标网络更新的频率 - 增加以获得更稳定训练 -AGGREGATE_STATS_EVERY = 5 # 更频繁地记录统计信息 +AGGREGATE_STATS_EVERY = 10 # 计算和聚合统计信息(如平均得分、奖励)的频率 SHOW_PREVIEW = False # 是否显示预览窗口 - 测试时设为False以显示CARLA主窗口 -IM_WIDTH = 160 # 减少图像尺寸,加快处理速度 +IM_WIDTH = 640 # 预览或模拟中捕获图像的宽度 -IM_HEIGHT = 120 # 减少图像尺寸,加快处理速度 +IM_HEIGHT = 480 # 预览或模拟中捕获图像的高度 SLOW_COUNTER = 330 @@ -73,17 +73,17 @@ SUCCESSFUL_THRESHOLD = 3 # 成功阈值 -LEARNING_RATE = 0.0001 # 增加学习率,加速收敛 -# 优化器的学习率 +LEARNING_RATE = 0.00005 +# 优化器的学习率 - 降低以获得更稳定训练 # PER (优先经验回放) 参数 -PER_ALPHA = 0.7 # 增加优先级影响 +PER_ALPHA = 0.6 # 优先级程度 (0 = 均匀采样, 1 = 完全优先级) -PER_BETA_START = 0.5 +PER_BETA_START = 0.4 # 重要性采样权重起始值 -PER_BETA_FRAMES = 50000 # 减少beta增长时间 +PER_BETA_FRAMES = 100000 # beta线性增长的帧数 # 训练策略参数 @@ -97,9 +97,9 @@ # 是否使用模仿学习(预训练) # 多目标优化权重(这些权重会自动调整) -SAFETY_WEIGHT = 0.50 # 提高安全权重 -EFFICIENCY_WEIGHT = 0.20 -COMFORT_WEIGHT = 0.15 +SAFETY_WEIGHT = 0.40 +EFFICIENCY_WEIGHT = 0.25 +COMFORT_WEIGHT = 0.20 RULE_FOLLOWING_WEIGHT = 0.15 # 课程学习参数 @@ -107,8 +107,4 @@ # 课程学习阶段数量 CURRICULUM_SUCCESS_THRESHOLDS = [0.3, 0.5, 0.7, 0.85, 0.9] -# 每个阶段进入下一阶段所需的成功率阈值 - -# 新增:反应时间参数 -REACTION_TIME_PENALTY = 0.1 # 对反应迟钝的惩罚系数 -PROACTIVE_REWARD = 0.5 # 对提前行动的奖励 \ No newline at end of file +# 每个阶段进入下一阶段所需的成功率阈值 \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Model.py b/src/Unmanned_vehicle_AD_DQN/Model.py index b1c7d7f4ed..19daa0e80c 100644 --- a/src/Unmanned_vehicle_AD_DQN/Model.py +++ b/src/Unmanned_vehicle_AD_DQN/Model.py @@ -10,9 +10,8 @@ import matplotlib.pyplot as plt from collections import deque from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Concatenate, Conv2D, AveragePooling2D, Activation, \ - Flatten, Dropout, BatchNormalization, MaxPooling2D, Multiply, Add, Lambda, Subtract, Reshape, LayerNormalization, \ - SpatialDropout2D, SeparableConv2D -from tensorflow.keras.optimizers import Adam, RMSprop + Flatten, Dropout, BatchNormalization, MaxPooling2D, Multiply, Add, Lambda, Subtract +from tensorflow.keras.optimizers import Adam from tensorflow.keras.models import Sequential, Model from tensorflow.keras.callbacks import TensorBoard import tensorflow as tf @@ -38,7 +37,7 @@ LEARNING_RATE = 0.00005 IM_HEIGHT = 480 IM_WIDTH = 640 - MODEL_NAME = "YY_Optimized_v2" + MODEL_NAME = "YY_Optimized" # 自定义TensorBoard类 @@ -73,54 +72,6 @@ def update_stats(self, **stats): self.writer.flush() -# 注意力机制模块 -class AttentionModule: - @staticmethod - def channel_attention(input_feature, ratio=8): - """通道注意力机制""" - channel_axis = -1 - channel = input_feature.shape[channel_axis] - - shared_layer_one = Dense(channel//ratio, activation='relu', kernel_initializer='he_normal', - use_bias=True, bias_initializer='zeros') - shared_layer_two = Dense(channel, kernel_initializer='he_normal', use_bias=True, - bias_initializer='zeros') - - avg_pool = GlobalAveragePooling2D()(input_feature) - avg_pool = Reshape((1, 1, channel))(avg_pool) - avg_pool = shared_layer_one(avg_pool) - avg_pool = shared_layer_two(avg_pool) - - max_pool = tf.reduce_max(input_feature, axis=[1, 2], keepdims=True) - max_pool = shared_layer_one(max_pool) - max_pool = shared_layer_two(max_pool) - - cbam_feature = Add()([avg_pool, max_pool]) - cbam_feature = Activation('sigmoid')(cbam_feature) - - return Multiply()([input_feature, cbam_feature]) - - @staticmethod - def spatial_attention(input_feature): - """空间注意力机制""" - avg_pool = tf.reduce_mean(input_feature, axis=3, keepdims=True) - max_pool = tf.reduce_max(input_feature, axis=3, keepdims=True) - concat = Concatenate(axis=3)([avg_pool, max_pool]) - attention = Conv2D(1, kernel_size=7, padding='same', activation='sigmoid', - kernel_initializer='he_normal', use_bias=False)(concat) - - return Multiply()([input_feature, attention]) - - @staticmethod - def cbam_block(input_feature, ratio=8): - """完整的CBAM注意力模块""" - # 通道注意力 - x = AttentionModule.channel_attention(input_feature, ratio) - # 空间注意力 - x = AttentionModule.spatial_attention(x) - return x - - # DQN智能体类 - 升级版(整合训练策略) class DQNAgent: def __init__(self, use_dueling=True, use_per=True, use_curriculum=True, use_multi_objective=True): @@ -159,10 +110,6 @@ def __init__(self, use_dueling=True, use_per=True, use_curriculum=True, use_mult self.multi_objective_optimizer = None self.imitation_manager = None - # 新增:反应时间追踪 - self.reaction_times = deque(maxlen=100) - self.last_obstacle_distance = float('inf') - def setup_training_strategies(self, env=None): """设置训练策略组件""" if self.use_curriculum and env: @@ -180,47 +127,44 @@ def setup_training_strategies(self, env=None): self.imitation_manager = ImitationLearningManager() def create_model(self): - """创建标准深度Q网络模型 - 优化版""" + """创建标准深度Q网络模型""" # 使用函数式API inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) - # 第一卷积块 - 使用深度可分离卷积加快计算 - x = SeparableConv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) + # 第一卷积块 - 使用较小的卷积核 + x = Conv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) # 第二卷积块 - x = SeparableConv2D(64, (3, 3), padding='same')(x) + x = Conv2D(64, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) # 第三卷积块 - x = SeparableConv2D(128, (3, 3), padding='same')(x) + x = Conv2D(128, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 注意力机制 - 增强对障碍物的关注 - x = AttentionModule.cbam_block(x) - - # 第四卷积块 - 增加一层特征提取 - x = SeparableConv2D(256, (3, 3), padding='same')(x) - x = Activation('relu')(x) - x = BatchNormalization()(x) - x = SpatialDropout2D(0.1)(x) + # 空间注意力机制 + attention = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(x) + x = Multiply()([x, attention]) # 展平层 x = Flatten()(x) - # 全连接层 - 优化网络结构 + # 全连接层 - 减少神经元数量 x = Dense(256, activation='relu')(x) x = Dropout(0.3)(x) x = Dense(128, activation='relu')(x) x = Dropout(0.3)(x) x = Dense(64, activation='relu')(x) x = Dropout(0.2)(x) + x = Dense(32, activation='relu')(x) + x = Dropout(0.1)(x) # 输出层 - 5个动作 outputs = Dense(5, activation='linear')(x) @@ -228,43 +172,36 @@ def create_model(self): # 创建模型 model = Model(inputs=inputs, outputs=outputs) - # 编译模型 - 使用自适应学习率优化器 - model.compile(loss="huber", - optimizer=Adam(learning_rate=LEARNING_RATE, clipnorm=1.0), - metrics=["mae"]) + # 编译模型 + model.compile(loss="huber", optimizer=Adam(learning_rate=LEARNING_RATE), metrics=["mae"]) return model def create_dueling_model(self): - """创建Dueling DQN模型架构 - 优化版""" + """创建Dueling DQN模型架构""" inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) # 共享的特征提取层 # 第一卷积块 - x = SeparableConv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) + x = Conv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) # 第二卷积块 - x = SeparableConv2D(64, (3, 3), padding='same')(x) + x = Conv2D(64, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) # 第三卷积块 - x = SeparableConv2D(128, (3, 3), padding='same')(x) + x = Conv2D(128, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 注意力机制 - x = AttentionModule.cbam_block(x) - - # 第四卷积块 - x = SeparableConv2D(256, (3, 3), padding='same')(x) - x = Activation('relu')(x) - x = BatchNormalization()(x) - x = SpatialDropout2D(0.1)(x) + # 空间注意力机制 + attention = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(x) + x = Multiply()([x, attention]) # 展平层 x = Flatten()(x) @@ -293,25 +230,13 @@ def create_dueling_model(self): model = Model(inputs=inputs, outputs=q_values) # 编译模型 - model.compile(loss="huber", - optimizer=Adam(learning_rate=LEARNING_RATE, clipnorm=1.0), - metrics=["mae"]) + model.compile(loss="huber", optimizer=Adam(learning_rate=LEARNING_RATE), metrics=["mae"]) return model - def update_replay_memory(self, transition, reaction_time=None): + def update_replay_memory(self, transition): """更新经验回放缓冲区""" # transition = (当前状态, 动作, 奖励, 新状态, 完成标志) - if reaction_time is not None: - # 记录反应时间 - self.reaction_times.append(reaction_time) - - # 如果反应时间过长,增加惩罚 - if reaction_time > 1.0: # 反应时间超过1秒 - transition = list(transition) - transition[2] -= 0.5 # 减少奖励 - transition = tuple(transition) - if self.use_per: # PER: 初始添加时使用最大优先级 self.replay_buffer.add(transition, error=1.0) # 初始误差设为1.0 @@ -319,7 +244,7 @@ def update_replay_memory(self, transition, reaction_time=None): self.replay_memory.append(transition) def minibatch_chooser(self): - """改进的经验采样策略 - 优先采样危险情况""" + """改进的经验采样策略""" if self.use_per: # PER采样 if len(self.replay_buffer) < MIN_REPLAY_MEMORY_SIZE: @@ -328,41 +253,34 @@ def minibatch_chooser(self): indices, samples, weights = self.replay_buffer.sample(MINIBATCH_SIZE) return samples else: - # 标准采样 - 增加危险经验的权重 + # 标准采样 if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: return random.sample(self.replay_memory, min(len(self.replay_memory), MINIBATCH_SIZE)) # 分类经验样本 - danger_samples = [] # 接近碰撞的经验 - collision_samples = [] # 碰撞经验 - positive_samples = [] # 成功避障经验 - neutral_samples = [] # 中性经验 + positive_samples = [] # 高奖励经验 + negative_samples = [] # 负奖励/碰撞经验 + neutral_samples = [] # 中性奖励经验 for sample in self.replay_memory: - state, action, reward, next_state, done = sample + _, _, reward, _, done = sample - if done and reward < -5: # 碰撞经验 - collision_samples.append(sample) - elif reward < -2 and not done: # 危险但未碰撞 - danger_samples.append(sample) - elif reward > 2: # 积极经验(成功避障) + if done and reward < -5: # 碰撞或严重错误 + negative_samples.append(sample) + elif reward > 1: # 积极经验 positive_samples.append(sample) else: # 中性经验 neutral_samples.append(sample) - # 平衡采样 - 增加危险经验的权重 + # 平衡采样 batch = [] - # 采样碰撞经验 (25%) - num_collision = min(len(collision_samples), MINIBATCH_SIZE // 4) - batch.extend(random.sample(collision_samples, num_collision)) + # 采样负经验 (20%) + num_negative = min(len(negative_samples), MINIBATCH_SIZE // 5) + batch.extend(random.sample(negative_samples, num_negative)) - # 采样危险经验 (30%) - num_danger = min(len(danger_samples), MINIBATCH_SIZE * 3 // 10) - batch.extend(random.sample(danger_samples, num_danger)) - - # 采样成功避障经验 (25%) - num_positive = min(len(positive_samples), MINIBATCH_SIZE // 4) + # 采样正经验 (30%) + num_positive = min(len(positive_samples), MINIBATCH_SIZE // 3) batch.extend(random.sample(positive_samples, num_positive)) # 用中性经验补全批次 @@ -471,10 +389,4 @@ def train_in_loop(self): def get_qs(self, state): """获取状态的Q值""" - return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255, verbose=0)[0] - - def get_average_reaction_time(self): - """获取平均反应时间""" - if len(self.reaction_times) > 0: - return np.mean(self.reaction_times) - return 0 \ No newline at end of file + return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255, verbose=0)[0] \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Test.py b/src/Unmanned_vehicle_AD_DQN/Test.py index b90d26188f..ccd53b468c 100644 --- a/src/Unmanned_vehicle_AD_DQN/Test.py +++ b/src/Unmanned_vehicle_AD_DQN/Test.py @@ -66,7 +66,7 @@ def select_best_model(model_files, preferred_keywords=None, excluded_keywords=No return None if preferred_keywords is None: - preferred_keywords = ["best", "advanced", "dueling_per", "v2"] + preferred_keywords = ["best", "advanced", "dueling_per"] if excluded_keywords is None: excluded_keywords = ["min", "avg", "final"] # 排除统计文件 @@ -151,22 +151,16 @@ def get_safe_action_advanced(model, state, env, previous_action, uncertainty_thr qs[1] *= 1.2 # 提高保持倾向 qs[2] *= 1.3 # 提高加速倾向 - # 2. 行人避障优先级 - 增强版 + # 2. 行人避障优先级 if hasattr(env, 'suggested_action') and env.suggested_action is not None: - # 根据距离调整优先级 - min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) - if min_ped_distance < 8.0: # 危险距离 - qs[env.suggested_action] += 5.0 # 大幅提高建议动作的Q值 - print(f"🚨 紧急避让: 执行动作 {env.suggested_action}, 距离: {min_ped_distance:.1f}m") - elif min_ped_distance < 12.0: # 预警距离 - qs[env.suggested_action] += 2.0 - print(f"⚠️ 安全避让: 执行动作 {env.suggested_action}, 距离: {min_ped_distance:.1f}m") + qs[env.suggested_action] += 3.0 # 大幅提高建议动作的Q值 + print(f"🚨 安全避让: 执行动作 {env.suggested_action}") env.suggested_action = None # 3. 防止过度转向 if hasattr(env, 'same_steer_counter') and env.same_steer_counter > 2: if previous_action in [3, 4]: - qs[previous_action] -= 2.0 # 降低连续同向转向的倾向 + qs[previous_action] -= 1.5 # 降低连续同向转向的倾向 # 4. 动作平滑性 if previous_action in [3, 4]: # 转向动作 @@ -183,24 +177,18 @@ def get_safe_action_advanced(model, state, env, previous_action, uncertainty_thr elif abs(vehicle_rotation) > 30: # 方向偏差大 # 鼓励向相反方向转向以回正 if vehicle_rotation > 0: # 偏左,鼓励右转 - qs[4] += 1.5 + qs[4] += 1.0 else: # 偏右,鼓励左转 - qs[3] += 1.5 + qs[3] += 1.0 - # 6. 紧急情况处理 - 增强版 + # 6. 紧急情况处理 min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) if min_ped_distance < 5.0: # 紧急避让距离 # 大幅调整Q值以确保安全 - qs[0] += 3.0 # 紧急制动 + qs[0] += 2.0 # 紧急制动 if min_ped_distance < 3.0: # 极危险 qs[2] = -float('inf') # 禁止加速 - qs[1] = -float('inf') # 禁止保持 - print("⚠️ 紧急制动! 危险距离!") - - # 7. 考虑运动检测结果 - if hasattr(env, 'motion_detected') and env.motion_detected: - # 如果检测到运动,提高警惕 - qs[0] += 0.5 # 鼓励减速观察 + print("⚠️ 紧急制动!") # 选择动作 action = np.argmax(qs) @@ -237,9 +225,6 @@ def run_test_episode(model, env, episode_num, use_advanced_safety=True): previous_action = 1 fps_counter = deque(maxlen=30) - # 反应时间统计 - reaction_times = [] - # 运行episode max_steps = SECONDS_PER_EPISODE * 60 @@ -257,12 +242,6 @@ def run_test_episode(model, env, episode_num, use_advanced_safety=True): previous_action = action - # 记录反应开始时间(如果有障碍物) - if hasattr(env, 'obstacle_detected_time') and env.obstacle_detected_time is not None: - if hasattr(env, 'reaction_start_time') and env.reaction_start_time is not None: - reaction_time = time.time() - env.reaction_start_time - reaction_times.append(reaction_time) - # 执行动作 new_state, reward, done, _ = env.step(action) @@ -281,14 +260,10 @@ def run_test_episode(model, env, episode_num, use_advanced_safety=True): velocity = env.vehicle.get_velocity() speed_kmh = 3.6 * np.linalg.norm([velocity.x, velocity.y, velocity.z]) - # 获取最近行人距离 - min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) - status = "✅" if reward > 0 else "⚠️" if reward < -1 else "➡️" print(f"{status} 步数: {step_count:4d} | FPS: {fps:4.1f} | " - f"速度: {speed_kmh:5.1f} km/h | 行人距离: {min_ped_distance:5.1f}m | " - f"奖励: {reward:6.2f} | 累计: {total_reward:7.2f}") + f"速度: {speed_kmh:5.1f} km/h | 奖励: {reward:6.2f} | 累计: {total_reward:7.2f}") if done: break @@ -300,14 +275,10 @@ def run_test_episode(model, env, episode_num, use_advanced_safety=True): success = total_reward > 5 result = "成功" if success else "失败" - # 计算平均反应时间 - avg_reaction_time = np.mean(reaction_times) if reaction_times else 0 - print(f"\nEpisode {episode_num} 结果: {result}") print(f"总步数: {step_count}, 总奖励: {total_reward:.2f}") - print(f"平均反应时间: {avg_reaction_time:.2f}秒") - return success, total_reward, step_count, avg_reaction_time + return success, total_reward, step_count def load_model_with_fallback(model_path): @@ -403,19 +374,17 @@ def comprehensive_model_evaluation(model_path, num_episodes=5): 'successes': 0, 'total_rewards': [], 'episode_lengths': [], - 'reaction_times': [], 'start_time': time.time() } try: for episode in range(1, num_episodes + 1): - success, reward, length, avg_rt = run_test_episode(model, env, episode, use_advanced_safety=True) + success, reward, length = run_test_episode(model, env, episode, use_advanced_safety=True) if success: results['successes'] += 1 results['total_rewards'].append(reward) results['episode_lengths'].append(length) - results['reaction_times'].append(avg_rt) # 短暂暂停 time.sleep(1) @@ -438,7 +407,6 @@ def comprehensive_model_evaluation(model_path, num_episodes=5): results['avg_length'] = np.mean(results['episode_lengths']) results['max_reward'] = max(results['total_rewards']) results['min_reward'] = min(results['total_rewards']) - results['avg_reaction_time'] = np.mean(results['reaction_times']) if results['reaction_times'] else 0 # 显示评估报告 print(f"\n{'='*60}") @@ -449,7 +417,6 @@ def comprehensive_model_evaluation(model_path, num_episodes=5): print(f"成功率: {results.get('success_rate', 0):.1f}%") print(f"平均奖励: {results.get('avg_reward', 0):.2f}") print(f"平均步数: {results.get('avg_length', 0):.1f}") - print(f"平均反应时间: {results.get('avg_reaction_time', 0):.2f}秒") print(f"最佳表现: {results.get('max_reward', 0):.2f}") print(f"最差表现: {results.get('min_reward', 0):.2f}") print(f"总测试时间: {results.get('total_time', 0):.1f}秒") @@ -533,7 +500,7 @@ def interactive_model_selection(model_files): def main(): """主函数 - 自动查找和测试模型""" print(f"\n{'='*60}") - print("自动驾驶模型测试系统 - 优化版") + print("自动驾驶模型测试系统") print(f"{'='*60}") # 显示当前脚本所在目录 diff --git a/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py b/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py index 6f35c41a38..cb5e611506 100644 --- a/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py +++ b/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py @@ -16,118 +16,82 @@ def __init__(self, env): self.current_stage = 0 self.stage_thresholds = [0.3, 0.5, 0.7, 0.85] # 成功率阈值 self.stage_configs = [ - # 阶段0: 入门 - 更早引入行人,但数量少速度慢 + # 阶段0: 入门 { - 'pedestrian_cross': 3, # 增加一点难度 - 'pedestrian_normal': 2, - 'pedestrian_speed_min': 0.3, # 降低行人速度 - 'pedestrian_speed_max': 0.8, - 'max_episode_steps': 900, # 减少最大步数,加速训练 - 'success_threshold': 0.3, - 'difficulty_name': '简单' + 'pedestrian_cross': 2, # 十字路口行人数量(减少) + 'pedestrian_normal': 1, # 普通路段行人数量(减少) + 'pedestrian_speed_min': 0.5, # 行人最低速度 + 'pedestrian_speed_max': 1.0, # 行人最高速度 + 'max_episode_steps': 1200, # 最大步数 (20秒 * 60FPS) + 'success_threshold': 0.3 # 进入下一阶段成功率 }, # 阶段1: 初级 { - 'pedestrian_cross': 5, - 'pedestrian_normal': 3, - 'pedestrian_speed_min': 0.5, - 'pedestrian_speed_max': 1.0, - 'max_episode_steps': 1200, - 'success_threshold': 0.5, - 'difficulty_name': '初级' - }, - # 阶段2: 中级 - 增加反应时间训练 - { - 'pedestrian_cross': 7, - 'pedestrian_normal': 4, + 'pedestrian_cross': 4, # 逐步增加 + 'pedestrian_normal': 2, 'pedestrian_speed_min': 0.7, 'pedestrian_speed_max': 1.3, - 'max_episode_steps': 1500, - 'success_threshold': 0.6, # 提高阈值 - 'difficulty_name': '中级' + 'max_episode_steps': 1800, # 30秒 + 'success_threshold': 0.5 }, - # 阶段3: 高级 + # 阶段2: 中级 { - 'pedestrian_cross': 9, - 'pedestrian_normal': 5, - 'pedestrian_speed_min': 0.9, - 'pedestrian_speed_max': 1.6, - 'max_episode_steps': 1800, - 'success_threshold': 0.7, - 'difficulty_name': '高级' + 'pedestrian_cross': 6, + 'pedestrian_normal': 3, + 'pedestrian_speed_min': 0.8, + 'pedestrian_speed_max': 1.5, + 'max_episode_steps': 2400, # 40秒 + 'success_threshold': 0.7 }, - # 阶段4: 专家 (正常难度) + # 阶段3: 高级 (正常难度) { - 'pedestrian_cross': 10, - 'pedestrian_normal': 6, + 'pedestrian_cross': 8, + 'pedestrian_normal': 4, 'pedestrian_speed_min': 1.0, 'pedestrian_speed_max': 2.0, - 'max_episode_steps': 2400, - 'success_threshold': 0.8, - 'difficulty_name': '专家' + 'max_episode_steps': 3600, # 60秒 + 'success_threshold': 0.85 }, - # 阶段5: 大师 (挑战) + # 阶段4: 专家 (挑战) { - 'pedestrian_cross': 12, - 'pedestrian_normal': 8, + 'pedestrian_cross': 10, # 适当减少 + 'pedestrian_normal': 5, 'pedestrian_speed_min': 1.2, 'pedestrian_speed_max': 2.5, - 'max_episode_steps': 3000, - 'success_threshold': 0.85, - 'difficulty_name': '大师' + 'max_episode_steps': 3600, + 'success_threshold': 0.9 } ] # 训练历史 self.success_history = deque(maxlen=20) # 记录最近20轮的成功情况 self.reward_history = deque(maxlen=50) # 记录最近50轮的奖励 - self.reaction_time_history = deque(maxlen=50) # 记录反应时间 - def update_stage(self, success, reward, reaction_time=None): + def update_stage(self, success, reward): """更新训练阶段""" # 记录历史 self.success_history.append(1 if success else 0) self.reward_history.append(reward) - if reaction_time is not None: - self.reaction_time_history.append(reaction_time) # 计算最近成功率 if len(self.success_history) >= 10: success_rate = sum(self.success_history) / len(self.success_history) avg_reward = np.mean(self.reward_history) if self.reward_history else 0 - # 每20轮打印一次 - if len(self.success_history) % 20 == 0: - stage_info = self.get_current_config() - print(f"课程学习 - 阶段: {self.current_stage}({stage_info['difficulty_name']}), " - f"成功率: {success_rate:.2f}, 平均奖励: {avg_reward:.2f}") - if self.reaction_time_history: - avg_rt = np.mean(self.reaction_time_history) - print(f" 平均反应时间: {avg_rt:.2f}秒") + # 减少打印频率 + if len(self.success_history) % 10 == 0: + print(f"课程学习 - 当前阶段: {self.current_stage}, 成功率: {success_rate:.2f}, 平均奖励: {avg_reward:.2f}") # 检查是否可以进入下一阶段 if self.current_stage < len(self.stage_configs) - 1: next_stage_threshold = self.stage_configs[self.current_stage]['success_threshold'] - - # 不仅要看成功率,还要看反应时间(如果可用) - can_advance = success_rate >= next_stage_threshold and avg_reward > 5 - if can_advance and self.reaction_time_history: - avg_rt = np.mean(self.reaction_time_history) - # 要求反应时间小于1秒 - if avg_rt < 1.0: - self.current_stage += 1 - print(f"🎉 课程学习: 进阶到阶段 {self.current_stage}!") - print(f" 新配置: {self.stage_configs[self.current_stage]['difficulty_name']}") - return True - elif can_advance: + if success_rate >= next_stage_threshold and avg_reward > 5: self.current_stage += 1 print(f"🎉 课程学习: 进阶到阶段 {self.current_stage}!") - print(f" 新配置: {self.stage_configs[self.current_stage]['difficulty_name']}") return True - # 如果表现太差或反应时间太长,退回上一阶段 - if self.current_stage > 0 and (success_rate < 0.2 or - (self.reaction_time_history and np.mean(self.reaction_time_history) > 2.0)): + # 如果表现太差,退回上一阶段 + if self.current_stage > 0 and success_rate < 0.2: self.current_stage -= 1 print(f"⚠️ 课程学习: 退回阶段 {self.current_stage}") return True @@ -149,28 +113,23 @@ class MultiObjectiveOptimizer: def __init__(self): # 定义优化目标及其权重(可动态调整) self.objectives = { - 'reaction_time': { - 'weight': 0.25, # 新增:反应时间权重 - 'description': '快速反应避障', - 'metrics': ['reaction_time', 'proactive_actions'] - }, 'safety': { - 'weight': 0.30, + 'weight': 0.35, # 稍微降低安全权重 'description': '安全避障和避免碰撞', 'metrics': ['collision_avoidance', 'pedestrian_distance'] }, 'efficiency': { - 'weight': 0.25, + 'weight': 0.30, # 提高效率权重 'description': '快速到达目的地', 'metrics': ['progress_speed', 'total_time'] }, 'comfort': { - 'weight': 0.15, + 'weight': 0.20, 'description': '平稳驾驶体验', 'metrics': ['smoothness', 'steering_changes'] }, 'rule_following': { - 'weight': 0.05, + 'weight': 0.15, 'description': '遵守交通规则', 'metrics': ['lane_keeping', 'speed_limit'] } @@ -178,7 +137,6 @@ def __init__(self): # 指标跟踪 self.metrics_history = { - 'reaction_time': [], 'safety': [], 'efficiency': [], 'comfort': [], @@ -198,25 +156,13 @@ def compute_composite_reward(self, metrics): # 记录指标历史 self.metrics_history[obj_name].append(normalized_value) - # 特殊奖励/惩罚项 + # 特殊惩罚项 if metrics.get('collision', False): - composite -= 10 # 增加碰撞惩罚 + composite -= 8 # 减少碰撞惩罚 if metrics.get('off_road', False): - composite -= 5 # 增加偏离道路惩罚 + composite -= 3 # 减少偏离道路惩罚 if metrics.get('dangerous_action', False): - composite -= 3 # 增加危险动作惩罚 - - # 新增:反应时间相关奖励/惩罚 - if 'reaction_time' in metrics: - rt = metrics['reaction_time'] - if rt < 0.5: # 快速反应 - composite += 2 - elif rt > 1.5: # 反应太慢 - composite -= 3 - - # 新增:主动避障奖励 - if metrics.get('proactive_action', False): - composite += 1.5 + composite -= 2 # 减少危险动作惩罚 return composite @@ -224,11 +170,10 @@ def _normalize_metric(self, value, metric_name): """归一化指标值到[0, 1]范围""" # 不同指标的归一化方式不同 normalization_rules = { - 'reaction_time': lambda x: max(0, 1 - x/3), # 反应时间越短越好 - 'safety': lambda x: min(max(x / 10, 0), 1), - 'efficiency': lambda x: min(max(x / 100, 0), 1), - 'comfort': lambda x: min(max((x + 5) / 10, 0), 1), - 'rule_following': lambda x: min(max(x, 0), 1) + 'safety': lambda x: min(max(x / 10, 0), 1), # 假设安全分满分10 + 'efficiency': lambda x: min(max(x / 100, 0), 1), # 效率分满分100 + 'comfort': lambda x: min(max((x + 5) / 10, 0), 1), # 舒适度[-5, 5] -> [0, 1] + 'rule_following': lambda x: min(max(x, 0), 1) # 规则遵循度[0, 1] } if metric_name in normalization_rules: @@ -251,7 +196,7 @@ def adjust_weights(self, performance_feedback): # 如果最差目标表现低于阈值,增加其权重 if recent_performance[worst_obj] < 0.3: - adjustment = 0.03 # 调整幅度 + adjustment = 0.02 # 减少调整幅度 self.objectives[worst_obj]['weight'] += adjustment self.objectives[best_obj]['weight'] -= adjustment @@ -278,7 +223,7 @@ def get_performance_report(self): return report -# 模仿学习管理器(保持不变,略作修改) +# 模仿学习管理器 class ImitationLearningManager: def __init__(self, expert_data_path=None): self.expert_data_path = expert_data_path @@ -299,11 +244,232 @@ def load_expert_data(self, path): except Exception as e: print(f"加载专家数据失败: {e}") return False + + def collect_expert_demonstration(self, env, num_episodes=10): + """收集专家示范数据(可以手动控制或使用规则控制器)""" + print(f"开始收集专家示范数据 ({num_episodes}个episodes)...") + + demonstrations = [] + + for episode in range(num_episodes): + print(f"收集专家示范 Episode {episode + 1}/{num_episodes}") + + state = env.reset(episode) + done = False + episode_data = [] + + step_count = 0 + max_steps = 60 * 60 # 最大60秒 + + while not done and step_count < max_steps: + # 这里可以使用规则控制器或手动控制 + # 示例:简单的规则控制器 + action = self._rule_based_controller(env) + + new_state, reward, done, _ = env.step(action) + + # 保存示范数据 + episode_data.append({ + 'state': state.copy(), + 'action': action, + 'reward': reward, + 'next_state': new_state.copy(), + 'done': done + }) + + state = new_state + step_count += 1 + + demonstrations.extend(episode_data) + env.cleanup_actors() + + # 保存专家数据 + self.expert_data = demonstrations + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + save_path = f"expert_data_{timestamp}.pkl" + + with open(save_path, 'wb') as f: + pickle.dump(demonstrations, f) + + print(f"专家示范数据已保存到: {save_path}, 共 {len(demonstrations)} 条记录") + return True + + def _rule_based_controller(self, env): + """基于规则的控制器(作为专家示范)""" + # 获取车辆状态 + vehicle_location = env.vehicle.get_location() + velocity = env.vehicle.get_velocity() + speed_kmh = 3.6 * math.sqrt(velocity.x**2 + velocity.y**2) + + # 简单规则:保持速度在20-35 km/h,避免障碍物 + if speed_kmh < 20: + return 2 # 加速 + elif speed_kmh > 35: + return 0 # 减速 + else: + # 检查前方障碍物 + has_obstacle_ahead = self._check_obstacle_ahead(env) + if has_obstacle_ahead: + return 0 # 减速 + else: + return 1 # 保持 + + return 1 # 默认保持 + + def _check_obstacle_ahead(self, env): + """检查前方是否有障碍物(简化版本)""" + # 这里可以添加更复杂的障碍物检测逻辑 + # 暂时返回False + return False + + def pretrain_with_behavioral_cloning(self, model, epochs=20): + """使用行为克隆进行预训练""" + if not self.expert_data: + print("没有专家数据可用,跳过预训练") + return model + + print(f"开始行为克隆预训练 ({epochs}个epochs)...") + + # 准备训练数据 + states = [] + actions = [] + + for demo in self.expert_data: + states.append(demo['state']) + actions.append(demo['action']) + + # 将状态归一化 + states = np.array(states) / 255.0 + + # 将动作转换为one-hot编码 + actions_onehot = tf.keras.utils.to_categorical(actions, num_classes=5) + + # 备份原始编译设置 + original_loss = model.loss + original_optimizer = model.optimizer + original_metrics = model.metrics_names + + # 重新编译模型用于分类任务 + model.compile( + optimizer=Adam(learning_rate=0.0001), + loss='categorical_crossentropy', + metrics=['accuracy'] + ) + + # 训练模型模仿专家行为 + history = model.fit( + states, actions_onehot, + batch_size=32, + epochs=epochs, + validation_split=0.2, + verbose=1 + ) + + print(f"预训练完成 - 最终准确率: {history.history['accuracy'][-1]:.3f}") + + # 恢复原始编译设置 + model.compile( + optimizer=original_optimizer, + loss=original_loss, + metrics=original_metrics + ) + + self.is_pretrained = True + return model + + def train_with_dagger(self, model, env, iterations=5, episodes_per_iter=5): + """使用DAgger算法进行训练""" + print(f"开始DAgger训练 ({iterations}次迭代,每次{episodes_per_iter}个episodes)...") + + aggregated_data = self.expert_data.copy() + + for iteration in range(iterations): + print(f"\nDAgger 迭代 {iteration + 1}/{iterations}") + + # 使用当前策略收集数据 + new_demos = [] + + for episode in range(episodes_per_iter): + print(f" 收集数据 Episode {episode + 1}/{episodes_per_iter}") + + state = env.reset(episode) + done = False + step_count = 0 + max_steps = 60 * 60 + + while not done and step_count < max_steps: + # 使用当前策略选择动作 + qs = model.predict(np.array(state).reshape(-1, *state.shape) / 255, verbose=0)[0] + action = np.argmax(qs) + + # 执行动作 + new_state, reward, done, _ = env.step(action) + + # 专家纠正(这里可以添加专家纠正逻辑) + # 如果策略动作与专家建议不同,使用专家动作 + expert_action = self._rule_based_controller(env) + + # 保存数据(使用专家纠正后的动作) + new_demos.append({ + 'state': state.copy(), + 'action': expert_action, # 使用专家动作 + 'reward': reward, + 'next_state': new_state.copy(), + 'done': done + }) + + state = new_state + step_count += 1 + + env.cleanup_actors() + + # 合并数据 + aggregated_data.extend(new_demos) + + # 在合并数据上重新训练 + states = [d['state'] for d in aggregated_data] + actions = [d['action'] for d in aggregated_data] + + states = np.array(states) / 255.0 + actions_onehot = tf.keras.utils.to_categorical(actions, num_classes=5) + + # 备份原始编译设置 + original_loss = model.loss + original_optimizer = model.optimizer + original_metrics = model.metrics_names + + # 重新编译用于分类 + model.compile( + optimizer=Adam(learning_rate=0.0001), + loss='categorical_crossentropy', + metrics=['accuracy'] + ) + + # 训练模型 + history = model.fit( + states, actions_onehot, + batch_size=32, + epochs=10, + validation_split=0.1, + verbose=0 + ) + + # 恢复原始编译设置 + model.compile( + optimizer=original_optimizer, + loss=original_loss, + metrics=original_metrics + ) + + print(f" 训练完成 - 准确率: {history.history['accuracy'][-1]:.3f}") + + print("DAgger训练完成!") + return model -# 优先经验回放缓冲区(增加对危险经验的优先级) +# 优先经验回放缓冲区 class PrioritizedReplayBuffer: - def __init__(self, max_size=15000, alpha=0.7, beta_start=0.5, beta_frames=50000): + def __init__(self, max_size=10000, alpha=0.6, beta_start=0.4, beta_frames=100000): self.max_size = max_size self.alpha = alpha # 优先级程度 (0 = 均匀采样, 1 = 完全优先级) self.beta_start = beta_start # 重要性采样权重起始值 @@ -328,11 +494,6 @@ def add(self, experience, error=None): else: priority = (abs(error) + 1e-5) ** self.alpha - # 如果是危险经验(负奖励较大),增加优先级 - reward = experience[2] - if reward < -2: # 危险经验 - priority *= 1.5 - self.buffer.append(experience) self.priorities.append(priority) diff --git a/src/Unmanned_vehicle_AD_DQN/main.py b/src/Unmanned_vehicle_AD_DQN/main.py index f69fb91625..e627c3006d 100644 --- a/src/Unmanned_vehicle_AD_DQN/main.py +++ b/src/Unmanned_vehicle_AD_DQN/main.py @@ -1,4 +1,4 @@ -# main_fixed.py - 修复模型保存问题的版本 +# main.py import glob import os import sys @@ -26,27 +26,6 @@ # 从Hyperparameters导入所有参数 from Hyperparameters import * -def ensure_models_directory(): - """确保models目录存在""" - if not os.path.exists('models'): - os.makedirs('models') - print("✅ 已创建 models 目录") - return 'models' - -def save_model_with_retry(model, filepath, max_retries=3): - """带重试机制的模型保存""" - for attempt in range(max_retries): - try: - model.save(filepath) - print(f"✅ 模型保存成功: {os.path.basename(filepath)}") - return True - except Exception as e: - print(f"⚠️ 保存失败 (尝试 {attempt+1}/{max_retries}): {e}") - time.sleep(1) - - print(f"❌ 无法保存模型: {filepath}") - return False - def extended_reward_calculation(env, action, reward, done, step_info): """ 扩展的奖励计算函数,用于多目标优化 @@ -59,23 +38,7 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 计算多目标指标 metrics = {} - # 1. 反应时间指标 - reaction_time = 0 - if hasattr(env, 'obstacle_detected_time') and env.obstacle_detected_time is not None: - if hasattr(env, 'reaction_start_time') and env.reaction_start_time is not None: - reaction_time = time.time() - env.reaction_start_time - - metrics['reaction_time'] = reaction_time - - # 2. 主动避障指标 - proactive_action = False - if hasattr(env, 'suggested_action') and env.suggested_action is not None: - if action == env.suggested_action: - proactive_action = True - - metrics['proactive_action'] = proactive_action - - # 3. 安全性指标 - 基于最近行人距离 + # 1. 安全性指标 - 基于最近行人距离 min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) safety_score = 0 if min_ped_distance < 100: @@ -92,12 +55,12 @@ def extended_reward_calculation(env, action, reward, done, step_info): metrics['safety'] = safety_score - # 4. 效率指标 - 基于进度 + # 2. 效率指标 - 基于进度 progress = (vehicle_location.x + 81) / 236.0 # 从-81到155 efficiency_score = progress * 100 # 进度百分比 metrics['efficiency'] = efficiency_score - # 5. 舒适度指标 - 基于转向平滑性 + # 3. 舒适度指标 - 基于转向平滑性 comfort_score = 5 # 默认舒适 if hasattr(env, 'last_action') and env.last_action in [3, 4]: @@ -112,7 +75,7 @@ def extended_reward_calculation(env, action, reward, done, step_info): metrics['comfort'] = comfort_score - # 6. 规则遵循指标 - 基于速度 + # 4. 规则遵循指标 - 基于速度 rule_score = 0.3 # 默认较低分数 if 20 <= speed_kmh <= 35: # 理想速度范围 @@ -126,11 +89,11 @@ def extended_reward_calculation(env, action, reward, done, step_info): metrics['rule_following'] = rule_score - # 7. 特殊事件 + # 5. 特殊事件 metrics['collision'] = len(getattr(env, 'collision_history', [])) > 0 metrics['off_road'] = vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30 - # 8. 危险动作检测 + # 6. 危险动作检测 if speed_kmh > 40 and action in [3, 4]: # 高速急转 metrics['dangerous_action'] = True else: @@ -142,14 +105,19 @@ def extended_reward_calculation(env, action, reward, done, step_info): FPS = 60 # 帧率 ep_rewards = [-200] # 存储每轮奖励 - # 确保models目录存在 - models_dir = ensure_models_directory() - # GPU内存配置 gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) tf.compat.v1.keras.backend.set_session( tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) + # 创建模型保存目录 + if not os.path.isdir('models'): + os.makedirs('models') + + # 创建专家数据目录 + if not os.path.isdir('expert_data'): + os.makedirs('expert_data') + # 创建智能体和环境 agent = DQNAgent( use_dueling=True, @@ -163,6 +131,30 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 设置训练策略 agent.setup_training_strategies(env) + # 可选:使用模仿学习进行预训练 + use_imitation_pretraining = False # 设置为True启用模仿学习预训练 + + if use_imitation_pretraining: + print("=" * 60) + print("开始模仿学习预训练阶段") + print("=" * 60) + + # 检查是否有现有的专家数据 + expert_files = glob.glob("expert_data/*.pkl") + if expert_files: + # 使用最新的专家数据 + latest_expert = max(expert_files, key=os.path.getctime) + agent.imitation_manager.load_expert_data(latest_expert) + else: + # 收集新的专家数据 + print("未找到专家数据,开始收集...") + agent.imitation_manager.collect_expert_demonstration(env, num_episodes=5) + + # 使用行为克隆进行预训练 + agent.model = agent.imitation_manager.pretrain_with_behavioral_cloning(agent.model, epochs=15) + agent.target_model.set_weights(agent.model.get_weights()) + print("模仿学习预训练完成!") + # 启动训练线程并等待训练初始化完成 trainer_thread = Thread(target=agent.train_in_loop, daemon=True) trainer_thread.start() @@ -186,7 +178,6 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 多目标统计 multi_obj_stats = { - 'reaction_time': [], 'safety': [], 'efficiency': [], 'comfort': [], @@ -196,9 +187,6 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 课程学习阶段记录 curriculum_stages = [] - # 反应时间统计 - reaction_time_stats = [] - # 迭代训练轮次 epds = [] for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'): @@ -209,7 +197,7 @@ def extended_reward_calculation(env, action, reward, done, step_info): if agent.curriculum_manager: config = agent.curriculum_manager.get_current_config() if episode % 50 == 0: # 每50轮打印一次 - print(f"课程学习 - 阶段 {agent.curriculum_manager.current_stage}({config['difficulty_name']}): " + print(f"课程学习 - 阶段 {agent.curriculum_manager.current_stage}: " f"行人(十字路口={config['pedestrian_cross']}, 普通={config['pedestrian_normal']})") curriculum_stages.append(agent.curriculum_manager.current_stage) @@ -219,7 +207,6 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 多目标指标记录 episode_metrics = { - 'reaction_time': [], 'safety': [], 'efficiency': [], 'comfort': [], @@ -247,18 +234,7 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 从Q网络获取动作(利用) qs = agent.get_qs(current_state) action = np.argmax(qs) - - # 如果有建议的避让动作,考虑采纳 - if hasattr(env, 'suggested_action') and env.suggested_action is not None: - # 检查建议动作的Q值 - suggested_q = qs[env.suggested_action] - current_best_q = qs[action] - - # 如果建议动作的Q值接近最佳动作,采纳建议 - if suggested_q > current_best_q * 0.8: - action = env.suggested_action - - if episode % 100 == 0 and step % 100 == 0: # 减少打印频率 + if episode % 50 == 0 and step % 30 == 0: # 减少打印频率 print(f'Ep {episode} Step {step}: Q值 [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}, {qs[3]:>5.2f}, {qs[4]:>5.2f}] 动作: {action}') else: # 随机选择动作(探索) @@ -269,12 +245,6 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 执行动作并获取结果 new_state, reward, done, _ = env.step(action) - # 计算反应时间 - reaction_time = 0 - if hasattr(env, 'obstacle_detected_time') and env.obstacle_detected_time is not None: - if hasattr(env, 'reaction_start_time') and env.reaction_start_time is not None: - reaction_time = time.time() - env.reaction_start_time - # 计算多目标指标 if agent.multi_objective_optimizer: step_info = {'step': step, 'action': action} @@ -291,9 +261,8 @@ def extended_reward_calculation(env, action, reward, done, step_info): score += reward # 累加奖励 - # 更新经验回放(带反应时间信息) - agent.update_replay_memory((current_state, action, reward, new_state, done), - reaction_time=reaction_time) + # 更新经验回放 + agent.update_replay_memory((current_state, action, reward, new_state, done)) current_state = new_state # 更新当前状态 step += 1 @@ -303,68 +272,40 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 本轮结束 - 销毁所有actor env.cleanup_actors() - - # 计算平均反应时间 - if episode_metrics['reaction_time']: - avg_reaction_time = np.mean([rt for rt in episode_metrics['reaction_time'] if rt > 0]) - reaction_time_stats.append(avg_reaction_time) - + # 计算本轮平均指标 avg_metrics = {} for key, values in episode_metrics.items(): if values: - # 过滤掉零值(无反应时) - if key == 'reaction_time': - filtered_values = [v for v in values if v > 0] - avg_metrics[key] = np.mean(filtered_values) if filtered_values else 0 - else: - avg_metrics[key] = np.mean(values) + avg_metrics[key] = np.mean(values) # 记录到统计中 if key in multi_obj_stats: multi_obj_stats[key].append(avg_metrics[key]) - # 更新课程学习(带反应时间) + # 更新课程学习 success = score > 5 # 成功完成的阈值 - avg_rt = avg_metrics.get('reaction_time', 0) if agent.curriculum_manager: - stage_changed = agent.curriculum_manager.update_stage(success, score, avg_rt) + stage_changed = agent.curriculum_manager.update_stage(success, score) if stage_changed: print(f"课程学习阶段已更新: {agent.curriculum_manager.current_stage}") - - # 阶段变化时保存模型 - model_path = f'{models_dir}/{MODEL_NAME}_stage_{agent.curriculum_manager.current_stage}_ep_{episode}.model' - save_model_with_retry(agent.model, model_path) # 更新多目标优化器权重 if agent.multi_objective_optimizer and episode % 20 == 0: agent.multi_objective_optimizer.adjust_weights(avg_metrics) + if episode % 100 == 0: + print(agent.multi_objective_optimizer.get_performance_report()) # 更新成功计数 if success: success_count += 1 - # ============================================ - # 修复:简化模型保存条件 - # ============================================ - - # 1. 定期保存模型(每10轮) - if episode % 10 == 0: - model_path = f'{models_dir}/{MODEL_NAME}_ep{episode}_score{score:.1f}.model' - save_model_with_retry(agent.model, model_path) - - # 2. 保存最佳模型(如果比之前好) + # 动态保存最佳模型 if score > best_score: best_score = score - model_path = f'{models_dir}/{MODEL_NAME}_best_ep{episode}_score{score:.1f}.model' - save_model_with_retry(agent.model, model_path) - print(f"🏆 新的最佳模型: Episode {episode}, 得分: {score:.2f}") - - # 3. 保存里程碑模型(每50轮) - if episode % 50 == 0: - model_path = f'{models_dir}/{MODEL_NAME}_milestone_ep{episode}.model' - save_model_with_retry(agent.model, model_path) - print(f"🎯 里程碑模型: Episode {episode}") - + model_suffix = f"advanced_best_{score:.2f}" + agent.model.save(f'models/{MODEL_NAME}_{model_suffix}.model') + print(f"新的最佳模型已保存: 得分={score:.2f}") + # 记录得分统计 scores.append(score) avg_scores.append(np.mean(scores[-10:]) if len(scores) >= 10 else np.mean(scores)) @@ -393,24 +334,23 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 添加多目标指标 if agent.multi_objective_optimizer: - for obj in ['reaction_time', 'safety', 'efficiency', 'comfort', 'rule_following']: + for obj in ['safety', 'efficiency', 'comfort', 'rule_following']: if multi_obj_stats[obj]: recent_avg = np.mean(multi_obj_stats[obj][-AGGREGATE_STATS_EVERY:]) if len(multi_obj_stats[obj]) >= AGGREGATE_STATS_EVERY else np.mean(multi_obj_stats[obj]) stats_dict[f'{obj}_score'] = recent_avg - # 添加反应时间统计 - if reaction_time_stats: - avg_rt = np.mean(reaction_time_stats[-AGGREGATE_STATS_EVERY:]) if len(reaction_time_stats) >= AGGREGATE_STATS_EVERY else np.mean(reaction_time_stats) - stats_dict['reaction_time'] = avg_rt - agent.tensorboard.update_stats(**stats_dict) + # 保存模型,仅当最小奖励达到设定值时 + if min_reward >= MIN_REWARD and (episode not in epds): + model_suffix = f"advanced_{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min" + agent.model.save(f'models/{MODEL_NAME}_{model_suffix}__{int(time.time())}.model') + epds.append(episode) # 打印训练信息 if episode % 10 == 0: # 每10轮打印一次 - avg_rt = np.mean(reaction_time_stats[-10:]) if len(reaction_time_stats) >= 10 else 0 - info_str = f'轮次: {episode:3d}, 得分: {score:6.2f}, 成功: {success_count:3d}, 反应时间: {avg_rt:.2f}s' + info_str = f'轮次: {episode:3d}, 得分: {score:6.2f}, 成功: {success_count:3d}' if agent.curriculum_manager: info_str += f', 阶段: {agent.curriculum_manager.current_stage}' print(info_str) @@ -424,57 +364,116 @@ def extended_reward_calculation(env, action, reward, done, step_info): agent.terminate = True trainer_thread.join() - # ============================================ - # 修复:始终保存最终模型 - # ============================================ - # 保存最终模型 - final_model_path = f'{models_dir}/{MODEL_NAME}_final_ep{EPISODES}_avg{np.mean(scores):.1f}.model' - save_model_with_retry(agent.model, final_model_path) - print(f"✅ 最终模型已保存: {final_model_path}") - - # 同时保存目标网络 - final_target_path = f'{models_dir}/{MODEL_NAME}_target_final.model' - save_model_with_retry(agent.target_model, final_target_path) - print(f"✅ 目标网络已保存: {final_target_path}") - - # 保存训练统计数据 - training_stats = { - 'scores': scores, - 'avg_scores': avg_scores, - 'multi_obj_stats': multi_obj_stats, - 'reaction_time_stats': reaction_time_stats, - 'curriculum_stages': curriculum_stages, - 'final_scores': { - 'max': max(scores) if scores else 0, - 'avg': np.mean(scores) if scores else 0, - 'min': min(scores) if scores else 0, + if len(scores) > 0: + final_max_reward = max(scores[-AGGREGATE_STATS_EVERY:] if len(scores) >= AGGREGATE_STATS_EVERY else scores) + final_avg_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:] if len(scores) >= AGGREGATE_STATS_EVERY else scores) + final_min_reward = min(scores[-AGGREGATE_STATS_EVERY:] if len(scores) >= AGGREGATE_STATS_EVERY else scores) + + model_suffix = "advanced_final" + agent.model.save( + f'models/{MODEL_NAME}_{model_suffix}__{final_max_reward:_>7.2f}max_{final_avg_reward:_>7.2f}avg_{final_min_reward:_>7.2f}min__{int(time.time())}.model') + + # 保存训练统计数据 + training_stats = { + 'scores': scores, + 'avg_scores': avg_scores, + 'multi_obj_stats': multi_obj_stats, + 'curriculum_stages': curriculum_stages, + 'final_scores': { + 'max': final_max_reward, + 'avg': final_avg_reward, + 'min': final_min_reward + } } - } - - stats_file = f'training_stats_{int(time.time())}.pkl' - with open(stats_file, 'wb') as f: - pickle.dump(training_stats, f) - print(f"📊 训练统计数据已保存到: {stats_file}") + + stats_file = f'training_stats_{int(time.time())}.pkl' + with open(stats_file, 'wb') as f: + pickle.dump(training_stats, f) + print(f"训练统计数据已保存到: {stats_file}") + + # 绘制训练曲线 + fig, axes = plt.subplots(3, 2, figsize=(15, 15)) + + # 1. 得分曲线 + axes[0, 0].plot(scores, label='每轮得分', alpha=0.6, linewidth=1) + axes[0, 0].plot(avg_scores, label='平均得分(最近10轮)', linewidth=2, color='red') + axes[0, 0].set_ylabel('得分') + axes[0, 0].set_xlabel('训练轮次') + axes[0, 0].set_title('训练进度 - 得分曲线') + axes[0, 0].legend() + axes[0, 0].grid(True, alpha=0.3) + + # 2. 探索率衰减曲线 + eps_values = [max(MIN_EPSILON, 1.0 * (EPSILON_DECAY ** i)) for i in range(len(scores))] + axes[0, 1].plot(eps_values, color='red', linewidth=2) + axes[0, 1].set_ylabel('探索率 (ε)') + axes[0, 1].set_xlabel('训练轮次') + axes[0, 1].set_title('探索率衰减曲线') + axes[0, 1].grid(True, alpha=0.3) + + # 3. PER缓冲区大小 + if per_stats['buffer_size']: + axes[1, 0].plot(per_stats['buffer_size'], color='green', linewidth=2) + axes[1, 0].axhline(y=REPLAY_MEMORY_SIZE, color='r', linestyle='--', alpha=0.5, label='最大容量') + axes[1, 0].set_ylabel('缓冲区大小') + axes[1, 0].set_xlabel('训练轮次') + axes[1, 0].set_title('PER缓冲区使用情况') + axes[1, 0].legend() + axes[1, 0].grid(True, alpha=0.3) + + # 4. 课程学习阶段变化 + if curriculum_stages: + axes[1, 1].plot(curriculum_stages, color='purple', linewidth=2, drawstyle='steps-post') + axes[1, 1].set_ylabel('课程学习阶段') + axes[1, 1].set_xlabel('训练轮次') + axes[1, 1].set_title('课程学习阶段变化') + axes[1, 1].grid(True, alpha=0.3) + + # 5. 多目标指标 + if multi_obj_stats['safety']: + colors = ['blue', 'green', 'orange', 'purple'] + for i, (key, values) in enumerate(multi_obj_stats.items()): + if values: + # 计算滑动平均 + window = 10 + if len(values) >= window: + smoothed = np.convolve(values, np.ones(window)/window, mode='valid') + axes[2, 0].plot(range(len(smoothed)), smoothed, label=key, color=colors[i], alpha=0.7) + else: + axes[2, 0].plot(values, label=key, color=colors[i], alpha=0.7) + + axes[2, 0].set_ylabel('分数') + axes[2, 0].set_xlabel('训练轮次') + axes[2, 0].set_title('多目标优化指标') + axes[2, 0].legend() + axes[2, 0].grid(True, alpha=0.3) + + # 6. 成功率统计 + success_rates = [] + for i in range(len(scores)): + window = scores[max(0, i-9):i+1] + success_rate = sum(1 for s in window if s > 5) / len(window) * 100 + success_rates.append(success_rate) + + axes[2, 1].plot(success_rates, color='darkred', linewidth=2) + axes[2, 1].axhline(y=80, color='g', linestyle='--', alpha=0.5, label='目标成功率80%') + axes[2, 1].set_ylabel('成功率 (%)') + axes[2, 1].set_xlabel('训练轮次') + axes[2, 1].set_title('最近10轮成功率') + axes[2, 1].legend() + axes[2, 1].grid(True, alpha=0.3) + + plt.suptitle('高级训练策略 - 综合训练报告', fontsize=16, fontweight='bold') + plt.tight_layout() + plt.show() - # 打印最终统计 print("\n" + "="*60) print("训练完成!") print("="*60) print(f"最终统计:") print(f" 总轮次: {EPISODES}") - print(f" 最佳得分: {max(scores) if scores else 0:.2f}") - print(f" 平均得分: {np.mean(scores) if scores else 0:.2f}") + print(f" 最佳得分: {best_score:.2f}") + print(f" 平均得分: {np.mean(scores):.2f}") print(f" 成功率: {(success_count/EPISODES)*100:.1f}%") - print(f" 平均反应时间: {np.mean(reaction_time_stats) if reaction_time_stats else 0:.2f}秒") - print(f" 最终探索率: {Hyperparameters.EPSILON:.4f}") - - # 显示保存的模型文件 - print(f"\n已保存的模型文件:") - model_files = glob.glob(f'{models_dir}/*.model') - if model_files: - for model_file in sorted(model_files, key=os.path.getmtime): - file_size = os.path.getsize(model_file) / (1024 * 1024) # MB - print(f" 📁 {os.path.basename(model_file)} ({file_size:.1f} MB)") - else: - print(" ⚠️ 没有找到模型文件,请检查保存路径和权限") \ No newline at end of file + print(f" 最终探索率: {Hyperparameters.EPSILON:.4f}") \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/requirements.txt b/src/Unmanned_vehicle_AD_DQN/requirements.txt index 44823773cd..33f0018cf2 100644 --- a/src/Unmanned_vehicle_AD_DQN/requirements.txt +++ b/src/Unmanned_vehicle_AD_DQN/requirements.txt @@ -3,4 +3,4 @@ numpy==1.19.5 tensorflow==2.6.0 tqdm==4.67.1 matplotlib==3.5.3 -opencv-python==4.12.0.88 +opencv-python==4.12.0.88 \ No newline at end of file