diff --git a/src/Unmanned_vehicle_AD_DQN/Environment.py b/src/Unmanned_vehicle_AD_DQN/Environment.py index a9d245b04c..0a8394be95 100644 --- a/src/Unmanned_vehicle_AD_DQN/Environment.py +++ b/src/Unmanned_vehicle_AD_DQN/Environment.py @@ -7,6 +7,7 @@ import numpy as np import cv2 import math +from collections import deque import carla from carla import ColorConverter @@ -17,8 +18,8 @@ except ImportError: # 如果在导入Hyperparameters时出错,使用默认值 SHOW_PREVIEW = False - IM_WIDTH = 640 - IM_HEIGHT = 480 + IM_WIDTH = 160 # 修改默认值 + IM_HEIGHT = 120 # 修改默认值 class CarEnv: @@ -40,6 +41,13 @@ def __init__(self): self.episode_start_time = None # 每轮开始时间 self.last_ped_distance = float('inf') # 上次最近行人距离 self.current_episode = 1 # 当前episode编号 + self.obstacle_detected_time = None # 障碍物检测到的时间 + self.reaction_start_time = None # 反应开始时间 + self.proactive_action_count = 0 # 主动避障动作计数 + + # 新增:多帧缓存用于运动检测 + self.frame_buffer = deque(maxlen=3) + self.motion_detected = False # 加载世界和蓝图 try: @@ -255,6 +263,11 @@ def reset(self, episode=1): self.last_action = 1 self.episode_start_time = time.time() self.last_ped_distance = float('inf') + self.obstacle_detected_time = None + self.reaction_start_time = None + self.proactive_action_count = 0 + self.frame_buffer.clear() + self.motion_detected = False # 设置车辆生成点 spawn_point = carla.Transform() @@ -366,6 +379,31 @@ def process_img(self, image): processed_image = np.array(image.raw_data) processed_image = processed_image.reshape((self.im_height, self.im_width, 4)) processed_image = processed_image[:, :, :3] # 移除alpha通道 + + # 添加简单的图像增强 + # 增加对比度以突出障碍物 + processed_image = cv2.convertScaleAbs(processed_image, alpha=1.2, beta=10) + + # 存储到帧缓冲区用于运动检测 + if len(self.frame_buffer) == 0: + self.frame_buffer.append(processed_image.copy()) + else: + # 简单的运动检测 + if len(self.frame_buffer) >= 2: + prev_frame = self.frame_buffer[-1] + gray_prev = cv2.cvtColor(prev_frame, cv2.COLOR_RGB2GRAY) + gray_current = cv2.cvtColor(processed_image, cv2.COLOR_RGB2GRAY) + + # 计算帧间差异 + diff = cv2.absdiff(gray_prev, gray_current) + _, diff_thresh = cv2.threshold(diff, 25, 255, cv2.THRESH_BINARY) + + # 如果差异足够大,检测到运动 + motion_pixels = np.sum(diff_thresh > 0) + motion_ratio = motion_pixels / (self.im_width * self.im_height) + self.motion_detected = motion_ratio > 0.01 # 1%的像素变化 + + self.frame_buffer.append(processed_image.copy()) # 显示预览(如果启用) if self.SHOW_CAM: @@ -377,7 +415,7 @@ def process_img(self, image): print(f"处理图像时出错: {e}") def reward(self, speed_kmh, current_steer): - """增强的奖励函数 - 特别强调行人避障""" + """增强的奖励函数 - 特别强调行人避障和快速反应""" reward = 0 done = False @@ -385,19 +423,19 @@ def reward(self, speed_kmh, current_steer): vehicle_location = self.vehicle.get_location() vehicle_rotation = self.vehicle.get_transform().rotation.yaw - # 1. 道路保持奖励(重要的基础) + # 1. 道路保持奖励 heading_error = abs(vehicle_rotation) if heading_error < 5: # 完美保持方向 - reward += 0.8 # 略微降低权重,给行人避障更多空间 + reward += 0.5 elif heading_error < 15: # 良好保持 - reward += 0.4 + reward += 0.2 elif heading_error < 30: # 可接受 - reward += 0.1 + reward += 0.05 else: # 方向偏差过大 - reward -= 0.3 * (heading_error / 30.0) # 降低惩罚强度 + reward -= 0.2 * (heading_error / 30.0) - # 2. 行人避障(最高优先级) - 增强奖励机制 + # 2. 行人避障(最高优先级) - 增强反应机制 min_ped_distance = float('inf') closest_pedestrian = None @@ -422,14 +460,24 @@ def reward(self, speed_kmh, current_steer): if active_pedestrians == 0: min_ped_distance = float('inf') - # 行人距离分级奖励 - 增强避障激励 + # 行人检测和反应时间记录 + current_time = time.time() + + # 检测到行人进入危险区域 if min_ped_distance < 100: # 只考虑100米内的行人 + if self.obstacle_detected_time is None: + self.obstacle_detected_time = current_time + self.reaction_start_time = current_time + + # 行人距离分级奖励/惩罚 - 强调提前反应 if min_ped_distance < 3.0: # 紧急避让距离 - reward -= 8.0 # 增加惩罚 + reward -= 15.0 # 大幅增加惩罚 done = True - print(f"Episode {self.current_episode}: 与行人距离过近 ({min_ped_distance:.1f}m)!") + print(f"Episode {self.current_episode}: 与行人碰撞! 距离: {min_ped_distance:.1f}m") + elif min_ped_distance < 5.0: # 危险距离 - reward -= 3.0 # 增加惩罚 + reward -= 5.0 # 增加惩罚 + # 计算避让方向 if closest_pedestrian: ped_y = closest_pedestrian.get_location().y @@ -439,58 +487,93 @@ def reward(self, speed_kmh, current_steer): self.suggested_action = 4 # 右转 else: # 行人在右侧 self.suggested_action = 3 # 左转 + + # 反应时间惩罚 + if self.reaction_start_time: + reaction_time = current_time - self.reaction_start_time + if reaction_time > 0.5: # 反应超过0.5秒 + reward -= 2.0 * (reaction_time - 0.5) + elif min_ped_distance < 8.0: # 预警距离 - reward -= 0.8 + reward -= 1.5 + + # 计算避让方向(提前建议) + if closest_pedestrian: + ped_y = closest_pedestrian.get_location().y + veh_y = vehicle_location.y + if ped_y < veh_y: # 行人在左侧 + self.suggested_action = 4 # 右转 + else: # 行人在右侧 + self.suggested_action = 3 # 左转 + + # 提前反应的奖励 + if self.last_ped_distance < 12.0 and min_ped_distance > self.last_ped_distance: + # 如果上次距离更近,这次距离更远了,说明正在成功避让 + reward += 1.0 + self.proactive_action_count += 1 + elif min_ped_distance < 12.0: # 安全距离 - reward += 0.5 # 增加安全距离奖励 - else: # 非常安全 - reward += 0.2 - - # 3. 成功避障奖励 - 当成功避开行人后给予额外奖励 - if self.last_ped_distance < 8.0 and min_ped_distance > self.last_ped_distance: - # 如果上次距离危险,这次距离更远了,说明成功避让 - reward += 0.3 + # 检测到行人但距离安全,给予轻微奖励 + reward += 0.3 + + # 如果检测到运动,给予额外奖励 + if self.motion_detected: + reward += 0.2 + + # 提前规划奖励 + if self.last_ped_distance < 8.0: + # 从危险距离进入安全距离,成功避障 + reward += 2.0 + + else: + # 没有检测到行人,重置反应计时 + self.obstacle_detected_time = None + self.reaction_start_time = None self.last_ped_distance = min_ped_distance # 保存当前距离 - # 4. 速度奖励(平衡避障和前进) + # 3. 速度奖励(平衡避障和前进) if 20 <= speed_kmh <= 35: # 理想速度区间 - reward += 0.4 - elif 10 <= speed_kmh < 20: # 较慢但安全(避障时可能需要减速) - reward += 0.2 + reward += 0.5 + elif 10 <= speed_kmh < 20: # 较慢但安全 + reward += 0.3 elif 35 < speed_kmh <= 40: # 稍快 reward += 0.1 elif speed_kmh > 40: # 过快,在行人环境中危险 - reward -= 0.5 # 增加惩罚 + reward -= 1.0 # 增加惩罚 else: # 停车或极慢 - reward -= 0.05 # 轻微惩罚 + reward -= 0.1 # 轻微惩罚 - # 5. 转向平滑性奖励(避障时可能需要适当转向) - steer_penalty = abs(current_steer) * 0.3 # 降低惩罚,给避障转向更多空间 + # 4. 转向平滑性奖励 + steer_penalty = abs(current_steer) * 0.2 # 降低惩罚,给避障转向更多空间 reward -= steer_penalty - # 6. 碰撞检测 + # 5. 碰撞检测 if len(self.collision_history) != 0: - reward = -10 # 减少碰撞惩罚 + reward = -20 # 增加碰撞惩罚 done = True print(f"Episode {self.current_episode}: 发生碰撞!") - # 7. 进度奖励(次要于避障) + # 6. 进度奖励 progress = (vehicle_location.x + 81) / 236.0 # 从-81到155 - reward += progress * 0.2 # 降低进度权重 + reward += progress * 0.3 - # 8. 边界检查 + # 7. 边界检查 if vehicle_location.x > 155: # 成功到达终点 - reward += 15 # 减少到达终点的奖励 + reward += 20 # 增加到达终点的奖励 done = True print(f"Episode {self.current_episode}: 成功到达终点!") elif vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30: # 偏离道路 - reward -= 3 # 降低偏离惩罚 + reward -= 5 # 增加偏离惩罚 done = True print(f"Episode {self.current_episode}: 偏离道路!") + # 8. 主动避障奖励 + if self.proactive_action_count > 0: + reward += self.proactive_action_count * 0.5 + # 限制奖励范围 - reward = max(min(reward, 20), -15) + reward = max(min(reward, 30), -25) return reward, done @@ -510,32 +593,46 @@ def step(self, action): brake = 0.0 steer = 0.0 - # 速度控制 + # 速度控制 - 优化动作响应 if action == 0: # 减速 throttle = 0.0 - brake = 0.5 + brake = 0.7 # 增加制动强度 elif action == 1: # 保持/轻微加速 - throttle = 0.3 + throttle = 0.4 # 增加油门 brake = 0.0 elif action == 2: # 加速 - throttle = 0.7 + throttle = 0.8 brake = 0.0 elif action == 3: # 左转 - throttle = 0.4 + throttle = 0.5 # 增加转向时的油门 brake = 0.0 - steer = -0.2 * speed_factor # 速度相关的转向幅度 + steer = -0.25 * speed_factor # 增加转向幅度 elif action == 4: # 右转 - throttle = 0.4 + throttle = 0.5 brake = 0.0 - steer = 0.2 * speed_factor # 速度相关的转向幅度 + steer = 0.25 * speed_factor # 增加转向幅度 + + # 如果有建议的避让动作,调整当前动作 + if self.suggested_action is not None: + # 如果当前动作与建议动作不同,调整转向幅度 + if self.suggested_action in [3, 4] and action in [3, 4]: + # 如果都是转向动作,但方向不同,增加建议方向的转向 + if self.suggested_action != action: + if self.suggested_action == 3: # 建议左转 + steer = -0.3 * speed_factor + else: # 建议右转 + steer = 0.3 * speed_factor + elif self.suggested_action == 0: # 建议减速 + brake = max(brake, 0.8) # 强制制动 + throttle = 0.0 # 限制连续同向转向 - 防止过度转向 if (action == 3 and self.last_action == 3) or (action == 4 and self.last_action == 4): self.same_steer_counter += 1 - if self.same_steer_counter > 3: # 连续3次同向转向后强制回正 - steer *= 0.5 # 减小转向幅度 - throttle *= 0.8 # 减速 - brake = 0.1 # 轻微制动 + if self.same_steer_counter > 2: # 连续2次同向转向后强制回正 + steer *= 0.3 # 大幅减小转向幅度 + throttle *= 0.6 # 减速 + brake = 0.2 # 轻微制动 else: self.same_steer_counter = 0 @@ -549,13 +646,13 @@ def step(self, action): steer=steer )) - # 等待物理更新 - time.sleep(0.05) + # 减少等待时间,加快反应 + time.sleep(0.03) # 从0.05减少到0.03 # 计算奖励和完成状态 reward, done = self.reward(speed_kmh, steer) # 限制极端奖励值 - reward = np.clip(reward, -15, 20) + reward = np.clip(reward, -25, 30) return self.front_camera, reward, done, None \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py index bf5be3a954..e701e4507e 100644 --- a/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py +++ b/src/Unmanned_vehicle_AD_DQN/Hyperparameters.py @@ -13,14 +13,14 @@ REWARD_OFFSET = -100 # 停止模拟的奖励阈值 -MIN_REPLAY_MEMORY_SIZE = 3_000 -# 开始训练前经验回放缓冲区的最小大小 - 增加以获得更稳定训练 +MIN_REPLAY_MEMORY_SIZE = 1_000 # 减少预热样本数,让训练更快开始 +# 开始训练前经验回放缓冲区的最小大小 -REPLAY_MEMORY_SIZE = 10_000 +REPLAY_MEMORY_SIZE = 15_000 # 增加经验回放容量 # 经验回放缓冲区的最大容量 -MINIBATCH_SIZE = 64 -# 每次训练从经验回放中采样的经验数量 - 增加批次大小 +MINIBATCH_SIZE = 128 # 增加批次大小 +# 每次训练从经验回放中采样的经验数量 PREDICTION_BATCH_SIZE = 1 # 预测阶段使用的批次大小 @@ -28,40 +28,40 @@ TRAINING_BATCH_SIZE = MINIBATCH_SIZE // 4 # 训练阶段使用的批次大小 -EPISODES = 800 # 从1000减少到800,因为减少了无行人阶段 +EPISODES = 500 # 增加训练轮次 # 智能体训练的总轮次数 SECONDS_PER_EPISODE = 60 # 每轮训练的秒数 -MIN_EPSILON = 0.01 +MIN_EPSILON = 0.05 # 增加最小探索率,保持一定的探索 # 最小探索率 EPSILON = 1.0 # 初始探索率 -EPSILON_DECAY = 0.998 -# 探索率的衰减率 - 减缓衰减速度 +EPSILON_DECAY = 0.995 # 减缓衰减速度 +# 探索率的衰减率 -MODEL_NAME = "YY_Optimized" +MODEL_NAME = "YY_Optimized_v2" # 训练模型的名称标识 -MIN_REWARD = 5 +MIN_REWARD = -100 # 降低阈值,更容易保存模型 # 被认为是"良好"或"积极"经验的最小奖励值 -UPDATE_TARGET_EVERY = 20 -# 目标网络更新的频率 - 增加以获得更稳定训练 +UPDATE_TARGET_EVERY = 15 # 更频繁地更新目标网络 +# 目标网络更新的频率 -AGGREGATE_STATS_EVERY = 10 +AGGREGATE_STATS_EVERY = 5 # 更频繁地记录统计信息 # 计算和聚合统计信息(如平均得分、奖励)的频率 SHOW_PREVIEW = False # 是否显示预览窗口 - 测试时设为False以显示CARLA主窗口 -IM_WIDTH = 640 +IM_WIDTH = 160 # 减少图像尺寸,加快处理速度 # 预览或模拟中捕获图像的宽度 -IM_HEIGHT = 480 +IM_HEIGHT = 120 # 减少图像尺寸,加快处理速度 # 预览或模拟中捕获图像的高度 SLOW_COUNTER = 330 @@ -73,17 +73,17 @@ SUCCESSFUL_THRESHOLD = 3 # 成功阈值 -LEARNING_RATE = 0.00005 -# 优化器的学习率 - 降低以获得更稳定训练 +LEARNING_RATE = 0.0001 # 增加学习率,加速收敛 +# 优化器的学习率 # PER (优先经验回放) 参数 -PER_ALPHA = 0.6 +PER_ALPHA = 0.7 # 增加优先级影响 # 优先级程度 (0 = 均匀采样, 1 = 完全优先级) -PER_BETA_START = 0.4 +PER_BETA_START = 0.5 # 重要性采样权重起始值 -PER_BETA_FRAMES = 100000 +PER_BETA_FRAMES = 50000 # 减少beta增长时间 # beta线性增长的帧数 # 训练策略参数 @@ -97,9 +97,9 @@ # 是否使用模仿学习(预训练) # 多目标优化权重(这些权重会自动调整) -SAFETY_WEIGHT = 0.40 -EFFICIENCY_WEIGHT = 0.25 -COMFORT_WEIGHT = 0.20 +SAFETY_WEIGHT = 0.50 # 提高安全权重 +EFFICIENCY_WEIGHT = 0.20 +COMFORT_WEIGHT = 0.15 RULE_FOLLOWING_WEIGHT = 0.15 # 课程学习参数 @@ -107,4 +107,8 @@ # 课程学习阶段数量 CURRICULUM_SUCCESS_THRESHOLDS = [0.3, 0.5, 0.7, 0.85, 0.9] -# 每个阶段进入下一阶段所需的成功率阈值 \ No newline at end of file +# 每个阶段进入下一阶段所需的成功率阈值 + +# 新增:反应时间参数 +REACTION_TIME_PENALTY = 0.1 # 对反应迟钝的惩罚系数 +PROACTIVE_REWARD = 0.5 # 对提前行动的奖励 \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Model.py b/src/Unmanned_vehicle_AD_DQN/Model.py index 19daa0e80c..b1c7d7f4ed 100644 --- a/src/Unmanned_vehicle_AD_DQN/Model.py +++ b/src/Unmanned_vehicle_AD_DQN/Model.py @@ -10,8 +10,9 @@ import matplotlib.pyplot as plt from collections import deque from tensorflow.keras.layers import Dense, GlobalAveragePooling2D, Input, Concatenate, Conv2D, AveragePooling2D, Activation, \ - Flatten, Dropout, BatchNormalization, MaxPooling2D, Multiply, Add, Lambda, Subtract -from tensorflow.keras.optimizers import Adam + Flatten, Dropout, BatchNormalization, MaxPooling2D, Multiply, Add, Lambda, Subtract, Reshape, LayerNormalization, \ + SpatialDropout2D, SeparableConv2D +from tensorflow.keras.optimizers import Adam, RMSprop from tensorflow.keras.models import Sequential, Model from tensorflow.keras.callbacks import TensorBoard import tensorflow as tf @@ -37,7 +38,7 @@ LEARNING_RATE = 0.00005 IM_HEIGHT = 480 IM_WIDTH = 640 - MODEL_NAME = "YY_Optimized" + MODEL_NAME = "YY_Optimized_v2" # 自定义TensorBoard类 @@ -72,6 +73,54 @@ def update_stats(self, **stats): self.writer.flush() +# 注意力机制模块 +class AttentionModule: + @staticmethod + def channel_attention(input_feature, ratio=8): + """通道注意力机制""" + channel_axis = -1 + channel = input_feature.shape[channel_axis] + + shared_layer_one = Dense(channel//ratio, activation='relu', kernel_initializer='he_normal', + use_bias=True, bias_initializer='zeros') + shared_layer_two = Dense(channel, kernel_initializer='he_normal', use_bias=True, + bias_initializer='zeros') + + avg_pool = GlobalAveragePooling2D()(input_feature) + avg_pool = Reshape((1, 1, channel))(avg_pool) + avg_pool = shared_layer_one(avg_pool) + avg_pool = shared_layer_two(avg_pool) + + max_pool = tf.reduce_max(input_feature, axis=[1, 2], keepdims=True) + max_pool = shared_layer_one(max_pool) + max_pool = shared_layer_two(max_pool) + + cbam_feature = Add()([avg_pool, max_pool]) + cbam_feature = Activation('sigmoid')(cbam_feature) + + return Multiply()([input_feature, cbam_feature]) + + @staticmethod + def spatial_attention(input_feature): + """空间注意力机制""" + avg_pool = tf.reduce_mean(input_feature, axis=3, keepdims=True) + max_pool = tf.reduce_max(input_feature, axis=3, keepdims=True) + concat = Concatenate(axis=3)([avg_pool, max_pool]) + attention = Conv2D(1, kernel_size=7, padding='same', activation='sigmoid', + kernel_initializer='he_normal', use_bias=False)(concat) + + return Multiply()([input_feature, attention]) + + @staticmethod + def cbam_block(input_feature, ratio=8): + """完整的CBAM注意力模块""" + # 通道注意力 + x = AttentionModule.channel_attention(input_feature, ratio) + # 空间注意力 + x = AttentionModule.spatial_attention(x) + return x + + # DQN智能体类 - 升级版(整合训练策略) class DQNAgent: def __init__(self, use_dueling=True, use_per=True, use_curriculum=True, use_multi_objective=True): @@ -110,6 +159,10 @@ def __init__(self, use_dueling=True, use_per=True, use_curriculum=True, use_mult self.multi_objective_optimizer = None self.imitation_manager = None + # 新增:反应时间追踪 + self.reaction_times = deque(maxlen=100) + self.last_obstacle_distance = float('inf') + def setup_training_strategies(self, env=None): """设置训练策略组件""" if self.use_curriculum and env: @@ -127,44 +180,47 @@ def setup_training_strategies(self, env=None): self.imitation_manager = ImitationLearningManager() def create_model(self): - """创建标准深度Q网络模型""" + """创建标准深度Q网络模型 - 优化版""" # 使用函数式API inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) - # 第一卷积块 - 使用较小的卷积核 - x = Conv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) + # 第一卷积块 - 使用深度可分离卷积加快计算 + x = SeparableConv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) # 第二卷积块 - x = Conv2D(64, (3, 3), padding='same')(x) + x = SeparableConv2D(64, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) # 第三卷积块 - x = Conv2D(128, (3, 3), padding='same')(x) + x = SeparableConv2D(128, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 空间注意力机制 - attention = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(x) - x = Multiply()([x, attention]) + # 注意力机制 - 增强对障碍物的关注 + x = AttentionModule.cbam_block(x) + + # 第四卷积块 - 增加一层特征提取 + x = SeparableConv2D(256, (3, 3), padding='same')(x) + x = Activation('relu')(x) + x = BatchNormalization()(x) + x = SpatialDropout2D(0.1)(x) # 展平层 x = Flatten()(x) - # 全连接层 - 减少神经元数量 + # 全连接层 - 优化网络结构 x = Dense(256, activation='relu')(x) x = Dropout(0.3)(x) x = Dense(128, activation='relu')(x) x = Dropout(0.3)(x) x = Dense(64, activation='relu')(x) x = Dropout(0.2)(x) - x = Dense(32, activation='relu')(x) - x = Dropout(0.1)(x) # 输出层 - 5个动作 outputs = Dense(5, activation='linear')(x) @@ -172,36 +228,43 @@ def create_model(self): # 创建模型 model = Model(inputs=inputs, outputs=outputs) - # 编译模型 - model.compile(loss="huber", optimizer=Adam(learning_rate=LEARNING_RATE), metrics=["mae"]) + # 编译模型 - 使用自适应学习率优化器 + model.compile(loss="huber", + optimizer=Adam(learning_rate=LEARNING_RATE, clipnorm=1.0), + metrics=["mae"]) return model def create_dueling_model(self): - """创建Dueling DQN模型架构""" + """创建Dueling DQN模型架构 - 优化版""" inputs = Input(shape=(IM_HEIGHT, IM_WIDTH, 3)) # 共享的特征提取层 # 第一卷积块 - x = Conv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) + x = SeparableConv2D(32, (3, 3), strides=(1, 1), padding='same')(inputs) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) # 第二卷积块 - x = Conv2D(64, (3, 3), padding='same')(x) + x = SeparableConv2D(64, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) # 第三卷积块 - x = Conv2D(128, (3, 3), padding='same')(x) + x = SeparableConv2D(128, (3, 3), padding='same')(x) x = Activation('relu')(x) x = BatchNormalization()(x) x = MaxPooling2D(pool_size=(2, 2))(x) - # 空间注意力机制 - attention = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(x) - x = Multiply()([x, attention]) + # 注意力机制 + x = AttentionModule.cbam_block(x) + + # 第四卷积块 + x = SeparableConv2D(256, (3, 3), padding='same')(x) + x = Activation('relu')(x) + x = BatchNormalization()(x) + x = SpatialDropout2D(0.1)(x) # 展平层 x = Flatten()(x) @@ -230,13 +293,25 @@ def create_dueling_model(self): model = Model(inputs=inputs, outputs=q_values) # 编译模型 - model.compile(loss="huber", optimizer=Adam(learning_rate=LEARNING_RATE), metrics=["mae"]) + model.compile(loss="huber", + optimizer=Adam(learning_rate=LEARNING_RATE, clipnorm=1.0), + metrics=["mae"]) return model - def update_replay_memory(self, transition): + def update_replay_memory(self, transition, reaction_time=None): """更新经验回放缓冲区""" # transition = (当前状态, 动作, 奖励, 新状态, 完成标志) + if reaction_time is not None: + # 记录反应时间 + self.reaction_times.append(reaction_time) + + # 如果反应时间过长,增加惩罚 + if reaction_time > 1.0: # 反应时间超过1秒 + transition = list(transition) + transition[2] -= 0.5 # 减少奖励 + transition = tuple(transition) + if self.use_per: # PER: 初始添加时使用最大优先级 self.replay_buffer.add(transition, error=1.0) # 初始误差设为1.0 @@ -244,7 +319,7 @@ def update_replay_memory(self, transition): self.replay_memory.append(transition) def minibatch_chooser(self): - """改进的经验采样策略""" + """改进的经验采样策略 - 优先采样危险情况""" if self.use_per: # PER采样 if len(self.replay_buffer) < MIN_REPLAY_MEMORY_SIZE: @@ -253,34 +328,41 @@ def minibatch_chooser(self): indices, samples, weights = self.replay_buffer.sample(MINIBATCH_SIZE) return samples else: - # 标准采样 + # 标准采样 - 增加危险经验的权重 if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE: return random.sample(self.replay_memory, min(len(self.replay_memory), MINIBATCH_SIZE)) # 分类经验样本 - positive_samples = [] # 高奖励经验 - negative_samples = [] # 负奖励/碰撞经验 - neutral_samples = [] # 中性奖励经验 + danger_samples = [] # 接近碰撞的经验 + collision_samples = [] # 碰撞经验 + positive_samples = [] # 成功避障经验 + neutral_samples = [] # 中性经验 for sample in self.replay_memory: - _, _, reward, _, done = sample + state, action, reward, next_state, done = sample - if done and reward < -5: # 碰撞或严重错误 - negative_samples.append(sample) - elif reward > 1: # 积极经验 + if done and reward < -5: # 碰撞经验 + collision_samples.append(sample) + elif reward < -2 and not done: # 危险但未碰撞 + danger_samples.append(sample) + elif reward > 2: # 积极经验(成功避障) positive_samples.append(sample) else: # 中性经验 neutral_samples.append(sample) - # 平衡采样 + # 平衡采样 - 增加危险经验的权重 batch = [] - # 采样负经验 (20%) - num_negative = min(len(negative_samples), MINIBATCH_SIZE // 5) - batch.extend(random.sample(negative_samples, num_negative)) + # 采样碰撞经验 (25%) + num_collision = min(len(collision_samples), MINIBATCH_SIZE // 4) + batch.extend(random.sample(collision_samples, num_collision)) - # 采样正经验 (30%) - num_positive = min(len(positive_samples), MINIBATCH_SIZE // 3) + # 采样危险经验 (30%) + num_danger = min(len(danger_samples), MINIBATCH_SIZE * 3 // 10) + batch.extend(random.sample(danger_samples, num_danger)) + + # 采样成功避障经验 (25%) + num_positive = min(len(positive_samples), MINIBATCH_SIZE // 4) batch.extend(random.sample(positive_samples, num_positive)) # 用中性经验补全批次 @@ -389,4 +471,10 @@ def train_in_loop(self): def get_qs(self, state): """获取状态的Q值""" - return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255, verbose=0)[0] \ No newline at end of file + return self.model.predict(np.array(state).reshape(-1, *state.shape) / 255, verbose=0)[0] + + def get_average_reaction_time(self): + """获取平均反应时间""" + if len(self.reaction_times) > 0: + return np.mean(self.reaction_times) + return 0 \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/Test.py b/src/Unmanned_vehicle_AD_DQN/Test.py index ccd53b468c..b90d26188f 100644 --- a/src/Unmanned_vehicle_AD_DQN/Test.py +++ b/src/Unmanned_vehicle_AD_DQN/Test.py @@ -66,7 +66,7 @@ def select_best_model(model_files, preferred_keywords=None, excluded_keywords=No return None if preferred_keywords is None: - preferred_keywords = ["best", "advanced", "dueling_per"] + preferred_keywords = ["best", "advanced", "dueling_per", "v2"] if excluded_keywords is None: excluded_keywords = ["min", "avg", "final"] # 排除统计文件 @@ -151,16 +151,22 @@ def get_safe_action_advanced(model, state, env, previous_action, uncertainty_thr qs[1] *= 1.2 # 提高保持倾向 qs[2] *= 1.3 # 提高加速倾向 - # 2. 行人避障优先级 + # 2. 行人避障优先级 - 增强版 if hasattr(env, 'suggested_action') and env.suggested_action is not None: - qs[env.suggested_action] += 3.0 # 大幅提高建议动作的Q值 - print(f"🚨 安全避让: 执行动作 {env.suggested_action}") + # 根据距离调整优先级 + min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) + if min_ped_distance < 8.0: # 危险距离 + qs[env.suggested_action] += 5.0 # 大幅提高建议动作的Q值 + print(f"🚨 紧急避让: 执行动作 {env.suggested_action}, 距离: {min_ped_distance:.1f}m") + elif min_ped_distance < 12.0: # 预警距离 + qs[env.suggested_action] += 2.0 + print(f"⚠️ 安全避让: 执行动作 {env.suggested_action}, 距离: {min_ped_distance:.1f}m") env.suggested_action = None # 3. 防止过度转向 if hasattr(env, 'same_steer_counter') and env.same_steer_counter > 2: if previous_action in [3, 4]: - qs[previous_action] -= 1.5 # 降低连续同向转向的倾向 + qs[previous_action] -= 2.0 # 降低连续同向转向的倾向 # 4. 动作平滑性 if previous_action in [3, 4]: # 转向动作 @@ -177,18 +183,24 @@ def get_safe_action_advanced(model, state, env, previous_action, uncertainty_thr elif abs(vehicle_rotation) > 30: # 方向偏差大 # 鼓励向相反方向转向以回正 if vehicle_rotation > 0: # 偏左,鼓励右转 - qs[4] += 1.0 + qs[4] += 1.5 else: # 偏右,鼓励左转 - qs[3] += 1.0 + qs[3] += 1.5 - # 6. 紧急情况处理 + # 6. 紧急情况处理 - 增强版 min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) if min_ped_distance < 5.0: # 紧急避让距离 # 大幅调整Q值以确保安全 - qs[0] += 2.0 # 紧急制动 + qs[0] += 3.0 # 紧急制动 if min_ped_distance < 3.0: # 极危险 qs[2] = -float('inf') # 禁止加速 - print("⚠️ 紧急制动!") + qs[1] = -float('inf') # 禁止保持 + print("⚠️ 紧急制动! 危险距离!") + + # 7. 考虑运动检测结果 + if hasattr(env, 'motion_detected') and env.motion_detected: + # 如果检测到运动,提高警惕 + qs[0] += 0.5 # 鼓励减速观察 # 选择动作 action = np.argmax(qs) @@ -225,6 +237,9 @@ def run_test_episode(model, env, episode_num, use_advanced_safety=True): previous_action = 1 fps_counter = deque(maxlen=30) + # 反应时间统计 + reaction_times = [] + # 运行episode max_steps = SECONDS_PER_EPISODE * 60 @@ -242,6 +257,12 @@ def run_test_episode(model, env, episode_num, use_advanced_safety=True): previous_action = action + # 记录反应开始时间(如果有障碍物) + if hasattr(env, 'obstacle_detected_time') and env.obstacle_detected_time is not None: + if hasattr(env, 'reaction_start_time') and env.reaction_start_time is not None: + reaction_time = time.time() - env.reaction_start_time + reaction_times.append(reaction_time) + # 执行动作 new_state, reward, done, _ = env.step(action) @@ -260,10 +281,14 @@ def run_test_episode(model, env, episode_num, use_advanced_safety=True): velocity = env.vehicle.get_velocity() speed_kmh = 3.6 * np.linalg.norm([velocity.x, velocity.y, velocity.z]) + # 获取最近行人距离 + min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) + status = "✅" if reward > 0 else "⚠️" if reward < -1 else "➡️" print(f"{status} 步数: {step_count:4d} | FPS: {fps:4.1f} | " - f"速度: {speed_kmh:5.1f} km/h | 奖励: {reward:6.2f} | 累计: {total_reward:7.2f}") + f"速度: {speed_kmh:5.1f} km/h | 行人距离: {min_ped_distance:5.1f}m | " + f"奖励: {reward:6.2f} | 累计: {total_reward:7.2f}") if done: break @@ -275,10 +300,14 @@ def run_test_episode(model, env, episode_num, use_advanced_safety=True): success = total_reward > 5 result = "成功" if success else "失败" + # 计算平均反应时间 + avg_reaction_time = np.mean(reaction_times) if reaction_times else 0 + print(f"\nEpisode {episode_num} 结果: {result}") print(f"总步数: {step_count}, 总奖励: {total_reward:.2f}") + print(f"平均反应时间: {avg_reaction_time:.2f}秒") - return success, total_reward, step_count + return success, total_reward, step_count, avg_reaction_time def load_model_with_fallback(model_path): @@ -374,17 +403,19 @@ def comprehensive_model_evaluation(model_path, num_episodes=5): 'successes': 0, 'total_rewards': [], 'episode_lengths': [], + 'reaction_times': [], 'start_time': time.time() } try: for episode in range(1, num_episodes + 1): - success, reward, length = run_test_episode(model, env, episode, use_advanced_safety=True) + success, reward, length, avg_rt = run_test_episode(model, env, episode, use_advanced_safety=True) if success: results['successes'] += 1 results['total_rewards'].append(reward) results['episode_lengths'].append(length) + results['reaction_times'].append(avg_rt) # 短暂暂停 time.sleep(1) @@ -407,6 +438,7 @@ def comprehensive_model_evaluation(model_path, num_episodes=5): results['avg_length'] = np.mean(results['episode_lengths']) results['max_reward'] = max(results['total_rewards']) results['min_reward'] = min(results['total_rewards']) + results['avg_reaction_time'] = np.mean(results['reaction_times']) if results['reaction_times'] else 0 # 显示评估报告 print(f"\n{'='*60}") @@ -417,6 +449,7 @@ def comprehensive_model_evaluation(model_path, num_episodes=5): print(f"成功率: {results.get('success_rate', 0):.1f}%") print(f"平均奖励: {results.get('avg_reward', 0):.2f}") print(f"平均步数: {results.get('avg_length', 0):.1f}") + print(f"平均反应时间: {results.get('avg_reaction_time', 0):.2f}秒") print(f"最佳表现: {results.get('max_reward', 0):.2f}") print(f"最差表现: {results.get('min_reward', 0):.2f}") print(f"总测试时间: {results.get('total_time', 0):.1f}秒") @@ -500,7 +533,7 @@ def interactive_model_selection(model_files): def main(): """主函数 - 自动查找和测试模型""" print(f"\n{'='*60}") - print("自动驾驶模型测试系统") + print("自动驾驶模型测试系统 - 优化版") print(f"{'='*60}") # 显示当前脚本所在目录 diff --git a/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py b/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py index cb5e611506..6f35c41a38 100644 --- a/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py +++ b/src/Unmanned_vehicle_AD_DQN/TrainingStrategies.py @@ -16,82 +16,118 @@ def __init__(self, env): self.current_stage = 0 self.stage_thresholds = [0.3, 0.5, 0.7, 0.85] # 成功率阈值 self.stage_configs = [ - # 阶段0: 入门 + # 阶段0: 入门 - 更早引入行人,但数量少速度慢 { - 'pedestrian_cross': 2, # 十字路口行人数量(减少) - 'pedestrian_normal': 1, # 普通路段行人数量(减少) - 'pedestrian_speed_min': 0.5, # 行人最低速度 - 'pedestrian_speed_max': 1.0, # 行人最高速度 - 'max_episode_steps': 1200, # 最大步数 (20秒 * 60FPS) - 'success_threshold': 0.3 # 进入下一阶段成功率 + 'pedestrian_cross': 3, # 增加一点难度 + 'pedestrian_normal': 2, + 'pedestrian_speed_min': 0.3, # 降低行人速度 + 'pedestrian_speed_max': 0.8, + 'max_episode_steps': 900, # 减少最大步数,加速训练 + 'success_threshold': 0.3, + 'difficulty_name': '简单' }, # 阶段1: 初级 { - 'pedestrian_cross': 4, # 逐步增加 - 'pedestrian_normal': 2, + 'pedestrian_cross': 5, + 'pedestrian_normal': 3, + 'pedestrian_speed_min': 0.5, + 'pedestrian_speed_max': 1.0, + 'max_episode_steps': 1200, + 'success_threshold': 0.5, + 'difficulty_name': '初级' + }, + # 阶段2: 中级 - 增加反应时间训练 + { + 'pedestrian_cross': 7, + 'pedestrian_normal': 4, 'pedestrian_speed_min': 0.7, 'pedestrian_speed_max': 1.3, - 'max_episode_steps': 1800, # 30秒 - 'success_threshold': 0.5 + 'max_episode_steps': 1500, + 'success_threshold': 0.6, # 提高阈值 + 'difficulty_name': '中级' }, - # 阶段2: 中级 + # 阶段3: 高级 { - 'pedestrian_cross': 6, - 'pedestrian_normal': 3, - 'pedestrian_speed_min': 0.8, - 'pedestrian_speed_max': 1.5, - 'max_episode_steps': 2400, # 40秒 - 'success_threshold': 0.7 + 'pedestrian_cross': 9, + 'pedestrian_normal': 5, + 'pedestrian_speed_min': 0.9, + 'pedestrian_speed_max': 1.6, + 'max_episode_steps': 1800, + 'success_threshold': 0.7, + 'difficulty_name': '高级' }, - # 阶段3: 高级 (正常难度) + # 阶段4: 专家 (正常难度) { - 'pedestrian_cross': 8, - 'pedestrian_normal': 4, + 'pedestrian_cross': 10, + 'pedestrian_normal': 6, 'pedestrian_speed_min': 1.0, 'pedestrian_speed_max': 2.0, - 'max_episode_steps': 3600, # 60秒 - 'success_threshold': 0.85 + 'max_episode_steps': 2400, + 'success_threshold': 0.8, + 'difficulty_name': '专家' }, - # 阶段4: 专家 (挑战) + # 阶段5: 大师 (挑战) { - 'pedestrian_cross': 10, # 适当减少 - 'pedestrian_normal': 5, + 'pedestrian_cross': 12, + 'pedestrian_normal': 8, 'pedestrian_speed_min': 1.2, 'pedestrian_speed_max': 2.5, - 'max_episode_steps': 3600, - 'success_threshold': 0.9 + 'max_episode_steps': 3000, + 'success_threshold': 0.85, + 'difficulty_name': '大师' } ] # 训练历史 self.success_history = deque(maxlen=20) # 记录最近20轮的成功情况 self.reward_history = deque(maxlen=50) # 记录最近50轮的奖励 + self.reaction_time_history = deque(maxlen=50) # 记录反应时间 - def update_stage(self, success, reward): + def update_stage(self, success, reward, reaction_time=None): """更新训练阶段""" # 记录历史 self.success_history.append(1 if success else 0) self.reward_history.append(reward) + if reaction_time is not None: + self.reaction_time_history.append(reaction_time) # 计算最近成功率 if len(self.success_history) >= 10: success_rate = sum(self.success_history) / len(self.success_history) avg_reward = np.mean(self.reward_history) if self.reward_history else 0 - # 减少打印频率 - if len(self.success_history) % 10 == 0: - print(f"课程学习 - 当前阶段: {self.current_stage}, 成功率: {success_rate:.2f}, 平均奖励: {avg_reward:.2f}") + # 每20轮打印一次 + if len(self.success_history) % 20 == 0: + stage_info = self.get_current_config() + print(f"课程学习 - 阶段: {self.current_stage}({stage_info['difficulty_name']}), " + f"成功率: {success_rate:.2f}, 平均奖励: {avg_reward:.2f}") + if self.reaction_time_history: + avg_rt = np.mean(self.reaction_time_history) + print(f" 平均反应时间: {avg_rt:.2f}秒") # 检查是否可以进入下一阶段 if self.current_stage < len(self.stage_configs) - 1: next_stage_threshold = self.stage_configs[self.current_stage]['success_threshold'] - if success_rate >= next_stage_threshold and avg_reward > 5: + + # 不仅要看成功率,还要看反应时间(如果可用) + can_advance = success_rate >= next_stage_threshold and avg_reward > 5 + if can_advance and self.reaction_time_history: + avg_rt = np.mean(self.reaction_time_history) + # 要求反应时间小于1秒 + if avg_rt < 1.0: + self.current_stage += 1 + print(f"🎉 课程学习: 进阶到阶段 {self.current_stage}!") + print(f" 新配置: {self.stage_configs[self.current_stage]['difficulty_name']}") + return True + elif can_advance: self.current_stage += 1 print(f"🎉 课程学习: 进阶到阶段 {self.current_stage}!") + print(f" 新配置: {self.stage_configs[self.current_stage]['difficulty_name']}") return True - # 如果表现太差,退回上一阶段 - if self.current_stage > 0 and success_rate < 0.2: + # 如果表现太差或反应时间太长,退回上一阶段 + if self.current_stage > 0 and (success_rate < 0.2 or + (self.reaction_time_history and np.mean(self.reaction_time_history) > 2.0)): self.current_stage -= 1 print(f"⚠️ 课程学习: 退回阶段 {self.current_stage}") return True @@ -113,23 +149,28 @@ class MultiObjectiveOptimizer: def __init__(self): # 定义优化目标及其权重(可动态调整) self.objectives = { + 'reaction_time': { + 'weight': 0.25, # 新增:反应时间权重 + 'description': '快速反应避障', + 'metrics': ['reaction_time', 'proactive_actions'] + }, 'safety': { - 'weight': 0.35, # 稍微降低安全权重 + 'weight': 0.30, 'description': '安全避障和避免碰撞', 'metrics': ['collision_avoidance', 'pedestrian_distance'] }, 'efficiency': { - 'weight': 0.30, # 提高效率权重 + 'weight': 0.25, 'description': '快速到达目的地', 'metrics': ['progress_speed', 'total_time'] }, 'comfort': { - 'weight': 0.20, + 'weight': 0.15, 'description': '平稳驾驶体验', 'metrics': ['smoothness', 'steering_changes'] }, 'rule_following': { - 'weight': 0.15, + 'weight': 0.05, 'description': '遵守交通规则', 'metrics': ['lane_keeping', 'speed_limit'] } @@ -137,6 +178,7 @@ def __init__(self): # 指标跟踪 self.metrics_history = { + 'reaction_time': [], 'safety': [], 'efficiency': [], 'comfort': [], @@ -156,13 +198,25 @@ def compute_composite_reward(self, metrics): # 记录指标历史 self.metrics_history[obj_name].append(normalized_value) - # 特殊惩罚项 + # 特殊奖励/惩罚项 if metrics.get('collision', False): - composite -= 8 # 减少碰撞惩罚 + composite -= 10 # 增加碰撞惩罚 if metrics.get('off_road', False): - composite -= 3 # 减少偏离道路惩罚 + composite -= 5 # 增加偏离道路惩罚 if metrics.get('dangerous_action', False): - composite -= 2 # 减少危险动作惩罚 + composite -= 3 # 增加危险动作惩罚 + + # 新增:反应时间相关奖励/惩罚 + if 'reaction_time' in metrics: + rt = metrics['reaction_time'] + if rt < 0.5: # 快速反应 + composite += 2 + elif rt > 1.5: # 反应太慢 + composite -= 3 + + # 新增:主动避障奖励 + if metrics.get('proactive_action', False): + composite += 1.5 return composite @@ -170,10 +224,11 @@ def _normalize_metric(self, value, metric_name): """归一化指标值到[0, 1]范围""" # 不同指标的归一化方式不同 normalization_rules = { - 'safety': lambda x: min(max(x / 10, 0), 1), # 假设安全分满分10 - 'efficiency': lambda x: min(max(x / 100, 0), 1), # 效率分满分100 - 'comfort': lambda x: min(max((x + 5) / 10, 0), 1), # 舒适度[-5, 5] -> [0, 1] - 'rule_following': lambda x: min(max(x, 0), 1) # 规则遵循度[0, 1] + 'reaction_time': lambda x: max(0, 1 - x/3), # 反应时间越短越好 + 'safety': lambda x: min(max(x / 10, 0), 1), + 'efficiency': lambda x: min(max(x / 100, 0), 1), + 'comfort': lambda x: min(max((x + 5) / 10, 0), 1), + 'rule_following': lambda x: min(max(x, 0), 1) } if metric_name in normalization_rules: @@ -196,7 +251,7 @@ def adjust_weights(self, performance_feedback): # 如果最差目标表现低于阈值,增加其权重 if recent_performance[worst_obj] < 0.3: - adjustment = 0.02 # 减少调整幅度 + adjustment = 0.03 # 调整幅度 self.objectives[worst_obj]['weight'] += adjustment self.objectives[best_obj]['weight'] -= adjustment @@ -223,7 +278,7 @@ def get_performance_report(self): return report -# 模仿学习管理器 +# 模仿学习管理器(保持不变,略作修改) class ImitationLearningManager: def __init__(self, expert_data_path=None): self.expert_data_path = expert_data_path @@ -244,232 +299,11 @@ def load_expert_data(self, path): except Exception as e: print(f"加载专家数据失败: {e}") return False - - def collect_expert_demonstration(self, env, num_episodes=10): - """收集专家示范数据(可以手动控制或使用规则控制器)""" - print(f"开始收集专家示范数据 ({num_episodes}个episodes)...") - - demonstrations = [] - - for episode in range(num_episodes): - print(f"收集专家示范 Episode {episode + 1}/{num_episodes}") - - state = env.reset(episode) - done = False - episode_data = [] - - step_count = 0 - max_steps = 60 * 60 # 最大60秒 - - while not done and step_count < max_steps: - # 这里可以使用规则控制器或手动控制 - # 示例:简单的规则控制器 - action = self._rule_based_controller(env) - - new_state, reward, done, _ = env.step(action) - - # 保存示范数据 - episode_data.append({ - 'state': state.copy(), - 'action': action, - 'reward': reward, - 'next_state': new_state.copy(), - 'done': done - }) - - state = new_state - step_count += 1 - - demonstrations.extend(episode_data) - env.cleanup_actors() - - # 保存专家数据 - self.expert_data = demonstrations - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - save_path = f"expert_data_{timestamp}.pkl" - - with open(save_path, 'wb') as f: - pickle.dump(demonstrations, f) - - print(f"专家示范数据已保存到: {save_path}, 共 {len(demonstrations)} 条记录") - return True - - def _rule_based_controller(self, env): - """基于规则的控制器(作为专家示范)""" - # 获取车辆状态 - vehicle_location = env.vehicle.get_location() - velocity = env.vehicle.get_velocity() - speed_kmh = 3.6 * math.sqrt(velocity.x**2 + velocity.y**2) - - # 简单规则:保持速度在20-35 km/h,避免障碍物 - if speed_kmh < 20: - return 2 # 加速 - elif speed_kmh > 35: - return 0 # 减速 - else: - # 检查前方障碍物 - has_obstacle_ahead = self._check_obstacle_ahead(env) - if has_obstacle_ahead: - return 0 # 减速 - else: - return 1 # 保持 - - return 1 # 默认保持 - - def _check_obstacle_ahead(self, env): - """检查前方是否有障碍物(简化版本)""" - # 这里可以添加更复杂的障碍物检测逻辑 - # 暂时返回False - return False - - def pretrain_with_behavioral_cloning(self, model, epochs=20): - """使用行为克隆进行预训练""" - if not self.expert_data: - print("没有专家数据可用,跳过预训练") - return model - - print(f"开始行为克隆预训练 ({epochs}个epochs)...") - - # 准备训练数据 - states = [] - actions = [] - - for demo in self.expert_data: - states.append(demo['state']) - actions.append(demo['action']) - - # 将状态归一化 - states = np.array(states) / 255.0 - - # 将动作转换为one-hot编码 - actions_onehot = tf.keras.utils.to_categorical(actions, num_classes=5) - - # 备份原始编译设置 - original_loss = model.loss - original_optimizer = model.optimizer - original_metrics = model.metrics_names - - # 重新编译模型用于分类任务 - model.compile( - optimizer=Adam(learning_rate=0.0001), - loss='categorical_crossentropy', - metrics=['accuracy'] - ) - - # 训练模型模仿专家行为 - history = model.fit( - states, actions_onehot, - batch_size=32, - epochs=epochs, - validation_split=0.2, - verbose=1 - ) - - print(f"预训练完成 - 最终准确率: {history.history['accuracy'][-1]:.3f}") - - # 恢复原始编译设置 - model.compile( - optimizer=original_optimizer, - loss=original_loss, - metrics=original_metrics - ) - - self.is_pretrained = True - return model - - def train_with_dagger(self, model, env, iterations=5, episodes_per_iter=5): - """使用DAgger算法进行训练""" - print(f"开始DAgger训练 ({iterations}次迭代,每次{episodes_per_iter}个episodes)...") - - aggregated_data = self.expert_data.copy() - - for iteration in range(iterations): - print(f"\nDAgger 迭代 {iteration + 1}/{iterations}") - - # 使用当前策略收集数据 - new_demos = [] - - for episode in range(episodes_per_iter): - print(f" 收集数据 Episode {episode + 1}/{episodes_per_iter}") - - state = env.reset(episode) - done = False - step_count = 0 - max_steps = 60 * 60 - - while not done and step_count < max_steps: - # 使用当前策略选择动作 - qs = model.predict(np.array(state).reshape(-1, *state.shape) / 255, verbose=0)[0] - action = np.argmax(qs) - - # 执行动作 - new_state, reward, done, _ = env.step(action) - - # 专家纠正(这里可以添加专家纠正逻辑) - # 如果策略动作与专家建议不同,使用专家动作 - expert_action = self._rule_based_controller(env) - - # 保存数据(使用专家纠正后的动作) - new_demos.append({ - 'state': state.copy(), - 'action': expert_action, # 使用专家动作 - 'reward': reward, - 'next_state': new_state.copy(), - 'done': done - }) - - state = new_state - step_count += 1 - - env.cleanup_actors() - - # 合并数据 - aggregated_data.extend(new_demos) - - # 在合并数据上重新训练 - states = [d['state'] for d in aggregated_data] - actions = [d['action'] for d in aggregated_data] - - states = np.array(states) / 255.0 - actions_onehot = tf.keras.utils.to_categorical(actions, num_classes=5) - - # 备份原始编译设置 - original_loss = model.loss - original_optimizer = model.optimizer - original_metrics = model.metrics_names - - # 重新编译用于分类 - model.compile( - optimizer=Adam(learning_rate=0.0001), - loss='categorical_crossentropy', - metrics=['accuracy'] - ) - - # 训练模型 - history = model.fit( - states, actions_onehot, - batch_size=32, - epochs=10, - validation_split=0.1, - verbose=0 - ) - - # 恢复原始编译设置 - model.compile( - optimizer=original_optimizer, - loss=original_loss, - metrics=original_metrics - ) - - print(f" 训练完成 - 准确率: {history.history['accuracy'][-1]:.3f}") - - print("DAgger训练完成!") - return model -# 优先经验回放缓冲区 +# 优先经验回放缓冲区(增加对危险经验的优先级) class PrioritizedReplayBuffer: - def __init__(self, max_size=10000, alpha=0.6, beta_start=0.4, beta_frames=100000): + def __init__(self, max_size=15000, alpha=0.7, beta_start=0.5, beta_frames=50000): self.max_size = max_size self.alpha = alpha # 优先级程度 (0 = 均匀采样, 1 = 完全优先级) self.beta_start = beta_start # 重要性采样权重起始值 @@ -494,6 +328,11 @@ def add(self, experience, error=None): else: priority = (abs(error) + 1e-5) ** self.alpha + # 如果是危险经验(负奖励较大),增加优先级 + reward = experience[2] + if reward < -2: # 危险经验 + priority *= 1.5 + self.buffer.append(experience) self.priorities.append(priority) diff --git a/src/Unmanned_vehicle_AD_DQN/main.py b/src/Unmanned_vehicle_AD_DQN/main.py index e627c3006d..f69fb91625 100644 --- a/src/Unmanned_vehicle_AD_DQN/main.py +++ b/src/Unmanned_vehicle_AD_DQN/main.py @@ -1,4 +1,4 @@ -# main.py +# main_fixed.py - 修复模型保存问题的版本 import glob import os import sys @@ -26,6 +26,27 @@ # 从Hyperparameters导入所有参数 from Hyperparameters import * +def ensure_models_directory(): + """确保models目录存在""" + if not os.path.exists('models'): + os.makedirs('models') + print("✅ 已创建 models 目录") + return 'models' + +def save_model_with_retry(model, filepath, max_retries=3): + """带重试机制的模型保存""" + for attempt in range(max_retries): + try: + model.save(filepath) + print(f"✅ 模型保存成功: {os.path.basename(filepath)}") + return True + except Exception as e: + print(f"⚠️ 保存失败 (尝试 {attempt+1}/{max_retries}): {e}") + time.sleep(1) + + print(f"❌ 无法保存模型: {filepath}") + return False + def extended_reward_calculation(env, action, reward, done, step_info): """ 扩展的奖励计算函数,用于多目标优化 @@ -38,7 +59,23 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 计算多目标指标 metrics = {} - # 1. 安全性指标 - 基于最近行人距离 + # 1. 反应时间指标 + reaction_time = 0 + if hasattr(env, 'obstacle_detected_time') and env.obstacle_detected_time is not None: + if hasattr(env, 'reaction_start_time') and env.reaction_start_time is not None: + reaction_time = time.time() - env.reaction_start_time + + metrics['reaction_time'] = reaction_time + + # 2. 主动避障指标 + proactive_action = False + if hasattr(env, 'suggested_action') and env.suggested_action is not None: + if action == env.suggested_action: + proactive_action = True + + metrics['proactive_action'] = proactive_action + + # 3. 安全性指标 - 基于最近行人距离 min_ped_distance = getattr(env, 'last_ped_distance', float('inf')) safety_score = 0 if min_ped_distance < 100: @@ -55,12 +92,12 @@ def extended_reward_calculation(env, action, reward, done, step_info): metrics['safety'] = safety_score - # 2. 效率指标 - 基于进度 + # 4. 效率指标 - 基于进度 progress = (vehicle_location.x + 81) / 236.0 # 从-81到155 efficiency_score = progress * 100 # 进度百分比 metrics['efficiency'] = efficiency_score - # 3. 舒适度指标 - 基于转向平滑性 + # 5. 舒适度指标 - 基于转向平滑性 comfort_score = 5 # 默认舒适 if hasattr(env, 'last_action') and env.last_action in [3, 4]: @@ -75,7 +112,7 @@ def extended_reward_calculation(env, action, reward, done, step_info): metrics['comfort'] = comfort_score - # 4. 规则遵循指标 - 基于速度 + # 6. 规则遵循指标 - 基于速度 rule_score = 0.3 # 默认较低分数 if 20 <= speed_kmh <= 35: # 理想速度范围 @@ -89,11 +126,11 @@ def extended_reward_calculation(env, action, reward, done, step_info): metrics['rule_following'] = rule_score - # 5. 特殊事件 + # 7. 特殊事件 metrics['collision'] = len(getattr(env, 'collision_history', [])) > 0 metrics['off_road'] = vehicle_location.x < -90 or abs(vehicle_location.y + 195) > 30 - # 6. 危险动作检测 + # 8. 危险动作检测 if speed_kmh > 40 and action in [3, 4]: # 高速急转 metrics['dangerous_action'] = True else: @@ -105,19 +142,14 @@ def extended_reward_calculation(env, action, reward, done, step_info): FPS = 60 # 帧率 ep_rewards = [-200] # 存储每轮奖励 + # 确保models目录存在 + models_dir = ensure_models_directory() + # GPU内存配置 gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=MEMORY_FRACTION) tf.compat.v1.keras.backend.set_session( tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options))) - # 创建模型保存目录 - if not os.path.isdir('models'): - os.makedirs('models') - - # 创建专家数据目录 - if not os.path.isdir('expert_data'): - os.makedirs('expert_data') - # 创建智能体和环境 agent = DQNAgent( use_dueling=True, @@ -131,30 +163,6 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 设置训练策略 agent.setup_training_strategies(env) - # 可选:使用模仿学习进行预训练 - use_imitation_pretraining = False # 设置为True启用模仿学习预训练 - - if use_imitation_pretraining: - print("=" * 60) - print("开始模仿学习预训练阶段") - print("=" * 60) - - # 检查是否有现有的专家数据 - expert_files = glob.glob("expert_data/*.pkl") - if expert_files: - # 使用最新的专家数据 - latest_expert = max(expert_files, key=os.path.getctime) - agent.imitation_manager.load_expert_data(latest_expert) - else: - # 收集新的专家数据 - print("未找到专家数据,开始收集...") - agent.imitation_manager.collect_expert_demonstration(env, num_episodes=5) - - # 使用行为克隆进行预训练 - agent.model = agent.imitation_manager.pretrain_with_behavioral_cloning(agent.model, epochs=15) - agent.target_model.set_weights(agent.model.get_weights()) - print("模仿学习预训练完成!") - # 启动训练线程并等待训练初始化完成 trainer_thread = Thread(target=agent.train_in_loop, daemon=True) trainer_thread.start() @@ -178,6 +186,7 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 多目标统计 multi_obj_stats = { + 'reaction_time': [], 'safety': [], 'efficiency': [], 'comfort': [], @@ -187,6 +196,9 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 课程学习阶段记录 curriculum_stages = [] + # 反应时间统计 + reaction_time_stats = [] + # 迭代训练轮次 epds = [] for episode in tqdm(range(1, EPISODES + 1), ascii=True, unit='episodes'): @@ -197,7 +209,7 @@ def extended_reward_calculation(env, action, reward, done, step_info): if agent.curriculum_manager: config = agent.curriculum_manager.get_current_config() if episode % 50 == 0: # 每50轮打印一次 - print(f"课程学习 - 阶段 {agent.curriculum_manager.current_stage}: " + print(f"课程学习 - 阶段 {agent.curriculum_manager.current_stage}({config['difficulty_name']}): " f"行人(十字路口={config['pedestrian_cross']}, 普通={config['pedestrian_normal']})") curriculum_stages.append(agent.curriculum_manager.current_stage) @@ -207,6 +219,7 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 多目标指标记录 episode_metrics = { + 'reaction_time': [], 'safety': [], 'efficiency': [], 'comfort': [], @@ -234,7 +247,18 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 从Q网络获取动作(利用) qs = agent.get_qs(current_state) action = np.argmax(qs) - if episode % 50 == 0 and step % 30 == 0: # 减少打印频率 + + # 如果有建议的避让动作,考虑采纳 + if hasattr(env, 'suggested_action') and env.suggested_action is not None: + # 检查建议动作的Q值 + suggested_q = qs[env.suggested_action] + current_best_q = qs[action] + + # 如果建议动作的Q值接近最佳动作,采纳建议 + if suggested_q > current_best_q * 0.8: + action = env.suggested_action + + if episode % 100 == 0 and step % 100 == 0: # 减少打印频率 print(f'Ep {episode} Step {step}: Q值 [{qs[0]:>5.2f}, {qs[1]:>5.2f}, {qs[2]:>5.2f}, {qs[3]:>5.2f}, {qs[4]:>5.2f}] 动作: {action}') else: # 随机选择动作(探索) @@ -245,6 +269,12 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 执行动作并获取结果 new_state, reward, done, _ = env.step(action) + # 计算反应时间 + reaction_time = 0 + if hasattr(env, 'obstacle_detected_time') and env.obstacle_detected_time is not None: + if hasattr(env, 'reaction_start_time') and env.reaction_start_time is not None: + reaction_time = time.time() - env.reaction_start_time + # 计算多目标指标 if agent.multi_objective_optimizer: step_info = {'step': step, 'action': action} @@ -261,8 +291,9 @@ def extended_reward_calculation(env, action, reward, done, step_info): score += reward # 累加奖励 - # 更新经验回放 - agent.update_replay_memory((current_state, action, reward, new_state, done)) + # 更新经验回放(带反应时间信息) + agent.update_replay_memory((current_state, action, reward, new_state, done), + reaction_time=reaction_time) current_state = new_state # 更新当前状态 step += 1 @@ -272,40 +303,68 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 本轮结束 - 销毁所有actor env.cleanup_actors() - + + # 计算平均反应时间 + if episode_metrics['reaction_time']: + avg_reaction_time = np.mean([rt for rt in episode_metrics['reaction_time'] if rt > 0]) + reaction_time_stats.append(avg_reaction_time) + # 计算本轮平均指标 avg_metrics = {} for key, values in episode_metrics.items(): if values: - avg_metrics[key] = np.mean(values) + # 过滤掉零值(无反应时) + if key == 'reaction_time': + filtered_values = [v for v in values if v > 0] + avg_metrics[key] = np.mean(filtered_values) if filtered_values else 0 + else: + avg_metrics[key] = np.mean(values) # 记录到统计中 if key in multi_obj_stats: multi_obj_stats[key].append(avg_metrics[key]) - # 更新课程学习 + # 更新课程学习(带反应时间) success = score > 5 # 成功完成的阈值 + avg_rt = avg_metrics.get('reaction_time', 0) if agent.curriculum_manager: - stage_changed = agent.curriculum_manager.update_stage(success, score) + stage_changed = agent.curriculum_manager.update_stage(success, score, avg_rt) if stage_changed: print(f"课程学习阶段已更新: {agent.curriculum_manager.current_stage}") + + # 阶段变化时保存模型 + model_path = f'{models_dir}/{MODEL_NAME}_stage_{agent.curriculum_manager.current_stage}_ep_{episode}.model' + save_model_with_retry(agent.model, model_path) # 更新多目标优化器权重 if agent.multi_objective_optimizer and episode % 20 == 0: agent.multi_objective_optimizer.adjust_weights(avg_metrics) - if episode % 100 == 0: - print(agent.multi_objective_optimizer.get_performance_report()) # 更新成功计数 if success: success_count += 1 - # 动态保存最佳模型 + # ============================================ + # 修复:简化模型保存条件 + # ============================================ + + # 1. 定期保存模型(每10轮) + if episode % 10 == 0: + model_path = f'{models_dir}/{MODEL_NAME}_ep{episode}_score{score:.1f}.model' + save_model_with_retry(agent.model, model_path) + + # 2. 保存最佳模型(如果比之前好) if score > best_score: best_score = score - model_suffix = f"advanced_best_{score:.2f}" - agent.model.save(f'models/{MODEL_NAME}_{model_suffix}.model') - print(f"新的最佳模型已保存: 得分={score:.2f}") - + model_path = f'{models_dir}/{MODEL_NAME}_best_ep{episode}_score{score:.1f}.model' + save_model_with_retry(agent.model, model_path) + print(f"🏆 新的最佳模型: Episode {episode}, 得分: {score:.2f}") + + # 3. 保存里程碑模型(每50轮) + if episode % 50 == 0: + model_path = f'{models_dir}/{MODEL_NAME}_milestone_ep{episode}.model' + save_model_with_retry(agent.model, model_path) + print(f"🎯 里程碑模型: Episode {episode}") + # 记录得分统计 scores.append(score) avg_scores.append(np.mean(scores[-10:]) if len(scores) >= 10 else np.mean(scores)) @@ -334,23 +393,24 @@ def extended_reward_calculation(env, action, reward, done, step_info): # 添加多目标指标 if agent.multi_objective_optimizer: - for obj in ['safety', 'efficiency', 'comfort', 'rule_following']: + for obj in ['reaction_time', 'safety', 'efficiency', 'comfort', 'rule_following']: if multi_obj_stats[obj]: recent_avg = np.mean(multi_obj_stats[obj][-AGGREGATE_STATS_EVERY:]) if len(multi_obj_stats[obj]) >= AGGREGATE_STATS_EVERY else np.mean(multi_obj_stats[obj]) stats_dict[f'{obj}_score'] = recent_avg + # 添加反应时间统计 + if reaction_time_stats: + avg_rt = np.mean(reaction_time_stats[-AGGREGATE_STATS_EVERY:]) if len(reaction_time_stats) >= AGGREGATE_STATS_EVERY else np.mean(reaction_time_stats) + stats_dict['reaction_time'] = avg_rt + agent.tensorboard.update_stats(**stats_dict) - # 保存模型,仅当最小奖励达到设定值时 - if min_reward >= MIN_REWARD and (episode not in epds): - model_suffix = f"advanced_{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min" - agent.model.save(f'models/{MODEL_NAME}_{model_suffix}__{int(time.time())}.model') - epds.append(episode) # 打印训练信息 if episode % 10 == 0: # 每10轮打印一次 - info_str = f'轮次: {episode:3d}, 得分: {score:6.2f}, 成功: {success_count:3d}' + avg_rt = np.mean(reaction_time_stats[-10:]) if len(reaction_time_stats) >= 10 else 0 + info_str = f'轮次: {episode:3d}, 得分: {score:6.2f}, 成功: {success_count:3d}, 反应时间: {avg_rt:.2f}s' if agent.curriculum_manager: info_str += f', 阶段: {agent.curriculum_manager.current_stage}' print(info_str) @@ -364,116 +424,57 @@ def extended_reward_calculation(env, action, reward, done, step_info): agent.terminate = True trainer_thread.join() + # ============================================ + # 修复:始终保存最终模型 + # ============================================ + # 保存最终模型 - if len(scores) > 0: - final_max_reward = max(scores[-AGGREGATE_STATS_EVERY:] if len(scores) >= AGGREGATE_STATS_EVERY else scores) - final_avg_reward = np.mean(scores[-AGGREGATE_STATS_EVERY:] if len(scores) >= AGGREGATE_STATS_EVERY else scores) - final_min_reward = min(scores[-AGGREGATE_STATS_EVERY:] if len(scores) >= AGGREGATE_STATS_EVERY else scores) - - model_suffix = "advanced_final" - agent.model.save( - f'models/{MODEL_NAME}_{model_suffix}__{final_max_reward:_>7.2f}max_{final_avg_reward:_>7.2f}avg_{final_min_reward:_>7.2f}min__{int(time.time())}.model') - - # 保存训练统计数据 - training_stats = { - 'scores': scores, - 'avg_scores': avg_scores, - 'multi_obj_stats': multi_obj_stats, - 'curriculum_stages': curriculum_stages, - 'final_scores': { - 'max': final_max_reward, - 'avg': final_avg_reward, - 'min': final_min_reward - } + final_model_path = f'{models_dir}/{MODEL_NAME}_final_ep{EPISODES}_avg{np.mean(scores):.1f}.model' + save_model_with_retry(agent.model, final_model_path) + print(f"✅ 最终模型已保存: {final_model_path}") + + # 同时保存目标网络 + final_target_path = f'{models_dir}/{MODEL_NAME}_target_final.model' + save_model_with_retry(agent.target_model, final_target_path) + print(f"✅ 目标网络已保存: {final_target_path}") + + # 保存训练统计数据 + training_stats = { + 'scores': scores, + 'avg_scores': avg_scores, + 'multi_obj_stats': multi_obj_stats, + 'reaction_time_stats': reaction_time_stats, + 'curriculum_stages': curriculum_stages, + 'final_scores': { + 'max': max(scores) if scores else 0, + 'avg': np.mean(scores) if scores else 0, + 'min': min(scores) if scores else 0, } - - stats_file = f'training_stats_{int(time.time())}.pkl' - with open(stats_file, 'wb') as f: - pickle.dump(training_stats, f) - print(f"训练统计数据已保存到: {stats_file}") - - # 绘制训练曲线 - fig, axes = plt.subplots(3, 2, figsize=(15, 15)) - - # 1. 得分曲线 - axes[0, 0].plot(scores, label='每轮得分', alpha=0.6, linewidth=1) - axes[0, 0].plot(avg_scores, label='平均得分(最近10轮)', linewidth=2, color='red') - axes[0, 0].set_ylabel('得分') - axes[0, 0].set_xlabel('训练轮次') - axes[0, 0].set_title('训练进度 - 得分曲线') - axes[0, 0].legend() - axes[0, 0].grid(True, alpha=0.3) - - # 2. 探索率衰减曲线 - eps_values = [max(MIN_EPSILON, 1.0 * (EPSILON_DECAY ** i)) for i in range(len(scores))] - axes[0, 1].plot(eps_values, color='red', linewidth=2) - axes[0, 1].set_ylabel('探索率 (ε)') - axes[0, 1].set_xlabel('训练轮次') - axes[0, 1].set_title('探索率衰减曲线') - axes[0, 1].grid(True, alpha=0.3) - - # 3. PER缓冲区大小 - if per_stats['buffer_size']: - axes[1, 0].plot(per_stats['buffer_size'], color='green', linewidth=2) - axes[1, 0].axhline(y=REPLAY_MEMORY_SIZE, color='r', linestyle='--', alpha=0.5, label='最大容量') - axes[1, 0].set_ylabel('缓冲区大小') - axes[1, 0].set_xlabel('训练轮次') - axes[1, 0].set_title('PER缓冲区使用情况') - axes[1, 0].legend() - axes[1, 0].grid(True, alpha=0.3) - - # 4. 课程学习阶段变化 - if curriculum_stages: - axes[1, 1].plot(curriculum_stages, color='purple', linewidth=2, drawstyle='steps-post') - axes[1, 1].set_ylabel('课程学习阶段') - axes[1, 1].set_xlabel('训练轮次') - axes[1, 1].set_title('课程学习阶段变化') - axes[1, 1].grid(True, alpha=0.3) - - # 5. 多目标指标 - if multi_obj_stats['safety']: - colors = ['blue', 'green', 'orange', 'purple'] - for i, (key, values) in enumerate(multi_obj_stats.items()): - if values: - # 计算滑动平均 - window = 10 - if len(values) >= window: - smoothed = np.convolve(values, np.ones(window)/window, mode='valid') - axes[2, 0].plot(range(len(smoothed)), smoothed, label=key, color=colors[i], alpha=0.7) - else: - axes[2, 0].plot(values, label=key, color=colors[i], alpha=0.7) - - axes[2, 0].set_ylabel('分数') - axes[2, 0].set_xlabel('训练轮次') - axes[2, 0].set_title('多目标优化指标') - axes[2, 0].legend() - axes[2, 0].grid(True, alpha=0.3) - - # 6. 成功率统计 - success_rates = [] - for i in range(len(scores)): - window = scores[max(0, i-9):i+1] - success_rate = sum(1 for s in window if s > 5) / len(window) * 100 - success_rates.append(success_rate) - - axes[2, 1].plot(success_rates, color='darkred', linewidth=2) - axes[2, 1].axhline(y=80, color='g', linestyle='--', alpha=0.5, label='目标成功率80%') - axes[2, 1].set_ylabel('成功率 (%)') - axes[2, 1].set_xlabel('训练轮次') - axes[2, 1].set_title('最近10轮成功率') - axes[2, 1].legend() - axes[2, 1].grid(True, alpha=0.3) - - plt.suptitle('高级训练策略 - 综合训练报告', fontsize=16, fontweight='bold') - plt.tight_layout() - plt.show() + } + stats_file = f'training_stats_{int(time.time())}.pkl' + with open(stats_file, 'wb') as f: + pickle.dump(training_stats, f) + print(f"📊 训练统计数据已保存到: {stats_file}") + + # 打印最终统计 print("\n" + "="*60) print("训练完成!") print("="*60) print(f"最终统计:") print(f" 总轮次: {EPISODES}") - print(f" 最佳得分: {best_score:.2f}") - print(f" 平均得分: {np.mean(scores):.2f}") + print(f" 最佳得分: {max(scores) if scores else 0:.2f}") + print(f" 平均得分: {np.mean(scores) if scores else 0:.2f}") print(f" 成功率: {(success_count/EPISODES)*100:.1f}%") - print(f" 最终探索率: {Hyperparameters.EPSILON:.4f}") \ No newline at end of file + print(f" 平均反应时间: {np.mean(reaction_time_stats) if reaction_time_stats else 0:.2f}秒") + print(f" 最终探索率: {Hyperparameters.EPSILON:.4f}") + + # 显示保存的模型文件 + print(f"\n已保存的模型文件:") + model_files = glob.glob(f'{models_dir}/*.model') + if model_files: + for model_file in sorted(model_files, key=os.path.getmtime): + file_size = os.path.getsize(model_file) / (1024 * 1024) # MB + print(f" 📁 {os.path.basename(model_file)} ({file_size:.1f} MB)") + else: + print(" ⚠️ 没有找到模型文件,请检查保存路径和权限") \ No newline at end of file diff --git a/src/Unmanned_vehicle_AD_DQN/requirements.txt b/src/Unmanned_vehicle_AD_DQN/requirements.txt index 33f0018cf2..44823773cd 100644 --- a/src/Unmanned_vehicle_AD_DQN/requirements.txt +++ b/src/Unmanned_vehicle_AD_DQN/requirements.txt @@ -3,4 +3,4 @@ numpy==1.19.5 tensorflow==2.6.0 tqdm==4.67.1 matplotlib==3.5.3 -opencv-python==4.12.0.88 \ No newline at end of file +opencv-python==4.12.0.88