From 15b9b813797968e662e1a8938c3cb93e738a7371 Mon Sep 17 00:00:00 2001 From: SmallBlueE Date: Sun, 15 Jun 2025 09:51:35 +0800 Subject: [PATCH 1/2] Throttle UI updates to episode end --- tempControl.html | 101 ++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/tempControl.html b/tempControl.html index 183af1b..95331d9 100644 --- a/tempControl.html +++ b/tempControl.html @@ -1977,35 +1977,9 @@

📋 系统日志

]; // 改进的奖励函数 - 接收水温和目标温度 - const reward = this.ddpgAgent.calculateReward(nextWaterTemp, episodeTarget, heatingPower, lastWaterTemp, step); - rewardTrace.push(reward); - this.log(`📈 [第${episode + 1}轮, 第${step + 1}步] 奖励: ${reward.toFixed(2)}`); - - if (this.trainCurveChart.data.datasets.length < 3) { - this.trainCurveChart.data.datasets.push({ - label: '每步奖励', - data: rewardTrace, - borderColor: '#E91E63', - backgroundColor: 'rgba(233, 30, 99, 0.1)', - fill: false, - tension: 0.1, - yAxisID: 'reward', - borderWidth: 1, - pointRadius: 0 - }); - } else { - this.trainCurveChart.data.datasets[2].data = rewardTrace; - } - if (!this.trainCurveChart.options.scales.reward) { - this.trainCurveChart.options.scales.reward = { - type: 'linear', - position: 'right', - title: { display: true, text: '奖励值' }, - beginAtZero: false, - grid: { drawOnChartArea: false } - }; - } - this.trainCurveChart.update('none'); + const reward = this.ddpgAgent.calculateReward(nextWaterTemp, episodeTarget, heatingPower, lastWaterTemp, step); + // 收集奖励，但不在每步更新图表 + rewardTrace.push(reward); // 改进的记忆函数，传入更多温度信息 this.ddpgAgent.remember(state, heatingPower, reward, nextState, false, waterTemp, episodeTarget, nextWaterTemp); @@ -2093,14 +2067,43 @@

📋 系统日志

this.lossChart.update('none'); } - if (this.tdLossChart) { - this.tdLossChart.data.labels.push(episode + 1); - this.tdLossChart.data.datasets[0].data.push(Math.abs(this.ddpgAgent.actorLoss || 0)); - this.tdLossChart.data.datasets[1].data.push(this.ddpgAgent.criticLoss || 0); - this.tdLossChart.update('none'); - } - - // 检查训练稳定性并可能重置探索 + if (this.tdLossChart) { + this.tdLossChart.data.labels.push(episode + 1); + this.tdLossChart.data.datasets[0].data.push(Math.abs(this.ddpgAgent.actorLoss || 0)); + this.tdLossChart.data.datasets[1].data.push(this.ddpgAgent.criticLoss || 0); + this.tdLossChart.update('none'); + } + + // 回合结束后更新奖励曲线 + if (this.trainCurveChart) { + if (this.trainCurveChart.data.datasets.length < 3) { + this.trainCurveChart.data.datasets.push({ + label: '每步奖励', + data: rewardTrace, + borderColor: '#E91E63', + backgroundColor: 'rgba(233, 30, 99, 0.1)', + fill: false, + tension: 0.1, + yAxisID: 'reward', + borderWidth: 1, + pointRadius: 0 + }); + } else { + this.trainCurveChart.data.datasets[2].data = rewardTrace; + } + if (!this.trainCurveChart.options.scales.reward) { + this.trainCurveChart.options.scales.reward = { + type: 'linear', + position: 'right', + title: { display: true, text: '奖励值' }, + beginAtZero: false, + grid: { drawOnChartArea: false } + }; + } + this.trainCurveChart.update('none'); + } + + // 检查训练稳定性并可能重置探索 const resetExploration = this.ddpgAgent.checkTrainingStability(totalReward); if (resetExploration) { this.log(`🔄 第${episode + 1}轮: 重置探索策略，增大噪声至 ${(this.ddpgAgent.noiseScale * 100).toFixed(1)}%`, 'warning'); @@ -2128,19 +2131,17 @@

📋 系统日志

`平均: ${avgReward.toFixed(1)}` ); - if (episode % 5 === 0 || episode === nEpisodes - 1) { - // 计算温度误差指标 - const avgError = tempErrors.reduce((a, b) => a + b, 0) / tempErrors.length; - const maxError = Math.max(...tempErrors); - - this.log( - `📈 第${episode + 1}轮: 奖励=${totalReward.toFixed(1)}, ` + - `目标温度=${episodeTarget.toFixed(1)}°C, ` + - `噪声=${(this.ddpgAgent.noiseScale * 100).toFixed(1)}%, ` + - `平均误差=${avgError.toFixed(2)}°C, ` + - `到达目标用时=${stepsToTarget !== null ? stepsToTarget : '未达'}步` - ); - } + // 计算温度误差指标 + const avgError = tempErrors.reduce((a, b) => a + b, 0) / tempErrors.length; + const maxError = Math.max(...tempErrors); + + this.log( + `📈 第${episode + 1}轮: 奖励=${totalReward.toFixed(1)}, ` + + `目标温度=${episodeTarget.toFixed(1)}°C, ` + + `噪声=${(this.ddpgAgent.noiseScale * 100).toFixed(1)}%, ` + + `平均误差=${avgError.toFixed(2)}°C, ` + + `到达目标用时=${stepsToTarget !== null ? stepsToTarget : '未达'}步` + ); // 每 5 轮或最后一轮，快速评估一次升温曲线 if (episode % 5 === 0 || episode === nEpisodes - 1) { From cab2c49e06a24c42a574e7c8de28638c8b09ef55 Mon Sep 17 00:00:00 2001 From: SmallBlueE Date: Sun, 15 Jun 2025 10:03:50 +0800 Subject: [PATCH 2/2] Remove step-level reward plotting --- tempControl.html | 49 +++--------------------------------------------- 1 file changed, 3 insertions(+), 46 deletions(-) diff --git a/tempControl.html b/tempControl.html index 95331d9..cba45f2 100644 --- a/tempControl.html +++ b/tempControl.html @@ -1432,7 +1432,6 @@

📋 系统日志

episodeRewards: [] }; - this.stepSinceLastDraw = 0; } setupCharts() { @@ -1817,14 +1816,7 @@

📋 系统日志

this.trainCurveChart.data.datasets[1].data = tempTrace; this.trainCurveChart.data.datasets[2].data = rewardTrace; - const CHART_UPDATE_INTERVAL = 1000; // 每 20 步才 redraw - - this.stepSinceLastDraw++; - if (this.stepSinceLastDraw >= CHART_UPDATE_INTERVAL || step === episodeEnd) { - this.stepSinceLastDraw = 0; - this.trainCurveChart.update('none'); - - } + this.trainCurveChart.update('none'); } @@ -1879,12 +1871,8 @@

📋 系统日志

deltaT: parseFloat(document.getElementById('deltaT').value) }; - const rewardTrace = []; - - this.stepSinceLastDraw = 0; - - // 禁止训练期间使用固定 targetTemp - config.targetTemp = null; + // 禁止训练期间使用固定 targetTemp + config.targetTemp = null; // 动态计算合理的目标温度范围，基于环境温度和最大功率 const minTemp = Math.max(25, config.roomTemp + 5); // 至少高于室温5度 @@ -1978,8 +1966,6 @@

📋 系统日志

// 改进的奖励函数 - 接收水温和目标温度 const reward = this.ddpgAgent.calculateReward(nextWaterTemp, episodeTarget, heatingPower, lastWaterTemp, step); - // 收集奖励，但不在每步更新图表 - rewardTrace.push(reward); // 改进的记忆函数，传入更多温度信息 this.ddpgAgent.remember(state, heatingPower, reward, nextState, false, waterTemp, episodeTarget, nextWaterTemp); @@ -2074,35 +2060,6 @@

📋 系统日志

this.tdLossChart.update('none'); } - // 回合结束后更新奖励曲线 - if (this.trainCurveChart) { - if (this.trainCurveChart.data.datasets.length < 3) { - this.trainCurveChart.data.datasets.push({ - label: '每步奖励', - data: rewardTrace, - borderColor: '#E91E63', - backgroundColor: 'rgba(233, 30, 99, 0.1)', - fill: false, - tension: 0.1, - yAxisID: 'reward', - borderWidth: 1, - pointRadius: 0 - }); - } else { - this.trainCurveChart.data.datasets[2].data = rewardTrace; - } - if (!this.trainCurveChart.options.scales.reward) { - this.trainCurveChart.options.scales.reward = { - type: 'linear', - position: 'right', - title: { display: true, text: '奖励值' }, - beginAtZero: false, - grid: { drawOnChartArea: false } - }; - } - this.trainCurveChart.update('none'); - } - // 检查训练稳定性并可能重置探索 const resetExploration = this.ddpgAgent.checkTrainingStability(totalReward); if (resetExploration) {