Add residual scaling for stability in forward pass

Introduced a fixed residual scaling factor to stabilize LN-free training. Updated residual connections and feed-forward networks to incorporate the scaling factor.

Author: liemark

models/model.py

@@ -67,6 +67,8 @@ def __init__(self, num_units, embed_dim=256, num_heads=4, num_layers=4, dropout=
             nn.init.xavier_uniform_(self.enemy_attentions[-1].in_proj_weight)
             nn.init.xavier_uniform_(self.friend_attentions[-1].in_proj_weight)
             # self.norm.append(nn.LayerNorm(embed_dim))
+        # 为了稳定 LN-free 训练，引入固定的残差缩放因子
+        self.res_scale = num_layers ** -0.5
 
     def forward(self, left_signs, left_counts, right_signs, right_counts):
         # 提取TopK特征（怪物 + 场地）
@@ -125,12 +127,12 @@ def forward(self, left_signs, left_counts, right_signs, right_counts):
             )
 
             # 残差连接
-            left_feat = left_feat + delta_left
-            right_feat = right_feat + delta_right
+            left_feat = left_feat + delta_left * self.res_scale
+            right_feat = right_feat + delta_right * self.res_scale
 
             # FFN
-            left_feat = left_feat + self.enemy_ffn[i](left_feat)
-            right_feat = right_feat + self.enemy_ffn[i](right_feat)
+            left_feat = left_feat + self.enemy_ffn[i](left_feat) * self.res_scale
+            right_feat = right_feat + self.enemy_ffn[i](right_feat) * self.res_scale
 
             # 友方注意力
             delta_left, _ = self.friend_attentions[i](
@@ -149,12 +151,12 @@ def forward(self, left_signs, left_counts, right_signs, right_counts):
             )
 
             # 残差连接
-            left_feat = left_feat + delta_left
-            right_feat = right_feat + delta_right
+            left_feat = left_feat + delta_left * self.res_scale
+            right_feat = right_feat + delta_right * self.res_scale
 
             # FFN
-            left_feat = left_feat + self.friend_ffn[i](left_feat)
-            right_feat = right_feat + self.friend_ffn[i](right_feat)
+            left_feat = left_feat + self.friend_ffn[i](left_feat) * self.res_scale
+            right_feat = right_feat + self.friend_ffn[i](right_feat) * self.res_scale
             # 不要归一化，数值嵌入本质是用模长表示战斗力，归一化后数值只能体现成角度变化
             # left_feat = self.norm[i](left_feat)
             # right_feat = self.norm[i](right_feat)