fix(rl): credit hit reward to the action that caused the collision

src/PingPongAI.AI/Agents/AIReinforcementAgent.cs

@@ -117,6 +117,24 @@ public void RegisterReward(double reward)
             _episodeBuffer[_episodeBuffer.Count - 1].Reward += reward;
         }
 
+        // Attaches a reward to the action recorded one tick earlier rather
+        // than the most recent one. Required when the simulator's update
+        // order makes a consequence observable only on the next tick:
+        // GameSimulator.Update runs UpdateBall (collision detection, sets
+        // paddle.HasHitBall) BEFORE UpdatePaddles applies the current
+        // frame's action. So when HasHitBall is true in frame N, the
+        // collision was caused by the paddle position established by
+        // action N-1, not the action just buffered in Decide() this frame.
+        // Crediting +1 to Count-1 would attribute the only positive signal
+        // to a random unrelated action — the policy never learns.
+        public void RegisterRewardForPreviousAction(double reward)
+        {
+            if (_episodeBuffer.Count < 2)
+                return;
+
+            _episodeBuffer[_episodeBuffer.Count - 2].Reward += reward;
+        }
+
         // Closes the current ralli: attaches the terminal reward to the
         // last step, computes discounted returns backward, and runs one
         // policy-gradient pass per step.

src/PingPongAI.App/MainWindow.xaml.cs

@@ -239,8 +239,11 @@ private void UpdateRL(IPongAgent agent, GameState previousState, GameState curre
                 : currentState.RightPaddle;
 
             // Step reward: +1 every frame the paddle hit the ball.
+            // Credit the *previous* action because UpdateBall (which sets
+            // HasHitBall) runs before UpdatePaddles inside GameSimulator —
+            // the collision used paddle position from the prior tick.
             if (paddle.HasHitBall)
-                rl.RegisterReward(+1.0);
+                rl.RegisterRewardForPreviousAction(+1.0);
 
             // Ralli end: score changed between the snapshot and the current state.
             bool leftScored = currentState.LeftScore > previousState.LeftScore;