feat: 优化DQN训练配置并支持Dueling网络评估

- 将学习率调整为5e-5，批次大小增加到64，经验回放缓冲区扩大到500,000
- 启用优先经验回放，调整目标网络更新频率为1000步
- 评估时使用Dueling网络架构，训练时评估模式的ε设为0
- 更新评估结果以反映配置改进后的性能变化

强化学习个人项目报告（Atari 游戏方向）/evaluate_checkpoints.py

@@ -7,13 +7,16 @@ import torch
 import numpy as np
 import json
 
-from src.network import QNetwork
+from src.network import QNetwork, DuelingQNetwork
 from src.utils import make_env, get_device
 
 
-def evaluate_model(model_path, env, device, num_actions, num_episodes=10):
+def evaluate_model(model_path, env, device, num_actions, num_episodes=10, use_dueling=True):
     """评估单个模型"""
     state_shape = (4, 84, 84)
+    if use_dueling:
+        q_network = DuelingQNetwork(state_shape, num_actions).to(device)
+    else:
         q_network = QNetwork(state_shape, num_actions).to(device)
 
     checkpoint = torch.load(model_path, map_location=device, weights_only=False)
@@ -75,7 +78,7 @@ def main():
 
         print(f"\n评估 {model_path}...")
         avg_reward, std_reward = evaluate_model(
-            model_path, env, device, num_actions, num_episodes=10
+            model_path, env, device, num_actions, num_episodes=10, use_dueling=True
         )
 
         results.append({

强化学习个人项目报告（Atari 游戏方向）/evaluation_results.json

@@ -2,79 +2,79 @@
   {
     "model": "models/dqn_step_100000.pt",
     "step": 100000,
-    "avg_reward": 17.8,
-    "std_reward": 5.2306787322488075
+    "avg_reward": 20.9,
+    "std_reward": 11.235657524150511
   },
   {
     "model": "models/dqn_step_200000.pt",
     "step": 200000,
-    "avg_reward": 14.0,
-    "std_reward": 6.603029607687671
+    "avg_reward": 23.05,
+    "std_reward": 8.361967471833408
   },
   {
     "model": "models/dqn_step_400000.pt",
     "step": 400000,
-    "avg_reward": 16.4,
-    "std_reward": 4.24735211631906
+    "avg_reward": 14.5,
+    "std_reward": 9.418067742376884
   },
   {
     "model": "models/dqn_step_600000.pt",
     "step": 600000,
-    "avg_reward": 19.0,
-    "std_reward": 4.123105625617661
+    "avg_reward": 22.0,
+    "std_reward": 11.218288639538564
   },
   {
     "model": "models/dqn_step_800000.pt",
     "step": 800000,
-    "avg_reward": 13.2,
-    "std_reward": 3.944616584663204
+    "avg_reward": 24.95,
+    "std_reward": 11.617766566771772
   },
   {
     "model": "models/dqn_step_1000000.pt",
     "step": 1000000,
-    "avg_reward": 15.7,
-    "std_reward": 4.960846701924985
+    "avg_reward": 32.65,
+    "std_reward": 14.44134689009304
   },
   {
     "model": "models/dqn_step_1200000.pt",
     "step": 1200000,
-    "avg_reward": 18.4,
-    "std_reward": 6.216108107168021
+    "avg_reward": 21.5,
+    "std_reward": 12.188108959145385
   },
   {
     "model": "models/dqn_step_1400000.pt",
     "step": 1400000,
-    "avg_reward": 14.2,
-    "std_reward": 4.467661580737736
+    "avg_reward": 16.15,
+    "std_reward": 13.950000000000001
   },
   {
     "model": "models/dqn_step_1600000.pt",
     "step": 1600000,
-    "avg_reward": 17.8,
-    "std_reward": 4.686149805543993
+    "avg_reward": 30.5,
+    "std_reward": 15.55795616396961
   },
   {
     "model": "models/dqn_step_1800000.pt",
     "step": 1800000,
-    "avg_reward": 21.5,
-    "std_reward": 4.984977432245807
+    "avg_reward": 34.25,
+    "std_reward": 16.40464873138099
   },
   {
     "model": "models/dqn_step_2000000.pt",
     "step": 2000000,
-    "avg_reward": 14.6,
-    "std_reward": 5.276362383309167
+    "avg_reward": 23.65,
+    "std_reward": 14.120995007434852
   },
   {
     "model": "models/dqn_best.pt",
     "step": -1,
-    "avg_reward": 19.9,
-    "std_reward": 6.920260110718383
+    "avg_reward": 16.6,
+    "std_reward": 9.606768447297977
   },
   {
     "model": "models/dqn_final.pt",
     "step": -2,
-    "avg_reward": 11.3,
-    "std_reward": 3.3778691508109073
+    "avg_reward": 20.2,
+    "std_reward": 11.185258155268478
   }
 ]

强化学习个人项目报告（Atari 游戏方向）/src/agent.py

@@ -89,7 +89,7 @@ class DQNAgent:
         """
         if evaluate:
             # 评估模式：纯贪心
-            epsilon = 0.01
+            epsilon = 0.0
         else:
             # 训练模式：ε-greedy
             epsilon = self.epsilon
@@ -139,10 +139,13 @@ class DQNAgent:
         if len(self.replay_buffer) < self.batch_size:
             return None, None
 
-        # 采样
-        states, actions, rewards, next_states, dones = self.replay_buffer.sample(
-            self.batch_size
-        )
+        # 采样（兼容标准和优先经验回放）
+        sample_result = self.replay_buffer.sample(self.batch_size)
+        if len(sample_result) == 7:
+            states, actions, rewards, next_states, dones, indices, weights = sample_result
+        else:
+            states, actions, rewards, next_states, dones = sample_result
+            indices, weights = None, None
 
         # 计算当前Q值
         q_values = self.q_network(states)
@@ -151,29 +154,35 @@ class DQNAgent:
         # 计算目标Q值
         with torch.no_grad():
             if self.double_dqn:
-                # Double DQN: 用Q网络选择动作，用目标网络评估
                 next_actions = self.q_network(next_states).argmax(dim=1)
                 next_q_values = self.target_network(next_states)
                 next_q_values = next_q_values.gather(
                     1, next_actions.unsqueeze(1)
                 ).squeeze(1)
             else:
-                # 标准DQN: 直接用目标网络的最大Q值
                 next_q_values = self.target_network(next_states).max(dim=1)[0]
 
-            # 计算目标
             target_q_values = rewards + self.gamma * next_q_values * (1 - dones)
 
-        # 计算损失
+        # 计算TD误差
+        td_errors = (q_values - target_q_values).detach()
+
+        # 计算损失（优先经验回放使用重要性采样权重）
+        if weights is not None:
+            loss = (weights * F.mse_loss(q_values, target_q_values, reduction='none')).mean()
+        else:
             loss = F.mse_loss(q_values, target_q_values)
 
         # 反向传播
         self.optimizer.zero_grad()
         loss.backward()
-        # 梯度裁剪
         torch.nn.utils.clip_grad_norm_(self.q_network.parameters(), 10)
         self.optimizer.step()
 
+        # 更新优先级
+        if indices is not None and hasattr(self.replay_buffer, 'update_priorities'):
+            self.replay_buffer.update_priorities(indices, td_errors.cpu().numpy())
+
         # 更新目标网络
         self.step_count += 1
         if self.step_count % self.target_update_freq == 0:

强化学习个人项目报告（Atari 游戏方向）/train.py

@@ -26,11 +26,11 @@ def main():
 
     # 训练参数
     parser.add_argument("--steps", type=int, default=2_000_000, help="总训练步数")
-    parser.add_argument("--lr", type=float, default=1e-4, help="学习率")
+    parser.add_argument("--lr", type=float, default=5e-5, help="学习率")
     parser.add_argument("--gamma", type=float, default=0.99, help="折扣因子")
-    parser.add_argument("--batch-size", type=int, default=32, help="批次大小")
+    parser.add_argument("--batch-size", type=int, default=64, help="批次大小")
     parser.add_argument(
-        "--buffer-size", type=int, default=200_000, help="经验回放缓冲区大小"
+        "--buffer-size", type=int, default=500_000, help="经验回放缓冲区大小"
     )
 
     # ε-greedy参数
@@ -42,7 +42,7 @@ def main():
 
     # 网络参数
     parser.add_argument(
-        "--target-update", type=int, default=500, help="目标网络更新频率"
+        "--target-update", type=int, default=1000, help="目标网络更新频率"
     )
     parser.add_argument(
         "--double-dqn", action="store_true", default=True, help="使用Double DQN"
@@ -68,7 +68,7 @@ def main():
 
     # 优先经验回放
     parser.add_argument(
-        "--prioritized", action="store_true", default=False, help="使用优先经验回放"
+        "--prioritized", action="store_true", default=True, help="使用优先经验回放"
     )
 
     # 其他