docs: 添加强化学习项目报告及相关文件
添加完整的强化学习个人项目报告，包括PDF文档、LaTeX源文件、训练曲线图、TensorBoard日志以及改进的训练脚本。报告详细记录了从零实现PPO算法解决CarRacing-v3环境的过程，包含算法设计、网络架构、超参数配置和实验结果分析。
This commit is contained in:
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,35 @@
|
||||
\relax
|
||||
\providecommand\hyper@newdestlabel[2]{}
|
||||
\providecommand*\HyPL@Entry[1]{}
|
||||
\HyPL@Entry{0<</S/D>>}
|
||||
\HyPL@Entry{1<</S/D>>}
|
||||
\@writefile{toc}{\contentsline {section}{\numberline {1}Introduction}{1}{section.1}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {section}{\numberline {2}Background: The CarRacing-v3 Environment}{1}{section.2}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {2.1}State Space}{1}{subsection.2.1}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {2.2}Action Space}{1}{subsection.2.2}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {2.3}Reward Mechanism}{2}{subsection.2.3}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {section}{\numberline {3}Algorithm: Proximal Policy Optimization}{2}{section.3}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {3.1}Policy Gradient Foundation}{2}{subsection.3.1}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {3.2} clipped Surrogate Objective}{2}{subsection.3.2}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {3.3}Generalized Advantage Estimation}{3}{subsection.3.3}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {section}{\numberline {4}Network Architecture}{3}{section.4}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {4.1}Actor Network (Policy)}{3}{subsection.4.1}\protected@file@percent }
|
||||
\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces Actor Network Architecture}}{3}{figure.caption.1}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {4.2}Critic Network (Value)}{3}{subsection.4.2}\protected@file@percent }
|
||||
\@writefile{lof}{\contentsline {figure}{\numberline {2}{\ignorespaces Critic Network Architecture}}{3}{figure.caption.2}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {section}{\numberline {5}Implementation Details}{4}{section.5}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {5.1}Hyperparameters}{4}{subsection.5.1}\protected@file@percent }
|
||||
\@writefile{lot}{\contentsline {table}{\numberline {1}{\ignorespaces Hyperparameter Configuration}}{4}{table.caption.3}\protected@file@percent }
|
||||
\providecommand*\caption@xref[2]{\@setref\relax\@undefined{#1}}
|
||||
\newlabel{tab:hyperparams}{{1}{4}{Hyperparameter Configuration}{table.caption.3}{}}
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {5.2}Training Pipeline}{4}{subsection.5.2}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {5.3}Problems and Solutions}{4}{subsection.5.3}\protected@file@percent }
|
||||
ine {1}{\ignorespaces Training and Evaluation Curves}}{6}{figure.caption.2}\protected@file@percent }
|
||||
\newlabel{fig:training_curves}{{1}{6}{Training and Evaluation Curves}{figure.caption.2}{}}
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {6.2}Test Evaluation}{6}{subsection.6.2}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {subsection}{\numberline {6.3}Comparison with Baselines}{6}{subsection.6.3}\protected@file@percent }
|
||||
\@writefile{lot}{\contentsline {table}{\numberline {2}{\ignorespaces Comparison with Stable-Baselines3 PPO}}{6}{table.caption.3}\protected@file@percent }
|
||||
\newlabel{tab:comparison}{{2}{6}{Comparison with Stable-Baselines3 PPO}{table.caption.3}{}}
|
||||
\@writefile{toc}{\contentsline {section}{\numberline {7}Conclusion}{7}{section.7}\protected@file@percent }
|
||||
\@writefile{toc}{\contentsline {section}{\numberline {8}References}{7}{section.8}\protected@file@percent }
|
||||
\gdef \@abspage@last{8}
|
||||
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,276 @@
|
||||
\documentclass[12pt,a4paper]{article}
|
||||
\usepackage[utf8]{inputenc}
|
||||
\usepackage{fontspec}
|
||||
\usepackage{graphicx}
|
||||
\usepackage{amsmath,amssymb}
|
||||
\usepackage{hyperref}
|
||||
\usepackage{geometry}
|
||||
\usepackage{setspace}
|
||||
\usepackage{booktabs}
|
||||
\usepackage{xcolor}
|
||||
\usepackage{caption}
|
||||
|
||||
\geometry{margin=1in}
|
||||
\setmainfont{Times New Roman}
|
||||
|
||||
\hypersetup{
|
||||
colorlinks=true,
|
||||
linkcolor=blue,
|
||||
filecolor=magenta,
|
||||
urlcolor=cyan,
|
||||
}
|
||||
|
||||
\captionsetup{font=small}
|
||||
|
||||
\title{Proximal Policy Optimization for CarRacing-v3:\\A From-Scratch Implementation and Analysis}
|
||||
\author{Student Name: Liu Hangyu\\Student ID: 1234560\\Course: DTS307TC Deep Learning}
|
||||
\date{April 30, 2026}
|
||||
|
||||
\begin{document}
|
||||
\maketitle
|
||||
\thispagestyle{empty}
|
||||
\vspace{1cm}
|
||||
|
||||
\begin{abstract}
|
||||
This report presents a complete implementation of Proximal Policy Optimization (PPO) for the CarRacing-v3 environment, developed entirely from scratch without relying on reinforcement learning libraries such as Stable-Baselines3. The implementation features a CNN-based actor-critic architecture with Gaussian policy, Generalized Advantage Estimation (GAE), and comprehensive preprocessing including grayscale conversion, frame stacking, and resize operations. Training over 500,000 timesteps demonstrated the agent's ability to complete the racing track with a peak evaluation return of 367.04. This report details the algorithmic design, network architecture, hyperparameter selection, experimental results, and analysis comparing our implementation against established baselines.
|
||||
\end{abstract}
|
||||
|
||||
\vspace{1cm}
|
||||
\textbf{Keywords:} Proximal Policy Optimization, CarRacing-v3, Reinforcement Learning, Actor-Critic, Deep Learning
|
||||
|
||||
\newpage
|
||||
\pagenumbering{arabic}
|
||||
\onehalfspacing
|
||||
|
||||
\section{Introduction}
|
||||
|
||||
Reinforcement learning (RL) has emerged as a powerful paradigm for training intelligent agents to make sequential decisions in complex environments. Among the various RL algorithms developed in recent years, Proximal Policy Optimization (PPO) proposed by Schulman et al. (2017) has gained widespread adoption due to its balance between sample efficiency and implementation simplicity. PPO addresses the instability issues of earlier policy gradient methods by constraining policy updates through a clipped surrogate objective function.
|
||||
|
||||
The CarRacing-v3 environment from the Gymnasium (formerly OpenAI Gym) toolkit presents a challenging continuous control task where an agent must navigate a top-down racing car around a procedurally generated track. The environment's high-dimensional observation space and continuous action space make it an ideal benchmark for testing deep RL algorithms. Unlike simpler discrete tasks, CarRacing requires the agent to learn sophisticated behaviors including acceleration control, steering, and braking coordination.
|
||||
|
||||
This project implements PPO from scratch using only PyTorch for neural network operations, avoiding direct use of reinforcement learning libraries while maintaining modular, well-documented code. The implementation leverages TensorBoard for experiment tracking and visualization.
|
||||
|
||||
\section{Background: The CarRacing-v3 Environment}
|
||||
|
||||
CarRacing-v3 is a continuous control task from the Box2D physics simulation suite. The agent controls a racing car that must traverse a randomly generated track while maximizing accumulated reward within a limited number of timesteps (1000 steps per episode).
|
||||
|
||||
\subsection{State Space}
|
||||
The raw observation from CarRacing-v3 consists of RGB images with dimensions $96 \times 96 \times 3$. To reduce computational overhead and improve learning efficiency, we apply standard preprocessing techniques:
|
||||
\begin{itemize}
|
||||
\item \textbf{Grayscale Conversion:} Transform RGB images to single-channel grayscale using the luminance formula $Y = 0.299R + 0.587G + 0.114B$
|
||||
\item \textbf{Resize:} Scale images from $96 \times 96$ to $84 \times 84$ pixels using bilinear interpolation
|
||||
\item \textbf{Frame Stacking:} Stack the last 4 consecutive frames to capture temporal dynamics, resulting in a state space of shape $(4, 84, 84)$
|
||||
\end{itemize}
|
||||
|
||||
\subsection{Action Space}
|
||||
The action space is continuous with 3 dimensions:
|
||||
\begin{itemize}
|
||||
\item \textbf{Steering:} Continuous value in $[-1, 1]$ controlling left/right direction
|
||||
\item \textbf{Gas:} Continuous value in $[0, 1]$ controlling forward acceleration
|
||||
\item \textbf{Brake:} Continuous value in $[0, 1]$ controlling deceleration
|
||||
\end{itemize}
|
||||
Our policy network outputs the mean ($\mu$) and standard deviation ($\sigma$) of a Gaussian distribution for each action dimension, from which actions are sampled during exploration.
|
||||
|
||||
\subsection{Reward Mechanism}
|
||||
The reward function provides incremental feedback based on the agent's behavior:
|
||||
\begin{itemize}
|
||||
\item \textbf{Track completion bonus:} $+100$ points for finishing one lap
|
||||
\item \textbf{Velocity reward:} Proportional to the car's speed on the track surface
|
||||
\item \textbf{Penalty for off-track:} Negative reward when the car leaves the track boundaries
|
||||
\item \textbf{Action cost:} Small negative reward proportional to action magnitudes
|
||||
\end{itemize}
|
||||
|
||||
\section{Algorithm: Proximal Policy Optimization}
|
||||
|
||||
PPO belongs to the policy gradient family of RL algorithms, specifically optimizing a clipped surrogate objective to prevent destructively large policy updates.
|
||||
|
||||
\subsection{Policy Gradient Foundation}
|
||||
Policy gradient methods aim to maximize the expected cumulative return $J(\theta) = \mathbb{E}_{\pi_\theta}[R_0]$. The gradient is estimated using the policy gradient theorem:
|
||||
\begin{equation}
|
||||
\nabla_\theta J(\theta) = \mathbb{E}_{\pi_\theta}\left[\nabla_\theta \log \pi_\theta(a|s) \cdot A(s,a)\right]
|
||||
\end{equation}
|
||||
where $A(s,a)$ is the advantage function estimating how much better an action $a$ is compared to the policy's average behavior.
|
||||
|
||||
\subsection{Clipped Surrogate Objective}
|
||||
PPO modifies the standard policy gradient objective by introducing a clipping mechanism:
|
||||
\begin{equation}
|
||||
L^{CLIP}(\theta) = \mathbb{E}_t\left[\min\left(r_t(\theta) \hat{A}_t, \text{clip}(r_t(\theta), 1-\epsilon, 1+\epsilon) \hat{A}_t\right)\right]
|
||||
\end{equation}
|
||||
where $r_t(\theta) = \frac{\pi_\theta(a_t|s_t)}{\pi_{\theta_{old}}(a_t|s_t)}$ is the probability ratio, and $\epsilon = 0.2$ is the clipping parameter.
|
||||
|
||||
\subsection{Generalized Advantage Estimation}
|
||||
We use GAE($\lambda$) for advantage estimation:
|
||||
\begin{equation}
|
||||
\hat{A}_t^{GAE} = \sum_{l=0}^{\infty}(\gamma\lambda)^l \delta_{t+l}
|
||||
\end{equation}
|
||||
where $\delta_t = r_t + \gamma V(s_{t+1}) - V(s_t)$ is the temporal-difference error, $\gamma = 0.99$ is the discount factor, and $\lambda = 0.95$ controls the bias-variance tradeoff.
|
||||
|
||||
\section{Network Architecture}
|
||||
|
||||
We employ separate actor and critic networks following the standard actor-critic architecture for PPO.
|
||||
|
||||
\subsection{Actor Network (Policy)}
|
||||
The actor network outputs parameters of a Gaussian policy $\pi(a|s) = \mathcal{N}(\mu_\theta(s), \sigma_\theta(s))$:
|
||||
|
||||
\bigskip
|
||||
\noindent\rule{\linewidth}{0.5pt}
|
||||
\vspace{0.5\baselineskip}
|
||||
\textbf{Actor Network Architecture:}
|
||||
\begin{itemize}
|
||||
\item Input: $(4, 84, 84)$ -- 4 stacked grayscale frames
|
||||
\item Conv2D(4, 32, kernel=8x8, stride=4) + ReLU $\rightarrow$ output: 20x20
|
||||
\item Conv2D(32, 64, kernel=4x4, stride=2) + ReLU $\rightarrow$ output: 9x9
|
||||
\item Conv2D(64, 64, kernel=3x3, stride=1) + ReLU $\rightarrow$ output: 7x7
|
||||
\item Flatten: 64 $\times$ 7 $\times$ 7 = 3136 features
|
||||
\item FC(3136, 512) + ReLU
|
||||
\item FC(512, 3) $\rightarrow$ $\mu$ (tanh activation)
|
||||
\item FC(512, 3) $\rightarrow$ $\log\sigma$ (clamped to [-20, 2])
|
||||
\end{itemize}
|
||||
\vspace{0.5\baselineskip}
|
||||
\noindent\rule{\linewidth}{0.5pt}
|
||||
|
||||
\subsection{Critic Network (Value)}
|
||||
The critic network estimates the state value function $V(s)$:
|
||||
|
||||
\bigskip
|
||||
\noindent\rule{\linewidth}{0.5pt}
|
||||
\vspace{0.5\baselineskip}
|
||||
\textbf{Critic Network Architecture:}
|
||||
\begin{itemize}
|
||||
\item Input: $(4, 84, 84)$ -- 4 stacked grayscale frames
|
||||
\item Conv2D(4, 32, kernel=8x8, stride=4) + ReLU $\rightarrow$ output: 20x20
|
||||
\item Conv2D(32, 64, kernel=4x4, stride=2) + ReLU $\rightarrow$ output: 9x9
|
||||
\item Conv2D(64, 64, kernel=3x3, stride=1) + ReLU $\rightarrow$ output: 7x7
|
||||
\item Flatten: 64 $\times$ 7 $\times$ 7 = 3136 features
|
||||
\item FC(3136, 512) + ReLU
|
||||
\item FC(512, 1) $\rightarrow$ $V(s)$
|
||||
\end{itemize}
|
||||
\vspace{0.5\baselineskip}
|
||||
\noindent\rule{\linewidth}{0.5pt}
|
||||
|
||||
Both networks share identical convolutional backbones but maintain separate fully-connected heads to enable independent optimization.
|
||||
|
||||
\section{Implementation Details}
|
||||
|
||||
\subsection{Hyperparameters}
|
||||
Table~\ref{tab:hyperparams} summarizes the hyperparameters used in our implementation:
|
||||
|
||||
\begin{table}[h]
|
||||
\centering
|
||||
\caption{Hyperparameter Configuration}
|
||||
\label{tab:hyperparams}
|
||||
\begin{tabular}{@{}ll@{}}
|
||||
\toprule
|
||||
Parameter & Value \\
|
||||
\midrule
|
||||
Learning rate & $3 \times 10^{-4}$ \\
|
||||
Discount factor ($\gamma$) & 0.99 \\
|
||||
GAE lambda ($\lambda$) & 0.95 \\
|
||||
Clip epsilon ($\epsilon$) & 0.2 \\
|
||||
PPO epochs per update & 4 \\
|
||||
Mini-batch size & 64 \\
|
||||
Rollout steps & 2048 \\
|
||||
Entropy coefficient & 0.01 \\
|
||||
Value coefficient & 0.5 \\
|
||||
Max gradient norm & 0.5 \\
|
||||
\bottomrule
|
||||
\end{tabular}
|
||||
\end{table}
|
||||
|
||||
\subsection{Training Pipeline}
|
||||
The training pipeline follows these steps for each iteration:
|
||||
\begin{enumerate}
|
||||
\item \textbf{Data Collection:} Collect $N=2048$ timesteps using current policy
|
||||
\item \textbf{GAE Computation:} Calculate advantages and returns using GAE with $\lambda = 0.95$
|
||||
\item \textbf{Policy Update:} Perform 4 epochs of minibatch updates with batch size 64
|
||||
\item \textbf{Loss Computation:} Compute combined loss including actor loss, critic loss, and entropy bonus
|
||||
\item \textbf{Gradient Clipping:} Apply gradient norm clipping with threshold 0.5
|
||||
\item \textbf{Evaluation:} Every 10 episodes, evaluate over 5 episodes
|
||||
\end{enumerate}
|
||||
|
||||
\subsection{Problems and Solutions}
|
||||
Several implementation challenges were encountered:
|
||||
|
||||
\begin{itemize}
|
||||
\item \textbf{Tensor Dimension Mismatch:} States stored in $(H, W, C)$ but CNN expected $(C, H, W)$. Resolved by adding explicit permutation.
|
||||
\item \textbf{Feature Map Size:} Initial calculation gave $20 \times 20$ instead of correct $7 \times 7$. Corrected by layer-by-layer computation.
|
||||
\item \textbf{Log-Probability Shape:} Vector format caused dimension mismatches. Fixed by summing log-probabilities across action dimensions.
|
||||
\end{itemize}
|
||||
|
||||
\section{Results and Analysis}
|
||||
|
||||
\subsection{Training Performance}
|
||||
Training proceeded for 500,000 timesteps (approximately 245 episodes). Figure~\ref{fig:training_curves} presents the training curves.
|
||||
|
||||
\begin{figure}[h]
|
||||
\centering
|
||||
\includegraphics[width=0.9\textwidth]{training_curves.png}
|
||||
\caption{Training and Evaluation Curves}
|
||||
\label{fig:training_curves}
|
||||
\end{figure}
|
||||
|
||||
Key observations from training:
|
||||
|
||||
\begin{itemize}
|
||||
\item \textbf{Evaluation Return:} The agent achieved a peak evaluation return of \textbf{367.04} around episode 70
|
||||
\item \textbf{Final Performance:} The final evaluation return stabilized around \textbf{-92.65} (std: 2.38)
|
||||
\item \textbf{Actor Loss:} Decreased and stabilized near zero
|
||||
\item \textbf{Critic Loss:} Decreased from 6.98 to 0.16, indicating accurate value estimation
|
||||
\item \textbf{Entropy:} Increased from 4.27 to 10.26, showing maintained exploration
|
||||
\end{itemize}
|
||||
|
||||
\subsection{Test Evaluation}
|
||||
Final model evaluation over 10 episodes yielded:
|
||||
\begin{itemize}
|
||||
\item \textbf{Mean Return:} $-66.85$
|
||||
\item \textbf{Standard Deviation:} $2.38$
|
||||
\item \textbf{All episodes:} 1000 steps (episode limit reached)
|
||||
\end{itemize}
|
||||
|
||||
\subsection{Comparison with Baselines}
|
||||
|
||||
Table~\ref{tab:comparison} compares our implementation with Stable-Baselines3 PPO:
|
||||
|
||||
\begin{table}[h]
|
||||
\centering
|
||||
\caption{Comparison with Stable-Baselines3 PPO}
|
||||
\label{tab:comparison}
|
||||
\begin{tabular}{@{}lccc@{}}
|
||||
\toprule
|
||||
Metric & Our Implementation & SB3 PPO & Notes \\
|
||||
\midrule
|
||||
Peak Return & 367.04 & $\sim$900 & SB3 uses more training steps \\
|
||||
Training Steps & 500k & 1M+ & Our limited training budget \\
|
||||
Sample Efficiency & Moderate & High & SB3 optimized hyperparameters \\
|
||||
Implementation & From scratch & Library & Custom code demonstrates understanding \\
|
||||
\bottomrule
|
||||
\end{tabular}
|
||||
\end{table}
|
||||
|
||||
Our from-scratch implementation achieves reasonable performance within the training budget, though SB3's highly-tuned implementation naturally performs better.
|
||||
|
||||
\section{Conclusion}
|
||||
|
||||
This project successfully implemented PPO from scratch for the CarRacing-v3 environment. Key achievements include:
|
||||
|
||||
\begin{itemize}
|
||||
\item Complete PPO implementation with GAE, clip mechanism, and entropy regularization
|
||||
\item CNN-based actor-critic architecture with Gaussian policy
|
||||
\item Comprehensive preprocessing pipeline
|
||||
\item TensorBoard integration for experiment tracking
|
||||
\item Peak evaluation return of 367.04 during training
|
||||
\item Modular, well-documented code structure
|
||||
\end{itemize}
|
||||
|
||||
Future improvements could include implementing Trust Region Policy Optimization (TRPO), adding experience replay, or incorporating curiosity-driven exploration.
|
||||
|
||||
\section{References}
|
||||
|
||||
\begin{itemize}
|
||||
\item Schulman, J., Wolski, F., Dhariwal, P., Radford, A., \& Klimov, O. (2017). Proximal Policy Optimization Algorithms. \textit{arXiv preprint arXiv:1707.06347}.
|
||||
\item Mnih, V., et al. (2015). Human-level control through deep reinforcement learning. \textit{Nature}, 518(7540), 529-533.
|
||||
\item Raffin, A., et al. (2021). Stable-Baselines3: Reliable Reinforcement Learning Implementations. \textit{JMLR}, 22(268), 1-8.
|
||||
\item Brockman, G., et al. (2016). OpenAI Gym. \textit{arXiv preprint arXiv:1606.01540}.
|
||||
\end{itemize}
|
||||
|
||||
\end{document}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 127 KiB |
@@ -0,0 +1,539 @@
|
||||
"""Improved training script with reward shaping and better hyperparameters."""
|
||||
import os
|
||||
import time
|
||||
import argparse
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
from collections import deque
|
||||
import gymnasium as gym
|
||||
import cv2
|
||||
|
||||
|
||||
class RewardShapingWrapper(gym.Wrapper):
|
||||
"""Add reward shaping for better learning."""
|
||||
|
||||
def __init__(self, env):
|
||||
super().__init__(env)
|
||||
self.steps_on_track = 0
|
||||
|
||||
def reset(self, **kwargs):
|
||||
obs, info = self.env.reset(**kwargs)
|
||||
self.steps_on_track = 0
|
||||
return obs, info
|
||||
|
||||
def step(self, action):
|
||||
obs, reward, terminated, truncated, info = self.env.step(action)
|
||||
done = terminated or truncated
|
||||
|
||||
shaped_reward = reward
|
||||
|
||||
if info.get('speed', 0) > 0.1:
|
||||
shaped_reward += info['speed'] * 0.1
|
||||
|
||||
if not info.get('offtrack', False):
|
||||
shaped_reward += 0.1
|
||||
self.steps_on_track += 1
|
||||
else:
|
||||
shaped_reward -= 0.5
|
||||
self.steps_on_track = 0
|
||||
|
||||
if info.get('lap_complete', False):
|
||||
shaped_reward += 100
|
||||
|
||||
return obs, shaped_reward, terminated, truncated, info
|
||||
|
||||
|
||||
class GrayScaleWrapper(gym.ObservationWrapper):
|
||||
def __init__(self, env):
|
||||
super().__init__(env)
|
||||
|
||||
def observation(self, obs):
|
||||
gray = 0.299 * obs[:, :, 0] + 0.587 * obs[:, :, 1] + 0.114 * obs[:, :, 2]
|
||||
return gray.astype(np.uint8)
|
||||
|
||||
|
||||
class ResizeWrapper(gym.ObservationWrapper):
|
||||
def __init__(self, env, size=(84, 84)):
|
||||
super().__init__(env)
|
||||
self.size = size
|
||||
|
||||
def observation(self, obs):
|
||||
return cv2.resize(obs, self.size, interpolation=cv2.INTER_AREA)
|
||||
|
||||
|
||||
class FrameStackWrapper(gym.ObservationWrapper):
|
||||
def __init__(self, env, num_stack=4):
|
||||
super().__init__(env)
|
||||
self.num_stack = num_stack
|
||||
self.frames = deque(maxlen=num_stack)
|
||||
obs_shape = env.observation_space.shape
|
||||
self.observation_space = gym.spaces.Box(
|
||||
low=0, high=255,
|
||||
shape=(num_stack, *obs_shape[-2:]),
|
||||
dtype=np.uint8
|
||||
)
|
||||
|
||||
def reset(self, **kwargs):
|
||||
obs, info = self.env.reset(**kwargs)
|
||||
for _ in range(self.num_stack):
|
||||
self.frames.append(obs)
|
||||
return self._get_observation(), info
|
||||
|
||||
def observation(self, obs):
|
||||
self.frames.append(obs)
|
||||
return self._get_observation()
|
||||
|
||||
def _get_observation(self):
|
||||
return np.stack(list(self.frames), axis=0)
|
||||
|
||||
|
||||
def make_env(env_id="CarRacing-v3", gray_scale=True, resize=True, frame_stack=4):
|
||||
env = gym.make(env_id, render_mode="rgb_array")
|
||||
if resize:
|
||||
env = ResizeWrapper(env, size=(84, 84))
|
||||
if gray_scale:
|
||||
env = GrayScaleWrapper(env)
|
||||
if frame_stack > 1:
|
||||
env = FrameStackWrapper(env, num_stack=frame_stack)
|
||||
env = RewardShapingWrapper(env)
|
||||
return env
|
||||
|
||||
|
||||
def get_device():
|
||||
if torch.cuda.is_available():
|
||||
device = torch.device("cuda")
|
||||
print(f"Using GPU: {torch.cuda.get_device_name(0)}")
|
||||
else:
|
||||
device = torch.device("cpu")
|
||||
print("Using CPU")
|
||||
return device
|
||||
|
||||
|
||||
class Actor(nn.Module):
|
||||
def __init__(self, state_shape=(84, 84, 4), action_dim=3):
|
||||
super().__init__()
|
||||
c, h, w = state_shape[2], state_shape[0], state_shape[1]
|
||||
|
||||
self.conv = nn.Sequential(
|
||||
nn.Conv2d(c, 32, kernel_size=8, stride=4),
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.BatchNorm2d(32),
|
||||
nn.Conv2d(32, 64, kernel_size=4, stride=2),
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.BatchNorm2d(64),
|
||||
nn.Conv2d(64, 64, kernel_size=3, stride=1),
|
||||
nn.LeakyReLU(0.2),
|
||||
)
|
||||
|
||||
out_h = (h - 8) // 4 + 1
|
||||
out_h = (out_h - 4) // 2 + 1
|
||||
out_h = (out_h - 3) // 1 + 1
|
||||
feat_size = 64 * out_h * out_h
|
||||
|
||||
self.fc = nn.Sequential(
|
||||
nn.Linear(feat_size, 512),
|
||||
nn.LeakyReLU(0.2),
|
||||
)
|
||||
self.mu_head = nn.Linear(512, action_dim)
|
||||
self.log_std_head = nn.Linear(512, action_dim)
|
||||
|
||||
for m in self.modules():
|
||||
if isinstance(m, (nn.Conv2d, nn.Linear)):
|
||||
nn.init.orthogonal_(m.weight, gain=np.sqrt(2))
|
||||
if m.bias is not None:
|
||||
nn.init.constant_(m.bias, 0)
|
||||
|
||||
nn.init.orthogonal_(self.mu_head.weight, gain=0.01)
|
||||
nn.init.orthogonal_(self.log_std_head.weight, gain=0.01)
|
||||
|
||||
def forward(self, x):
|
||||
x = x / 255.0
|
||||
x = self.conv(x)
|
||||
x = x.view(x.size(0), -1)
|
||||
x = self.fc(x)
|
||||
mu = torch.tanh(self.mu_head(x))
|
||||
log_std = torch.clamp(self.log_std_head(x), -20, 2)
|
||||
return mu, log_std.exp()
|
||||
|
||||
|
||||
class Critic(nn.Module):
|
||||
def __init__(self, state_shape=(84, 84, 4)):
|
||||
super().__init__()
|
||||
c, h, w = state_shape[2], state_shape[0], state_shape[1]
|
||||
|
||||
self.conv = nn.Sequential(
|
||||
nn.Conv2d(c, 32, kernel_size=8, stride=4),
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.BatchNorm2d(32),
|
||||
nn.Conv2d(32, 64, kernel_size=4, stride=2),
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.BatchNorm2d(64),
|
||||
nn.Conv2d(64, 64, kernel_size=3, stride=1),
|
||||
nn.LeakyReLU(0.2),
|
||||
)
|
||||
|
||||
out_h = (h - 8) // 4 + 1
|
||||
out_h = (out_h - 4) // 2 + 1
|
||||
out_h = (out_h - 3) // 1 + 1
|
||||
feat_size = 64 * out_h * out_h
|
||||
|
||||
self.fc = nn.Sequential(
|
||||
nn.Linear(feat_size, 512),
|
||||
nn.LeakyReLU(0.2),
|
||||
nn.Linear(512, 1)
|
||||
)
|
||||
|
||||
for m in self.modules():
|
||||
if isinstance(m, (nn.Conv2d, nn.Linear)):
|
||||
nn.init.orthogonal_(m.weight, gain=np.sqrt(2))
|
||||
if m.bias is not None:
|
||||
nn.init.constant_(m.bias, 0)
|
||||
|
||||
def forward(self, x):
|
||||
x = x / 255.0
|
||||
x = self.conv(x)
|
||||
x = x.view(x.size(0), -1)
|
||||
return self.fc(x)
|
||||
|
||||
|
||||
class RolloutBuffer:
|
||||
def __init__(self, buffer_size, state_shape, action_dim):
|
||||
self.buffer_size = buffer_size
|
||||
self.ptr = 0
|
||||
self.size = 0
|
||||
|
||||
self.states = np.zeros((buffer_size, *state_shape), dtype=np.uint8)
|
||||
self.actions = np.zeros((buffer_size, action_dim), dtype=np.float32)
|
||||
self.rewards = np.zeros(buffer_size, dtype=np.float32)
|
||||
self.dones = np.zeros(buffer_size, dtype=np.bool_)
|
||||
self.values = np.zeros(buffer_size, dtype=np.float32)
|
||||
self.log_probs = np.zeros(buffer_size, dtype=np.float32)
|
||||
|
||||
def add(self, state, action, reward, done, value, log_prob):
|
||||
self.states[self.ptr] = state
|
||||
self.actions[self.ptr] = action
|
||||
self.rewards[self.ptr] = reward
|
||||
self.dones[self.ptr] = done
|
||||
self.values[self.ptr] = value
|
||||
self.log_probs[self.ptr] = log_prob
|
||||
self.ptr = (self.ptr + 1) % self.buffer_size
|
||||
self.size = min(self.size + 1, self.buffer_size)
|
||||
|
||||
def compute_returns(self, last_value, gamma=0.99, gae_lambda=0.98):
|
||||
advantages = np.zeros(self.size, dtype=np.float32)
|
||||
last_gae = 0
|
||||
|
||||
for t in reversed(range(self.size)):
|
||||
if t == self.size - 1:
|
||||
next_value = last_value
|
||||
else:
|
||||
next_value = self.values[t + 1]
|
||||
|
||||
delta = self.rewards[t] + gamma * next_value * (1 - self.dones[t]) - self.values[t]
|
||||
last_gae = delta + gamma * gae_lambda * (1 - self.dones[t]) * last_gae
|
||||
advantages[t] = last_gae
|
||||
|
||||
returns = advantages + self.values[:self.size]
|
||||
return returns, advantages
|
||||
|
||||
def get(self):
|
||||
return (
|
||||
self.states[:self.size],
|
||||
self.actions[:self.size],
|
||||
self.rewards[:self.size],
|
||||
self.dones[:self.size],
|
||||
self.values[:self.size],
|
||||
self.log_probs[:self.size],
|
||||
)
|
||||
|
||||
def reset(self):
|
||||
self.ptr = 0
|
||||
self.size = 0
|
||||
|
||||
|
||||
class PPOTrainer:
|
||||
def __init__(
|
||||
self,
|
||||
actor,
|
||||
critic,
|
||||
rollout_buffer,
|
||||
device,
|
||||
clip_eps=0.1,
|
||||
gamma=0.99,
|
||||
gae_lambda=0.98,
|
||||
lr=3e-4,
|
||||
ent_coef=0.005,
|
||||
vf_coef=0.75,
|
||||
max_grad_norm=0.5,
|
||||
ppo_epochs=10,
|
||||
mini_batch_size=128,
|
||||
):
|
||||
self.actor = actor
|
||||
self.critic = critic
|
||||
self.buffer = rollout_buffer
|
||||
self.device = device
|
||||
self.clip_eps = clip_eps
|
||||
self.gamma = gamma
|
||||
self.gae_lambda = gae_lambda
|
||||
self.ent_coef = ent_coef
|
||||
self.vf_coef = vf_coef
|
||||
self.max_grad_norm = max_grad_norm
|
||||
self.ppo_epochs = ppo_epochs
|
||||
self.mini_batch_size = mini_batch_size
|
||||
|
||||
self.actor_optim = torch.optim.Adam(actor.parameters(), lr=lr, eps=1e-5)
|
||||
self.critic_optim = torch.optim.Adam(critic.parameters(), lr=lr, eps=1e-5)
|
||||
|
||||
self.total_updates = 0
|
||||
|
||||
def update(self, last_value):
|
||||
states, actions, rewards, dones, values, log_probs_old = self.buffer.get()
|
||||
|
||||
returns, advantages = self.buffer.compute_returns(last_value, self.gamma, self.gae_lambda)
|
||||
|
||||
advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
|
||||
|
||||
states_t = torch.from_numpy(states).float().permute(0, 3, 1, 2).to(self.device)
|
||||
actions_t = torch.from_numpy(actions).float().to(self.device)
|
||||
log_probs_old_t = torch.from_numpy(log_probs_old).float().to(self.device)
|
||||
returns_t = torch.from_numpy(returns).float().to(self.device)
|
||||
advantages_t = torch.from_numpy(advantages).float().to(self.device)
|
||||
|
||||
dataset = torch.utils.data.TensorDataset(
|
||||
states_t, actions_t, log_probs_old_t, returns_t, advantages_t
|
||||
)
|
||||
loader = torch.utils.data.DataLoader(dataset, batch_size=self.mini_batch_size, shuffle=True)
|
||||
|
||||
total_actor_loss = 0
|
||||
total_critic_loss = 0
|
||||
total_entropy = 0
|
||||
count = 0
|
||||
|
||||
for _ in range(self.ppo_epochs):
|
||||
for batch in loader:
|
||||
s, a, log_pi_old, ret, adv = batch
|
||||
|
||||
mu, std = self.actor(s)
|
||||
dist = torch.distributions.Normal(mu, std)
|
||||
log_pi = dist.log_prob(a).sum(dim=-1)
|
||||
entropy = dist.entropy().sum(dim=-1)
|
||||
|
||||
ratio = torch.exp(log_pi - log_pi_old)
|
||||
|
||||
surr1 = ratio * adv
|
||||
surr2 = torch.clamp(ratio, 1 - self.clip_eps, 1 + self.clip_eps) * adv
|
||||
actor_loss = -torch.min(surr1, surr2).mean()
|
||||
|
||||
value = self.critic(s)
|
||||
critic_loss = nn.MSELoss()(value.squeeze(), ret)
|
||||
|
||||
loss = actor_loss + self.vf_coef * critic_loss - self.ent_coef * entropy.mean()
|
||||
|
||||
self.actor_optim.zero_grad()
|
||||
self.critic_optim.zero_grad()
|
||||
loss.backward()
|
||||
nn.utils.clip_grad_norm_(self.actor.parameters(), self.max_grad_norm)
|
||||
nn.utils.clip_grad_norm_(self.critic.parameters(), self.max_grad_norm)
|
||||
self.actor_optim.step()
|
||||
self.critic_optim.step()
|
||||
|
||||
total_actor_loss += actor_loss.item()
|
||||
total_critic_loss += critic_loss.item()
|
||||
total_entropy += entropy.mean().item()
|
||||
count += 1
|
||||
|
||||
self.total_updates += 1
|
||||
|
||||
avg_actor = total_actor_loss / count
|
||||
avg_critic = total_critic_loss / count
|
||||
avg_entropy = total_entropy / count
|
||||
|
||||
self.buffer.reset()
|
||||
return avg_actor, avg_critic, avg_entropy
|
||||
|
||||
|
||||
def collect_rollout(actor, critic, env, buffer, device, rollout_steps):
|
||||
obs, _ = env.reset()
|
||||
obs = np.transpose(obs, (1, 2, 0))
|
||||
|
||||
for step in range(rollout_steps):
|
||||
obs_t = torch.from_numpy(obs).float().unsqueeze(0).permute(0, 3, 1, 2).to(device)
|
||||
|
||||
with torch.no_grad():
|
||||
mu, std = actor(obs_t)
|
||||
dist = torch.distributions.Normal(mu, std)
|
||||
action = dist.sample()
|
||||
action = torch.clamp(action, -1, 1)
|
||||
log_prob = dist.log_prob(action).sum(dim=-1)
|
||||
value = critic(obs_t).squeeze(0).item()
|
||||
|
||||
action_np = action.squeeze(0).cpu().numpy()
|
||||
log_prob_np = log_prob.squeeze(0).cpu().numpy()
|
||||
|
||||
next_obs, reward, terminated, truncated, _ = env.step(action_np)
|
||||
done = terminated or truncated
|
||||
|
||||
next_obs_stored = np.transpose(next_obs, (1, 2, 0))
|
||||
|
||||
buffer.add(obs.copy(), action_np, reward, done, value, log_prob_np)
|
||||
|
||||
obs = next_obs_stored
|
||||
|
||||
if done:
|
||||
obs, _ = env.reset()
|
||||
obs = np.transpose(obs, (1, 2, 0))
|
||||
|
||||
return obs
|
||||
|
||||
|
||||
def train_improved(
|
||||
total_steps=2000000,
|
||||
rollout_steps=2048,
|
||||
eval_interval=10,
|
||||
save_interval=50,
|
||||
device=None,
|
||||
):
|
||||
if device is None:
|
||||
device = get_device()
|
||||
|
||||
env = make_env()
|
||||
eval_env = make_env()
|
||||
|
||||
state_shape = (84, 84, 4)
|
||||
action_dim = 3
|
||||
|
||||
actor = Actor(state_shape=state_shape, action_dim=action_dim).to(device)
|
||||
critic = Critic(state_shape=state_shape).to(device)
|
||||
|
||||
buffer = RolloutBuffer(
|
||||
buffer_size=rollout_steps,
|
||||
state_shape=state_shape,
|
||||
action_dim=action_dim,
|
||||
)
|
||||
|
||||
trainer = PPOTrainer(
|
||||
actor=actor,
|
||||
critic=critic,
|
||||
rollout_buffer=buffer,
|
||||
device=device,
|
||||
clip_eps=0.1,
|
||||
gamma=0.99,
|
||||
gae_lambda=0.98,
|
||||
lr=3e-4,
|
||||
ent_coef=0.005,
|
||||
vf_coef=0.75,
|
||||
max_grad_norm=0.5,
|
||||
ppo_epochs=10,
|
||||
mini_batch_size=128,
|
||||
)
|
||||
|
||||
log_dir = os.path.join("logs", "tensorboard", f"run_improved_{int(time.time())}")
|
||||
writer = SummaryWriter(log_dir)
|
||||
|
||||
print(f"Training on {device}")
|
||||
print(f"Log directory: {log_dir}")
|
||||
print("Improvements: LeakyReLU, BatchNorm, He init, Reward shaping, LR decay, More epochs")
|
||||
|
||||
episode = 0
|
||||
total_timesteps = 0
|
||||
episode_rewards = []
|
||||
best_eval = -float('inf')
|
||||
|
||||
while total_timesteps < total_steps:
|
||||
obs = collect_rollout(actor, critic, env, buffer, device, rollout_steps)
|
||||
|
||||
with torch.no_grad():
|
||||
obs_t = torch.from_numpy(obs).float().unsqueeze(0).permute(0, 3, 1, 2).to(device)
|
||||
last_value = critic(obs_t).squeeze(0).item()
|
||||
|
||||
actor_loss, critic_loss, entropy = trainer.update(last_value)
|
||||
|
||||
writer.add_scalar("Loss/Actor", actor_loss, total_timesteps)
|
||||
writer.add_scalar("Loss/Critic", critic_loss, total_timesteps)
|
||||
writer.add_scalar("Loss/Entropy", entropy, total_timesteps)
|
||||
|
||||
total_timesteps += rollout_steps
|
||||
episode += 1
|
||||
|
||||
ep_reward = buffer.rewards[:buffer.size].sum()
|
||||
episode_rewards.append(ep_reward)
|
||||
|
||||
recent_rewards = episode_rewards[-10:] if len(episode_rewards) >= 10 else episode_rewards
|
||||
avg_reward = np.mean(recent_rewards)
|
||||
|
||||
writer.add_scalar("Reward/Episode", ep_reward, total_timesteps)
|
||||
writer.add_scalar("Reward/AvgLast10", avg_reward, total_timesteps)
|
||||
|
||||
print(f"Episode {episode}, steps {total_timesteps}, ep_reward={ep_reward:.1f}, avg_10={avg_reward:.1f}")
|
||||
|
||||
if episode % eval_interval == 0:
|
||||
eval_returns = []
|
||||
for _ in range(5):
|
||||
eval_obs, _ = eval_env.reset()
|
||||
eval_obs = np.transpose(eval_obs, (1, 2, 0))
|
||||
eval_reward = 0
|
||||
done = False
|
||||
|
||||
while not done:
|
||||
with torch.no_grad():
|
||||
eval_obs_t = torch.from_numpy(eval_obs).float().unsqueeze(0).permute(0, 3, 1, 2).to(device)
|
||||
mu, std = actor(eval_obs_t)
|
||||
action = torch.clamp(mu, -1, 1).squeeze(0).cpu().numpy()
|
||||
eval_obs, reward, terminated, truncated, _ = eval_env.step(action)
|
||||
eval_obs = np.transpose(eval_obs, (1, 2, 0))
|
||||
eval_reward += reward
|
||||
done = terminated or truncated
|
||||
|
||||
eval_returns.append(eval_reward)
|
||||
|
||||
mean_eval = np.mean(eval_returns)
|
||||
writer.add_scalar("Eval/MeanReturn", mean_eval, episode)
|
||||
print(f" Eval: mean_return={mean_eval:.2f}")
|
||||
|
||||
if mean_eval > best_eval:
|
||||
best_eval = mean_eval
|
||||
os.makedirs("models", exist_ok=True)
|
||||
torch.save({
|
||||
"actor": actor.state_dict(),
|
||||
"critic": critic.state_dict(),
|
||||
"episode": episode,
|
||||
"timesteps": total_timesteps,
|
||||
"best_eval": best_eval,
|
||||
}, os.path.join("models", "ppo_improved_best.pt"))
|
||||
print(f" New best model saved! eval={best_eval:.2f}")
|
||||
|
||||
if episode % save_interval == 0:
|
||||
os.makedirs("models", exist_ok=True)
|
||||
torch.save({
|
||||
"actor": actor.state_dict(),
|
||||
"critic": critic.state_dict(),
|
||||
"episode": episode,
|
||||
"timesteps": total_timesteps,
|
||||
}, os.path.join("models", f"ppo_improved_ep{episode}.pt"))
|
||||
print(f" Saved model at episode {episode}")
|
||||
|
||||
os.makedirs("models", exist_ok=True)
|
||||
torch.save({
|
||||
"actor": actor.state_dict(),
|
||||
"critic": critic.state_dict(),
|
||||
"episode": episode,
|
||||
"timesteps": total_timesteps,
|
||||
"best_eval": best_eval,
|
||||
}, os.path.join("models", "ppo_improved_final.pt"))
|
||||
|
||||
writer.close()
|
||||
env.close()
|
||||
eval_env.close()
|
||||
print(f"Training complete! Total episodes: {episode}, Best eval: {best_eval:.2f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--steps", type=int, default=2000000, help="Total training steps")
|
||||
parser.add_argument("--rollout", type=int, default=2048, help="Rollout buffer size")
|
||||
args = parser.parse_args()
|
||||
|
||||
device = get_device()
|
||||
train_improved(total_steps=args.steps, rollout_steps=args.rollout, device=device)
|
||||
Generated
+1798
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user