easy-rl/codes/DQN/main.ipynb

In [1]:

import sys,os
from pathlib import Path
curr_path = str(Path().absolute())
parent_path = str(Path().absolute().parent)
sys.path.append(parent_path) # add current terminal path to sys.path
import gym
import torch
import datetime
from DQN.agent import DQN
from common.plot import plot_rewards
from common.utils import save_results

In [2]:

SEQUENCE = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") # 获取当前时间
SAVED_MODEL_PATH = curr_path+"/saved_model/"+SEQUENCE+'/' # 生成保存的模型路径
if not os.path.exists(curr_path+"/saved_model/"): # 检测是否存在文件夹
    os.mkdir(curr_path+"/saved_model/")
if not os.path.exists(SAVED_MODEL_PATH): # 检测是否存在文件夹
    os.mkdir(SAVED_MODEL_PATH)
RESULT_PATH = curr_path+"/results/"+SEQUENCE+'/' # 存储reward的路径
if not os.path.exists(curr_path+"/results/"): # 检测是否存在文件夹
    os.mkdir(curr_path+"/results/")
if not os.path.exists(RESULT_PATH): # 检测是否存在文件夹
    os.mkdir(RESULT_PATH)

In [3]:

class DQNConfig:
    def __init__(self):
        self.algo = "DQN" # 算法名称
        self.gamma = 0.99
        self.epsilon_start = 0.95 # e-greedy策略的初始epsilon
        self.epsilon_end = 0.01
        self.epsilon_decay = 200
        self.lr = 0.01 # 学习率
        self.memory_capacity = 800 # Replay Memory容量
        self.batch_size = 64
        self.train_eps = 300 # 训练的episode数目
        self.train_steps = 200 # 训练每个episode的最大长度
        self.target_update = 2 # target net的更新频率
        self.eval_eps = 20 # 测试的episode数目
        self.eval_steps = 200 # 测试每个episode的最大长度
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # 检测gpu
        self.hidden_dim = 128 # 神经网络隐藏层维度

In [4]:

def train(cfg,env,agent):
    print('Start to train !')
    rewards = []
    ma_rewards = [] # 滑动平均的reward
    ep_steps = []
    for i_episode in range(cfg.train_eps):
        state = env.reset() # reset环境状态
        ep_reward = 0
        for i_step in range(cfg.train_steps):
            action = agent.choose_action(state) # 根据当前环境state选择action
            next_state, reward, done, _ = env.step(action) # 更新环境参数
            ep_reward += reward
            agent.memory.push(state, action, reward, next_state, done) # 将state等这些transition存入memory
            state = next_state # 跳转到下一个状态
            agent.update() # 每步更新网络
            if done:
                break
        # 更新target network，复制DQN中的所有weights and biases
        if i_episode % cfg.target_update == 0:
            agent.target_net.load_state_dict(agent.policy_net.state_dict())
        print('Episode:{}/{}, Reward:{}, Steps:{}, Done:{}'.format(i_episode+1,cfg.train_eps,ep_reward,i_step+1,done))
        ep_steps.append(i_step)
        rewards.append(ep_reward)
        # 计算滑动窗口的reward
        if ma_rewards:
            ma_rewards.append(
                0.9*ma_rewards[-1]+0.1*ep_reward)
        else:
            ma_rewards.append(ep_reward)   
    print('Complete training！')
    return rewards,ma_rewards

In [5]:

cfg = DQNConfig()
env = gym.make('CartPole-v0')
env.seed(1)
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.n
agent = DQN(state_dim,action_dim,cfg)
rewards,ma_rewards = train(cfg,env,agent)
agent.save(path=SAVED_MODEL_PATH)
save_results(rewards,ma_rewards,tag='train',path=RESULT_PATH)
plot_rewards(rewards,ma_rewards,tag="train",algo = cfg.algo,path=RESULT_PATH)

Start to train !
Episode:1/300, Reward:41.0, Steps:41, Done:True
Episode:2/300, Reward:23.0, Steps:23, Done:True
Episode:3/300, Reward:19.0, Steps:19, Done:True
Episode:4/300, Reward:17.0, Steps:17, Done:True
Episode:5/300, Reward:14.0, Steps:14, Done:True
Episode:6/300, Reward:15.0, Steps:15, Done:True
Episode:7/300, Reward:10.0, Steps:10, Done:True
Episode:8/300, Reward:23.0, Steps:23, Done:True
Episode:9/300, Reward:14.0, Steps:14, Done:True
Episode:10/300, Reward:9.0, Steps:9, Done:True
Episode:11/300, Reward:9.0, Steps:9, Done:True
Episode:12/300, Reward:9.0, Steps:9, Done:True
Episode:13/300, Reward:9.0, Steps:9, Done:True
Episode:14/300, Reward:10.0, Steps:10, Done:True
Episode:15/300, Reward:10.0, Steps:10, Done:True
Episode:16/300, Reward:12.0, Steps:12, Done:True
Episode:17/300, Reward:10.0, Steps:10, Done:True
Episode:18/300, Reward:10.0, Steps:10, Done:True
Episode:19/300, Reward:9.0, Steps:9, Done:True
Episode:20/300, Reward:10.0, Steps:10, Done:True
Episode:21/300, Reward:8.0, Steps:8, Done:True
Episode:22/300, Reward:10.0, Steps:10, Done:True
Episode:23/300, Reward:10.0, Steps:10, Done:True
Episode:24/300, Reward:13.0, Steps:13, Done:True
Episode:25/300, Reward:10.0, Steps:10, Done:True
Episode:26/300, Reward:9.0, Steps:9, Done:True
Episode:27/300, Reward:9.0, Steps:9, Done:True
Episode:28/300, Reward:9.0, Steps:9, Done:True
Episode:29/300, Reward:12.0, Steps:12, Done:True
Episode:30/300, Reward:9.0, Steps:9, Done:True
Episode:31/300, Reward:9.0, Steps:9, Done:True
Episode:32/300, Reward:10.0, Steps:10, Done:True
Episode:33/300, Reward:11.0, Steps:11, Done:True
Episode:34/300, Reward:12.0, Steps:12, Done:True
Episode:35/300, Reward:8.0, Steps:8, Done:True
Episode:36/300, Reward:10.0, Steps:10, Done:True
Episode:37/300, Reward:9.0, Steps:9, Done:True
Episode:38/300, Reward:10.0, Steps:10, Done:True
Episode:39/300, Reward:10.0, Steps:10, Done:True
Episode:40/300, Reward:10.0, Steps:10, Done:True
Episode:41/300, Reward:9.0, Steps:9, Done:True
Episode:42/300, Reward:10.0, Steps:10, Done:True
Episode:43/300, Reward:10.0, Steps:10, Done:True
Episode:44/300, Reward:10.0, Steps:10, Done:True
Episode:45/300, Reward:9.0, Steps:9, Done:True
Episode:46/300, Reward:22.0, Steps:22, Done:True
Episode:47/300, Reward:74.0, Steps:74, Done:True
Episode:48/300, Reward:13.0, Steps:13, Done:True
Episode:49/300, Reward:29.0, Steps:29, Done:True
Episode:50/300, Reward:56.0, Steps:56, Done:True
Episode:51/300, Reward:74.0, Steps:74, Done:True
Episode:52/300, Reward:85.0, Steps:85, Done:True
Episode:53/300, Reward:72.0, Steps:72, Done:True
Episode:54/300, Reward:114.0, Steps:114, Done:True
Episode:55/300, Reward:97.0, Steps:97, Done:True
Episode:56/300, Reward:101.0, Steps:101, Done:True
Episode:57/300, Reward:104.0, Steps:104, Done:True
Episode:58/300, Reward:58.0, Steps:58, Done:True
Episode:59/300, Reward:11.0, Steps:11, Done:True
Episode:60/300, Reward:56.0, Steps:56, Done:True
Episode:61/300, Reward:74.0, Steps:74, Done:True
Episode:62/300, Reward:51.0, Steps:51, Done:True
Episode:63/300, Reward:113.0, Steps:113, Done:True
Episode:64/300, Reward:48.0, Steps:48, Done:True
Episode:65/300, Reward:97.0, Steps:97, Done:True
Episode:66/300, Reward:59.0, Steps:59, Done:True
Episode:67/300, Reward:200.0, Steps:200, Done:True
Episode:68/300, Reward:67.0, Steps:67, Done:True
Episode:69/300, Reward:200.0, Steps:200, Done:True
Episode:70/300, Reward:45.0, Steps:45, Done:True
Episode:71/300, Reward:48.0, Steps:48, Done:True
Episode:72/300, Reward:90.0, Steps:90, Done:True
Episode:73/300, Reward:47.0, Steps:47, Done:True
Episode:74/300, Reward:94.0, Steps:94, Done:True
Episode:75/300, Reward:107.0, Steps:107, Done:True
Episode:76/300, Reward:12.0, Steps:12, Done:True
Episode:77/300, Reward:30.0, Steps:30, Done:True
Episode:78/300, Reward:62.0, Steps:62, Done:True
Episode:79/300, Reward:64.0, Steps:64, Done:True
Episode:80/300, Reward:41.0, Steps:41, Done:True
Episode:81/300, Reward:67.0, Steps:67, Done:True
Episode:82/300, Reward:45.0, Steps:45, Done:True
Episode:83/300, Reward:130.0, Steps:130, Done:True
Episode:84/300, Reward:50.0, Steps:50, Done:True
Episode:85/300, Reward:51.0, Steps:51, Done:True
Episode:86/300, Reward:67.0, Steps:67, Done:True
Episode:87/300, Reward:37.0, Steps:37, Done:True
Episode:88/300, Reward:41.0, Steps:41, Done:True
Episode:89/300, Reward:54.0, Steps:54, Done:True
Episode:90/300, Reward:93.0, Steps:93, Done:True
Episode:91/300, Reward:71.0, Steps:71, Done:True
Episode:92/300, Reward:102.0, Steps:102, Done:True
Episode:93/300, Reward:55.0, Steps:55, Done:True
Episode:94/300, Reward:73.0, Steps:73, Done:True
Episode:95/300, Reward:61.0, Steps:61, Done:True
Episode:96/300, Reward:16.0, Steps:16, Done:True
Episode:97/300, Reward:61.0, Steps:61, Done:True
Episode:98/300, Reward:79.0, Steps:79, Done:True
Episode:99/300, Reward:76.0, Steps:76, Done:True
Episode:100/300, Reward:32.0, Steps:32, Done:True
Episode:101/300, Reward:95.0, Steps:95, Done:True
Episode:102/300, Reward:83.0, Steps:83, Done:True
Episode:103/300, Reward:41.0, Steps:41, Done:True
Episode:104/300, Reward:30.0, Steps:30, Done:True
Episode:105/300, Reward:83.0, Steps:83, Done:True
Episode:106/300, Reward:95.0, Steps:95, Done:True
Episode:107/300, Reward:104.0, Steps:104, Done:True
Episode:108/300, Reward:98.0, Steps:98, Done:True
Episode:109/300, Reward:109.0, Steps:109, Done:True
Episode:110/300, Reward:63.0, Steps:63, Done:True
Episode:111/300, Reward:98.0, Steps:98, Done:True
Episode:112/300, Reward:105.0, Steps:105, Done:True
Episode:113/300, Reward:99.0, Steps:99, Done:True
Episode:114/300, Reward:200.0, Steps:200, Done:True
Episode:115/300, Reward:200.0, Steps:200, Done:True
Episode:116/300, Reward:47.0, Steps:47, Done:True
Episode:117/300, Reward:98.0, Steps:98, Done:True
Episode:118/300, Reward:200.0, Steps:200, Done:True
Episode:119/300, Reward:52.0, Steps:52, Done:True
Episode:120/300, Reward:55.0, Steps:55, Done:True
Episode:121/300, Reward:200.0, Steps:200, Done:True
Episode:122/300, Reward:200.0, Steps:200, Done:True
Episode:123/300, Reward:200.0, Steps:200, Done:True
Episode:124/300, Reward:200.0, Steps:200, Done:True
Episode:125/300, Reward:200.0, Steps:200, Done:True
Episode:126/300, Reward:40.0, Steps:40, Done:True
Episode:127/300, Reward:42.0, Steps:42, Done:True
Episode:128/300, Reward:101.0, Steps:101, Done:True
Episode:129/300, Reward:200.0, Steps:200, Done:True
Episode:130/300, Reward:70.0, Steps:70, Done:True
Episode:131/300, Reward:175.0, Steps:175, Done:True
Episode:132/300, Reward:90.0, Steps:90, Done:True
Episode:133/300, Reward:81.0, Steps:81, Done:True
Episode:134/300, Reward:61.0, Steps:61, Done:True
Episode:135/300, Reward:74.0, Steps:74, Done:True
Episode:136/300, Reward:68.0, Steps:68, Done:True
Episode:137/300, Reward:50.0, Steps:50, Done:True
Episode:138/300, Reward:51.0, Steps:51, Done:True
Episode:139/300, Reward:99.0, Steps:99, Done:True
Episode:140/300, Reward:87.0, Steps:87, Done:True
Episode:141/300, Reward:94.0, Steps:94, Done:True
Episode:142/300, Reward:51.0, Steps:51, Done:True
Episode:143/300, Reward:200.0, Steps:200, Done:True
Episode:144/300, Reward:55.0, Steps:55, Done:True
Episode:145/300, Reward:200.0, Steps:200, Done:True
Episode:146/300, Reward:57.0, Steps:57, Done:True
Episode:147/300, Reward:129.0, Steps:129, Done:True
Episode:148/300, Reward:74.0, Steps:74, Done:True
Episode:149/300, Reward:108.0, Steps:108, Done:True
Episode:150/300, Reward:63.0, Steps:63, Done:True
Episode:151/300, Reward:200.0, Steps:200, Done:True
Episode:152/300, Reward:103.0, Steps:103, Done:True
Episode:153/300, Reward:129.0, Steps:129, Done:True
Episode:154/300, Reward:77.0, Steps:77, Done:True
Episode:155/300, Reward:129.0, Steps:129, Done:True
Episode:156/300, Reward:200.0, Steps:200, Done:True
Episode:157/300, Reward:181.0, Steps:181, Done:True
Episode:158/300, Reward:200.0, Steps:200, Done:True
Episode:159/300, Reward:136.0, Steps:136, Done:True
Episode:160/300, Reward:200.0, Steps:200, Done:True
Episode:161/300, Reward:181.0, Steps:181, Done:True
Episode:162/300, Reward:120.0, Steps:120, Done:True
Episode:163/300, Reward:190.0, Steps:190, Done:True
Episode:164/300, Reward:200.0, Steps:200, Done:True
Episode:165/300, Reward:200.0, Steps:200, Done:True
Episode:166/300, Reward:200.0, Steps:200, Done:True
Episode:167/300, Reward:200.0, Steps:200, Done:True
Episode:168/300, Reward:200.0, Steps:200, Done:True
Episode:169/300, Reward:200.0, Steps:200, Done:True
Episode:170/300, Reward:89.0, Steps:89, Done:True
Episode:171/300, Reward:74.0, Steps:74, Done:True
Episode:172/300, Reward:200.0, Steps:200, Done:True
Episode:173/300, Reward:200.0, Steps:200, Done:True
Episode:174/300, Reward:200.0, Steps:200, Done:True
Episode:175/300, Reward:200.0, Steps:200, Done:True
Episode:176/300, Reward:93.0, Steps:93, Done:True
Episode:177/300, Reward:139.0, Steps:139, Done:True
Episode:178/300, Reward:78.0, Steps:78, Done:True
Episode:179/300, Reward:200.0, Steps:200, Done:True
Episode:180/300, Reward:200.0, Steps:200, Done:True
Episode:181/300, Reward:200.0, Steps:200, Done:True
Episode:182/300, Reward:200.0, Steps:200, Done:True
Episode:183/300, Reward:200.0, Steps:200, Done:True
Episode:184/300, Reward:200.0, Steps:200, Done:True
Episode:185/300, Reward:200.0, Steps:200, Done:True
Episode:186/300, Reward:200.0, Steps:200, Done:True
Episode:187/300, Reward:200.0, Steps:200, Done:True
Episode:188/300, Reward:200.0, Steps:200, Done:True
Episode:189/300, Reward:200.0, Steps:200, Done:True
Episode:190/300, Reward:200.0, Steps:200, Done:True
Episode:191/300, Reward:200.0, Steps:200, Done:True
Episode:192/300, Reward:200.0, Steps:200, Done:True
Episode:193/300, Reward:190.0, Steps:190, Done:True
Episode:194/300, Reward:200.0, Steps:200, Done:True
Episode:195/300, Reward:200.0, Steps:200, Done:True
Episode:196/300, Reward:200.0, Steps:200, Done:True
Episode:197/300, Reward:200.0, Steps:200, Done:True
Episode:198/300, Reward:200.0, Steps:200, Done:True
Episode:199/300, Reward:200.0, Steps:200, Done:True
Episode:200/300, Reward:200.0, Steps:200, Done:True
Episode:201/300, Reward:200.0, Steps:200, Done:True
Episode:202/300, Reward:200.0, Steps:200, Done:True
Episode:203/300, Reward:67.0, Steps:67, Done:True
Episode:204/300, Reward:200.0, Steps:200, Done:True
Episode:205/300, Reward:200.0, Steps:200, Done:True
Episode:206/300, Reward:200.0, Steps:200, Done:True
Episode:207/300, Reward:200.0, Steps:200, Done:True
Episode:208/300, Reward:200.0, Steps:200, Done:True
Episode:209/300, Reward:200.0, Steps:200, Done:True
Episode:210/300, Reward:200.0, Steps:200, Done:True
Episode:211/300, Reward:200.0, Steps:200, Done:True
Episode:212/300, Reward:200.0, Steps:200, Done:True
Episode:213/300, Reward:200.0, Steps:200, Done:True
Episode:214/300, Reward:200.0, Steps:200, Done:True
Episode:215/300, Reward:200.0, Steps:200, Done:True
Episode:216/300, Reward:200.0, Steps:200, Done:True
Episode:217/300, Reward:200.0, Steps:200, Done:True
Episode:218/300, Reward:44.0, Steps:44, Done:True
Episode:219/300, Reward:200.0, Steps:200, Done:True
Episode:220/300, Reward:200.0, Steps:200, Done:True
Episode:221/300, Reward:200.0, Steps:200, Done:True
Episode:222/300, Reward:200.0, Steps:200, Done:True
Episode:223/300, Reward:200.0, Steps:200, Done:True
Episode:224/300, Reward:200.0, Steps:200, Done:True
Episode:225/300, Reward:200.0, Steps:200, Done:True
Episode:226/300, Reward:200.0, Steps:200, Done:True
Episode:227/300, Reward:200.0, Steps:200, Done:True
Episode:228/300, Reward:200.0, Steps:200, Done:True
Episode:229/300, Reward:200.0, Steps:200, Done:True
Episode:230/300, Reward:200.0, Steps:200, Done:True
Episode:231/300, Reward:200.0, Steps:200, Done:True
Episode:232/300, Reward:200.0, Steps:200, Done:True
Episode:233/300, Reward:200.0, Steps:200, Done:True
Episode:234/300, Reward:200.0, Steps:200, Done:True
Episode:235/300, Reward:200.0, Steps:200, Done:True
Episode:236/300, Reward:200.0, Steps:200, Done:True
Episode:237/300, Reward:200.0, Steps:200, Done:True
Episode:238/300, Reward:200.0, Steps:200, Done:True
Episode:239/300, Reward:200.0, Steps:200, Done:True
Episode:240/300, Reward:200.0, Steps:200, Done:True
Episode:241/300, Reward:200.0, Steps:200, Done:True
Episode:242/300, Reward:126.0, Steps:126, Done:True
Episode:243/300, Reward:200.0, Steps:200, Done:True
Episode:244/300, Reward:200.0, Steps:200, Done:True
Episode:245/300, Reward:200.0, Steps:200, Done:True
Episode:246/300, Reward:200.0, Steps:200, Done:True
Episode:247/300, Reward:200.0, Steps:200, Done:True
Episode:248/300, Reward:118.0, Steps:118, Done:True
Episode:249/300, Reward:200.0, Steps:200, Done:True
Episode:250/300, Reward:200.0, Steps:200, Done:True
Episode:251/300, Reward:99.0, Steps:99, Done:True
Episode:252/300, Reward:145.0, Steps:145, Done:True
Episode:253/300, Reward:200.0, Steps:200, Done:True
Episode:254/300, Reward:200.0, Steps:200, Done:True
Episode:255/300, Reward:200.0, Steps:200, Done:True
Episode:256/300, Reward:200.0, Steps:200, Done:True
Episode:257/300, Reward:130.0, Steps:130, Done:True
Episode:258/300, Reward:170.0, Steps:170, Done:True
Episode:259/300, Reward:200.0, Steps:200, Done:True
Episode:260/300, Reward:200.0, Steps:200, Done:True
Episode:261/300, Reward:200.0, Steps:200, Done:True
Episode:262/300, Reward:200.0, Steps:200, Done:True
Episode:263/300, Reward:200.0, Steps:200, Done:True
Episode:264/300, Reward:200.0, Steps:200, Done:True
Episode:265/300, Reward:200.0, Steps:200, Done:True
Episode:266/300, Reward:200.0, Steps:200, Done:True
Episode:267/300, Reward:200.0, Steps:200, Done:True
Episode:268/300, Reward:200.0, Steps:200, Done:True
Episode:269/300, Reward:200.0, Steps:200, Done:True
Episode:270/300, Reward:200.0, Steps:200, Done:True
Episode:271/300, Reward:200.0, Steps:200, Done:True
Episode:272/300, Reward:135.0, Steps:135, Done:True
Episode:273/300, Reward:200.0, Steps:200, Done:True
Episode:274/300, Reward:200.0, Steps:200, Done:True
Episode:275/300, Reward:200.0, Steps:200, Done:True
Episode:276/300, Reward:200.0, Steps:200, Done:True
Episode:277/300, Reward:200.0, Steps:200, Done:True
Episode:278/300, Reward:200.0, Steps:200, Done:True
Episode:279/300, Reward:200.0, Steps:200, Done:True
Episode:280/300, Reward:200.0, Steps:200, Done:True
Episode:281/300, Reward:200.0, Steps:200, Done:True
Episode:282/300, Reward:200.0, Steps:200, Done:True
Episode:283/300, Reward:200.0, Steps:200, Done:True
Episode:284/300, Reward:200.0, Steps:200, Done:True
Episode:285/300, Reward:200.0, Steps:200, Done:True
Episode:286/300, Reward:200.0, Steps:200, Done:True
Episode:287/300, Reward:200.0, Steps:200, Done:True
Episode:288/300, Reward:200.0, Steps:200, Done:True
Episode:289/300, Reward:200.0, Steps:200, Done:True
Episode:290/300, Reward:200.0, Steps:200, Done:True
Episode:291/300, Reward:200.0, Steps:200, Done:True
Episode:292/300, Reward:200.0, Steps:200, Done:True
Episode:293/300, Reward:200.0, Steps:200, Done:True
Episode:294/300, Reward:200.0, Steps:200, Done:True
Episode:295/300, Reward:200.0, Steps:200, Done:True
Episode:296/300, Reward:200.0, Steps:200, Done:True
Episode:297/300, Reward:200.0, Steps:200, Done:True
Episode:298/300, Reward:200.0, Steps:200, Done:True
Episode:299/300, Reward:200.0, Steps:200, Done:True
Episode:300/300, Reward:200.0, Steps:200, Done:True
Complete training！
results saved!