110 KiB
110 KiB
In [1]:
import sys,os
from pathlib import Path
curr_path = str(Path().absolute())
parent_path = str(Path().absolute().parent)
sys.path.append(parent_path) # add current terminal path to sys.path
import gym
import torch
import numpy as np
import datetime
from HierarchicalDQN.agent import HierarchicalDQN
from common.plot import plot_rewards
from common.utils import save_resultsIn [2]:
SEQUENCE = datetime.datetime.now().strftime(
"%Y%m%d-%H%M%S") # obtain current time
SAVED_MODEL_PATH = curr_path+"/saved_model/"+SEQUENCE+'/' # path to save model
if not os.path.exists(curr_path+"/saved_model/"):
os.mkdir(curr_path+"/saved_model/")
if not os.path.exists(SAVED_MODEL_PATH):
os.mkdir(SAVED_MODEL_PATH)
RESULT_PATH = curr_path+"/results/"+SEQUENCE+'/' # path to save rewards
if not os.path.exists(curr_path+"/results/"):
os.mkdir(curr_path+"/results/")
if not os.path.exists(RESULT_PATH):
os.mkdir(RESULT_PATH)In [3]:
class HierarchicalDQNConfig:
def __init__(self):
self.algo = "H-DQN" # name of algo
self.gamma = 0.95
self.epsilon_start = 1 # start epsilon of e-greedy policy
self.epsilon_end = 0.01
self.epsilon_decay = 500
self.lr = 0.0001 # learning rate
self.memory_capacity = 20000 # Replay Memory capacity
self.batch_size = 64
self.train_eps = 300 # 训练的episode数目
self.target_update = 2 # target net的更新频率
self.eval_eps = 20 # 测试的episode数目
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu") # 检测gpu
self.hidden_dim = 256 # dimension of hidden layerIn [4]:
def train(cfg, env, agent):
print('Start to train !')
rewards = []
ma_rewards = [] # moveing average reward
for i_episode in range(cfg.train_eps):
state = env.reset()
done = False
ep_reward = 0
while not done:
goal = agent.set_goal(state)
onehot_goal = agent.to_onehot(goal)
meta_state = state
extrinsic_reward = 0
while not done and goal != np.argmax(state):
goal_state = np.concatenate([state, onehot_goal])
action = agent.choose_action(goal_state)
next_state, reward, done, _ = env.step(action)
ep_reward += reward
extrinsic_reward += reward
intrinsic_reward = 1.0 if goal == np.argmax(
next_state) else 0.0
agent.memory.push(goal_state, action, intrinsic_reward, np.concatenate(
[next_state, onehot_goal]), done)
state = next_state
agent.update()
agent.meta_memory.push(meta_state, goal, extrinsic_reward, state, done)
print('Episode:{}/{}, Reward:{}'.format(i_episode+1, cfg.train_eps, ep_reward))
rewards.append(ep_reward)
if ma_rewards:
ma_rewards.append(
0.9*ma_rewards[-1]+0.1*ep_reward)
else:
ma_rewards.append(ep_reward)
print('Complete training！')
return rewards, ma_rewardsIn [5]:
env = gym.make('CartPole-v0')
env.seed(1)
cfg = HierarchicalDQNConfig()
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.n
agent = HierarchicalDQN(state_dim, action_dim, cfg)
rewards, ma_rewards = train(cfg, env, agent)
agent.save(path=SAVED_MODEL_PATH)
save_results(rewards, ma_rewards, tag='train', path=RESULT_PATH)
plot_rewards(rewards, ma_rewards, tag="train",
algo=cfg.algo, path=RESULT_PATH)Start to train ! Episode:1/300, Reward:25.0 Episode:2/300, Reward:26.0 Episode:3/300, Reward:23.0 Episode:4/300, Reward:19.0 Episode:5/300, Reward:23.0 Episode:6/300, Reward:21.0 Episode:7/300, Reward:21.0 Episode:8/300, Reward:22.0 Episode:9/300, Reward:15.0 Episode:10/300, Reward:12.0 Episode:11/300, Reward:39.0 Episode:12/300, Reward:42.0 Episode:13/300, Reward:79.0 Episode:14/300, Reward:54.0 Episode:15/300, Reward:28.0 Episode:16/300, Reward:85.0 Episode:17/300, Reward:46.0 Episode:18/300, Reward:37.0 Episode:19/300, Reward:45.0 Episode:20/300, Reward:79.0 Episode:21/300, Reward:80.0 Episode:22/300, Reward:154.0 Episode:23/300, Reward:74.0 Episode:24/300, Reward:129.0 Episode:25/300, Reward:185.0 Episode:26/300, Reward:200.0 Episode:27/300, Reward:115.0 Episode:28/300, Reward:104.0 Episode:29/300, Reward:200.0 Episode:30/300, Reward:118.0 Episode:31/300, Reward:200.0 Episode:32/300, Reward:200.0 Episode:33/300, Reward:83.0 Episode:34/300, Reward:75.0 Episode:35/300, Reward:46.0 Episode:36/300, Reward:96.0 Episode:37/300, Reward:78.0 Episode:38/300, Reward:150.0 Episode:39/300, Reward:147.0 Episode:40/300, Reward:74.0 Episode:41/300, Reward:137.0 Episode:42/300, Reward:182.0 Episode:43/300, Reward:200.0 Episode:44/300, Reward:200.0 Episode:45/300, Reward:200.0 Episode:46/300, Reward:184.0 Episode:47/300, Reward:200.0 Episode:48/300, Reward:200.0 Episode:49/300, Reward:200.0 Episode:50/300, Reward:61.0 Episode:51/300, Reward:9.0 Episode:52/300, Reward:9.0 Episode:53/300, Reward:200.0 Episode:54/300, Reward:200.0 Episode:55/300, Reward:200.0 Episode:56/300, Reward:200.0 Episode:57/300, Reward:200.0 Episode:58/300, Reward:200.0 Episode:59/300, Reward:200.0 Episode:60/300, Reward:167.0 Episode:61/300, Reward:200.0 Episode:62/300, Reward:200.0 Episode:63/300, Reward:200.0 Episode:64/300, Reward:200.0 Episode:65/300, Reward:200.0 Episode:66/300, Reward:200.0 Episode:67/300, Reward:200.0 Episode:68/300, Reward:200.0 Episode:69/300, Reward:197.0 Episode:70/300, Reward:200.0 Episode:71/300, Reward:200.0 Episode:72/300, Reward:200.0 Episode:73/300, Reward:200.0 Episode:74/300, Reward:200.0 Episode:75/300, Reward:200.0 Episode:76/300, Reward:200.0 Episode:77/300, Reward:200.0 Episode:78/300, Reward:200.0 Episode:79/300, Reward:200.0 Episode:80/300, Reward:200.0 Episode:81/300, Reward:181.0 Episode:82/300, Reward:200.0 Episode:83/300, Reward:200.0 Episode:84/300, Reward:200.0 Episode:85/300, Reward:200.0 Episode:86/300, Reward:200.0 Episode:87/300, Reward:200.0 Episode:88/300, Reward:200.0 Episode:89/300, Reward:200.0 Episode:90/300, Reward:200.0 Episode:91/300, Reward:200.0 Episode:92/300, Reward:200.0 Episode:93/300, Reward:200.0 Episode:94/300, Reward:200.0 Episode:95/300, Reward:200.0 Episode:96/300, Reward:200.0 Episode:97/300, Reward:200.0 Episode:98/300, Reward:200.0 Episode:99/300, Reward:192.0 Episode:100/300, Reward:183.0 Episode:101/300, Reward:200.0 Episode:102/300, Reward:200.0 Episode:103/300, Reward:200.0 Episode:104/300, Reward:200.0 Episode:105/300, Reward:200.0 Episode:106/300, Reward:200.0 Episode:107/300, Reward:200.0 Episode:108/300, Reward:200.0 Episode:109/300, Reward:200.0 Episode:110/300, Reward:200.0 Episode:111/300, Reward:200.0 Episode:112/300, Reward:200.0 Episode:113/300, Reward:200.0 Episode:114/300, Reward:200.0 Episode:115/300, Reward:200.0 Episode:116/300, Reward:200.0 Episode:117/300, Reward:200.0 Episode:118/300, Reward:200.0 Episode:119/300, Reward:200.0 Episode:120/300, Reward:196.0 Episode:121/300, Reward:200.0 Episode:122/300, Reward:200.0 Episode:123/300, Reward:200.0 Episode:124/300, Reward:200.0 Episode:125/300, Reward:200.0 Episode:126/300, Reward:189.0 Episode:127/300, Reward:193.0 Episode:128/300, Reward:200.0 Episode:129/300, Reward:200.0 Episode:130/300, Reward:193.0 Episode:131/300, Reward:183.0 Episode:132/300, Reward:183.0 Episode:133/300, Reward:200.0 Episode:134/300, Reward:200.0 Episode:135/300, Reward:200.0 Episode:136/300, Reward:200.0 Episode:137/300, Reward:200.0 Episode:138/300, Reward:200.0 Episode:139/300, Reward:100.0 Episode:140/300, Reward:118.0 Episode:141/300, Reward:99.0 Episode:142/300, Reward:185.0 Episode:143/300, Reward:41.0 Episode:144/300, Reward:11.0 Episode:145/300, Reward:9.0 Episode:146/300, Reward:152.0 Episode:147/300, Reward:155.0 Episode:148/300, Reward:181.0 Episode:149/300, Reward:197.0 Episode:150/300, Reward:200.0 Episode:151/300, Reward:200.0 Episode:152/300, Reward:200.0 Episode:153/300, Reward:200.0 Episode:154/300, Reward:200.0 Episode:155/300, Reward:200.0 Episode:156/300, Reward:123.0 Episode:157/300, Reward:11.0 Episode:158/300, Reward:8.0 Episode:159/300, Reward:9.0 Episode:160/300, Reward:10.0 Episode:161/300, Reward:9.0 Episode:162/300, Reward:10.0 Episode:163/300, Reward:9.0 Episode:164/300, Reward:9.0 Episode:165/300, Reward:10.0 Episode:166/300, Reward:9.0 Episode:167/300, Reward:9.0 Episode:168/300, Reward:9.0 Episode:169/300, Reward:9.0 Episode:170/300, Reward:10.0 Episode:171/300, Reward:9.0 Episode:172/300, Reward:9.0 Episode:173/300, Reward:11.0 Episode:174/300, Reward:11.0 Episode:175/300, Reward:10.0 Episode:176/300, Reward:9.0 Episode:177/300, Reward:10.0 Episode:178/300, Reward:8.0 Episode:179/300, Reward:9.0 Episode:180/300, Reward:9.0 Episode:181/300, Reward:10.0 Episode:182/300, Reward:10.0 Episode:183/300, Reward:9.0 Episode:184/300, Reward:10.0 Episode:185/300, Reward:10.0 Episode:186/300, Reward:13.0 Episode:187/300, Reward:16.0 Episode:188/300, Reward:117.0 Episode:189/300, Reward:13.0 Episode:190/300, Reward:16.0 Episode:191/300, Reward:11.0 Episode:192/300, Reward:11.0 Episode:193/300, Reward:13.0 Episode:194/300, Reward:13.0 Episode:195/300, Reward:9.0 Episode:196/300, Reward:20.0 Episode:197/300, Reward:12.0 Episode:198/300, Reward:10.0 Episode:199/300, Reward:14.0 Episode:200/300, Reward:12.0 Episode:201/300, Reward:14.0 Episode:202/300, Reward:12.0 Episode:203/300, Reward:11.0 Episode:204/300, Reward:10.0 Episode:205/300, Reward:13.0 Episode:206/300, Reward:10.0 Episode:207/300, Reward:10.0 Episode:208/300, Reward:13.0 Episode:209/300, Reward:9.0 Episode:210/300, Reward:11.0 Episode:211/300, Reward:14.0 Episode:212/300, Reward:10.0 Episode:213/300, Reward:20.0 Episode:214/300, Reward:12.0 Episode:215/300, Reward:13.0 Episode:216/300, Reward:17.0 Episode:217/300, Reward:17.0 Episode:218/300, Reward:11.0 Episode:219/300, Reward:15.0 Episode:220/300, Reward:26.0 Episode:221/300, Reward:73.0 Episode:222/300, Reward:44.0 Episode:223/300, Reward:48.0 Episode:224/300, Reward:102.0 Episode:225/300, Reward:162.0 Episode:226/300, Reward:123.0 Episode:227/300, Reward:200.0 Episode:228/300, Reward:200.0 Episode:229/300, Reward:120.0 Episode:230/300, Reward:173.0 Episode:231/300, Reward:138.0 Episode:232/300, Reward:106.0 Episode:233/300, Reward:193.0 Episode:234/300, Reward:117.0 Episode:235/300, Reward:120.0 Episode:236/300, Reward:98.0 Episode:237/300, Reward:98.0 Episode:238/300, Reward:200.0 Episode:239/300, Reward:96.0 Episode:240/300, Reward:170.0 Episode:241/300, Reward:107.0 Episode:242/300, Reward:107.0 Episode:243/300, Reward:200.0 Episode:244/300, Reward:128.0 Episode:245/300, Reward:165.0 Episode:246/300, Reward:168.0 Episode:247/300, Reward:200.0 Episode:248/300, Reward:200.0 Episode:249/300, Reward:200.0 Episode:250/300, Reward:200.0 Episode:251/300, Reward:200.0 Episode:252/300, Reward:200.0 Episode:253/300, Reward:200.0 Episode:254/300, Reward:200.0 Episode:255/300, Reward:200.0 Episode:256/300, Reward:200.0 Episode:257/300, Reward:164.0 Episode:258/300, Reward:200.0 Episode:259/300, Reward:190.0 Episode:260/300, Reward:185.0 Episode:261/300, Reward:200.0 Episode:262/300, Reward:200.0 Episode:263/300, Reward:200.0 Episode:264/300, Reward:200.0 Episode:265/300, Reward:168.0 Episode:266/300, Reward:200.0 Episode:267/300, Reward:200.0 Episode:268/300, Reward:200.0 Episode:269/300, Reward:200.0 Episode:270/300, Reward:200.0 Episode:271/300, Reward:200.0 Episode:272/300, Reward:200.0 Episode:273/300, Reward:200.0 Episode:274/300, Reward:200.0 Episode:275/300, Reward:188.0 Episode:276/300, Reward:200.0 Episode:277/300, Reward:177.0 Episode:278/300, Reward:200.0 Episode:279/300, Reward:200.0 Episode:280/300, Reward:200.0 Episode:281/300, Reward:200.0 Episode:282/300, Reward:200.0 Episode:283/300, Reward:200.0 Episode:284/300, Reward:189.0 Episode:285/300, Reward:200.0 Episode:286/300, Reward:200.0 Episode:287/300, Reward:200.0 Episode:288/300, Reward:200.0 Episode:289/300, Reward:200.0 Episode:290/300, Reward:200.0 Episode:291/300, Reward:200.0 Episode:292/300, Reward:200.0 Episode:293/300, Reward:200.0 Episode:294/300, Reward:200.0 Episode:295/300, Reward:200.0 Episode:296/300, Reward:200.0 Episode:297/300, Reward:200.0 Episode:298/300, Reward:200.0 Episode:299/300, Reward:200.0 Episode:300/300, Reward:200.0 Complete training！ results saved!
