56 lines
1.6 KiB
Python
56 lines
1.6 KiB
Python
#!/usr/bin/env python
|
|
# coding=utf-8
|
|
'''
|
|
Author: JiangJi
|
|
Email: johnjim0816@gmail.com
|
|
Date: 2021-05-03 22:16:08
|
|
LastEditor: JiangJi
|
|
LastEditTime: 2022-07-20 23:54:40
|
|
Discription:
|
|
Environment:
|
|
'''
|
|
import torch
|
|
import torch.optim as optim
|
|
import torch.nn as nn
|
|
import torch.nn.functional as F
|
|
from torch.distributions import Categorical
|
|
|
|
class ActorCritic(nn.Module):
|
|
''' A2C网络模型，包含一个Actor和Critic
|
|
'''
|
|
def __init__(self, input_dim, output_dim, hidden_dim):
|
|
super(ActorCritic, self).__init__()
|
|
self.critic = nn.Sequential(
|
|
nn.Linear(input_dim, hidden_dim),
|
|
nn.ReLU(),
|
|
nn.Linear(hidden_dim, 1)
|
|
)
|
|
|
|
self.actor = nn.Sequential(
|
|
nn.Linear(input_dim, hidden_dim),
|
|
nn.ReLU(),
|
|
nn.Linear(hidden_dim, output_dim),
|
|
nn.Softmax(dim=1),
|
|
)
|
|
|
|
def forward(self, x):
|
|
value = self.critic(x)
|
|
probs = self.actor(x)
|
|
dist = Categorical(probs)
|
|
return dist, value
|
|
class A2C:
|
|
''' A2C算法
|
|
'''
|
|
def __init__(self,n_states,n_actions,cfg) -> None:
|
|
self.gamma = cfg.gamma
|
|
self.device = torch.device(cfg.device)
|
|
self.model = ActorCritic(n_states, n_actions, cfg.hidden_size).to(self.device)
|
|
self.optimizer = optim.Adam(self.model.parameters())
|
|
|
|
def compute_returns(self,next_value, rewards, masks):
|
|
R = next_value
|
|
returns = []
|
|
for step in reversed(range(len(rewards))):
|
|
R = rewards[step] + self.gamma * R * masks[step]
|
|
returns.insert(0, R)
|
|
return returns |