OPENAI-Baeslines-详解（三）-DDPG中文

network, env,
seed=None,
 
 # 总步数 和总回合数 只能存在一个
 # 若两个都不存在，那么epoch为500 
total_timesteps=None,             # 总步数
nb_epochs=None,                   # 总回合数 
nb_epoch_cycles=20,               # 每个回合的循环运行次数
nb_rollout_steps=100,             # 每个回合与环境交互的次数
nb_train_steps=50,                # 每个回合训练次数
nb_eval_steps=100,                # 每个回合的评估次数
eval_env=None, 			    # 在每个回合训练完成之后，开始测试环境的步数。

render=False,                     # 是否显示交互
render_eval=False,
noise_type='adaptive-param_0.2',

for epoch in range(nb_epochs):   
    for cycle in range(nb_epoch_cycles):

        # reset环境
        if nenvs > 1:     
            agent.reset()
        for t_rollout in range(nb_rollout_steps):   
            # 输出动作
            action, q, _, _ = agent.step(obs, apply_noise=True, compute_Q=True)
         	  # 动作都是归一化在-1到1之间
            new_obs, r, done, info = env.step(max_action * action)  

            t += 1
            if rank == 0 and render:
                env.render()
            episode_reward += r
            episode_step += 1

            # 存进memory
            epoch_actions.append(action)
            epoch_qs.append(q)
            agent.store_transition(obs, action, r, new_obs, done) 
            # 新旧 状态更新
            obs = new_obs
            for d in range(len(done)):  # 对每一个agent进行reset
                if done[d]:
                    if nenvs == 1:
                        agent.reset()

for t_train in range(nb_train_steps):
   	# 噪声更新
	if memory.nb_entries >= batch_size and t_train % param_noise_adaption_interval == 0:
      distance = agent.adapt_param_noise()
      epoch_adaptive_distances.append(distance)
     # agent 训练
   	cl, al = agent.train()

target_actor = copy(actor)          
target_actor.name = 'target_actor'
self.target_actor = target_actor

target_critic = copy(critic)
target_critic.name = 'target_critic'
self.target_critic = target_critic

# actor
self.actor_tf = actor(normalized_obs0)
# critic 输入中的动作位置 为placeholder  
self.normalized_critic_tf = critic(normalized_obs0, self.actions)
self.critic_tf = denormalize(tf.clip_by_value(self.normalized_critic_tf,self.return_range[0], self.return_range[1]), self.ret_rms)

# critic 输入中的动作位置 为actor的输出
self.normalized_critic_with_actor_tf = critic(normalized_obs0, self.actor_tf, reuse=True)
self.critic_with_actor_tf =denormalize(tf.clip_by_value(self.normalized_critic_with_actor_tf, self.return_range[0], self.return_range[1]), self.ret_rms)

# target Q值计算
Q_obs1 = denormalize(target_critic(normalized_obs1, target_actor(normalized_obs1)), self.ret_rms)
self.target_Q = self.rewards + (1. - self.terminals1) * gamma * Q_obs1

def step(self, obs, apply_noise=True, compute_Q=True):
	feed_dict = {self.obs0: U.adjust_shape(self.obs0, [obs])}# 送入数据
	# 利用网络计算动作和Q值 
      action, q = self.sess.run([actor_tf, self.critic_with_actor_tf],	feed_dict=feed_dict)  
      # 之后是为了增加噪声
      noise = self.action_noise()
	action += noise
      action = np.clip(action, self.action_range[0], self.action_range[1])

def setup_actor_optimizer(self):
	self.actor_loss = -tf.reduce_mean(self.critic_with_actor_tf)# Q值
	self.actor_grads = U.flatgrad(self.actor_loss, self.actor.trainable_vars, clip_norm=self.clip_norm) # 计算梯度
      self.actor_optimizer = MpiAdam(var_list=self.actor.trainable_vars,beta1=0.9, beta2=0.999, epsilon=1e-08)

def setup_critic_optimizer(self):
 	normalized_critic_target_tf = tf.clip_by_value(normalize(self.critic_target, self.ret_rms), self.return_range[0], self.return_range[1])
	self.critic_loss = tf.reduce_mean(tf.square(self.normalized_critic_tf - normalized_critic_target_tf))
	# 187-196 在这里会对critic的loss 增加 l2 约束。
      self.critic_grads = U.flatgrad(self.critic_loss, self.critic.trainable_vars, clip_norm=self.clip_norm)  # 计算梯度 
      self.critic_optimizer = MpiAdam(var_list=self.critic.trainable_vars,
            beta1=0.9, beta2=0.999, epsilon=1e-08) # 反向训练      

def train(self):
	# 经验池随机采样
	batch = self.memory.sample(batch_size=self.batch_size)
	ops = [self.actor_grads, self.actor_loss, self.critic_grads, self.critic_loss]
      # 根据采样数据重新计算Q值等。
      actor_grads, actor_loss, critic_grads, critic_loss = self.sess.run(ops, feed_dict={
            self.obs0: batch['obs0'],
            self.actions: batch['actions'],
            self.critic_target: target_Q,
        })
      # 训练328 行
	self.actor_optimizer.update(actor_grads, stepsize=self.actor_lr)
      self.critic_optimizer.update(critic_grads, stepsize=self.critic_lr)

def setup_param_noise(self, normalized_obs0):
	param_noise_actor = copy(self.actor)
     self.perturbed_actor_tf = param_noise_actor(normalized_obs0)
     

#50 行
def get_perturbed_actor_updates(actor, perturbed_actor, param_noise_stddev):
	# 增加均值为零 方差为param_noise_stddev的 高斯噪声
	updates.append(tf.assign(perturbed_var, var + tf.random_normal(tf.shape(var), mean=0., stddev=param_noise_stddev)))

#259行 
if self.param_noise is not None and apply_noise:
    actor_tf = self.perturbed_actor_tf  # 注意 这里只选择了参数固定的噪声。
else:
    actor_tf = self.actor_tf

# 初始化 将所有网络初始化、优化器初始化、硬更新一次target网络
def initialize(self, sess):
# 软更新target_net
def update_target_net(self):

# 通过从 数据库中采样数据并得到所有结果的函数
def setup_stats(self):
def get_stats(self):