# BipedalWalker-v2 one-step actor-critic agent trains slow

I'm trying to solve the OpenAI BipedalWalker-v2 by using a one-step actor-critic agent. I'm implementing the solution using python and tensorflow. My question is whether the code is slow because of the nature of the task or because the code is inefficient, or both.

I'm following this pseudo-code taken from the Book Reinforcement Learning An Introduction by Richard S. Sutton and Andrew G. Barto.

# My Agent Class

import tensorflow as tf
import numpy as np
import gym
import matplotlib.pyplot as plt

class agent_episodic_continuous_action():
def __init__(self, lr,gamma,sample_variance, s_size,a_size,dist_type):
self.gamma = gamma
self.I = 1
self.delta = 1
self.dist_type = dist_type
self.is_actor_brain_present = False
self.is_critic_brain_present = False
self.s_size = s_size
self.state_in= tf.placeholder(shape=[None,s_size],dtype=tf.float32)
self.a_size=a_size
self.cov = tf.eye(a_size)
self.reduction = sample_variance #0.01
self.weights_actor ={}
self.biases_actor ={}
self.weights_critic ={}
self.biases_critic ={}
self.time_step_info = {'s':0,'a':0,'r':0,'s1':0,'d':0}
self.reward = tf.placeholder(shape=[None,1],dtype=tf.float32)
if a_size > 1:
self.action_holder = tf.placeholder(shape=[None,a_size],dtype=tf.float32)
else:
self.action_holder = tf.placeholder(shape=[None],dtype=tf.float32)

def save_model(self,path,sess):
self.saver.save(sess, path)

self.saver.restore(sess, path)

def weights_init_actor(self,hidd_layer,mean,stddev):
num_input = self.s_size
num_output = self.a_size
n_hidden_1 = hidd_layer[0]
num_hidden_layers = len(hidd_layer)
self.weights_actor['h_{0}'.format(0)] = tf.Variable(tf.random_normal([num_input, n_hidden_1],mean=mean,stddev=stddev),name='actor')
self.biases_actor['b_{0}'.format(0)] = tf.Variable(tf.random_normal([n_hidden_1],mean=mean,stddev=stddev),name='actor')
for i in range(num_hidden_layers):
if i < num_hidden_layers-1:
num_input = n_hidden_1
n_hidden_1 = hidd_layer[i+1]
self.weights_actor['h_{0}'.format(i+1)] = tf.Variable(tf.random_normal([num_input, n_hidden_1],mean=mean,stddev=stddev),name='actor')
self.biases_actor['b_{0}'.format(i+1)] = tf.Variable(tf.random_normal([n_hidden_1],mean=mean,stddev=stddev),name='actor')
else:
self.weights_actor['h_{0}'.format("out")] = tf.Variable(tf.random_normal([n_hidden_1, num_output],mean=mean,stddev=stddev),name='actor')
self.biases_actor['b_{0}'.format("out")] = tf.Variable(tf.random_normal([num_output],mean=mean,stddev=stddev),name='actor')

def weights_init_critic(self,hidd_layer,mean,stddev):
num_input = self.s_size
# num_output = self.a_size
num_output = 1
n_hidden_1 = hidd_layer[0]
num_hidden_layers = len(hidd_layer)
self.weights_critic['h_{0}'.format(0)] = tf.Variable(tf.random_normal([num_input, n_hidden_1],mean=mean,stddev=stddev),name='critic')
self.biases_critic['b_{0}'.format(0)] = tf.Variable(tf.random_normal([n_hidden_1],mean=mean,stddev=stddev),name='critic')
for i in range(num_hidden_layers):
if i < num_hidden_layers-1:
num_input = n_hidden_1
n_hidden_1 = hidd_layer[i+1]
self.weights_critic['h_{0}'.format(i+1)] = tf.Variable(tf.random_normal([num_input, n_hidden_1],mean=mean,stddev=stddev),name='critic')
self.biases_critic['b_{0}'.format(i+1)] = tf.Variable(tf.random_normal([n_hidden_1],mean=mean,stddev=stddev),name='critic')
else:
self.weights_critic['h_{0}'.format("out")] = tf.Variable(tf.random_normal([n_hidden_1, num_output],mean=mean,stddev=stddev),name='critic')
self.biases_critic['b_{0}'.format("out")] = tf.Variable(tf.random_normal([num_output],mean=mean,stddev=stddev),name='critic')

def create_actor_brain(self,hidd_layer,hidd_act_fn,output_act_fn,mean,stddev):
self.is_actor_brain_present =  True
self.weights_init_actor(hidd_layer,mean,stddev)
num_hidden_layers = len(hidd_layer)

if hidd_act_fn == "relu":
for i in range(num_hidden_layers):
if i < num_hidden_layers-1:
else:
if output_act_fn == "linear":
elif output_act_fn == "sopftmax":

self.output_actor = layer_out
self.actor_tvar_num = (num_hidden_layers+1)*2

def create_critic_brain(self,hidd_layer,hidd_act_fn,output_act_fn,mean,stddev):
if self.is_actor_brain_present:
self.weights_init_critic(hidd_layer,mean,stddev)
num_hidden_layers = len(hidd_layer)
if hidd_act_fn == "relu":
for i in range(num_hidden_layers):
if i < num_hidden_layers-1:
else:
if output_act_fn == "linear":
elif output_act_fn == "sopftmax":
self.output_critic = layer_out
self.critic_tvar_num = (num_hidden_layers+1)*2
self.is_critic_brain_present = True
else:

def critic(self):
return self.output_critic

def get_delta(self,sess):
self.delta = self.time_step_info['r'] + (not self.time_step_info['d'])*self.gamma*sess.run(self.critic(),feed_dict={self.state_in:self.time_step_info['s1']}) - sess.run(self.critic(),feed_dict={self.state_in:self.time_step_info['s']})

def normal_dist_prob(self):
cov_inv = 1/float(self.reduction)
y = tf.reduce_sum(tf.square((self.time_step_info['a']-self.output_actor))*tf.ones([1,self.a_size])*cov_inv,1)
Z = (2*np.pi)**(0.5*4)*(self.reduction**self.a_size)**(0.5)
pdf = tf.exp(-0.5*y)/Z
return pdf

def create_actor_loss(self):
self.actor_loss = -tf.log(self.normal_dist_prob())

def create_critic_loss(self):
self.critic_loss = -self.critic()

def sample_action(self,sess,state):
state = np.array([state])
mean= sess.run([self.output_actor],feed_dict={self.state_in:state})
sample = np.random.multivariate_normal(mean[0][0],np.eye(self.a_size)*self.reduction)
return sample

def update_actor_weights(self):
return self.update_actor_batch

def update_critic_weights(self):
return self.update_critic_batch

def update_I(self):
self.I = self.I*self.gamma

def reset_I(self):
self.I = 1

def update_time_step_info(self,s,a,r,s1,d):

self.time_step_info['s'] = s
self.time_step_info['a'] = a
self.time_step_info['r'] = r
self.time_step_info['s1'] = s1
self.time_step_info['d'] = d

def shuffle_memories(self):
np.random.shuffle(self.episode_history)

def create_graph_connections(self):
if self.is_actor_brain_present and self.is_critic_brain_present:
# self.create_pi_dist()
self.normal_dist_prob()
self.create_actor_loss()
self.create_critic_loss()
self.tvars = tf.trainable_variables()
self.update_actor_weights()
self.update_critic_weights()
self.saver = tf.train.Saver()
else:
print("initialize actor and critic brains first")

self.init = tf.global_variables_initializer()

def bound_actions(self,sess,state,lower_limit,uper_limit):
action = self.sample_action(sess,state)
bounded_action = np.copy(action)
for i,act in enumerate(action):

if act < lower_limit[i]:
bounded_action[i] = lower_limit[i]
elif act > uper_limit[i]:
bounded_action[i]= uper_limit[i]
return bounded_action

# Agent instantiation

tf.reset_default_graph()
agent= agent_episodic_continuous_action(1e-3,0.7,0.02,s_size=24,a_size=4,dist_type="normal")
agent.create_actor_brain([12,5],"relu","linear",0.0,0.14)
agent.create_critic_brain([12,5],"relu","linear",0.0,0.14)
agent.create_graph_connections()

path = "/home/diego/Desktop/Study/RL/projects/models/biped/model.ckt"

env = gym.make('BipedalWalker-v2')
uper_action_limit = env.action_space.high
lower_action_limit = env.action_space.low

total_returns=[]

# Training loops

with tf.Session() as sess:
try:
sess.run(agent.init)
for i in range(30):
agent.reset_I()
s = env.reset()
d = False
print(i)
while not d:
a=agent.bound_actions(sess,s,lower_action_limit,uper_action_limit)
s1,r,d,_ = env.step(a)
env.render()
agent.update_time_step_info([s],[a],[r],[s1],d)
agent.get_delta(sess)
sess.run(agent.update_critic_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})
sess.run(agent.update_actor_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})
agent.update_I()
s = s1
total_returns.append(r)
except Exception as e:
print(e)

env.close()
plt.plot(r)
plt.show

# Edit: Update

I managed to locate what are the lines that slow down (and eventually break) the code, these are:

sess.run(agent.update_critic_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})
sess.run(agent.update_actor_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})

Why is this happening? first the speed is good, then it starts getting slower and slower.

# Edit #2:

There is a memory leak in the lines:

sess.run(agent.update_critic_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})
sess.run(agent.update_actor_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})

The TensorFlow graph is getting bigger with each iteration. I still can't find the leak. Why is this happening?

sess.graph.finalize()

I was able to track down the source of the problem.

The code was slow because the Tensorflow graph was getting bigger after each iteration. The cause of this was the 2 lines mentioned on the Edit #2. These two lines are:

sess.run(agent.update_critic_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})
sess.run(agent.update_actor_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})

These two lines execute their corresponding functions:

agent.update_critic_weights()
agent.update_actor_weights()

each of these functions were adding a new element to the graph each time they were called:

and