3
\$\begingroup\$

I'm trying to solve the OpenAI BipedalWalker-v2 by using a one-step actor-critic agent. I'm implementing the solution using python and tensorflow. My question is whether the code is slow because of the nature of the task or because the code is inefficient, or both.

I'm following this pseudo-code taken from the Book Reinforcement Learning An Introduction by Richard S. Sutton and Andrew G. Barto.

enter image description here

My Agent Class

import tensorflow as tf
import numpy as np
import gym
import matplotlib.pyplot as plt

class agent_episodic_continuous_action():
    def __init__(self, lr,gamma,sample_variance, s_size,a_size,dist_type):
        self.gamma = gamma
        self.I = 1
        self.delta = 1
        self.dist_type = dist_type
        self.is_actor_brain_present = False
        self.is_critic_brain_present = False
        self.s_size = s_size
        self.state_in= tf.placeholder(shape=[None,s_size],dtype=tf.float32)
        self.a_size=a_size
        self.cov = tf.eye(a_size)
        self.reduction = sample_variance #0.01
        self.weights_actor ={}
        self.biases_actor ={}
        self.weights_critic ={}
        self.biases_critic ={}   
        self.time_step_info = {'s':0,'a':0,'r':0,'s1':0,'d':0}
        self.reward = tf.placeholder(shape=[None,1],dtype=tf.float32)
        if a_size > 1:
            self.action_holder = tf.placeholder(shape=[None,a_size],dtype=tf.float32)
        else:
            self.action_holder = tf.placeholder(shape=[None],dtype=tf.float32)
        self.gradient_holders = []
        self.optimizer = tf.train.AdamOptimizer(learning_rate=lr)

    def save_model(self,path,sess):
        self.saver.save(sess, path)

    def load_model(self,path,sess):
        self.saver.restore(sess, path)

    def weights_init_actor(self,hidd_layer,mean,stddev):   
        num_input = self.s_size
        num_output = self.a_size
        n_hidden_1 = hidd_layer[0]
        num_hidden_layers = len(hidd_layer)        
        self.weights_actor['h_{0}'.format(0)] = tf.Variable(tf.random_normal([num_input, n_hidden_1],mean=mean,stddev=stddev),name='actor')
        self.biases_actor['b_{0}'.format(0)] = tf.Variable(tf.random_normal([n_hidden_1],mean=mean,stddev=stddev),name='actor') 
        for i in range(num_hidden_layers):
            if i < num_hidden_layers-1:
                num_input = n_hidden_1
                n_hidden_1 = hidd_layer[i+1]
                self.weights_actor['h_{0}'.format(i+1)] = tf.Variable(tf.random_normal([num_input, n_hidden_1],mean=mean,stddev=stddev),name='actor')
                self.biases_actor['b_{0}'.format(i+1)] = tf.Variable(tf.random_normal([n_hidden_1],mean=mean,stddev=stddev),name='actor')
            else:
                self.weights_actor['h_{0}'.format("out")] = tf.Variable(tf.random_normal([n_hidden_1, num_output],mean=mean,stddev=stddev),name='actor')  
                self.biases_actor['b_{0}'.format("out")] = tf.Variable(tf.random_normal([num_output],mean=mean,stddev=stddev),name='actor')    

    def weights_init_critic(self,hidd_layer,mean,stddev):   
        num_input = self.s_size
       # num_output = self.a_size
        num_output = 1
        n_hidden_1 = hidd_layer[0]
        num_hidden_layers = len(hidd_layer)        
        self.weights_critic['h_{0}'.format(0)] = tf.Variable(tf.random_normal([num_input, n_hidden_1],mean=mean,stddev=stddev),name='critic')
        self.biases_critic['b_{0}'.format(0)] = tf.Variable(tf.random_normal([n_hidden_1],mean=mean,stddev=stddev),name='critic') 
        for i in range(num_hidden_layers):
            if i < num_hidden_layers-1:
                num_input = n_hidden_1
                n_hidden_1 = hidd_layer[i+1]
                self.weights_critic['h_{0}'.format(i+1)] = tf.Variable(tf.random_normal([num_input, n_hidden_1],mean=mean,stddev=stddev),name='critic')
                self.biases_critic['b_{0}'.format(i+1)] = tf.Variable(tf.random_normal([n_hidden_1],mean=mean,stddev=stddev),name='critic')
            else:
                self.weights_critic['h_{0}'.format("out")] = tf.Variable(tf.random_normal([n_hidden_1, num_output],mean=mean,stddev=stddev),name='critic')  
                self.biases_critic['b_{0}'.format("out")] = tf.Variable(tf.random_normal([num_output],mean=mean,stddev=stddev),name='critic')    

    def create_actor_brain(self,hidd_layer,hidd_act_fn,output_act_fn,mean,stddev):        
        self.is_actor_brain_present =  True
        self.weights_init_actor(hidd_layer,mean,stddev)        
        num_hidden_layers = len(hidd_layer)   

        if hidd_act_fn == "relu":
            layer_h = tf.nn.relu(tf.add(tf.matmul(self.state_in, self.weights_actor['h_0']), self.biases_actor['b_0']))
            for i in range(num_hidden_layers):
                if i < num_hidden_layers-1:          
                    layer_h = tf.nn.relu(tf.add(tf.matmul(layer_h, self.weights_actor['h_{0}'.format(i+1)]), self.biases_actor['b_{0}'.format(i+1)]))
                else:
                    if output_act_fn == "linear":
                        layer_out = tf.add(tf.matmul(layer_h, self.weights_actor['h_{0}'.format("out")]), self.biases_actor['b_{0}'.format("out")])
                    elif output_act_fn == "sopftmax":
                        layer_out = tf.nn.softmax(tf.add(tf.matmul(layer_h, self.weights_actor['h_{0}'.format("out")]), self.biases_actor['b_{0}'.format("out")]))

            self.output_actor = layer_out
            self.actor_tvar_num = (num_hidden_layers+1)*2

    def create_critic_brain(self,hidd_layer,hidd_act_fn,output_act_fn,mean,stddev):
        if self.is_actor_brain_present:            
            self.weights_init_critic(hidd_layer,mean,stddev)        
            num_hidden_layers = len(hidd_layer)          
            if hidd_act_fn == "relu":      
                layer_h = tf.nn.relu(tf.add(tf.matmul(self.state_in, self.weights_critic['h_0']), self.biases_critic['b_0']))
                for i in range(num_hidden_layers):
                    if i < num_hidden_layers-1:          
                        layer_h = tf.nn.relu(tf.add(tf.matmul(layer_h, self.weights_critic['h_{0}'.format(i+1)]), self.biases_critic['b_{0}'.format(i+1)]))
                    else:
                        if output_act_fn == "linear":
                            layer_out = tf.add(tf.matmul(layer_h, self.weights_critic['h_{0}'.format("out")]), self.biases_critic['b_{0}'.format("out")])
                        elif output_act_fn == "sopftmax":
                            layer_out = tf.nn.softmax(tf.add(tf.matmul(layer_h, self.weights_critic['h_{0}'.format("out")]), self.biases_critic['b_{0}'.format("out")]))                      
                self.output_critic = layer_out
                self.critic_tvar_num = (num_hidden_layers+1)*2   
                self.is_critic_brain_present = True                
        else:
            print("please create actor brain first")

    def critic(self):        
        return self.output_critic

    def get_delta(self,sess):        
        self.delta = self.time_step_info['r'] + (not self.time_step_info['d'])*self.gamma*sess.run(self.critic(),feed_dict={self.state_in:self.time_step_info['s1']}) - sess.run(self.critic(),feed_dict={self.state_in:self.time_step_info['s']})

    def normal_dist_prob(self):
        cov_inv = 1/float(self.reduction)
        y = tf.reduce_sum(tf.square((self.time_step_info['a']-self.output_actor))*tf.ones([1,self.a_size])*cov_inv,1)
        Z = (2*np.pi)**(0.5*4)*(self.reduction**self.a_size)**(0.5)
        pdf = tf.exp(-0.5*y)/Z
        return pdf            

    def create_actor_loss(self):
        self.actor_loss = -tf.log(self.normal_dist_prob())

    def create_critic_loss(self):
        self.critic_loss = -self.critic()       

    def sample_action(self,sess,state):
        state = np.array([state])
        mean= sess.run([self.output_actor],feed_dict={self.state_in:state})
        sample = np.random.multivariate_normal(mean[0][0],np.eye(self.a_size)*self.reduction)
        return sample

    def calculate_actor_loss_gradient(self):
        self.actor_gradients = tf.gradients(self.actor_loss,self.tvars[:self.actor_tvar_num])
        self.actor_gradients = self.I*self.delta*self.actor_gradients

    def calculate_critic_loss_gradient(self):
        self.critic_gradients = tf.gradients(self.critic_loss,self.tvars[self.actor_tvar_num:])
        self.critic_gradients = self.delta*self.critic_gradients


    def update_actor_weights(self):
        self.update_actor_batch = self.optimizer.apply_gradients(zip(self.actor_gradients,self.tvars[:self.actor_tvar_num]))   
        return self.update_actor_batch

    def update_critic_weights(self):
        self.update_critic_batch = self.optimizer.apply_gradients(zip(self.critic_gradients,self.tvars[self.actor_tvar_num:]))   
        return self.update_critic_batch

    def update_I(self):
        self.I = self.I*self.gamma

    def reset_I(self):
        self.I = 1

    def update_time_step_info(self,s,a,r,s1,d):

        self.time_step_info['s'] = s
        self.time_step_info['a'] = a        
        self.time_step_info['r'] = r       
        self.time_step_info['s1'] = s1
        self.time_step_info['d'] = d        

    def shuffle_memories(self):
        np.random.shuffle(self.episode_history)

    def create_graph_connections(self):
        if self.is_actor_brain_present and self.is_critic_brain_present:
           # self.create_pi_dist()
            self.normal_dist_prob()
            self.create_actor_loss()
            self.create_critic_loss()
            self.tvars = tf.trainable_variables()
            self.calculate_actor_loss_gradient()
            self.calculate_critic_loss_gradient()
            self.update_actor_weights()
            self.update_critic_weights()
            self.saver = tf.train.Saver()
        else:
            print("initialize actor and critic brains first")

        self.init = tf.global_variables_initializer()

    def bound_actions(self,sess,state,lower_limit,uper_limit):
        action = self.sample_action(sess,state)
        bounded_action = np.copy(action)
        for i,act in enumerate(action):

            if act < lower_limit[i]:
                bounded_action[i] = lower_limit[i]
            elif act > uper_limit[i]:
                bounded_action[i]= uper_limit[i]
        return bounded_action       

Agent instantiation

tf.reset_default_graph()
agent= agent_episodic_continuous_action(1e-3,0.7,0.02,s_size=24,a_size=4,dist_type="normal")
agent.create_actor_brain([12,5],"relu","linear",0.0,0.14)
agent.create_critic_brain([12,5],"relu","linear",0.0,0.14)
agent.create_graph_connections()

path = "/home/diego/Desktop/Study/RL/projects/models/biped/model.ckt"

env = gym.make('BipedalWalker-v2')
uper_action_limit = env.action_space.high
lower_action_limit = env.action_space.low

total_returns=[]

Training loops

with tf.Session() as sess:
    try:
        sess.run(agent.init)
        #agent_2.load_model(path,sess)        
        for i in range(30): 
            agent.reset_I()
            s = env.reset()    
            d = False
            print(i)
            while not d:
                a=agent.bound_actions(sess,s,lower_action_limit,uper_action_limit)  
                s1,r,d,_ = env.step(a)
                env.render()
                agent.update_time_step_info([s],[a],[r],[s1],d)                 
                agent.get_delta(sess)
                sess.run(agent.update_critic_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})
                sess.run(agent.update_actor_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})
                agent.update_I()  
                s = s1
                total_returns.append(r)
    except Exception as e:
        print(e)

env.close()        
plt.plot(r)
plt.show   

Edit: Update

I managed to locate what are the lines that slow down (and eventually break) the code, these are:

      sess.run(agent.update_critic_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})
        sess.run(agent.update_actor_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})

Why is this happening? first the speed is good, then it starts getting slower and slower.

Edit #2:

There is a memory leak in the lines:

 sess.run(agent.update_critic_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})
 sess.run(agent.update_actor_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})

The TensorFlow graph is getting bigger with each iteration. I still can't find the leak. Why is this happening?

\$\endgroup\$
1
\$\begingroup\$

By adding the line:

sess.graph.finalize()

I was able to track down the source of the problem.

The code was slow because the Tensorflow graph was getting bigger after each iteration. The cause of this was the 2 lines mentioned on the Edit #2. These two lines are:

sess.run(agent.update_critic_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})
sess.run(agent.update_actor_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})

These two lines execute their corresponding functions:

agent.update_critic_weights()
agent.update_actor_weights()

each of these functions were adding a new element to the graph each time they were called:

self.update_actor_batch = self.optimizer.apply_gradients(zip(self.actor_gradients,self.tvars[:self.actor_tvar_num]))

and

self.update_critic_batch = self.optimizer.apply_gradients(zip(self.critic_gradients,self.tvars[self.actor_tvar_num:]))

thus, if we want to fix the problem, instead of passing the function to the sess.run we can pass the element. The final solution is shown below:

change these two lines:

sess.run(agent.update_critic_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})
sess.run(agent.update_actor_weights(),feed_dict={agent.state_in:agent.time_step_info['s']})

for this line:

sess.run([agent.update_critic_batch,agent.update_actor_batch],feed_dict={agent.state_in:agent.time_step_info['s']})
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.