5
\$\begingroup\$

This algorithm is a convolutional deep neural network used for image recognition.

I used the MNIST data set, which is a bunch of images from 0 to 9. As of right now, I have trained this image with 500 iterations (about 2 hours) using a regular gradient descent algorithm.

My cost/error, which was calculated using cross entropy, is a little over 1.3 for both training and test set. (Cost was surprisingly slightly less for test set than training set.)

Overall this error correctly guesses the number on any given image a little over 50% of the time. (I assume I can easily surpass that with a longer training phase.)

I am simply looking for any pointers from tensor flow/ML experts on how I can improve my efficiency.

import numpy as np
import tensorflow as tf
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
x_trains = mnist.train.images
x_trains = np.reshape(x_trains,[-1,28,28,1])
y_trains = mnist.train.labels

x_test = mnist.test.images
x_test = np.reshape(x_test,[-1,28,28,1])
y_test = mnist.test.labels

"""create an instance of the class with two required arguments, image and labels"""
"""images are the training images to be used. labels are the images respective labels"""

class convolution():
    # ensure input_var is in 4 dimensions
    def __init__(self, images, labels):

    self.images = images
    self.labels = labels

    """these are the placeholders for our two input arguments directly above"""
    self.inputs = tf.placeholder(tf.float32, name = 'inputs')
    self.outputs = tf.placeholder(tf.float32, name = 'outputs')

    self.reuse = None        
    self.conv1()       

"""our first convolution of our images"""    
def conv1(self, padding = 'VALID'):

    with tf.variable_scope('convolutions'):
        self.filters = tf.get_variable("filters", dtype=tf.float32, 
        initializer=tf.random_normal([4,4,1,2], mean = 0.3, stddev=0.5))

        self.data = tf.nn.convolution(self.inputs, self.filters, padding)           
        """makes all negative numbers 0"""
        self.data = tf.nn.relu(self.data) 

        self.pool1([1,3,2,1], [1,3,2,1])    
"""1st pool layer. Value is our output of conv1, kernel_size is size of,
our window to average strides is how many neurons(pixels) our kernel will move"""
def pool1(self, kernal_size, strides, padding = 'SAME'):

    self.data = tf.nn.avg_pool(self.data, kernal_size, strides, padding)

    self.conv2()   

"""second convolution"""    
def conv2(self, padding = 'VALID'):
    self.filters2 = tf.get_variable("filters2", dtype=tf.float32, 
    initializer=tf.random_normal([4,4,2,1], mean = 0.5, stddev=0.5)) 

    self.data = tf.nn.convolution(self.data, self.filters2, padding)
    self.data = tf.nn.relu(self.data)

    self.pool2([1,2,2,1],[1,2,2,1])

"""second pool layer"""
def pool2(self, kernal_size, strides, padding = 'SAME'):

    self.data = tf.nn.avg_pool(self.data, kernal_size, strides, padding)

    self.flatten()      

"""flattens our array of pixels into a vector, this puts our data in standard "features
form", and enables matrix multiplication linear regression function on our features/weights"""    
def flatten(self):
    self.data = tf.contrib.layers.flatten(self.data)
    self.fully_connected()

def fully_connected(self):
    with tf.variable_scope('hidden_layers'):
        """create a matrix of weights where Y axis(first int) is == to the shape of our
        X axis, and X axis(second int) produces the amount of output nodes we want"""

        """we need to create a session here so we can save the 1st index of our data, and
        plug it into our self.weights variable below to ensure propper matrix multiplication"""
        with tf.Session() as session:
            session.run(self.filters.initializer)
            session.run(self.filters2.initializer)
            self.feed_dict = {self.inputs : self.images, self.outputs : self.labels}    
            shapes = session.run(self.data, self.feed_dict)
            shapes = shapes.shape[1]

        """create first set of weights to be matrix mutlipled by features"""

        self.weights = tf.get_variable("weights", shape=[shapes, 14],
        initializer=tf.contrib.layers.xavier_initializer())                             

        """create first bias to be added to matrix multipliation problem"""
        self.bias = tf.get_variable('bias',dtype = tf.float32, 
        initializer = tf.random_normal([1]))

        """first hidden layer is calculated. equation is simple linear MX + B"""
        self.hidden_layer = tf.matmul(self.data, self.weights) + self.bias
        """non linear RELU activation function"""
        self.data = tf.nn.relu(self.hidden_layer)

        self.fully_connected2()
def fully_connected2(self):

    """This does the same exact equation with different weights/updated variable as 
    the previous fully_connected function"""

    self.weights2 = tf.get_variable("weights2", shape=[14, 12],
    initializer=tf.contrib.layers.xavier_initializer())  
    self.bias2 = tf.get_variable('bias2',dtype = tf.float32, 
    initializer = tf.random_normal([1]))
    self.hidden_layer2 = tf.matmul(self.data, self.weights2, name = 'hidden_layer2') + self.bias2

    self.data = tf.nn.relu(self.hidden_layer2)

    self.fully_connected3()

def fully_connected3(self):
    """This does the same exact equation with different weights/updated variable as 
    the previous fully_connected function"""

    self.weights3 = tf.get_variable("weights3", shape=[12, 10],
    initializer=tf.contrib.layers.xavier_initializer())  

    self.bias3 = tf.get_variable('bias3', dtype = tf.float32, 
    initializer = tf.random_normal([1]))    
    self.hidden_layer3 = tf.matmul(self.data, self.weights3) + self.bias3

    self.initialize_and_train()


def initialize_and_train(self):
    """this method is specificaly for testing phase. code below this is for training"""
    self.probabilities = tf.nn.softmax(self.hidden_layer3,name = 'test_probabilities')

    """calulates 10 probabilities based off of our input nodes, than calculates the error
    using cross entropy function. logits are the values to be used as input to softmax"""
    self.error = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
            logits = self.hidden_layer3, labels = self.outputs, name = 'error'))
    """initialize all of our variables with acutal numbers"""
    with tf.Session() as session:
        session.run(self.filters.initializer)
        session.run(self.filters2.initializer)
        session.run(self.weights.initializer)
        session.run(self.weights2.initializer)
        session.run(self.bias.initializer)
        session.run(self.bias2.initializer)
        session.run(self.weights3.initializer)
        session.run(self.bias3.initializer)
        """create gradient descent function"""
        self.train = tf.train.GradientDescentOptimizer(0.1).minimize(self.error)

        """this for loop runs gradient descent and prints error every 20th iteration"""
        for i in range(500):
            session.run(self.train, self.feed_dict)
            iteration = i+1

            if iteration % 20 == 0: 
                print(session.run(self.weights[10]))
                print('#', iteration, 'error is:', session.run(self.error, self.feed_dict))
        """save the final results of our weights/filter variables as outputfile"""
        self.saver = tf.train.Saver() 
        self.saver.save(session, 'outputfile')



 #This function is separate from the above class, and should be 
 #ran after the above class and all of its methods have been ran
 """run test data with this funtion after algo has been trained"""  
def restore(x,y): 
    with tf.Session() as ses:
        """import the graph that we saved after training"""
        saver = tf.train.import_meta_graph('outputfile.meta')
        saver.restore(ses,tf.train.latest_checkpoint('./'))

        """create instance of get_default_graph, which gets gets the specific 
    tensors in a graph"""
    graph = tf.get_default_graph()
    """get placeholders, than feed our test examples into our placeholders"""
    w1 = graph.get_tensor_by_name("inputs:0")
    w2 = graph.get_tensor_by_name('outputs:0')
    feed_dict = {w1 : x, w2 : y}        

    """restore self.error and self.probability function and then runs said functions"""
    restored_probability = graph.get_tensor_by_name('convolutions/hidden_layers/test_probabilities:0')
    restored_error = graph.get_tensor_by_name('convolutions/hidden_layers/Mean:0')

    """only run this to look at small populations of my test set, gives you actual probabilites"""
    val = ses.run(restored_probability,feed_dict)
    print(ses.run(tf.as_string(val, scientific=None)))        

    """gives you the acutal labels of each corresponding image"""
    print(y_)
    """gives the cost function average."""
    print('errors are:', ses.run(restored_error,feed_dict))
\$\endgroup\$
  • \$\begingroup\$ I suggest you take a look this example. Keep it simple and flat. Also, class names should be written in CamelCase. \$\endgroup\$ – Adel Redjimi Jan 3 '18 at 15:47
  • \$\begingroup\$ This hardly suffices as even a comment, but if you're simply trying to implement a practice model, I can't imagine why you'd go the extra mile and build with vanilla TF. Take advantage of Keras (unless you've already done this in Keras and want to dig deeper). Doing so would allow you to get to the meat of the problem (fine-tuning) quicker instead of worrying about mechanicals \$\endgroup\$ – Alex L Jul 25 '18 at 4:50

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.