A simple fully connected ANN module

Question

I've written a simple module that creates a fully connected neural network of any size. The arguments of the train function are list of tuples with a training example array first and an array containing its class second, list that contains a number of neurons in every layer including the input and the output layers, a learning rate and a number of epochs. To run a trained network I've written the run function, its arguments are an input and weights from the trained network. Since I'm a beginner to programming and machine learning I'll be very happy getting advice regarding computational efficiency and optimization.

import numpy as np
def weights_init(inSize,outSize): #initialize the weights
    return 2*np.random.random((inSize,outSize))-1

def Sigmoid(input, weights): #create a sigmoid layer and return a layer along with its derivative
    out = 1/(1+np.exp(-np.dot(input,weights)))
    derivative = out*(1-out)
    return out,derivative

def backProp(layers, weights, deriv, size, rate = 1): 
    derivative = deriv.pop()#get the cost function derivative
    #reverse all the lists because we need to go backwards
    deriv = deriv[::-1] 
    layers = layers[::-1]
    weights = weights[::-1]
    new_weights=[]
    #backpopagate
    new_weights.append(weights[0]+(layers[1].T.dot(derivative*rate))) #this one does not fit well the algorithm inside for loop, so it's outside of it
    for i in range(len(size)-2):
        derivative = derivative.dot(weights[i].T)*deriv[i]
        new_weights.append(weights[i+1]+(layers[i+2].T.dot(derivative*rate)))
    return new_weights[::-1]

def train(input,size,rate=1,epochs=1): #train the network
    layers=[]
    weights=[]
    derivs=[] 
    for i in xrange(len(size)-1): #weights initialization
        weights.append(weights_init(size[i],size[i+1]))
    for i in xrange(epochs): #the training process
        for example, target in input:  #online learning
            layers.append(example) 
            for i in xrange(len(size)-1):
                layer, derivative = Sigmoid(layers[i],weights[i])#calculate the layer and itd derivative
                layers.append(layer)
                derivs.append(derivative)

            loss_deriv = target-layer[-1] #loss function

            derivs[-1] = loss_deriv*derivs[-1] #multiply the loss function by the final layer's derivative
            weights = backProp(layers,weights,derivs,size) #update the weights
            layers=[]
            derivs = []
    return weights   

def run(input,weights): #run a trained neural network
    layers=[input]
    for i in xrange(len(weights)):
        layer,derivative = Sigmoid(layers[i],weights[i])
        layers.append(layer)
    return layers

An example:

X = [(np.array([[0,0,1]]),np.array([[0]])),(    np.array([[0,1,1]]),np.array([[1]])), (np.array([[1,0,1]]),np.array([[1]])), (np.array([[1,1,1]]),np.array([[0]]))]

weights = train(X,[3,4,1],epochs=60000)
run(X[0][0],weights)

Answer 1

Without commenting on the algorithm for a lack of knowledge regarding neural networks, this answer provides some style suggestions. Take the first method:

def weights_init(inSize,outSize): #initialize the weights
    return 2*np.random.random((inSize,outSize))-1

In python it is against the PEP style guide to use camelCase for arguments. That comment should be dropped down a line. Place some spaces in between your arguments and operators so it can breath better:

def weights_init(in_size, out_size): 
    #initialize the weights
    return 2 * np.random.random((in_size, out_size)) - 1

Function names shouldn't be Uppercased.

def sigmoid(input, weights): 
    #create a sigmoid layer and return a layer along with its derivative
    out = 1 / (1 + np.exp(-np.dot(input, weights)))
    derivative = out * (1 - out)
    return out, derivative

Default arguments in the param signature shouldn't have a space between the equals sign.

Also it would be nice to add double line breaks for ease or reading.

def backProp(layers, weights, deriv, size, rate=1): 
    #get the cost function derivative
    derivative = deriv.pop()

    #reverse all the lists because we need to go backwards
    deriv = deriv[::-1] 
    layers = layers[::-1]
    weights = weights[::-1]
    new_weights=[]

    #backpopagate
    new_weights.append(weights[0] + (layers[1].T.dot(derivative * rate))) 

    #this one does not fit well the algorithm inside for loop, so it's outside of it
    for i in range(len(size) - 2):
        derivative = derivative.dot(weights[i].T) * deriv[i]
        new_weights.append(weights[i + 1] + (layers[i + 2].T.dot(derivative * rate)))
    return new_weights[::-1]

def train(input, size, rate=1, epochs=1): 
    #train the network
    layers=[]
    weights=[]
    derivs=[] 

    #weights initialization
    for i in xrange(len(size) - 1): 
        weights.append(weights_init(size[i], size[i + 1]))

    #the training process
    for i in xrange(epochs): 

        #online learning
        for example, target in input:  
            layers.append(example) 

            for i in xrange(len(size) - 1):
                #calculate the layer and itd derivative
                layer, derivative = sigmoid(layers[i], weights[i])

                layers.append(layer)
                derivs.append(derivative)

            #loss function
            loss_deriv = target-layer[-1] 

            #multiply the loss function by the final layer's derivative
            derivs[-1] = loss_deriv * derivs[-1]  

            #update the weights
            weights = backProp(layers, weights, derivs, size) 
            layers=[]
            derivs = []

    return weights